From 2e2e8153e2187b3db5013ac38a292f611cc05107 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Tue, 29 Sep 2026 11:43:26 -0500 Subject: [PATCH] feat(ugc): denoise=oidn traces icons' occlusion per pixel and denoises it (optional build) Intel Open Image Denoise 2 (Apache-2.0) behind the CMake option DLU_OIDN (off): an installed OIDN is used when found, else Intel's release package (pinned by hash) is downloaded and its libraries copied next to the servers. A denoiser only removes noise that differs from pixel to pixel; the icons' occlusion comes from the bake, per vertex, which it leaves as it is (checked: white noise 0.17 -> 0.006 relative spread, per-vertex blocks unchanged). So with denoise=oidn a model's icon is drawn from model.noao.nif (its colors before the bake) with its occlusion traced per pixel of the supersampled image (denoise_samples rays, default 4, with the bake's distance and strength and the ray backend), box filtered and denoised at the icon's size, guided by the colors and normals. The model keeps its baked occlusion. Icons drawn again from stored files use the stored model.noao.nif the same way. OIDN works on a thread of its own; its time is charged to the job's thread (UgcThrottle::Charge), so the CPU budget and the recorded CPU time include it. Check: configure with -DDLU_OIDN=ON; UgcServer --make-model x.lxfml out oidn and compare its icon.png with one made with off; an OFF build leaves icons as they were. Co-Authored-By: Claude Opus 5.5 --- dDashboardServer/routes/SettingsCatalog.cpp | 3 +- dUgcServer/CMakeLists.txt | 5 + dUgcServer/Processing/UgcJobs.cpp | 21 +- dUgcServer/Processing/UgcJobs.h | 5 +- dUgcServer/Processing/UgcProcessor.cpp | 12 +- dUgcServer/Processing/UgcThrottle.cpp | 12 ++ dUgcServer/Processing/UgcThrottle.h | 6 + dUgcServer/Render/UgcRender.cpp | 224 ++++++++++++++++++-- dUgcServer/Render/UgcRender.h | 10 +- dUgcServer/UgcServer.cpp | 6 +- resources/ugcconfig.ini | 5 +- tests/dUgcTests/UgcTests.cpp | 45 ++++ thirdparty/CMakeLists.txt | 29 +++ 13 files changed, 346 insertions(+), 37 deletions(-) diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index 70cab6234..0ff28b4d0 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -504,7 +504,8 @@ namespace { c.Add(Labels(Choice(UGC, "hsr_method", "How hidden faces are found", "toolbox: LU Toolbox's paths (above). fast: the model rendered from 42 directions around it, removing what shows in none; much faster, but it also removes faces seen only by bounced light (insides seen through openings, recesses). Staff can pick another for one make when making models again.", "toolbox", { "toolbox", "fast" }), { "LU Toolbox (paths)", "Fast (renders from around)" })); c.Add(Unit(Int(UGC, "hsr_fast_resolution", "Fast method's render size", "The size of each of the fast method's 42 renders; bigger keeps smaller visible faces.", "1024", 64, 4096), "pixels")); c.Add(Labels(Choice(UGC, "ray_backend", "Ray tracer", "What traces the rays of the hidden faces' paths and of the occlusion: builtin (the UGC server's own), embree (Intel Embree on the CPU) or hiprt (the GPU, when the server was built with it and has one; else embree). They give the same results but for rounding. Staff can pick another for one make when making models again.", "builtin", { "builtin", "embree", "hiprt" }), { "Built in", "Embree (CPU)", "HIPRT (GPU)" })); - c.Add(Labels(Choice(UGC, "denoise", "Denoise icons", "oidn: icons are cleaned up with Intel Open Image Denoise (when the server was built with it; else off).", "off", { "off", "oidn" }), { "Off", "Open Image Denoise" })); + c.Add(Labels(Choice(UGC, "denoise", "Denoise icons", "oidn (when the server was built with Intel Open Image Denoise; else off): a model's icon is drawn from its colors before the occlusion bake, with its occlusion traced per pixel with a few rays and the noise removed by the denoiser, instead of the baked occlusion. The model itself keeps its baked occlusion (a denoiser only works on images).", "off", { "off", "oidn" }), { "Off", "Open Image Denoise" })); + c.Add(Int(UGC, "denoise_samples", "Denoised occlusion rays per pixel", "Occlusion rays traced from each pixel of a denoised icon (before it is scaled down, so 16 times as many per icon pixel); fewer is faster and noisier.", "4", 1, 256)); c.Add(Bool(UGC, "bake_ao", "Darken hidden corners", "Ambient occlusion baked into the vertex colors.", true)); c.Add(Int(UGC, "ao_samples", "Occlusion rays per vertex", "", "64", 1, 1024)); c.Add(Float(UGC, "ao_distance", "Occlusion distance", "", "5", 0, 1000)); diff --git a/dUgcServer/CMakeLists.txt b/dUgcServer/CMakeLists.txt index 7bfdf1c93..d601ce58c 100644 --- a/dUgcServer/CMakeLists.txt +++ b/dUgcServer/CMakeLists.txt @@ -28,6 +28,11 @@ add_library(dUgc STATIC ${DUGC_SOURCES}) target_include_directories(dUgc PUBLIC "." "Bricks" "Formats" "Model" "Processing" "Render" "${PROJECT_SOURCE_DIR}/thirdparty/MD5" "${PROJECT_SOURCE_DIR}/thirdparty/nlohmann") target_link_libraries(dUgc PUBLIC dCommon glm::glm tinyxml2 MD5 PRIVATE embree) +# Intel Open Image Denoise, when built with DLU_OIDN (thirdparty/CMakeLists.txt): the denoise=oidn setting +if(DLU_OIDN) + target_link_libraries(dUgc PRIVATE OpenImageDenoise) + target_compile_definitions(dUgc PRIVATE DLU_OIDN) +endif() add_executable(UgcServer "UgcServer.cpp" "Processing/UgcProcessor.cpp" "Bricks/UgcCdClient.cpp") diff --git a/dUgcServer/Processing/UgcJobs.cpp b/dUgcServer/Processing/UgcJobs.cpp index 7309a4f7b..6bd23c594 100644 --- a/dUgcServer/Processing/UgcJobs.cpp +++ b/dUgcServer/Processing/UgcJobs.cpp @@ -36,8 +36,9 @@ namespace UgcJobs { namespace { // The icon files of a model, and false when nothing was drawn - bool AddIcon(UgcStorage::Files& files, const UgcModel::Model& model, const UgcRender::IconOptions& options, const std::vector* ao = nullptr) { - const auto icon = UgcRender::RenderIcon(model, options, ao); + bool AddIcon(UgcStorage::Files& files, const UgcModel::Model& model, const UgcRender::IconOptions& options, const std::vector* ao = nullptr, + const UgcModel::Model* plain = nullptr) { + const auto icon = UgcRender::RenderIcon(model, options, ao, plain); bool drawn = false; for (size_t i = 3; i < icon.rgba.size(); i += 4) drawn = drawn || icon.rgba[i] != 0; files["icon.png"] = UgcFormats::EncodePng(icon); @@ -150,10 +151,16 @@ namespace UgcJobs { } bool IconFromNif(const std::string& nif, const UgcRender::IconOptions& options, UgcStorage::Files& files, std::string& error, - const std::map& tagLooks, const std::set& overlayTags) { + const std::map& tagLooks, const std::set& overlayTags, const std::string* plainNif) { const auto readBack = NifFile::Parse(nif, 0, error); if (!readBack) return false; - AddIcon(files, UgcModel::FromNif(*readBack, tagLooks, overlayTags), options); + // The denoiser's guide: the model before its occlusion was baked in (model.noao.nif), when there is one + std::optional plain; + if (options.denoise != UgcRender::eDenoise::OFF && plainNif) { + std::string plainError; + if (const auto plainRead = NifFile::Parse(*plainNif, 0, plainError)) plain = UgcModel::FromNif(*plainRead, tagLooks, overlayTags); + } + AddIcon(files, UgcModel::FromNif(*readBack, tagLooks, overlayTags), options, nullptr, plain ? &*plain : nullptr); return true; } @@ -342,10 +349,12 @@ namespace UgcJobs { // Stored compressed only (the client downloads .gz; the dashboard's copies are inflated when asked for). The // LXFML is served from the database. AddDownload(outcome.files, "model.nif", nif); + std::string plainNif; { const std::vector opaque{ DivideByLook(preview.opaque, separate) }; const std::vector transparent{ transparentPieces[0] }; - outcome.files["model.noao.nif.gz"] = ZCompression::Gzip(UgcFormats::WriteLodNif("SceneNode_Model", groups(1, opaque, transparent))); + plainNif = UgcFormats::WriteLodNif("SceneNode_Model", groups(1, opaque, transparent)); + outcome.files["model.noao.nif.gz"] = ZCompression::Gzip(plainNif); } // The icon is drawn from the .nif just made (its most detailed LOD, read back like any client .nif), so it @@ -354,7 +363,7 @@ namespace UgcJobs { auto iconOptions = settings.icon; UgcIconParams::Apply(iconOptions, iconValues); std::string nifError; - if (!IconFromNif(nif, iconOptions, outcome.files, nifError, settings.shaders.TagLooks(), settings.shaders.OverlayTags())) { + if (!IconFromNif(nif, iconOptions, outcome.files, nifError, settings.shaders.TagLooks(), settings.shaders.OverlayTags(), &plainNif)) { outcome.error = "the .nif made can't be read back for the icon: " + nifError; return outcome; } diff --git a/dUgcServer/Processing/UgcJobs.h b/dUgcServer/Processing/UgcJobs.h index 5089d1ea4..116c0dc0e 100644 --- a/dUgcServer/Processing/UgcJobs.h +++ b/dUgcServer/Processing/UgcJobs.h @@ -101,9 +101,10 @@ namespace UgcJobs { // A model's icon files (icon.png, icon.dds download) drawn from its .nif (LOD 0), its metal and glow groups by // `tagLooks` (Shaders::TagLooks); false (and `error`) when the .nif can't be read - // (the groups drawn over others, `overlayTags` (Shaders::OverlayTags), left out) + // (the groups drawn over others, `overlayTags` (Shaders::OverlayTags), left out). `plainNif`: its model.noao.nif, the + // denoiser's guide when options.denoise is on. bool IconFromNif(const std::string& nif, const UgcRender::IconOptions& options, UgcStorage::Files& files, std::string& error, - const std::map& tagLooks = {}, const std::set& overlayTags = {}); + const std::map& tagLooks = {}, const std::set& overlayTags = {}, const std::string* plainNif = nullptr); /** * A model's stats.json after its icon was drawn again in `iconMs`: ms.icon becomes that and ms.total changes by the diff --git a/dUgcServer/Processing/UgcProcessor.cpp b/dUgcServer/Processing/UgcProcessor.cpp index 78f8ab426..4038b180f 100644 --- a/dUgcServer/Processing/UgcProcessor.cpp +++ b/dUgcServer/Processing/UgcProcessor.cpp @@ -242,7 +242,9 @@ void UgcProcessor::Worker() { const auto nif = m_Storage.ReadNif(Kind::MODEL, job.id, "model.nif"); auto options = settings.icon; UgcIconParams::Apply(options, job.iconValues); - outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, outcome.files, outcome.error, settings.shaders.TagLooks(), settings.shaders.OverlayTags()); + const auto plain = options.denoise != UgcRender::eDenoise::OFF ? m_Storage.ReadNif(Kind::MODEL, job.id, "model.noao.nif") : std::nullopt; + outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, outcome.files, outcome.error, settings.shaders.TagLooks(), settings.shaders.OverlayTags(), + plain ? &*plain : nullptr); if (!nif) outcome.error = "the model has no stored .nif yet"; } if (outcome.ok && outcome.files.contains("assembly.nif")) { @@ -279,7 +281,7 @@ void UgcProcessor::Worker() { continue; } const auto start = std::chrono::steady_clock::now(); - const double cpuStart = UgcThrottle::ThreadCpuSeconds(); + const double cpuStart = UgcThrottle::JobCpuSeconds(); Done done{ job.kind, job.id, job.attempts }; done.memoryEstimate = job.memory; done.iconOnly = job.iconOnly; @@ -290,7 +292,9 @@ void UgcProcessor::Worker() { auto options = settings.icon; UgcIconParams::Apply(options, job.iconValues); const auto iconStart = std::chrono::steady_clock::now(); - done.outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, done.outcome.files, done.outcome.error, settings.shaders.TagLooks(), settings.shaders.OverlayTags()); + const auto plain = options.denoise != UgcRender::eDenoise::OFF ? m_Storage.ReadNif(Kind::MODEL, job.id, "model.noao.nif") : std::nullopt; + done.outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, done.outcome.files, done.outcome.error, settings.shaders.TagLooks(), settings.shaders.OverlayTags(), + plain ? &*plain : nullptr); if (!nif) done.outcome.error = "no stored .nif"; // The make's time keeps its icon's: the stats get the new icon's time, the row the difference (Collect) const auto stats = done.outcome.ok ? m_Storage.ReadNif(Kind::MODEL, job.id, "stats.json") : std::nullopt; @@ -336,7 +340,7 @@ void UgcProcessor::Worker() { } done.outcome.files.clear(); done.milliseconds = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); - done.cpuMilliseconds = std::max(0.0, UgcThrottle::ThreadCpuSeconds() - cpuStart) * 1000.0; + done.cpuMilliseconds = std::max(0.0, UgcThrottle::JobCpuSeconds() - cpuStart) * 1000.0; try { UgcThrottle::Checkpoint(); } catch (const UgcThrottle::Cancelled&) { diff --git a/dUgcServer/Processing/UgcThrottle.cpp b/dUgcServer/Processing/UgcThrottle.cpp index 4bd104860..c35bdbe4b 100644 --- a/dUgcServer/Processing/UgcThrottle.cpp +++ b/dUgcServer/Processing/UgcThrottle.cpp @@ -22,6 +22,7 @@ namespace { std::atomic g_LastSleep{ 0 }; thread_local double t_LastCpu = -1.0; + thread_local double t_Charged = 0.0; // CPU seconds libraries used for this thread on threads of their own int64_t UnixMs() { return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); @@ -55,6 +56,17 @@ namespace UgcThrottle { return std::chrono::duration(std::chrono::steady_clock::now().time_since_epoch()).count(); } + void Charge(double seconds) { + if (!(seconds > 0.0)) return; + t_Charged += seconds; + // The next Checkpoint sees it as used since the last + if (t_LastCpu >= 0.0) t_LastCpu -= seconds; + } + + double JobCpuSeconds() { + return ThreadCpuSeconds() + t_Charged; + } + void Begin() { t_LastCpu = ThreadCpuSeconds(); } diff --git a/dUgcServer/Processing/UgcThrottle.h b/dUgcServer/Processing/UgcThrottle.h index 4a658bc72..0581a67fb 100644 --- a/dUgcServer/Processing/UgcThrottle.h +++ b/dUgcServer/Processing/UgcThrottle.h @@ -36,6 +36,12 @@ namespace UgcThrottle { // This thread's CPU time in seconds double ThreadCpuSeconds(); + // CPU time a library spent for this thread on threads of its own (Open Image Denoise's), counted as this thread's: + // by the budget at the next Checkpoint and in JobCpuSeconds + void Charge(double seconds); + // This thread's CPU time with what was charged to it, in seconds (a job's CPU time is the difference) + double JobCpuSeconds(); + // Parses "18-23" (from 18:00 to 23:59; may wrap past midnight, "22-6") into from and to; false when it isn't that bool ParseHours(const std::string& text, int& from, int& to); // Whether `hour` (0-23) is inside the range; never when from or to is negative diff --git a/dUgcServer/Render/UgcRender.cpp b/dUgcServer/Render/UgcRender.cpp index d80cb88fc..1d7de74ca 100644 --- a/dUgcServer/Render/UgcRender.cpp +++ b/dUgcServer/Render/UgcRender.cpp @@ -1,13 +1,20 @@ #include "UgcRender.h" #include +#include #include #include #include +#include +#include #include #include +#ifdef DLU_OIDN +#include +#endif + #include "UgcIconPose.h" #include "UgcPalette.h" #include "UgcRays.h" @@ -81,6 +88,64 @@ namespace { float PixelSize() const { return 2.0f * radius / resolution; } }; +#ifdef DLU_OIDN + // Open Image Denoise's CPU device for the thread, made the first time: one thread of its own (not all the cores), + // whose time Denoise charges to the asking thread (UgcThrottle::Charge); null when it can't be made + oidn::DeviceRef* OidnDevice() { + thread_local std::optional device; + thread_local bool tried = false; + if (!tried) { + tried = true; + auto made = oidn::newDevice(oidn::DeviceType::CPU); + if (made) { + made.set("numThreads", 1); + made.set("setAffinity", false); + made.commit(); + const char* message = nullptr; + if (made.getError(message) == oidn::Error::None) device = std::move(made); + } + } + return device ? &*device : nullptr; + } +#endif + + // Denoises `color` (linear, premultiplied, n x n) with Open Image Denoise's ray tracing filter, guided by `albedo` + // and `normals` (noise free); false (and `color` as it was) when it can't + bool Denoise(std::vector& color, const std::vector& albedo, const std::vector& normals, int n) { +#ifdef DLU_OIDN + auto* device = OidnDevice(); + if (!device || n <= 0) return false; + const size_t pixels = static_cast(n) * n; + std::vector input(pixels), output(pixels); + for (size_t i = 0; i < pixels; i++) input[i] = glm::vec3(color[i]); + auto filter = device->newFilter("RT"); + filter.setImage("color", input.data(), oidn::Format::Float3, n, n); + filter.setImage("albedo", const_cast(albedo.data()), oidn::Format::Float3, n, n); + filter.setImage("normal", const_cast(normals.data()), oidn::Format::Float3, n, n); + filter.setImage("output", output.data(), oidn::Format::Float3, n, n); + filter.set("hdr", true); + filter.set("cleanAux", true); + // It works on a thread of its own while this one waits: its time is this job's CPU time + const auto started = std::chrono::steady_clock::now(); + filter.commit(); + filter.execute(); + UgcThrottle::Charge(std::chrono::duration(std::chrono::steady_clock::now() - started).count()); + const char* message = nullptr; + if (device->getError(message) != oidn::Error::None) return false; + // The shape's outline and coverage stay as drawn + for (size_t i = 0; i < pixels; i++) { + if (color[i].a > 0.0f) color[i] = glm::vec4(glm::max(output[i], glm::vec3(0.0f)), color[i].a); + } + return true; +#else + (void)color; + (void)albedo; + (void)normals; + (void)n; + return false; +#endif + } + void Bounds(const UgcModel::Model& model, glm::vec3& center, float& radius) { glm::vec3 min{}, max{}; if (!model.Bounds(min, max)) { @@ -106,6 +171,9 @@ namespace UgcRender { } bool Available(eDenoise denoise) { +#ifdef DLU_OIDN + if (denoise == eDenoise::OIDN) return OidnDevice() != nullptr; +#endif return denoise == eDenoise::OFF; } @@ -267,14 +335,17 @@ namespace UgcRender { return ao; } - Image RenderIcon(const UgcModel::Model& source, const IconOptions& options, const std::vector* opaqueAo) { + Image RenderIcon(const UgcModel::Model& source, const IconOptions& options, const std::vector* opaqueAo, const UgcModel::Model* plain) { const int size = std::clamp(options.size, 8, 1024); const int supersample = std::clamp(options.supersample, 1, 8); const int n = size * supersample; Image image{ size, size, std::vector(static_cast(size) * size * 4, 0) }; if (source.Empty()) return image; - UgcModel::Model model = source; + // Denoised with the model before its bake: its occlusion is traced per pixel (noisy), then denoised + const bool denoise = options.denoise != eDenoise::OFF && Available(options.denoise); + const bool traced = denoise && plain && !plain->opaque.Empty() && options.denoiseSamples > 0 && options.bakedAo > 0.0f && options.ao.distance > 0.0f; + UgcModel::Model model = traced ? *plain : source; const auto rotation = UgcIconPose::ModelRotation(options.modelYawDegrees, options.modelPitchDegrees, options.modelRollDegrees) * options.modelRotation; model.opaque.Transform(rotation); model.transparent.Transform(rotation); @@ -300,7 +371,7 @@ namespace UgcRender { // Ambient occlusion darkens the world light (opaque bricks only, as they are what occludes) std::vector ao; - if (options.ao.enabled) { + if (options.ao.enabled && !traced) { ao = opaqueAo && opaqueAo->size() == model.opaque.positions.size() ? *opaqueAo : AmbientOcclusion(model.opaque, model.opaque, options.ao.distance, options.ao.samples, options.ao.rays); } @@ -343,7 +414,15 @@ namespace UgcRender { } const auto glitterAlpha = anyGlitter ? UgcGlitter::FleckAlpha(options.glitter) : std::vector{}; - const auto shade = [&](const UgcModel::Mesh& mesh, bool isOpaque, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) { + // A point's surface: its normal (towards the camera), its color before the light (glitter's flecks on it), where + // it is and how it looks + struct Surface { + glm::vec3 normal; + glm::vec4 base; + glm::vec3 position; + UgcModel::eLook look; + }; + const auto surface = [&](const UgcModel::Mesh& mesh, bool isOpaque, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) { glm::vec3 normal(0.0f, 1.0f, 0.0f); if (mesh.normals.size() == mesh.positions.size()) { normal = mesh.normals[i0] * w0 + mesh.normals[i1] * w1 + mesh.normals[i2] * w2; @@ -353,17 +432,7 @@ namespace UgcRender { } glm::vec4 base(0.63f, 0.63f, 0.63f, 1.0f); if (mesh.colors.size() == mesh.positions.size()) base = mesh.colors[i0] * w0 + mesh.colors[i1] * w1 + mesh.colors[i2] * w2; - const float occlusion = isOpaque && ao.size() == mesh.positions.size() ? ao[i0] * w0 + ao[i1] * w1 + ao[i2] * w2 : 1.0f; - const float strength = std::clamp(options.ao.strength, 0.0f, 1.0f); const auto position = mesh.positions[i0] * w0 + mesh.positions[i1] * w1 + mesh.positions[i2] * w2; - const float direct = std::max(0.0f, glm::dot(normal, light)); - const float sun = direct > 0.0f ? sunlit(position + normal * shadowBias) : 0.0f; - // Diffuse: the world's light (radiance `ambient`), a fill from the camera and the sun's (irradiances, over pi) - const float lighting = options.ambient * (1.0f - strength * (1.0f - occlusion)) + - (options.fill * std::max(0.0f, glm::dot(normal, toCamera)) + options.sunStrength * direct * sun) / 3.14159265f; - // The sun's highlight (Blinn-Phong), white, on top of the color - const float highlight = direct > 0.0f ? options.specular * options.sunStrength / 3.14159265f * sun * std::pow(std::max(0.0f, glm::dot(normal, halfway)), std::max(options.shininess, 1.0f)) : 0.0f; - const float exposure = std::max(options.exposure, 0.0f); const auto look = mesh.looks.size() == mesh.positions.size() && (isOpaque || mesh.looks[i0] == UgcModel::eLook::GLITTER) ? mesh.looks[i0] : UgcModel::eLook::PLASTIC; if (look == UgcModel::eLook::GLITTER) { // LEGO-AnimUV: lerp(vertex color, the texture's white, its alpha), then lit as plastic @@ -373,6 +442,25 @@ namespace UgcRender { const float fleck = UgcGlitter::Sample(glitterAlpha, uv); base = glm::vec4(glm::mix(glm::vec3(base), glm::vec3(1.0f), fleck), base.a); } + return Surface{ normal, base, position, look }; + }; + + const auto shade = [&](const UgcModel::Mesh& mesh, bool isOpaque, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) { + const auto point = surface(mesh, isOpaque, i0, i1, i2, w0, w1, w2); + const auto& normal = point.normal; + const auto& base = point.base; + const auto& position = point.position; + const auto look = point.look; + const float occlusion = isOpaque && ao.size() == mesh.positions.size() ? ao[i0] * w0 + ao[i1] * w1 + ao[i2] * w2 : 1.0f; + const float strength = std::clamp(options.ao.strength, 0.0f, 1.0f); + const float direct = std::max(0.0f, glm::dot(normal, light)); + const float sun = direct > 0.0f ? sunlit(position + normal * shadowBias) : 0.0f; + // Diffuse: the world's light (radiance `ambient`), a fill from the camera and the sun's (irradiances, over pi) + const float lighting = options.ambient * (1.0f - strength * (1.0f - occlusion)) + + (options.fill * std::max(0.0f, glm::dot(normal, toCamera)) + options.sunStrength * direct * sun) / 3.14159265f; + // The sun's highlight (Blinn-Phong), white, on top of the color + const float highlight = direct > 0.0f ? options.specular * options.sunStrength / 3.14159265f * sun * std::pow(std::max(0.0f, glm::dot(normal, halfway)), std::max(options.shininess, 1.0f)) : 0.0f; + const float exposure = std::max(options.exposure, 0.0f); if (look == UgcModel::eLook::PLASTIC || look == UgcModel::eLook::GLITTER) { return glm::vec4((ToLinear(base.r) * lighting + highlight) * exposure, (ToLinear(base.g) * lighting + highlight) * exposure, (ToLinear(base.b) * lighting + highlight) * exposure, std::clamp(base.a, 0.0f, 1.0f)); @@ -397,6 +485,12 @@ namespace UgcRender { }; // Opaque first, with the depth buffer + // Traced: each pixel's point, for the occlusion traced after + std::vector points, pointNormals; + if (traced) { + points.assign(color.size(), glm::vec3(0.0f)); + pointNormals.assign(color.size(), glm::vec3(0.0f)); + } { const auto& mesh = model.opaque; std::vector screen(mesh.positions.size()); @@ -409,9 +503,47 @@ namespace UgcRender { depth[index] = z; const auto shaded = shade(mesh, true, i0, i1, i2, w0, w1, w2); color[index] = glm::vec4(glm::vec3(shaded), 1.0f); + if (traced) { + const auto point = surface(mesh, true, i0, i1, i2, w0, w1, w2); + points[index] = point.position; + pointNormals[index] = point.normal; + } }); } } + // Traced: every opaque pixel's occlusion from a few rays (cosine weighted around its normal, a pattern turned + // per pixel, so the error differs from pixel to pixel as the denoiser expects), darkening it as the bake darkens + // the vertex colors (LU Toolbox's Bake Lighting: the color times 1 - strength x (1 - occlusion)) + if (traced) { + const auto scene = UgcRays::Make(options.ao.rays, model.opaque); + const auto count = static_cast(std::clamp(options.denoiseSamples, 1, 256)); + for (size_t index = 0; index < color.size(); index++) { + if ((index & 0x3FF) == 0) UgcThrottle::Checkpoint(); + if (color[index].a <= 0.0f) continue; + const auto& normal = pointNormals[index]; + const glm::vec3 helper = std::abs(normal.x) < 0.9f ? glm::vec3(1, 0, 0) : glm::vec3(0, 1, 0); + const auto tangent = glm::normalize(glm::cross(helper, normal)); + const auto bitangent = glm::cross(normal, tangent); + uint32_t hash = static_cast(index) * 0x9E3779B9u; + hash ^= hash >> 16; + hash *= 0x85EBCA6Bu; + hash ^= hash >> 13; + const float turn = static_cast(hash >> 8) / 16777216.0f; + const float shift = static_cast((hash * 0xC2B2AE35u) >> 8) / 16777216.0f; + const auto origin = points[index] + normal * 1e-3f; + uint32_t open = 0; + for (uint32_t i = 0; i < count; i++) { + const float u = std::fmod((i + shift) / static_cast(count), 1.0f); + const float phi = 2.0f * 3.14159265f * std::fmod(RadicalInverse(i) + turn, 1.0f); + const float r = std::sqrt(u), up = std::sqrt(std::max(0.0f, 1.0f - u)); + const auto direction = tangent * (r * std::cos(phi)) + bitangent * (r * std::sin(phi)) + normal * up; + if (!scene->Occluded(origin, direction, 1e-4f, options.ao.distance)) open++; + } + const float occlusion = static_cast(open) / static_cast(count); + const float lit = 1.0f - std::clamp(options.bakedAo, 0.0f, 1.0f) * (1.0f - occlusion); + color[index] = glm::vec4(glm::vec3(color[index]) * lit, color[index].a); + } + } // Then transparent triangles, farthest first, blended over it { @@ -435,15 +567,67 @@ namespace UgcRender { } } - // Box filter down to the icon's size + // Box filter down to the icon's size (linear, premultiplied) const float samples = static_cast(supersample * supersample); + const auto boxFilter = [&](const auto& full) { + using Pixel = typename std::decay_t::value_type; + std::vector small(static_cast(size) * size, Pixel(0.0f)); + for (int y = 0; y < size; y++) { + for (int x = 0; x < size; x++) { + Pixel sum(0.0f); + for (int sy = 0; sy < supersample; sy++) { + for (int sx = 0; sx < supersample; sx++) sum += full[static_cast(y * supersample + sy) * n + (x * supersample + sx)]; + } + sum /= samples; + small[static_cast(y) * size + x] = sum; + } + } + return small; + }; + auto filtered = boxFilter(color); + + // Denoised at the icon's size (the supersampling has averaged the pixels already; a denoiser's time grows with + // the pixels), guided by the colors before the light and the normals of the same view, which have no noise + if (denoise) { + std::vector albedo(static_cast(n) * n, glm::vec3(0.0f)), normals(static_cast(n) * n, glm::vec3(0.0f)); + // The colors before the bake when known (traced: the model drawn) + UgcModel::Model guide = plain && !traced ? *plain : UgcModel::Model{}; + if (plain && !traced) { + guide.opaque.Transform(rotation); + guide.transparent.Transform(rotation); + } + const auto& drawn = plain && !traced ? guide : model; + std::vector guideDepth(static_cast(n) * n, INF); + for (const bool isOpaque : { true, false }) { + const auto& mesh = isOpaque ? drawn.opaque : drawn.transparent; + std::vector screen(mesh.positions.size()); + for (size_t v = 0; v < mesh.positions.size(); v++) screen[v] = project(mesh.positions[v]); + for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) { + const auto i0 = mesh.indices[i], i1 = mesh.indices[i + 1], i2 = mesh.indices[i + 2]; + Rasterize(n, n, screen[i0], screen[i1], screen[i2], [&](int x, int y, float z, float w0, float w1, float w2) { + const size_t index = static_cast(y) * n + x; + // Transparent surfaces over the opaque ones, in any order (a guide needn't be exact) + if (z >= guideDepth[index]) return; + const auto point = surface(mesh, isOpaque, i0, i1, i2, w0, w1, w2); + const glm::vec3 linear(ToLinear(point.base.r), ToLinear(point.base.g), ToLinear(point.base.b)); + if (isOpaque) { + guideDepth[index] = z; + albedo[index] = linear; + normals[index] = point.normal; + } else { + const float alpha = std::clamp(point.base.a, 0.0f, 1.0f); + albedo[index] = glm::mix(albedo[index], linear, alpha); + normals[index] = glm::normalize(glm::mix(normals[index], point.normal, alpha) + glm::vec3(1e-6f)); + } + }); + } + } + Denoise(filtered, boxFilter(albedo), boxFilter(normals), size); + } + for (int y = 0; y < size; y++) { for (int x = 0; x < size; x++) { - glm::vec4 sum(0.0f); - for (int sy = 0; sy < supersample; sy++) { - for (int sx = 0; sx < supersample; sx++) sum += color[static_cast(y * supersample + sy) * n + (x * supersample + sx)]; - } - sum /= samples; + const auto& sum = filtered[static_cast(y) * size + x]; uint8_t* out = &image.rgba[(static_cast(y) * size + x) * 4]; if (sum.a <= 0.0f) continue; // sRGB, then the contrast around its middle grey diff --git a/dUgcServer/Render/UgcRender.h b/dUgcServer/Render/UgcRender.h index 3fe243139..4a4234f83 100644 --- a/dUgcServer/Render/UgcRender.h +++ b/dUgcServer/Render/UgcRender.h @@ -73,6 +73,8 @@ namespace UgcRender { float glowEmissive{ 1.0f }; // how far glowing shapes go from lit to their plain color (the glow_emissive setting) UgcGlitter::Params glitter; // the glitter's flecks (glitter_size, glitter_density), drawn where they are at the start eDenoise denoise{}; // the finished icon denoised (denoise); off when the build can't + int denoiseSamples{ 4 }; // denoised: occlusion rays per pixel (of the supersampled image) traced on the model before its bake + float bakedAo{ 1.0f }; // denoised: the bake's strength (ao_strength; 0 when bake_ao is off), for the traced occlusion }; // The model drawn from the icon's camera, framed to fit, on a transparent background. `opaqueAo`: the opaque mesh's @@ -82,7 +84,13 @@ namespace UgcRender { // sky-and-ground reflection tinted by the color and a highlight, sharp for polished metal and broad for brushed // steel (Polished Metal, Brushed Steel: an environment map tinted by the vertex color). GLITTER vertices (opaque or // transparent) get the glitter texture's white flecks over their color before the light (LEGO-AnimUV), still. - Image RenderIcon(const UgcModel::Model& model, const IconOptions& options, const std::vector* opaqueAo = nullptr); + // With options.denoise (and a build that has it) the image is denoised by Open Image Denoise, guided by the colors + // before the light and the normals (which have no noise). A denoiser only removes noise that differs from pixel to + // pixel, not the baked occlusion's (which is per vertex), so with `plain` (the same model with the colors it had + // before its occlusion was baked in) that is drawn instead, with its occlusion traced per pixel: options. + // denoiseSamples rays (options.ao's distance, strength and ray backend) from each pixel of the supersampled image, + // noisy, then denoised. Without `plain` it is only denoised. + Image RenderIcon(const UgcModel::Model& model, const IconOptions& options, const std::vector* opaqueAo = nullptr, const UgcModel::Model* plain = nullptr); /** * The fast hidden-face test (hsr_method=fast, what the UGC server did before it traced LU Toolbox's paths): the diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index 449d07b41..592b895ed 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -180,6 +180,8 @@ namespace { settings.icon.ao.distance = settings.ao.distance; settings.icon.ao.rays = settings.hsr.rays; settings.icon.denoise = UgcRender::ParseDenoise(Game::config->GetValue("denoise")).value_or(UgcRender::eDenoise::OFF); + settings.icon.denoiseSamples = std::clamp(Setting("denoise_samples", 4), 1, 256); + settings.icon.bakedAo = settings.ao.enabled ? std::clamp(settings.ao.strength, 0.0f, 1.0f) : 0.0f; settings.maxBricks = Setting("max_model_bricks", 0); return settings; } @@ -572,7 +574,7 @@ namespace { if (!library.LoadMaterials()) std::cerr << "Couldn't read Materials.xml from " << (res / "brickdb.zip") << "; bricks will be grey\n"; UgcJobs::Outcome outcome; const auto start = std::chrono::steady_clock::now(); - const double cpuStart = UgcThrottle::ThreadCpuSeconds(); + const double cpuStart = UgcThrottle::JobCpuSeconds(); if (mode == "--make-model") { const auto data = UgcBricks::ReadFile(input); if (!data) { @@ -599,7 +601,7 @@ namespace { for (const auto& [name, data] : outcome.files) { std::ofstream(output / name, std::ios::binary).write(data.data(), static_cast(data.size())); } - const double cpuMs = (UgcThrottle::ThreadCpuSeconds() - cpuStart) * 1000.0; + const double cpuMs = (UgcThrottle::JobCpuSeconds() - cpuStart) * 1000.0; std::cout << "Made " << outcome.files.size() << " files in " << ms << " ms (" << cpuMs << " ms CPU)" << (outcome.options.empty() ? "" : " with " + outcome.options) << (outcome.note.empty() ? "" : ": " + outcome.note) << "\n"; return EXIT_SUCCESS; diff --git a/resources/ugcconfig.ini b/resources/ugcconfig.ini index bba898f4c..c171aab5e 100644 --- a/resources/ugcconfig.ini +++ b/resources/ugcconfig.ini @@ -149,8 +149,11 @@ hsr_fast_resolution=1024 # hiprt (the GPU, when built with DLU_HIPRT and the machine has one; else embree). ray_backend=builtin -# denoise: off, or oidn (icons cleaned up with Intel Open Image Denoise, when built with DLU_OIDN; else off). +# denoise: off, or oidn (when built with DLU_OIDN; else off): a model's icon is drawn from its colors before the +# occlusion bake with its occlusion traced per pixel (denoise_samples rays from each pixel of the supersampled image) +# and denoised by Intel Open Image Denoise. The model keeps its baked occlusion. denoise=off +denoise_samples=4 # Bake ambient occlusion into the vertex colors: rays per vertex, how far they look, strength 0 (none) to 1, and how # much glowing colors add diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp index 10718006a..9a012514b 100644 --- a/tests/dUgcTests/UgcTests.cpp +++ b/tests/dUgcTests/UgcTests.cpp @@ -2111,6 +2111,51 @@ TEST(UgcProcessOptions, ParseApplyAndRecord) { EXPECT_EQ(stats["settings"]["denoise"], made.denoise); } +TEST(UgcRender, DenoisedIconsTraceTheOcclusionPerPixel) { + // A box standing on a floor, grey: denoised, the icon is drawn from the model before its bake with the occlusion + // traced per pixel, so the floor is darker by the box than away from it; with few rays it is about what many give + UgcModel::Model model; + AddBox(model.opaque, glm::vec3(-4.0f, -0.2f, -4.0f), glm::vec3(4.0f, 0.0f, 4.0f)); + AddBox(model.opaque, glm::vec3(-0.5f, 0.0f, -0.5f), glm::vec3(0.5f, 1.0f, 0.5f)); + UgcRender::IconOptions options; + options.size = 48; + options.supersample = 2; + options.pitchDegrees = 60.0f; + options.shadows = 0.0f; + options.ao.distance = 2.0f; + const auto normal = UgcRender::RenderIcon(model, options); + options.denoise = UgcRender::eDenoise::OIDN; + options.denoiseSamples = 2; + const auto few = UgcRender::RenderIcon(model, options, nullptr, &model); + if (!UgcRender::Available(UgcRender::eDenoise::OIDN)) { + EXPECT_EQ(few.rgba, normal.rgba); // without it the option does nothing + GTEST_SKIP() << "built without Open Image Denoise (DLU_OIDN)"; + } + options.denoiseSamples = 64; + const auto many = UgcRender::RenderIcon(model, options, nullptr, &model); + ASSERT_EQ(few.rgba.size(), normal.rgba.size()); + double difference = 0.0, count = 0.0; + for (size_t i = 0; i < few.rgba.size(); i += 4) { + ASSERT_EQ(few.rgba[i + 3], normal.rgba[i + 3]) << i; // the same outline + if (few.rgba[i + 3] != 255) continue; + difference += std::abs(static_cast(few.rgba[i]) - many.rgba[i]); + count++; + } + EXPECT_LT(difference / count, 4.0); + // Darker than the icon without occlusion only by the box (near the middle), not out at the floor's edges + size_t darkened = 0, darkenedFar = 0; + for (int y = 0; y < 48; y++) { + for (int x = 0; x < 48; x++) { + const size_t i = (static_cast(y) * 48 + x) * 4; + if (few.rgba[i + 3] != 255 || few.rgba[i] + 30 > normal.rgba[i]) continue; + darkened++; + if (std::abs(x - 24) > 14 || std::abs(y - 24) > 14) darkenedFar++; + } + } + EXPECT_GT(darkened, 20u); + EXPECT_EQ(darkenedFar, 0u); +} + TEST(UgcRays, NamesAndFallback) { for (const auto backend : { UgcRays::eBackend::BUILTIN, UgcRays::eBackend::EMBREE, UgcRays::eBackend::HIPRT }) { EXPECT_EQ(UgcRays::Parse(UgcRays::Name(backend)), backend); diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index fb3ecf3c6..ae34080eb 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -179,3 +179,32 @@ function(dlu_fetch_embree) endforeach() endfunction() dlu_fetch_embree() + +# Intel Open Image Denoise 2 (Apache-2.0), the UGC server's denoise=oidn: optional, off by default (it brings TBB and +# large neural network weights). An installed one is used when CMake finds it, else Intel's release package (prebuilt, +# with its TBB) is downloaded and its libraries are copied next to the servers. +option(DLU_OIDN "Build the UGC server with Intel Open Image Denoise (the denoise=oidn setting)" OFF) +if(DLU_OIDN) + # Its imported targets are used by the UGC server, in another directory + set(CMAKE_FIND_PACKAGE_TARGETS_GLOBAL ON) + find_package(OpenImageDenoise 2 CONFIG QUIET) + if(NOT OpenImageDenoise_FOUND) + if(WIN32) + set(DLU_OIDN_URL "https://github.com/RenderKit/oidn/releases/download/v2.4.1/oidn-2.4.1.x64.windows.zip") + set(DLU_OIDN_SHA256 "682d94ba57525ed177d73412e0ed903f576867bd048f830a5c6f63c56b25e8b8") + set(DLU_OIDN_LIBRARIES "bin/*.dll") + elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + set(DLU_OIDN_URL "https://github.com/RenderKit/oidn/releases/download/v2.4.1/oidn-2.4.1.x86_64.linux.tar.gz") + set(DLU_OIDN_SHA256 "b69ca2443a226ef692ca46bdc4f89995b1e99091f2665b906cbe07e9673e48cc") + set(DLU_OIDN_LIBRARIES "lib/*.so*") + else() + message(FATAL_ERROR "DLU_OIDN: install Open Image Denoise 2 (no release package for this platform)") + endif() + FetchContent_Declare(oidn URL ${DLU_OIDN_URL} URL_HASH SHA256=${DLU_OIDN_SHA256} DOWNLOAD_EXTRACT_TIMESTAMP TRUE) + FetchContent_MakeAvailable(oidn) + find_package(OpenImageDenoise 2 CONFIG REQUIRED PATHS "${oidn_SOURCE_DIR}" NO_DEFAULT_PATH) + file(GLOB DLU_OIDN_FILES "${oidn_SOURCE_DIR}/${DLU_OIDN_LIBRARIES}") + file(COPY ${DLU_OIDN_FILES} DESTINATION "${CMAKE_BINARY_DIR}") + endif() + message(STATUS "Open Image Denoise ${OpenImageDenoise_VERSION}: the UGC server can denoise icons") +endif()