diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index 70cab6234..0ff28b4d0 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -504,7 +504,8 @@ namespace { c.Add(Labels(Choice(UGC, "hsr_method", "How hidden faces are found", "toolbox: LU Toolbox's paths (above). fast: the model rendered from 42 directions around it, removing what shows in none; much faster, but it also removes faces seen only by bounced light (insides seen through openings, recesses). Staff can pick another for one make when making models again.", "toolbox", { "toolbox", "fast" }), { "LU Toolbox (paths)", "Fast (renders from around)" })); c.Add(Unit(Int(UGC, "hsr_fast_resolution", "Fast method's render size", "The size of each of the fast method's 42 renders; bigger keeps smaller visible faces.", "1024", 64, 4096), "pixels")); c.Add(Labels(Choice(UGC, "ray_backend", "Ray tracer", "What traces the rays of the hidden faces' paths and of the occlusion: builtin (the UGC server's own), embree (Intel Embree on the CPU) or hiprt (the GPU, when the server was built with it and has one; else embree). They give the same results but for rounding. Staff can pick another for one make when making models again.", "builtin", { "builtin", "embree", "hiprt" }), { "Built in", "Embree (CPU)", "HIPRT (GPU)" })); - c.Add(Labels(Choice(UGC, "denoise", "Denoise icons", "oidn: icons are cleaned up with Intel Open Image Denoise (when the server was built with it; else off).", "off", { "off", "oidn" }), { "Off", "Open Image Denoise" })); + c.Add(Labels(Choice(UGC, "denoise", "Denoise icons", "oidn (when the server was built with Intel Open Image Denoise; else off): a model's icon is drawn from its colors before the occlusion bake, with its occlusion traced per pixel with a few rays and the noise removed by the denoiser, instead of the baked occlusion. The model itself keeps its baked occlusion (a denoiser only works on images).", "off", { "off", "oidn" }), { "Off", "Open Image Denoise" })); + c.Add(Int(UGC, "denoise_samples", "Denoised occlusion rays per pixel", "Occlusion rays traced from each pixel of a denoised icon (before it is scaled down, so 16 times as many per icon pixel); fewer is faster and noisier.", "4", 1, 256)); c.Add(Bool(UGC, "bake_ao", "Darken hidden corners", "Ambient occlusion baked into the vertex colors.", true)); c.Add(Int(UGC, "ao_samples", "Occlusion rays per vertex", "", "64", 1, 1024)); c.Add(Float(UGC, "ao_distance", "Occlusion distance", "", "5", 0, 1000)); diff --git a/dUgcServer/CMakeLists.txt b/dUgcServer/CMakeLists.txt index 7bfdf1c93..d601ce58c 100644 --- a/dUgcServer/CMakeLists.txt +++ b/dUgcServer/CMakeLists.txt @@ -28,6 +28,11 @@ add_library(dUgc STATIC ${DUGC_SOURCES}) target_include_directories(dUgc PUBLIC "." "Bricks" "Formats" "Model" "Processing" "Render" "${PROJECT_SOURCE_DIR}/thirdparty/MD5" "${PROJECT_SOURCE_DIR}/thirdparty/nlohmann") target_link_libraries(dUgc PUBLIC dCommon glm::glm tinyxml2 MD5 PRIVATE embree) +# Intel Open Image Denoise, when built with DLU_OIDN (thirdparty/CMakeLists.txt): the denoise=oidn setting +if(DLU_OIDN) + target_link_libraries(dUgc PRIVATE OpenImageDenoise) + target_compile_definitions(dUgc PRIVATE DLU_OIDN) +endif() add_executable(UgcServer "UgcServer.cpp" "Processing/UgcProcessor.cpp" "Bricks/UgcCdClient.cpp") diff --git a/dUgcServer/Processing/UgcJobs.cpp b/dUgcServer/Processing/UgcJobs.cpp index 7309a4f7b..6bd23c594 100644 --- a/dUgcServer/Processing/UgcJobs.cpp +++ b/dUgcServer/Processing/UgcJobs.cpp @@ -36,8 +36,9 @@ namespace UgcJobs { namespace { // The icon files of a model, and false when nothing was drawn - bool AddIcon(UgcStorage::Files& files, const UgcModel::Model& model, const UgcRender::IconOptions& options, const std::vector* ao = nullptr) { - const auto icon = UgcRender::RenderIcon(model, options, ao); + bool AddIcon(UgcStorage::Files& files, const UgcModel::Model& model, const UgcRender::IconOptions& options, const std::vector* ao = nullptr, + const UgcModel::Model* plain = nullptr) { + const auto icon = UgcRender::RenderIcon(model, options, ao, plain); bool drawn = false; for (size_t i = 3; i < icon.rgba.size(); i += 4) drawn = drawn || icon.rgba[i] != 0; files["icon.png"] = UgcFormats::EncodePng(icon); @@ -150,10 +151,16 @@ namespace UgcJobs { } bool IconFromNif(const std::string& nif, const UgcRender::IconOptions& options, UgcStorage::Files& files, std::string& error, - const std::map& tagLooks, const std::set& overlayTags) { + const std::map& tagLooks, const std::set& overlayTags, const std::string* plainNif) { const auto readBack = NifFile::Parse(nif, 0, error); if (!readBack) return false; - AddIcon(files, UgcModel::FromNif(*readBack, tagLooks, overlayTags), options); + // The denoiser's guide: the model before its occlusion was baked in (model.noao.nif), when there is one + std::optional plain; + if (options.denoise != UgcRender::eDenoise::OFF && plainNif) { + std::string plainError; + if (const auto plainRead = NifFile::Parse(*plainNif, 0, plainError)) plain = UgcModel::FromNif(*plainRead, tagLooks, overlayTags); + } + AddIcon(files, UgcModel::FromNif(*readBack, tagLooks, overlayTags), options, nullptr, plain ? &*plain : nullptr); return true; } @@ -342,10 +349,12 @@ namespace UgcJobs { // Stored compressed only (the client downloads .gz; the dashboard's copies are inflated when asked for). The // LXFML is served from the database. AddDownload(outcome.files, "model.nif", nif); + std::string plainNif; { const std::vector opaque{ DivideByLook(preview.opaque, separate) }; const std::vector transparent{ transparentPieces[0] }; - outcome.files["model.noao.nif.gz"] = ZCompression::Gzip(UgcFormats::WriteLodNif("SceneNode_Model", groups(1, opaque, transparent))); + plainNif = UgcFormats::WriteLodNif("SceneNode_Model", groups(1, opaque, transparent)); + outcome.files["model.noao.nif.gz"] = ZCompression::Gzip(plainNif); } // The icon is drawn from the .nif just made (its most detailed LOD, read back like any client .nif), so it @@ -354,7 +363,7 @@ namespace UgcJobs { auto iconOptions = settings.icon; UgcIconParams::Apply(iconOptions, iconValues); std::string nifError; - if (!IconFromNif(nif, iconOptions, outcome.files, nifError, settings.shaders.TagLooks(), settings.shaders.OverlayTags())) { + if (!IconFromNif(nif, iconOptions, outcome.files, nifError, settings.shaders.TagLooks(), settings.shaders.OverlayTags(), &plainNif)) { outcome.error = "the .nif made can't be read back for the icon: " + nifError; return outcome; } diff --git a/dUgcServer/Processing/UgcJobs.h b/dUgcServer/Processing/UgcJobs.h index 5089d1ea4..116c0dc0e 100644 --- a/dUgcServer/Processing/UgcJobs.h +++ b/dUgcServer/Processing/UgcJobs.h @@ -101,9 +101,10 @@ namespace UgcJobs { // A model's icon files (icon.png, icon.dds download) drawn from its .nif (LOD 0), its metal and glow groups by // `tagLooks` (Shaders::TagLooks); false (and `error`) when the .nif can't be read - // (the groups drawn over others, `overlayTags` (Shaders::OverlayTags), left out) + // (the groups drawn over others, `overlayTags` (Shaders::OverlayTags), left out). `plainNif`: its model.noao.nif, the + // denoiser's guide when options.denoise is on. bool IconFromNif(const std::string& nif, const UgcRender::IconOptions& options, UgcStorage::Files& files, std::string& error, - const std::map& tagLooks = {}, const std::set& overlayTags = {}); + const std::map& tagLooks = {}, const std::set& overlayTags = {}, const std::string* plainNif = nullptr); /** * A model's stats.json after its icon was drawn again in `iconMs`: ms.icon becomes that and ms.total changes by the diff --git a/dUgcServer/Processing/UgcProcessor.cpp b/dUgcServer/Processing/UgcProcessor.cpp index 78f8ab426..4038b180f 100644 --- a/dUgcServer/Processing/UgcProcessor.cpp +++ b/dUgcServer/Processing/UgcProcessor.cpp @@ -242,7 +242,9 @@ void UgcProcessor::Worker() { const auto nif = m_Storage.ReadNif(Kind::MODEL, job.id, "model.nif"); auto options = settings.icon; UgcIconParams::Apply(options, job.iconValues); - outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, outcome.files, outcome.error, settings.shaders.TagLooks(), settings.shaders.OverlayTags()); + const auto plain = options.denoise != UgcRender::eDenoise::OFF ? m_Storage.ReadNif(Kind::MODEL, job.id, "model.noao.nif") : std::nullopt; + outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, outcome.files, outcome.error, settings.shaders.TagLooks(), settings.shaders.OverlayTags(), + plain ? &*plain : nullptr); if (!nif) outcome.error = "the model has no stored .nif yet"; } if (outcome.ok && outcome.files.contains("assembly.nif")) { @@ -279,7 +281,7 @@ void UgcProcessor::Worker() { continue; } const auto start = std::chrono::steady_clock::now(); - const double cpuStart = UgcThrottle::ThreadCpuSeconds(); + const double cpuStart = UgcThrottle::JobCpuSeconds(); Done done{ job.kind, job.id, job.attempts }; done.memoryEstimate = job.memory; done.iconOnly = job.iconOnly; @@ -290,7 +292,9 @@ void UgcProcessor::Worker() { auto options = settings.icon; UgcIconParams::Apply(options, job.iconValues); const auto iconStart = std::chrono::steady_clock::now(); - done.outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, done.outcome.files, done.outcome.error, settings.shaders.TagLooks(), settings.shaders.OverlayTags()); + const auto plain = options.denoise != UgcRender::eDenoise::OFF ? m_Storage.ReadNif(Kind::MODEL, job.id, "model.noao.nif") : std::nullopt; + done.outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, done.outcome.files, done.outcome.error, settings.shaders.TagLooks(), settings.shaders.OverlayTags(), + plain ? &*plain : nullptr); if (!nif) done.outcome.error = "no stored .nif"; // The make's time keeps its icon's: the stats get the new icon's time, the row the difference (Collect) const auto stats = done.outcome.ok ? m_Storage.ReadNif(Kind::MODEL, job.id, "stats.json") : std::nullopt; @@ -336,7 +340,7 @@ void UgcProcessor::Worker() { } done.outcome.files.clear(); done.milliseconds = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); - done.cpuMilliseconds = std::max(0.0, UgcThrottle::ThreadCpuSeconds() - cpuStart) * 1000.0; + done.cpuMilliseconds = std::max(0.0, UgcThrottle::JobCpuSeconds() - cpuStart) * 1000.0; try { UgcThrottle::Checkpoint(); } catch (const UgcThrottle::Cancelled&) { diff --git a/dUgcServer/Processing/UgcThrottle.cpp b/dUgcServer/Processing/UgcThrottle.cpp index 4bd104860..c35bdbe4b 100644 --- a/dUgcServer/Processing/UgcThrottle.cpp +++ b/dUgcServer/Processing/UgcThrottle.cpp @@ -22,6 +22,7 @@ namespace { std::atomic g_LastSleep{ 0 }; thread_local double t_LastCpu = -1.0; + thread_local double t_Charged = 0.0; // CPU seconds libraries used for this thread on threads of their own int64_t UnixMs() { return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); @@ -55,6 +56,17 @@ namespace UgcThrottle { return std::chrono::duration(std::chrono::steady_clock::now().time_since_epoch()).count(); } + void Charge(double seconds) { + if (!(seconds > 0.0)) return; + t_Charged += seconds; + // The next Checkpoint sees it as used since the last + if (t_LastCpu >= 0.0) t_LastCpu -= seconds; + } + + double JobCpuSeconds() { + return ThreadCpuSeconds() + t_Charged; + } + void Begin() { t_LastCpu = ThreadCpuSeconds(); } diff --git a/dUgcServer/Processing/UgcThrottle.h b/dUgcServer/Processing/UgcThrottle.h index 4a658bc72..0581a67fb 100644 --- a/dUgcServer/Processing/UgcThrottle.h +++ b/dUgcServer/Processing/UgcThrottle.h @@ -36,6 +36,12 @@ namespace UgcThrottle { // This thread's CPU time in seconds double ThreadCpuSeconds(); + // CPU time a library spent for this thread on threads of its own (Open Image Denoise's), counted as this thread's: + // by the budget at the next Checkpoint and in JobCpuSeconds + void Charge(double seconds); + // This thread's CPU time with what was charged to it, in seconds (a job's CPU time is the difference) + double JobCpuSeconds(); + // Parses "18-23" (from 18:00 to 23:59; may wrap past midnight, "22-6") into from and to; false when it isn't that bool ParseHours(const std::string& text, int& from, int& to); // Whether `hour` (0-23) is inside the range; never when from or to is negative diff --git a/dUgcServer/Render/UgcRender.cpp b/dUgcServer/Render/UgcRender.cpp index d80cb88fc..1d7de74ca 100644 --- a/dUgcServer/Render/UgcRender.cpp +++ b/dUgcServer/Render/UgcRender.cpp @@ -1,13 +1,20 @@ #include "UgcRender.h" #include +#include #include #include #include +#include +#include #include #include +#ifdef DLU_OIDN +#include +#endif + #include "UgcIconPose.h" #include "UgcPalette.h" #include "UgcRays.h" @@ -81,6 +88,64 @@ namespace { float PixelSize() const { return 2.0f * radius / resolution; } }; +#ifdef DLU_OIDN + // Open Image Denoise's CPU device for the thread, made the first time: one thread of its own (not all the cores), + // whose time Denoise charges to the asking thread (UgcThrottle::Charge); null when it can't be made + oidn::DeviceRef* OidnDevice() { + thread_local std::optional device; + thread_local bool tried = false; + if (!tried) { + tried = true; + auto made = oidn::newDevice(oidn::DeviceType::CPU); + if (made) { + made.set("numThreads", 1); + made.set("setAffinity", false); + made.commit(); + const char* message = nullptr; + if (made.getError(message) == oidn::Error::None) device = std::move(made); + } + } + return device ? &*device : nullptr; + } +#endif + + // Denoises `color` (linear, premultiplied, n x n) with Open Image Denoise's ray tracing filter, guided by `albedo` + // and `normals` (noise free); false (and `color` as it was) when it can't + bool Denoise(std::vector& color, const std::vector& albedo, const std::vector& normals, int n) { +#ifdef DLU_OIDN + auto* device = OidnDevice(); + if (!device || n <= 0) return false; + const size_t pixels = static_cast(n) * n; + std::vector input(pixels), output(pixels); + for (size_t i = 0; i < pixels; i++) input[i] = glm::vec3(color[i]); + auto filter = device->newFilter("RT"); + filter.setImage("color", input.data(), oidn::Format::Float3, n, n); + filter.setImage("albedo", const_cast(albedo.data()), oidn::Format::Float3, n, n); + filter.setImage("normal", const_cast(normals.data()), oidn::Format::Float3, n, n); + filter.setImage("output", output.data(), oidn::Format::Float3, n, n); + filter.set("hdr", true); + filter.set("cleanAux", true); + // It works on a thread of its own while this one waits: its time is this job's CPU time + const auto started = std::chrono::steady_clock::now(); + filter.commit(); + filter.execute(); + UgcThrottle::Charge(std::chrono::duration(std::chrono::steady_clock::now() - started).count()); + const char* message = nullptr; + if (device->getError(message) != oidn::Error::None) return false; + // The shape's outline and coverage stay as drawn + for (size_t i = 0; i < pixels; i++) { + if (color[i].a > 0.0f) color[i] = glm::vec4(glm::max(output[i], glm::vec3(0.0f)), color[i].a); + } + return true; +#else + (void)color; + (void)albedo; + (void)normals; + (void)n; + return false; +#endif + } + void Bounds(const UgcModel::Model& model, glm::vec3& center, float& radius) { glm::vec3 min{}, max{}; if (!model.Bounds(min, max)) { @@ -106,6 +171,9 @@ namespace UgcRender { } bool Available(eDenoise denoise) { +#ifdef DLU_OIDN + if (denoise == eDenoise::OIDN) return OidnDevice() != nullptr; +#endif return denoise == eDenoise::OFF; } @@ -267,14 +335,17 @@ namespace UgcRender { return ao; } - Image RenderIcon(const UgcModel::Model& source, const IconOptions& options, const std::vector* opaqueAo) { + Image RenderIcon(const UgcModel::Model& source, const IconOptions& options, const std::vector* opaqueAo, const UgcModel::Model* plain) { const int size = std::clamp(options.size, 8, 1024); const int supersample = std::clamp(options.supersample, 1, 8); const int n = size * supersample; Image image{ size, size, std::vector(static_cast(size) * size * 4, 0) }; if (source.Empty()) return image; - UgcModel::Model model = source; + // Denoised with the model before its bake: its occlusion is traced per pixel (noisy), then denoised + const bool denoise = options.denoise != eDenoise::OFF && Available(options.denoise); + const bool traced = denoise && plain && !plain->opaque.Empty() && options.denoiseSamples > 0 && options.bakedAo > 0.0f && options.ao.distance > 0.0f; + UgcModel::Model model = traced ? *plain : source; const auto rotation = UgcIconPose::ModelRotation(options.modelYawDegrees, options.modelPitchDegrees, options.modelRollDegrees) * options.modelRotation; model.opaque.Transform(rotation); model.transparent.Transform(rotation); @@ -300,7 +371,7 @@ namespace UgcRender { // Ambient occlusion darkens the world light (opaque bricks only, as they are what occludes) std::vector ao; - if (options.ao.enabled) { + if (options.ao.enabled && !traced) { ao = opaqueAo && opaqueAo->size() == model.opaque.positions.size() ? *opaqueAo : AmbientOcclusion(model.opaque, model.opaque, options.ao.distance, options.ao.samples, options.ao.rays); } @@ -343,7 +414,15 @@ namespace UgcRender { } const auto glitterAlpha = anyGlitter ? UgcGlitter::FleckAlpha(options.glitter) : std::vector{}; - const auto shade = [&](const UgcModel::Mesh& mesh, bool isOpaque, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) { + // A point's surface: its normal (towards the camera), its color before the light (glitter's flecks on it), where + // it is and how it looks + struct Surface { + glm::vec3 normal; + glm::vec4 base; + glm::vec3 position; + UgcModel::eLook look; + }; + const auto surface = [&](const UgcModel::Mesh& mesh, bool isOpaque, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) { glm::vec3 normal(0.0f, 1.0f, 0.0f); if (mesh.normals.size() == mesh.positions.size()) { normal = mesh.normals[i0] * w0 + mesh.normals[i1] * w1 + mesh.normals[i2] * w2; @@ -353,17 +432,7 @@ namespace UgcRender { } glm::vec4 base(0.63f, 0.63f, 0.63f, 1.0f); if (mesh.colors.size() == mesh.positions.size()) base = mesh.colors[i0] * w0 + mesh.colors[i1] * w1 + mesh.colors[i2] * w2; - const float occlusion = isOpaque && ao.size() == mesh.positions.size() ? ao[i0] * w0 + ao[i1] * w1 + ao[i2] * w2 : 1.0f; - const float strength = std::clamp(options.ao.strength, 0.0f, 1.0f); const auto position = mesh.positions[i0] * w0 + mesh.positions[i1] * w1 + mesh.positions[i2] * w2; - const float direct = std::max(0.0f, glm::dot(normal, light)); - const float sun = direct > 0.0f ? sunlit(position + normal * shadowBias) : 0.0f; - // Diffuse: the world's light (radiance `ambient`), a fill from the camera and the sun's (irradiances, over pi) - const float lighting = options.ambient * (1.0f - strength * (1.0f - occlusion)) + - (options.fill * std::max(0.0f, glm::dot(normal, toCamera)) + options.sunStrength * direct * sun) / 3.14159265f; - // The sun's highlight (Blinn-Phong), white, on top of the color - const float highlight = direct > 0.0f ? options.specular * options.sunStrength / 3.14159265f * sun * std::pow(std::max(0.0f, glm::dot(normal, halfway)), std::max(options.shininess, 1.0f)) : 0.0f; - const float exposure = std::max(options.exposure, 0.0f); const auto look = mesh.looks.size() == mesh.positions.size() && (isOpaque || mesh.looks[i0] == UgcModel::eLook::GLITTER) ? mesh.looks[i0] : UgcModel::eLook::PLASTIC; if (look == UgcModel::eLook::GLITTER) { // LEGO-AnimUV: lerp(vertex color, the texture's white, its alpha), then lit as plastic @@ -373,6 +442,25 @@ namespace UgcRender { const float fleck = UgcGlitter::Sample(glitterAlpha, uv); base = glm::vec4(glm::mix(glm::vec3(base), glm::vec3(1.0f), fleck), base.a); } + return Surface{ normal, base, position, look }; + }; + + const auto shade = [&](const UgcModel::Mesh& mesh, bool isOpaque, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) { + const auto point = surface(mesh, isOpaque, i0, i1, i2, w0, w1, w2); + const auto& normal = point.normal; + const auto& base = point.base; + const auto& position = point.position; + const auto look = point.look; + const float occlusion = isOpaque && ao.size() == mesh.positions.size() ? ao[i0] * w0 + ao[i1] * w1 + ao[i2] * w2 : 1.0f; + const float strength = std::clamp(options.ao.strength, 0.0f, 1.0f); + const float direct = std::max(0.0f, glm::dot(normal, light)); + const float sun = direct > 0.0f ? sunlit(position + normal * shadowBias) : 0.0f; + // Diffuse: the world's light (radiance `ambient`), a fill from the camera and the sun's (irradiances, over pi) + const float lighting = options.ambient * (1.0f - strength * (1.0f - occlusion)) + + (options.fill * std::max(0.0f, glm::dot(normal, toCamera)) + options.sunStrength * direct * sun) / 3.14159265f; + // The sun's highlight (Blinn-Phong), white, on top of the color + const float highlight = direct > 0.0f ? options.specular * options.sunStrength / 3.14159265f * sun * std::pow(std::max(0.0f, glm::dot(normal, halfway)), std::max(options.shininess, 1.0f)) : 0.0f; + const float exposure = std::max(options.exposure, 0.0f); if (look == UgcModel::eLook::PLASTIC || look == UgcModel::eLook::GLITTER) { return glm::vec4((ToLinear(base.r) * lighting + highlight) * exposure, (ToLinear(base.g) * lighting + highlight) * exposure, (ToLinear(base.b) * lighting + highlight) * exposure, std::clamp(base.a, 0.0f, 1.0f)); @@ -397,6 +485,12 @@ namespace UgcRender { }; // Opaque first, with the depth buffer + // Traced: each pixel's point, for the occlusion traced after + std::vector points, pointNormals; + if (traced) { + points.assign(color.size(), glm::vec3(0.0f)); + pointNormals.assign(color.size(), glm::vec3(0.0f)); + } { const auto& mesh = model.opaque; std::vector screen(mesh.positions.size()); @@ -409,9 +503,47 @@ namespace UgcRender { depth[index] = z; const auto shaded = shade(mesh, true, i0, i1, i2, w0, w1, w2); color[index] = glm::vec4(glm::vec3(shaded), 1.0f); + if (traced) { + const auto point = surface(mesh, true, i0, i1, i2, w0, w1, w2); + points[index] = point.position; + pointNormals[index] = point.normal; + } }); } } + // Traced: every opaque pixel's occlusion from a few rays (cosine weighted around its normal, a pattern turned + // per pixel, so the error differs from pixel to pixel as the denoiser expects), darkening it as the bake darkens + // the vertex colors (LU Toolbox's Bake Lighting: the color times 1 - strength x (1 - occlusion)) + if (traced) { + const auto scene = UgcRays::Make(options.ao.rays, model.opaque); + const auto count = static_cast(std::clamp(options.denoiseSamples, 1, 256)); + for (size_t index = 0; index < color.size(); index++) { + if ((index & 0x3FF) == 0) UgcThrottle::Checkpoint(); + if (color[index].a <= 0.0f) continue; + const auto& normal = pointNormals[index]; + const glm::vec3 helper = std::abs(normal.x) < 0.9f ? glm::vec3(1, 0, 0) : glm::vec3(0, 1, 0); + const auto tangent = glm::normalize(glm::cross(helper, normal)); + const auto bitangent = glm::cross(normal, tangent); + uint32_t hash = static_cast(index) * 0x9E3779B9u; + hash ^= hash >> 16; + hash *= 0x85EBCA6Bu; + hash ^= hash >> 13; + const float turn = static_cast(hash >> 8) / 16777216.0f; + const float shift = static_cast((hash * 0xC2B2AE35u) >> 8) / 16777216.0f; + const auto origin = points[index] + normal * 1e-3f; + uint32_t open = 0; + for (uint32_t i = 0; i < count; i++) { + const float u = std::fmod((i + shift) / static_cast(count), 1.0f); + const float phi = 2.0f * 3.14159265f * std::fmod(RadicalInverse(i) + turn, 1.0f); + const float r = std::sqrt(u), up = std::sqrt(std::max(0.0f, 1.0f - u)); + const auto direction = tangent * (r * std::cos(phi)) + bitangent * (r * std::sin(phi)) + normal * up; + if (!scene->Occluded(origin, direction, 1e-4f, options.ao.distance)) open++; + } + const float occlusion = static_cast(open) / static_cast(count); + const float lit = 1.0f - std::clamp(options.bakedAo, 0.0f, 1.0f) * (1.0f - occlusion); + color[index] = glm::vec4(glm::vec3(color[index]) * lit, color[index].a); + } + } // Then transparent triangles, farthest first, blended over it { @@ -435,15 +567,67 @@ namespace UgcRender { } } - // Box filter down to the icon's size + // Box filter down to the icon's size (linear, premultiplied) const float samples = static_cast(supersample * supersample); + const auto boxFilter = [&](const auto& full) { + using Pixel = typename std::decay_t::value_type; + std::vector small(static_cast(size) * size, Pixel(0.0f)); + for (int y = 0; y < size; y++) { + for (int x = 0; x < size; x++) { + Pixel sum(0.0f); + for (int sy = 0; sy < supersample; sy++) { + for (int sx = 0; sx < supersample; sx++) sum += full[static_cast(y * supersample + sy) * n + (x * supersample + sx)]; + } + sum /= samples; + small[static_cast(y) * size + x] = sum; + } + } + return small; + }; + auto filtered = boxFilter(color); + + // Denoised at the icon's size (the supersampling has averaged the pixels already; a denoiser's time grows with + // the pixels), guided by the colors before the light and the normals of the same view, which have no noise + if (denoise) { + std::vector albedo(static_cast(n) * n, glm::vec3(0.0f)), normals(static_cast(n) * n, glm::vec3(0.0f)); + // The colors before the bake when known (traced: the model drawn) + UgcModel::Model guide = plain && !traced ? *plain : UgcModel::Model{}; + if (plain && !traced) { + guide.opaque.Transform(rotation); + guide.transparent.Transform(rotation); + } + const auto& drawn = plain && !traced ? guide : model; + std::vector guideDepth(static_cast(n) * n, INF); + for (const bool isOpaque : { true, false }) { + const auto& mesh = isOpaque ? drawn.opaque : drawn.transparent; + std::vector screen(mesh.positions.size()); + for (size_t v = 0; v < mesh.positions.size(); v++) screen[v] = project(mesh.positions[v]); + for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) { + const auto i0 = mesh.indices[i], i1 = mesh.indices[i + 1], i2 = mesh.indices[i + 2]; + Rasterize(n, n, screen[i0], screen[i1], screen[i2], [&](int x, int y, float z, float w0, float w1, float w2) { + const size_t index = static_cast(y) * n + x; + // Transparent surfaces over the opaque ones, in any order (a guide needn't be exact) + if (z >= guideDepth[index]) return; + const auto point = surface(mesh, isOpaque, i0, i1, i2, w0, w1, w2); + const glm::vec3 linear(ToLinear(point.base.r), ToLinear(point.base.g), ToLinear(point.base.b)); + if (isOpaque) { + guideDepth[index] = z; + albedo[index] = linear; + normals[index] = point.normal; + } else { + const float alpha = std::clamp(point.base.a, 0.0f, 1.0f); + albedo[index] = glm::mix(albedo[index], linear, alpha); + normals[index] = glm::normalize(glm::mix(normals[index], point.normal, alpha) + glm::vec3(1e-6f)); + } + }); + } + } + Denoise(filtered, boxFilter(albedo), boxFilter(normals), size); + } + for (int y = 0; y < size; y++) { for (int x = 0; x < size; x++) { - glm::vec4 sum(0.0f); - for (int sy = 0; sy < supersample; sy++) { - for (int sx = 0; sx < supersample; sx++) sum += color[static_cast(y * supersample + sy) * n + (x * supersample + sx)]; - } - sum /= samples; + const auto& sum = filtered[static_cast(y) * size + x]; uint8_t* out = &image.rgba[(static_cast(y) * size + x) * 4]; if (sum.a <= 0.0f) continue; // sRGB, then the contrast around its middle grey diff --git a/dUgcServer/Render/UgcRender.h b/dUgcServer/Render/UgcRender.h index 3fe243139..4a4234f83 100644 --- a/dUgcServer/Render/UgcRender.h +++ b/dUgcServer/Render/UgcRender.h @@ -73,6 +73,8 @@ namespace UgcRender { float glowEmissive{ 1.0f }; // how far glowing shapes go from lit to their plain color (the glow_emissive setting) UgcGlitter::Params glitter; // the glitter's flecks (glitter_size, glitter_density), drawn where they are at the start eDenoise denoise{}; // the finished icon denoised (denoise); off when the build can't + int denoiseSamples{ 4 }; // denoised: occlusion rays per pixel (of the supersampled image) traced on the model before its bake + float bakedAo{ 1.0f }; // denoised: the bake's strength (ao_strength; 0 when bake_ao is off), for the traced occlusion }; // The model drawn from the icon's camera, framed to fit, on a transparent background. `opaqueAo`: the opaque mesh's @@ -82,7 +84,13 @@ namespace UgcRender { // sky-and-ground reflection tinted by the color and a highlight, sharp for polished metal and broad for brushed // steel (Polished Metal, Brushed Steel: an environment map tinted by the vertex color). GLITTER vertices (opaque or // transparent) get the glitter texture's white flecks over their color before the light (LEGO-AnimUV), still. - Image RenderIcon(const UgcModel::Model& model, const IconOptions& options, const std::vector* opaqueAo = nullptr); + // With options.denoise (and a build that has it) the image is denoised by Open Image Denoise, guided by the colors + // before the light and the normals (which have no noise). A denoiser only removes noise that differs from pixel to + // pixel, not the baked occlusion's (which is per vertex), so with `plain` (the same model with the colors it had + // before its occlusion was baked in) that is drawn instead, with its occlusion traced per pixel: options. + // denoiseSamples rays (options.ao's distance, strength and ray backend) from each pixel of the supersampled image, + // noisy, then denoised. Without `plain` it is only denoised. + Image RenderIcon(const UgcModel::Model& model, const IconOptions& options, const std::vector* opaqueAo = nullptr, const UgcModel::Model* plain = nullptr); /** * The fast hidden-face test (hsr_method=fast, what the UGC server did before it traced LU Toolbox's paths): the diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index 449d07b41..592b895ed 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -180,6 +180,8 @@ namespace { settings.icon.ao.distance = settings.ao.distance; settings.icon.ao.rays = settings.hsr.rays; settings.icon.denoise = UgcRender::ParseDenoise(Game::config->GetValue("denoise")).value_or(UgcRender::eDenoise::OFF); + settings.icon.denoiseSamples = std::clamp(Setting("denoise_samples", 4), 1, 256); + settings.icon.bakedAo = settings.ao.enabled ? std::clamp(settings.ao.strength, 0.0f, 1.0f) : 0.0f; settings.maxBricks = Setting("max_model_bricks", 0); return settings; } @@ -572,7 +574,7 @@ namespace { if (!library.LoadMaterials()) std::cerr << "Couldn't read Materials.xml from " << (res / "brickdb.zip") << "; bricks will be grey\n"; UgcJobs::Outcome outcome; const auto start = std::chrono::steady_clock::now(); - const double cpuStart = UgcThrottle::ThreadCpuSeconds(); + const double cpuStart = UgcThrottle::JobCpuSeconds(); if (mode == "--make-model") { const auto data = UgcBricks::ReadFile(input); if (!data) { @@ -599,7 +601,7 @@ namespace { for (const auto& [name, data] : outcome.files) { std::ofstream(output / name, std::ios::binary).write(data.data(), static_cast(data.size())); } - const double cpuMs = (UgcThrottle::ThreadCpuSeconds() - cpuStart) * 1000.0; + const double cpuMs = (UgcThrottle::JobCpuSeconds() - cpuStart) * 1000.0; std::cout << "Made " << outcome.files.size() << " files in " << ms << " ms (" << cpuMs << " ms CPU)" << (outcome.options.empty() ? "" : " with " + outcome.options) << (outcome.note.empty() ? "" : ": " + outcome.note) << "\n"; return EXIT_SUCCESS; diff --git a/resources/ugcconfig.ini b/resources/ugcconfig.ini index bba898f4c..c171aab5e 100644 --- a/resources/ugcconfig.ini +++ b/resources/ugcconfig.ini @@ -149,8 +149,11 @@ hsr_fast_resolution=1024 # hiprt (the GPU, when built with DLU_HIPRT and the machine has one; else embree). ray_backend=builtin -# denoise: off, or oidn (icons cleaned up with Intel Open Image Denoise, when built with DLU_OIDN; else off). +# denoise: off, or oidn (when built with DLU_OIDN; else off): a model's icon is drawn from its colors before the +# occlusion bake with its occlusion traced per pixel (denoise_samples rays from each pixel of the supersampled image) +# and denoised by Intel Open Image Denoise. The model keeps its baked occlusion. denoise=off +denoise_samples=4 # Bake ambient occlusion into the vertex colors: rays per vertex, how far they look, strength 0 (none) to 1, and how # much glowing colors add diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp index 10718006a..9a012514b 100644 --- a/tests/dUgcTests/UgcTests.cpp +++ b/tests/dUgcTests/UgcTests.cpp @@ -2111,6 +2111,51 @@ TEST(UgcProcessOptions, ParseApplyAndRecord) { EXPECT_EQ(stats["settings"]["denoise"], made.denoise); } +TEST(UgcRender, DenoisedIconsTraceTheOcclusionPerPixel) { + // A box standing on a floor, grey: denoised, the icon is drawn from the model before its bake with the occlusion + // traced per pixel, so the floor is darker by the box than away from it; with few rays it is about what many give + UgcModel::Model model; + AddBox(model.opaque, glm::vec3(-4.0f, -0.2f, -4.0f), glm::vec3(4.0f, 0.0f, 4.0f)); + AddBox(model.opaque, glm::vec3(-0.5f, 0.0f, -0.5f), glm::vec3(0.5f, 1.0f, 0.5f)); + UgcRender::IconOptions options; + options.size = 48; + options.supersample = 2; + options.pitchDegrees = 60.0f; + options.shadows = 0.0f; + options.ao.distance = 2.0f; + const auto normal = UgcRender::RenderIcon(model, options); + options.denoise = UgcRender::eDenoise::OIDN; + options.denoiseSamples = 2; + const auto few = UgcRender::RenderIcon(model, options, nullptr, &model); + if (!UgcRender::Available(UgcRender::eDenoise::OIDN)) { + EXPECT_EQ(few.rgba, normal.rgba); // without it the option does nothing + GTEST_SKIP() << "built without Open Image Denoise (DLU_OIDN)"; + } + options.denoiseSamples = 64; + const auto many = UgcRender::RenderIcon(model, options, nullptr, &model); + ASSERT_EQ(few.rgba.size(), normal.rgba.size()); + double difference = 0.0, count = 0.0; + for (size_t i = 0; i < few.rgba.size(); i += 4) { + ASSERT_EQ(few.rgba[i + 3], normal.rgba[i + 3]) << i; // the same outline + if (few.rgba[i + 3] != 255) continue; + difference += std::abs(static_cast(few.rgba[i]) - many.rgba[i]); + count++; + } + EXPECT_LT(difference / count, 4.0); + // Darker than the icon without occlusion only by the box (near the middle), not out at the floor's edges + size_t darkened = 0, darkenedFar = 0; + for (int y = 0; y < 48; y++) { + for (int x = 0; x < 48; x++) { + const size_t i = (static_cast(y) * 48 + x) * 4; + if (few.rgba[i + 3] != 255 || few.rgba[i] + 30 > normal.rgba[i]) continue; + darkened++; + if (std::abs(x - 24) > 14 || std::abs(y - 24) > 14) darkenedFar++; + } + } + EXPECT_GT(darkened, 20u); + EXPECT_EQ(darkenedFar, 0u); +} + TEST(UgcRays, NamesAndFallback) { for (const auto backend : { UgcRays::eBackend::BUILTIN, UgcRays::eBackend::EMBREE, UgcRays::eBackend::HIPRT }) { EXPECT_EQ(UgcRays::Parse(UgcRays::Name(backend)), backend); diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index fb3ecf3c6..ae34080eb 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -179,3 +179,32 @@ function(dlu_fetch_embree) endforeach() endfunction() dlu_fetch_embree() + +# Intel Open Image Denoise 2 (Apache-2.0), the UGC server's denoise=oidn: optional, off by default (it brings TBB and +# large neural network weights). An installed one is used when CMake finds it, else Intel's release package (prebuilt, +# with its TBB) is downloaded and its libraries are copied next to the servers. +option(DLU_OIDN "Build the UGC server with Intel Open Image Denoise (the denoise=oidn setting)" OFF) +if(DLU_OIDN) + # Its imported targets are used by the UGC server, in another directory + set(CMAKE_FIND_PACKAGE_TARGETS_GLOBAL ON) + find_package(OpenImageDenoise 2 CONFIG QUIET) + if(NOT OpenImageDenoise_FOUND) + if(WIN32) + set(DLU_OIDN_URL "https://github.com/RenderKit/oidn/releases/download/v2.4.1/oidn-2.4.1.x64.windows.zip") + set(DLU_OIDN_SHA256 "682d94ba57525ed177d73412e0ed903f576867bd048f830a5c6f63c56b25e8b8") + set(DLU_OIDN_LIBRARIES "bin/*.dll") + elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + set(DLU_OIDN_URL "https://github.com/RenderKit/oidn/releases/download/v2.4.1/oidn-2.4.1.x86_64.linux.tar.gz") + set(DLU_OIDN_SHA256 "b69ca2443a226ef692ca46bdc4f89995b1e99091f2665b906cbe07e9673e48cc") + set(DLU_OIDN_LIBRARIES "lib/*.so*") + else() + message(FATAL_ERROR "DLU_OIDN: install Open Image Denoise 2 (no release package for this platform)") + endif() + FetchContent_Declare(oidn URL ${DLU_OIDN_URL} URL_HASH SHA256=${DLU_OIDN_SHA256} DOWNLOAD_EXTRACT_TIMESTAMP TRUE) + FetchContent_MakeAvailable(oidn) + find_package(OpenImageDenoise 2 CONFIG REQUIRED PATHS "${oidn_SOURCE_DIR}" NO_DEFAULT_PATH) + file(GLOB DLU_OIDN_FILES "${oidn_SOURCE_DIR}/${DLU_OIDN_LIBRARIES}") + file(COPY ${DLU_OIDN_FILES} DESTINATION "${CMAKE_BINARY_DIR}") + endif() + message(STATUS "Open Image Denoise ${OpenImageDenoise_VERSION}: the UGC server can denoise icons") +endif()