Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-02 02:43:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ugc): processor=toolbox-blender in the UGC server

Browse Source 
The processor setting and toolbox_* settings (Blender, LU-Toolbox-
Standalone, add-ons folder, brick and work folders, device, threads,
timeout). Models asking for toolbox-blender are made by the Blender
worker when it can run, else natively, logged at start, when it changes
and per make. /status has the worker's state; --make-model takes
toolbox-blender too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-29 16:28:36 -05:00
parent 6ae6eac64e
commit 889cc79e94
4 changed files with 134 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
dUgcServer/Processing/UgcProcessor.cpp
View File

@@ -152,6 +152,20 @@ void UgcProcessor::Stop() {
if (thread.joinable()) thread.join();
}
m_Threads.clear(); if (m_FileThread.joinable()) m_FileThread.join();
// LU Toolbox's Blender goes with the workers
m_Toolbox.Stop();
}
void UgcProcessor::ConfigureToolbox(const UgcToolbox::Config& config) {
m_Toolbox.Configure(config);
auto problem = UgcToolbox::Problem(config);
std::lock_guard lock(m_Mutex);
m_ToolboxProblem = std::move(problem);
}
std::string UgcProcessor::ToolboxProblem() const {
std::lock_guard lock(m_Mutex);
return m_ToolboxProblem;
}
void UgcProcessor::Drain() {
@@ -188,6 +202,7 @@ void UgcProcessor::Worker() {
while (true) {
Job job;
UgcJobs::Settings settings;
std::string toolboxProblem;
int nice = 0;
{
std::unique_lock lock(m_Mutex);
@@ -213,6 +228,7 @@ void UgcProcessor::Worker() {
m_MemoryInUse += job.memory;
settings = m_Settings;
nice = m_Limits.nice;
toolboxProblem = m_ToolboxProblem;
}
if (nice != appliedNice) {
ApplyNice(nice);
@@ -304,9 +320,19 @@ void UgcProcessor::Worker() {
// The processing options staff picked for this make, over the settings (unknown ones: the settings')
UgcProcessOptions::Choice choice;
if (job.kind == Kind::MODEL && UgcProcessOptions::Parse(job.options, choice)) UgcJobs::ApplyOptions(settings, choice);
done.outcome = job.kind == Kind::MODEL
? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast<uint64_t>(job.id), job.iconValues)
: UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings);
// LU Toolbox in Blender when it's asked for and can run, else the UGC server's own pipeline (and why)
const bool toolbox = job.kind == Kind::MODEL &&
UgcToolbox::Resolve(settings.processor, toolboxProblem, done.fallback) == UgcProcessOptions::TOOLBOX_BLENDER;
if (toolbox) {
done.outcome = UgcJobs::ProcessModelToolbox(job.blob, m_Library, settings, m_Toolbox, static_cast<uint64_t>(job.id), job.iconValues);
} else if (job.kind == Kind::MODEL) {
done.outcome = UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast<uint64_t>(job.id), job.iconValues);
if (!done.fallback.empty()) {
done.outcome.note = "toolbox-blender can't be used (" + done.fallback + "): made natively" + (done.outcome.note.empty() ? "" : "; " + done.outcome.note);
}
} else {
done.outcome = UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings);
}
}
} catch (const UgcThrottle::Cancelled&) {
// Stopping: abandoned, not failed; nothing is written or recorded
@@ -498,6 +524,9 @@ void UgcProcessor::Record(const Done& done) {
if (const auto combo = m_ComboOf.find(done.id); combo != m_ComboOf.end()) Database::Get()->SetModularBuildCombination(done.id, combo->second);
}
m_Recent.erase({ done.kind, done.id });
if (!done.fallback.empty()) {
LOG("Model %llu asked for toolbox-blender, which can't be used (%s): made natively", static_cast<unsigned long long>(done.id), done.fallback.c_str());
}
if (done.outcome.empty) {
m_Empty++;
@@ -727,6 +756,8 @@ nlohmann::json UgcProcessor::Status() const {
status["evicted"] = m_Evicted;
status["storedBytes"] = m_StoredBytes;
status["maxStorageBytes"] = m_Config.maxStorageBytes;
// LU Toolbox's Blender (processor=toolbox-blender): whether it runs, what it reported, why it can't be used
status["toolbox"] = m_Toolbox.Status();
auto& recent = status["recent"] = nlohmann::json::array();
for (auto it = m_Log.rbegin(); it != m_Log.rend(); ++it) {
recent.push_back({ { "kind", KindName(it->kind) }, { "id", std::to_string(it->id) }, { "ok", it->ok },

10
dUgcServer/Processing/UgcProcessor.h
View File

@@ -21,6 +21,7 @@
#include "UgcIconParams.h"
#include "UgcJobs.h"
#include "UgcStorage.h"
#include "UgcToolbox.h"
/**
* The UGC server's queue. The main thread (Update) takes pending rows from the database, gathers each one's input
@@ -58,6 +59,12 @@ public:
// Main thread: new settings and limits (config reload)
void Configure(UgcJobs::Settings settings, Limits limits);
// Main thread: LU Toolbox's Blender settings (processor=toolbox-blender); a running Blender is started again with
// them before its next model
void ConfigureToolbox(const UgcToolbox::Config& config);
// Why toolbox-blender can't be used with the settings given (empty: it can)
std::string ToolboxProblem() const;
// Waits for the jobs that are running; queued ones are dropped (they stay pending in the database)
void Stop();
@@ -200,6 +207,7 @@ private:
bool iconOnly{};
double iconChangeMs{}; // an icon drawn again: how much longer it took than the icon made before
std::vector<Checksum> checksums; // of the files written that the client downloads as sd0
std::string fallback; // toolbox-blender was asked for but can't be used: why (made natively)
};
// Main thread: records the checksums of an item's files (ugc_file_checksums)
@@ -229,6 +237,8 @@ private:
UgcBricks::BrickLibrary& m_Library;
UgcJobs::Settings m_Settings; // guarded by m_Mutex (workers copy it per job)
Limits m_Limits; // guarded by m_Mutex
UgcToolbox::Worker m_Toolbox; // LU Toolbox's Blender, which the workers take turns on
std::string m_ToolboxProblem{ "not configured" }; // guarded by m_Mutex: why toolbox-blender can't be used (UgcToolbox::Problem)
mutable std::mutex m_Mutex;
std::condition_variable m_Wake;

71
dUgcServer/UgcServer.cpp
View File

@@ -181,9 +181,40 @@ namespace {
settings.icon.denoiseSamples = std::clamp(Setting<int32_t>("denoise_samples", 4), 1, 256);
settings.icon.bakedAo = settings.ao.enabled ? std::clamp(settings.ao.strength, 0.0f, 1.0f) : 0.0f;
settings.maxBricks = Setting<uint32_t>("max_model_bricks", 0);
// What makes the models: native, or toolbox-blender (LU Toolbox in Blender, when it can run: ReadToolboxConfig)
const auto processorName = Game::config->GetValue("processor");
if (UgcProcessOptions::Contains(UgcProcessOptions::PROCESSOR, processorName)) settings.processor = processorName;
return settings;
}
// A path setting: relative ones are next to the server binaries; empty is `fallback` (empty: none)
std::filesystem::path PathSetting(const std::string& key, const std::string& fallback = "") {
const auto value = Game::config->GetValue(key).empty() ? fallback : Game::config->GetValue(key);
if (value.empty()) return {};
std::filesystem::path path(value);
return path.is_relative() ? BinaryPathFinder::GetBinaryDir() / path : path;
}
std::filesystem::path ResPath();
// LU Toolbox in a headless Blender (processor=toolbox-blender, docs/UgcServer.md "LU Toolbox in Blender"): the
// external programs and folders it needs
UgcToolbox::Config ReadToolboxConfig() {
UgcToolbox::Config config;
config.blender = PathSetting("toolbox_blender");
config.standalone = PathSetting("toolbox_standalone_dir");
config.scripts = PathSetting("toolbox_scripts_dir");
config.worker = BinaryPathFinder::GetBinaryDir() / "ugc-toolbox" / "dlu_toolbox_worker.py";
config.brickdb = PathSetting("toolbox_brickdb_dir", "toolbox-brickdb");
config.work = PathSetting("toolbox_work_dir", "toolbox-work");
config.res = ResPath();
if (!Game::config->GetValue("toolbox_device").empty()) config.device = Game::config->GetValue("toolbox_device");
config.threads = std::clamp(Setting<uint32_t>("toolbox_threads", 4), 1u, 256u);
config.timeoutSeconds = std::clamp(Setting<uint32_t>("toolbox_timeout_seconds", 1800), 10u, 86400u);
config.nice = std::clamp(Setting<int32_t>("worker_nice", 0), 0, 19);
return config;
}
// The processing options the settings pick, and what is used instead when this build or machine can't
// (main thread: the GPU is set up here the first time)
void LogProcessingOptions(const UgcJobs::Settings& settings) {
@@ -195,6 +226,21 @@ namespace {
if (!UgcRender::Available(settings.icon.denoise)) LOG("denoise=%s can't be used (the server was built without DLU_OIDN): off instead", std::string(UgcRender::Name(settings.icon.denoise)).c_str());
}
// Whether LU Toolbox in Blender can be used, logged at start and when it changes (and the processor setting's
// fallback): every make that asks for it and can't have it logs why too
void LogToolbox(const UgcJobs::Settings& settings, const std::string& problem) {
static std::optional<std::string> logged;
if (logged && *logged == problem + settings.processor) return;
logged = problem + settings.processor;
if (problem.empty()) {
LOG("toolbox-blender can be used%s", settings.processor == UgcProcessOptions::TOOLBOX_BLENDER ? " and makes the models (processor=toolbox-blender)" : "");
} else if (settings.processor == UgcProcessOptions::TOOLBOX_BLENDER) {
LOG("processor=toolbox-blender can't be used (%s): models are made natively", problem.c_str());
} else {
LOG("toolbox-blender can't be used (%s)", problem.c_str());
}
}
UgcProcessor::Limits ReadLimits() {
UgcProcessor::Limits limits;
const auto cores = std::max<unsigned>(std::thread::hardware_concurrency(), 1);
@@ -575,7 +621,7 @@ namespace {
auto settings = ReadSettings();
UgcProcessOptions::Choice choice;
if (!UgcProcessOptions::Parse(options, choice)) {
std::cerr << "Unknown processing options \"" << options << "\" (ray backend embree, hiprt or embree-gpu; denoise off or oidn)\n";
std::cerr << "Unknown processing options \"" << options << "\" (ray backend embree, hiprt or embree-gpu; denoise off or oidn; processor native or toolbox-blender)\n";
return EXIT_FAILURE;
}
UgcJobs::ApplyOptions(settings, choice);
@@ -593,7 +639,19 @@ namespace {
std::cerr << "Can't read " << input << "\n";
return EXIT_FAILURE;
}
outcome = UgcJobs::ProcessModel(*data, library, settings);
// LU Toolbox in Blender when asked for and it can run (Blender is started for this model and stopped after)
UgcToolbox::Worker toolbox;
toolbox.Configure(ReadToolboxConfig());
std::string why;
if (UgcToolbox::Resolve(settings.processor, UgcToolbox::Problem(toolbox.GetConfig()), why) == UgcProcessOptions::TOOLBOX_BLENDER) {
// A file name has no object id: its name hashed names Blender's files
outcome = UgcJobs::ProcessModelToolbox(*data, library, settings, toolbox, UgcModularKey::StorageId(input));
toolbox.Stop();
// Blender's CPU time is charged to this thread (JobCpuSeconds below)
} else {
if (!why.empty()) std::cerr << "toolbox-blender can't be used (" << why << "): made natively\n";
outcome = UgcJobs::ProcessModel(*data, library, settings);
}
} else {
CDClientDatabase::Connect((BinaryPathFinder::GetBinaryDir() / "resServer/CDServer.sqlite").string());
UgcJobs::ModularInput modular;
@@ -633,7 +691,7 @@ int main(int argc, char** argv) {
Game::config = new dConfig("ugcconfig.ini");
// UgcServer --make-model <file.lxfml> <folder> [options] or --make-modular "1:4713+1:4714+1:4715" <folder> [options];
// options: processing options over the settings (UgcProcessOptions), e.g. embree fast
// options: processing options over the settings (UgcProcessOptions), e.g. embree oidn, or toolbox-blender
if (argc >= 4 && (std::string(argv[1]) == "--make-model" || std::string(argv[1]) == "--make-modular")) {
std::string options;
for (int i = 4; i < argc; i++) options += std::string(options.empty() ? "" : " ") + argv[i];
@@ -704,7 +762,9 @@ int main(int argc, char** argv) {
processorConfig.threads = threads > 0 ? threads : std::max<size_t>(std::thread::hardware_concurrency() / 2, 1);
UgcProcessor processor(processorConfig, storage, library, ReadSettings());
processor.Configure(ReadSettings(), ReadLimits());
processor.ConfigureToolbox(ReadToolboxConfig());
LogProcessingOptions(ReadSettings());
LogToolbox(ReadSettings(), processor.ToolboxProblem());
g_Processor = &processor;
// Sent with the traffic reports to the dashboard (Diagnostics)
TrafficStats::Local().SetGauge("workers_busy", [&processor] { return static_cast<double>(processor.Busy()); });
@@ -772,7 +832,10 @@ int main(int argc, char** argv) {
// Settings the dashboard changed arrive as a config reload; pick them up
if (now - lastConfigure >= std::chrono::seconds(5)) {
lastConfigure = now;
processor.Configure(ReadSettings(), ReadLimits());
const auto settings = ReadSettings();
processor.Configure(settings, ReadLimits());
processor.ConfigureToolbox(ReadToolboxConfig());
LogToolbox(settings, processor.ToolboxProblem());
}
}

23
resources/ugcconfig.ini
View File

@@ -149,6 +149,29 @@ embree_gpu_device=0
denoise=off
denoise_samples=4
# What makes the models: native (the UGC server's own pipeline) or toolbox-blender (LU Toolbox itself in a headless
# Blender, an external program the UGC server starts and keeps running; far slower, one model at a time). Staff can pick
# either for one make. Without Blender, LU Toolbox or niftools (the settings below) toolbox-blender can't be used: models
# are made natively and the UGC server logs why (docs/UgcServer.md, "LU Toolbox in Blender").
# toolbox_blender: the Blender executable (LU Toolbox is made for Blender 3.1).
# toolbox_standalone_dir: the LU-Toolbox-Standalone folder (lu_batch_driver.py).
# toolbox_scripts_dir: a Blender scripts folder whose addons/ has lu_toolbox and io_scene_niftools (empty: the add-ons
# installed in Blender's own user folder).
# toolbox_brickdb_dir, toolbox_work_dir: LU Toolbox's brick folder (made from the client's res the first time) and the
# folder of the model being made and Blender's log, relative to the server binaries.
# toolbox_device: what Cycles bakes on: cpu, cuda, optix, hip or auto. toolbox_threads: Blender's CPU threads (its CPU
# time counts against max_cpu_percent, which pauses it; worker_nice applies to it). toolbox_timeout_seconds: a model
# taking longer fails and Blender is started again.
processor=native
toolbox_blender=
toolbox_standalone_dir=
toolbox_scripts_dir=
toolbox_brickdb_dir=toolbox-brickdb
toolbox_work_dir=toolbox-work
toolbox_device=cpu
toolbox_threads=4
toolbox_timeout_seconds=1800
# Bake ambient occlusion into the vertex colors: rays per vertex, how far they look, strength 0 (none) to 1, and how
# much glowing colors add
bake_ao=1