Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-02 10:53:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ugc): LU Toolbox worker in a headless Blender

Browse Source 
UgcToolbox runs LU Toolbox itself in an external Blender that stays up
and makes one model after another (dlu_toolbox_worker.py runs
LU-Toolbox-Standalone's steps: import, Process Model, Bake Lighting,
niftools export). JSON lines over its stdin and original stdout; it is
restarted after a crash, a timeout, 100 models or new settings, runs at
worker_nice with toolbox_threads, and its CPU time is charged to the
worker, which pauses it (SIGSTOP) while the CPU budget is overdrawn.
ProcessModelToolbox reads LU Toolbox's .nif back, counts its triangles
and draws the icon from it. Tests: option parsing, the fallback, the
protocol framing, the worker with a stand-in Blender (crash, hang,
failure, give-up), and LU Toolbox itself when DLU_TEST_TOOLBOX_* are set.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-29 16:28:30 -05:00
parent eb201d7b86
commit 6ae6eac64e
13 changed files with 1585 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
dUgcServer/CMakeLists.txt
View File

@@ -4,6 +4,7 @@
# Render/ the software rasterizer, ambient occlusion, hidden-face removal, rays, icon parameters and pose
# Formats/ the files written for the client and the dashboard (NIF, DDS, PNG, sd0)
# Processing/ the job queue and workers, the CPU budget, stored files
# Toolbox/ processor=toolbox-blender: LU Toolbox itself in a headless Blender (an external program), its worker
# UgcServer.cpp the server: settings, the master link, HTTP routes
# The pure part of the UGC server (brick data, meshes, rendering, file formats, storage), shared with the tests
@@ -22,10 +23,13 @@ set(DUGC_SOURCES
"Render/UgcIconPose.cpp"
"Render/UgcRays.cpp"
"Render/UgcRender.cpp"
"Toolbox/UgcToolbox.cpp"
"Toolbox/UgcToolboxJob.cpp"
"Toolbox/UgcToolboxProtocol.cpp"
)
add_library(dUgc STATIC ${DUGC_SOURCES})
target_include_directories(dUgc PUBLIC "." "Bricks" "Formats" "Model" "Processing" "Render"
target_include_directories(dUgc PUBLIC "." "Bricks" "Formats" "Model" "Processing" "Render" "Toolbox"
"${PROJECT_SOURCE_DIR}/thirdparty/MD5" "${PROJECT_SOURCE_DIR}/thirdparty/nlohmann")
target_link_libraries(dUgc PUBLIC dCommon glm::glm tinyxml2 MD5 PRIVATE embree)
# Intel Open Image Denoise, when built with DLU_OIDN (thirdparty/CMakeLists.txt): the denoise=oidn setting
@@ -77,6 +81,9 @@ if(DLU_HIPRT)
target_compile_definitions(dUgc PRIVATE DLU_HIPRT DLU_HIPRT_INCLUDE_DIR="${DLU_HIPRT_INCLUDE_DIR}")
endif()
# The script Blender runs for processor=toolbox-blender, next to the servers (Blender is never linked in)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/Toolbox/dlu_toolbox_worker.py" "${PROJECT_BINARY_DIR}/ugc-toolbox/dlu_toolbox_worker.py" COPYONLY)
add_executable(UgcServer "UgcServer.cpp" "Processing/UgcProcessor.cpp" "Bricks/UgcCdClient.cpp")
target_include_directories(UgcServer PRIVATE

1
dUgcServer/Processing/UgcJobs.cpp
View File

@@ -179,6 +179,7 @@ namespace UgcJobs {
void ApplyOptions(Settings& settings, const UgcProcessOptions::Choice& choice) {
if (const auto rays = UgcRays::Parse(choice.rays)) settings.ao.rays = settings.icon.ao.rays = *rays;
if (const auto denoise = UgcRender::ParseDenoise(choice.denoise)) settings.icon.denoise = *denoise;
if (UgcProcessOptions::Contains(UgcProcessOptions::PROCESSOR, choice.processor)) settings.processor = choice.processor;
}
UgcProcessOptions::Choice MadeWith(const Settings& settings) {

18
dUgcServer/Processing/UgcJobs.h
View File

@@ -16,6 +16,10 @@
#include "UgcRender.h"
#include "UgcStorage.h"
namespace UgcToolbox {
class Worker;
}
/**
* The work a UGC worker thread does for one item: everything from the stored LXFML (or a modular build's modules) to
* the finished files. No database, network or CDClient: the main thread gathers the input and stores the result.
@@ -63,6 +67,9 @@ namespace UgcJobs {
UgcRender::AoOptions ao; // Bake Lighting (AO Only)
UgcRender::IconOptions icon; // from the icon_* settings (UgcIconParams); presets and overrides go over it
uint32_t maxBricks{}; // a model with more fails; 0: no limit
// processor: what makes a player model's files, UgcProcessOptions::PROCESSOR (native: ProcessModel;
// toolbox-blender: ProcessModelToolbox, when UgcToolbox can run, else native)
std::string processor{ UgcProcessOptions::NATIVE };
};
/**
@@ -98,6 +105,17 @@ namespace UgcJobs {
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed = 0,
const UgcIconParams::Values& iconValues = {});
/**
* A player model made by LU Toolbox itself in Blender (processor=toolbox-blender, docs/UgcServer.md): its .nif is
* LU Toolbox's (import, Process Model, Bake Lighting, niftools export), checked by reading it back; the icon is
* drawn from it by the icon renderer, as a native model's is from its .nif (not denoised: there is no .nif before
* the bake). stats.json has the bricks and triangles before (the bricks' own meshes) and after (the .nif's), and
* LU Toolbox's steps' times. Blender's CPU time is charged to the calling thread (UgcThrottle::Charge), which keeps
* the CPU budget by pausing Blender. `id` names the worker's files.
*/
Outcome ProcessModelToolbox(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, UgcToolbox::Worker& toolbox,
uint64_t id, const UgcIconParams::Values& iconValues = {});
// A model's icon files (icon.png, icon.dds download) drawn from its .nif (LOD 0), its metal and glow groups by
// `tagLooks` (Shaders::TagLooks); false (and `error`) when the .nif can't be read
// (the groups drawn over others, `overlayTags` (Shaders::OverlayTags), left out). `plainNif`: its model.noao.nif, the

11
dUgcServer/Processing/UgcThrottle.cpp
View File

@@ -75,6 +75,10 @@ namespace UgcThrottle {
bool IsCancelled() { return g_Cancel; }
void Checkpoint() {
Checkpoint({});
}
void Checkpoint(const std::function<void(bool)>& pausing) {
if (g_Cancel) throw Cancelled{};
const double budget = g_Budget;
if (budget <= 0.0) return;
@@ -99,10 +103,15 @@ namespace UgcThrottle {
g_LastSleep = UnixMs();
// In short sleeps, so a cancel isn't held up by a long wait
const auto until = std::chrono::steady_clock::now() + std::chrono::duration_cast<std::chrono::steady_clock::duration>(std::chrono::duration<double>(wait));
if (pausing) pausing(true);
while (std::chrono::steady_clock::now() < until) {
if (g_Cancel) throw Cancelled{};
if (g_Cancel) {
if (pausing) pausing(false);
throw Cancelled{};
}
std::this_thread::sleep_for(std::min<std::chrono::steady_clock::duration>(until - std::chrono::steady_clock::now(), std::chrono::milliseconds(100)));
}
if (pausing) pausing(false);
// Time asleep costs no CPU; don't count this call's own bookkeeping twice
t_LastCpu = ThreadCpuSeconds();
}

5
dUgcServer/Processing/UgcThrottle.h
View File

@@ -1,6 +1,7 @@
#pragma once
#include <cstdint>
#include <functional>
#include <string>
/**
@@ -18,6 +19,10 @@ namespace UgcThrottle {
// called, so a job stops in moments when the server shuts down.
void Checkpoint();
// The same, for work done by another process for this thread (LU Toolbox's Blender, charged with Charge first):
// `pausing` is called with true before the sleep and false after it, to stop that process meanwhile
void Checkpoint(const std::function<void(bool)>& pausing);
// Thrown by Checkpoint while cancelled: the job is abandoned, not failed (its row stays waiting)
struct Cancelled {};
// Stop (true) or allow (false) every job's work: set when the server stops, cleared when it starts

504
dUgcServer/Toolbox/UgcToolbox.cpp Normal file
View File

@@ -0,0 +1,504 @@
#include "UgcToolbox.h"
#include <algorithm>
#include <cstring>
#include <deque>
#include <fstream>
#include <sstream>
#include <thread>
#include "UgcKeys.h"
#if !defined(_WIN32)
#include <csignal>
#include <fcntl.h>
#include <poll.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#if defined(__linux__)
#include <sys/prctl.h>
#endif
extern char** environ;
#endif
namespace UgcToolbox {
namespace {
bool IsFile(const std::filesystem::path& path) {
std::error_code ec;
return std::filesystem::is_regular_file(path, ec);
}
bool IsDirectory(const std::filesystem::path& path) {
std::error_code ec;
return std::filesystem::is_directory(path, ec);
}
std::string Quote(const std::filesystem::path& path) { return "\"" + path.string() + "\""; }
double Since(std::chrono::steady_clock::time_point start) {
return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
}
// How long Blender may take to be ready: the first start unpacks the brick database and links the primitives
constexpr auto START_TIMEOUT = std::chrono::seconds(180);
// How often a waiting worker looks at Blender (the CPU budget's pace)
constexpr int PACE_MS = 100;
// After this many failed starts in a row, no new start is tried for RETRY_AFTER (each make falls back meanwhile)
constexpr uint32_t MAX_FAILED_STARTS = 3;
constexpr auto RETRY_AFTER = std::chrono::minutes(10);
}
std::string Problem(const Config& config) {
#if defined(_WIN32)
(void)config;
return "toolbox-blender runs Blender as a worker on Linux (and other POSIX systems) only for now";
#else
if (config.blender.empty()) return "toolbox_blender (the Blender executable) is not set";
if (!IsFile(config.blender) || access(config.blender.c_str(), X_OK) != 0) return "toolbox_blender " + Quote(config.blender) + " is not an executable file";
if (config.standalone.empty()) return "toolbox_standalone_dir (LU-Toolbox-Standalone) is not set";
if (!IsFile(config.standalone / "lu_batch_driver.py")) return "toolbox_standalone_dir " + Quote(config.standalone) + " has no lu_batch_driver.py";
if (!config.scripts.empty()) {
for (const auto* addon : { "lu_toolbox", "io_scene_niftools" }) {
if (!IsFile(config.scripts / "addons" / addon / "__init__.py")) return "toolbox_scripts_dir " + Quote(config.scripts) + " has no addons/" + addon;
}
}
if (!IsFile(config.worker)) return "the worker script " + Quote(config.worker) + " is missing (the build copies it next to the servers)";
if (std::find(std::begin(DEVICES), std::end(DEVICES), config.device) == std::end(DEVICES)) return "toolbox_device \"" + config.device + "\" isn't cpu, cuda, optix, hip or auto";
if (config.brickdb.empty()) return "toolbox_brickdb_dir is not set";
if (!IsDirectory(config.brickdb / "Assemblies") && !IsFile(config.res / "brickdb.zip")) return "the client's res folder " + Quote(config.res) + " has no brickdb.zip for LU Toolbox's brick folder";
if (config.work.empty()) return "toolbox_work_dir is not set";
return {};
#endif
}
std::string_view Resolve(std::string_view wanted, const std::string& problem, std::string& why) {
why.clear();
if (wanted != UgcProcessOptions::TOOLBOX_BLENDER) return UgcProcessOptions::NATIVE;
if (!problem.empty()) {
why = problem;
return UgcProcessOptions::NATIVE;
}
return UgcProcessOptions::TOOLBOX_BLENDER;
}
#if defined(_WIN32)
struct Worker::Process {};
Worker::~Worker() = default;
void Worker::Configure(const Config& config) { std::lock_guard lock(m_ConfigMutex); m_Config = config; }
Config Worker::GetConfig() const { std::lock_guard lock(m_ConfigMutex); return m_Config; }
Result Worker::Make(uint64_t, const std::string&, const std::vector<uint32_t>&, const std::function<void(double, const Pause&)>&) {
return { false, Problem(GetConfig()) };
}
void Worker::Stop() {}
bool Worker::StartLocked(std::string& error) { error = Problem(m_Config); return false; }
void Worker::StopLocked(bool) {}
std::string Worker::LogTail() const { return {}; }
void Worker::PublishStatus() {}
nlohmann::json Worker::Status() const { return { { "running", false }, { "problem", Problem(GetConfig()) } }; }
#else
struct Worker::Process {
pid_t pid{ -1 };
int toWorker{ -1 }; // its stdin
int fromWorker{ -1 }; // its replies (its original stdout)
UgcToolboxProtocol::LineReader reader;
std::filesystem::path log;
double lastCpu{};
bool paused{};
int status{}; // its exit status once reaped
bool exited{};
// Its CPU time (all threads) in seconds; 0 where it can't be read
double CpuSeconds() const {
#if defined(__linux__)
std::ifstream stat("/proc/" + std::to_string(pid) + "/stat");
std::string text;
std::getline(stat, text);
// The command is in parentheses and may hold spaces; utime and stime are fields 14 and 15
const auto close = text.rfind(')');
if (close == std::string::npos) return 0.0;
std::istringstream fields(text.substr(close + 2));
std::string field;
uint64_t utime = 0, stime = 0;
for (int i = 3; i <= 15 && fields >> field; i++) {
if (i == 14) utime = std::strtoull(field.c_str(), nullptr, 10);
if (i == 15) stime = std::strtoull(field.c_str(), nullptr, 10);
}
const auto ticks = sysconf(_SC_CLK_TCK);
return ticks > 0 ? static_cast<double>(utime + stime) / static_cast<double>(ticks) : 0.0;
#else
return 0.0;
#endif
}
bool Alive() {
if (exited) return false;
const auto reaped = waitpid(pid, &status, WNOHANG);
if (reaped == pid) exited = true;
return !exited;
}
std::string ExitReason() const {
if (!exited) return "it stopped answering";
if (WIFSIGNALED(status)) return "it was killed by signal " + std::to_string(WTERMSIG(status));
if (WIFEXITED(status)) return "it exited with code " + std::to_string(WEXITSTATUS(status));
return "it stopped";
}
void Pause(bool pause) {
if (pause == paused || exited) return;
kill(pid, pause ? SIGSTOP : SIGCONT);
paused = pause;
}
bool Send(const nlohmann::json& message) {
const auto line = UgcToolboxProtocol::Frame(message);
size_t sent = 0;
while (sent < line.size()) {
// A socket, so a worker that is gone gives an error instead of SIGPIPE
const auto n = send(toWorker, line.data() + sent, line.size() - sent, MSG_NOSIGNAL);
if (n < 0 && errno == EINTR) continue;
if (n <= 0) return false;
sent += static_cast<size_t>(n);
}
return true;
}
/**
* The next message, waiting at most until `deadline`; `tick` is called every PACE_MS meanwhile. nullopt when the
* time is up, the worker is gone (`gone` set) or closed its side.
*/
std::optional<nlohmann::json> Receive(std::chrono::steady_clock::time_point deadline, bool& gone, const std::function<void()>& tick) {
gone = false;
while (true) {
while (const auto line = reader.Next()) {
if (auto message = UgcToolboxProtocol::Parse(*line)) return message;
}
if (std::chrono::steady_clock::now() >= deadline) return std::nullopt;
pollfd fd{ fromWorker, POLLIN, 0 };
const auto ready = poll(&fd, 1, PACE_MS);
if (ready > 0) {
char buffer[4096];
const auto n = read(fromWorker, buffer, sizeof(buffer));
if (n > 0) {
reader.Feed(std::string_view(buffer, static_cast<size_t>(n)));
continue;
}
if (n == 0 || (errno != EINTR && errno != EAGAIN)) {
// Its side closed: it exited (or is about to)
for (int i = 0; i < 20 && Alive(); i++) std::this_thread::sleep_for(std::chrono::milliseconds(50));
gone = true;
return std::nullopt;
}
}
if (!Alive()) {
gone = true;
return std::nullopt;
}
if (tick) tick();
}
}
void Close() {
if (toWorker >= 0) close(toWorker);
if (fromWorker >= 0) close(fromWorker);
toWorker = fromWorker = -1;
}
};
Worker::~Worker() {
std::lock_guard lock(m_JobMutex);
StopLocked(false);
}
void Worker::Configure(const Config& config) {
std::lock_guard lock(m_ConfigMutex);
m_Config = config;
}
Config Worker::GetConfig() const {
std::lock_guard lock(m_ConfigMutex);
return m_Config;
}
std::string Worker::LogTail() const {
if (!m_Process) return {};
std::ifstream file(m_Process->log, std::ios::binary);
if (!file) return {};
file.seekg(0, std::ios::end);
const auto size = static_cast<std::streamoff>(file.tellg());
file.seekg(std::max<std::streamoff>(0, size - 4096));
std::string text((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
// The last few lines with something in them
std::deque<std::string> lines;
std::istringstream stream(text);
std::string line;
while (std::getline(stream, line)) {
if (line.find_first_not_of(" \t\r") == std::string::npos) continue;
lines.push_back(line);
if (lines.size() > 6) lines.pop_front();
}
std::string out;
for (const auto& kept : lines) out += (out.empty() ? "" : " | ") + kept;
return out.substr(0, 600);
}
bool Worker::StartLocked(std::string& error) {
m_Running = GetConfig();
const auto& config = m_Running;
if (error = Problem(config); !error.empty()) return false;
std::error_code ec;
std::filesystem::create_directories(config.work, ec);
if (ec) {
error = "can't make toolbox_work_dir " + Quote(config.work) + ": " + ec.message();
return false;
}
auto process = std::make_unique<Process>();
process->log = config.work / "blender.log";
int input[2]{ -1, -1 }, output[2]{ -1, -1 };
if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, input) != 0 || socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, output) != 0) {
error = std::string("socketpair: ") + std::strerror(errno);
for (const int fd : { input[0], input[1], output[0], output[1] }) if (fd >= 0) close(fd);
return false;
}
const int logFd = open(process->log.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
std::vector<std::string> args{ config.blender.string(), "-b", "--factory-startup", "-t", std::to_string(std::max<uint32_t>(config.threads, 1)),
"--python", config.worker.string(), "--", "--standalone", config.standalone.string(), "--brickdb", config.brickdb.string(),
"--res", config.res.string(), "--device", config.device };
std::vector<char*> argv;
for (auto& arg : args) argv.push_back(arg.data());
argv.push_back(nullptr);
// The environment, with BLENDER_USER_SCRIPTS when the add-ons are in a folder of their own (made before fork: the
// child may only make async-signal-safe calls)
std::vector<std::string> environment;
for (char** entry = environ; entry && *entry; entry++) {
if (!config.scripts.empty() && std::strncmp(*entry, "BLENDER_USER_SCRIPTS=", 21) == 0) continue;
environment.emplace_back(*entry);
}
if (!config.scripts.empty()) environment.push_back("BLENDER_USER_SCRIPTS=" + config.scripts.string());
std::vector<char*> envp;
for (auto& entry : environment) envp.push_back(entry.data());
envp.push_back(nullptr);
const int nice = config.nice;
rlimit files{};
const int maxFd = getrlimit(RLIMIT_NOFILE, &files) == 0 && files.rlim_cur != RLIM_INFINITY ? static_cast<int>(std::min<rlim_t>(files.rlim_cur, 65536)) : 4096;
const pid_t pid = fork();
if (pid == 0) {
// The child: only async-signal-safe calls until exec
dup2(input[1], 0);
dup2(output[1], 1);
if (logFd >= 0) dup2(logFd, 2);
// None of the server's sockets and files (its listening port above all) go to Blender
for (int fd = 3; fd < maxFd; fd++) close(fd);
#if defined(__linux__)
// Blender goes when the UGC server goes, however it goes
prctl(PR_SET_PDEATHSIG, SIGKILL);
#endif
if (nice > 0) setpriority(PRIO_PROCESS, 0, nice);
execve(argv[0], argv.data(), envp.data());
_exit(127);
}
close(input[1]);
close(output[1]);
if (logFd >= 0) close(logFd);
if (pid < 0) {
error = std::string("fork: ") + std::strerror(errno);
close(input[0]);
close(output[0]);
return false;
}
process->pid = pid;
process->toWorker = input[0];
process->fromWorker = output[0];
m_Process = process.release();
m_Jobs = 0;
m_Starts++;
bool gone = false;
const auto deadline = std::chrono::steady_clock::now() + START_TIMEOUT;
while (true) {
const auto message = m_Process->Receive(deadline, gone, {});
if (!message) {
error = gone ? "Blender stopped while starting (" + m_Process->ExitReason() + ")" : "Blender wasn't ready in " + std::to_string(START_TIMEOUT.count()) + " s";
break;
}
const auto type = UgcToolboxProtocol::TypeOf(*message);
if (type == UgcToolboxProtocol::eType::READY) {
m_Versions = "Blender " + message->value("blender", std::string("?")) + ", LU Toolbox " + message->value("toolbox", std::string("?")) +
", niftools " + message->value("niftools", std::string("?")) + ", " + message->value("device", std::string("?"));
m_FailedStarts = 0;
return true;
}
if (type == UgcToolboxProtocol::eType::FAILED) {
error = "the worker couldn't start: " + message->value("error", std::string("no reason"));
break;
}
}
const auto tail = LogTail();
if (!tail.empty()) error += " (Blender's log: " + tail + ")";
StopLocked(true);
return false;
}
void Worker::StopLocked(bool kill) {
if (!m_Process) return;
auto& process = *m_Process;
process.Pause(false);
if (!kill && process.Alive()) {
process.Send({ { "cmd", "quit" } });
for (int i = 0; i < 100 && process.Alive(); i++) std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
if (process.Alive()) {
::kill(process.pid, SIGKILL);
waitpid(process.pid, &process.status, 0);
process.exited = true;
}
process.Close();
delete m_Process;
m_Process = nullptr;
m_Jobs = 0;
}
void Worker::Stop() {
std::lock_guard lock(m_JobMutex);
StopLocked(false);
PublishStatus();
}
void Worker::PublishStatus() {
nlohmann::json status{ { "running", m_Process != nullptr }, { "pid", m_Process ? m_Process->pid : 0 }, { "models", m_Jobs }, { "starts", m_Starts },
{ "versions", m_Versions }, { "lastError", m_LastError } };
std::lock_guard lock(m_StatusMutex);
m_Status = std::move(status);
}
nlohmann::json Worker::Status() const {
const auto config = GetConfig();
std::lock_guard lock(m_StatusMutex);
auto status = m_Status;
status["problem"] = Problem(config);
status["device"] = config.device;
status["threads"] = config.threads;
return status;
}
Result Worker::Make(uint64_t id, const std::string& lxfml, const std::vector<uint32_t>& lods, const std::function<void(double, const Pause&)>& pace) {
Result result;
const auto asked = std::chrono::steady_clock::now();
std::unique_lock lock(m_JobMutex);
// Given up while waiting for Blender (the server stopping): before starting one
if (pace) pace(0.0, [](bool) {});
const auto config = GetConfig();
// New settings, or it made its share of models: a new Blender
if (m_Process && (!(config == m_Running) || (m_Running.restartAfterJobs > 0 && m_Jobs >= m_Running.restartAfterJobs) || !m_Process->Alive())) StopLocked(false);
if (!m_Process) {
if (m_FailedStarts >= MAX_FAILED_STARTS && std::chrono::steady_clock::now() < m_RetryAfter) {
result.error = "Blender failed to start " + std::to_string(m_FailedStarts) + " times; trying again later (" + m_LastError + ")";
return result;
}
std::string error;
result.started = true;
if (!StartLocked(error)) {
m_FailedStarts++;
if (m_FailedStarts >= MAX_FAILED_STARTS) m_RetryAfter = std::chrono::steady_clock::now() + RETRY_AFTER;
m_LastError = error;
result.error = "Blender couldn't be started: " + error;
PublishStatus();
return result;
}
PublishStatus();
}
result.waitedMs = Since(asked);
result.blender = m_Versions;
auto& process = *m_Process;
const auto base = m_Running.work / ("model-" + std::to_string(id));
const auto input = base.string() + ".lxfml", output = base.string() + ".nif";
std::error_code ec;
std::filesystem::remove(output, ec);
{
std::ofstream file(input, std::ios::binary);
file.write(lxfml.data(), static_cast<std::streamsize>(lxfml.size()));
if (!file) {
result.error = "can't write " + input;
return result;
}
}
const auto cleanUp = [&] {
std::filesystem::remove(input, ec);
std::filesystem::remove(output, ec);
};
const auto requestId = m_NextId++;
if (!process.Send(UgcToolboxProtocol::MakeRequest(requestId, input, output, lods))) {
result.error = "Blender isn't taking work (" + process.ExitReason() + ")";
StopLocked(true);
cleanUp();
PublishStatus();
return result;
}
const double cpuStart = process.CpuSeconds();
process.lastCpu = cpuStart;
const Pause pause = [&process](bool paused) { process.Pause(paused); };
const auto tick = [&] {
if (!pace) return;
const double cpu = process.CpuSeconds();
const double used = std::max(0.0, cpu - process.lastCpu);
process.lastCpu = cpu;
pace(used, pause);
};
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(std::max<uint32_t>(m_Running.timeoutSeconds, 1));
try {
bool gone = false;
while (true) {
const auto message = process.Receive(deadline, gone, tick);
if (!message) {
result.blenderCpuSeconds = std::max(0.0, process.lastCpu - cpuStart);
const auto tail = LogTail();
if (gone) {
result.error = "Blender stopped while making the model (" + process.ExitReason() + ")";
} else {
result.error = "LU Toolbox took longer than toolbox_timeout_seconds (" + std::to_string(m_Running.timeoutSeconds) + " s)";
}
if (!tail.empty()) result.error += "; Blender's log: " + tail;
StopLocked(true);
break;
}
const auto done = UgcToolboxProtocol::ParseDone(*message);
if (!done || done->id != requestId) continue;
process.Pause(false);
result.blenderCpuSeconds = std::max(0.0, process.CpuSeconds() - cpuStart);
m_Jobs++;
result.ms = nlohmann::json::object();
for (const auto& [name, ms] : done->ms) result.ms[name] = ms;
if (!done->ok) {
result.error = "LU Toolbox failed: " + done->error;
break;
}
std::ifstream file(output, std::ios::binary);
result.nif.assign(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>());
if (result.nif.empty()) {
result.error = "LU Toolbox wrote no .nif";
break;
}
result.ok = true;
break;
}
} catch (...) {
// Given up (the server stopping): Blender stops too, mid-model
StopLocked(true);
cleanUp();
PublishStatus();
throw;
}
if (!result.ok) m_LastError = result.error;
cleanUp();
PublishStatus();
return result;
}
#endif
}

118
dUgcServer/Toolbox/UgcToolbox.h Normal file
View File

@@ -0,0 +1,118 @@
#pragma once
#include <chrono>
#include <cstdint>
#include <filesystem>
#include <functional>
#include <mutex>
#include <string>
#include <vector>
#include "json.hpp"
#include "UgcToolboxProtocol.h"
/**
* The processor=toolbox-blender option (docs/UgcServer.md, "LU Toolbox in Blender"): models made by LU Toolbox itself
* in a headless Blender, an external program the UGC server starts and talks to (never linked in). One Blender stays up
* and makes one model after another (dlu_toolbox_worker.py, which runs LU-Toolbox-Standalone's steps); it is started
* when first needed, again after it crashes or hangs, and after `restartAfterJobs` models. The UGC workers take turns
* on it. Linux and other POSIX systems only for now.
*/
namespace UgcToolbox {
struct Config {
std::filesystem::path blender; // toolbox_blender: the Blender executable
std::filesystem::path standalone; // toolbox_standalone_dir: LU-Toolbox-Standalone (lu_batch_driver.py)
std::filesystem::path scripts; // toolbox_scripts_dir: a Blender scripts folder whose addons/ has lu_toolbox and
// io_scene_niftools (empty: the add-ons installed in Blender's own user folder)
std::filesystem::path worker; // dlu_toolbox_worker.py (next to the servers, ugc-toolbox/)
std::filesystem::path brickdb; // toolbox_brickdb_dir: LU Toolbox's brick folder, made from `res` by the worker
std::filesystem::path res; // the client's res folder
std::filesystem::path work; // toolbox_work_dir: the LXFML and .nif of the model being made, Blender's log
std::string device{ "cpu" }; // toolbox_device: cpu, cuda, optix, hip or auto (Cycles, for the bakes)
uint32_t threads{ 4 }; // toolbox_threads: Blender's threads (-t)
uint32_t timeoutSeconds{ 1800 }; // toolbox_timeout_seconds: a model taking longer fails and Blender is restarted
uint32_t restartAfterJobs{ 100 }; // Blender is started again after this many models (0: never)
int nice{}; // worker_nice
bool operator==(const Config&) const = default;
};
inline constexpr std::string_view DEVICES[] = { "cpu", "cuda", "optix", "hip", "auto" };
/**
* Why toolbox-blender can't be used with `config` (empty: it can, as far as can be told without starting Blender): a
* missing setting, program, folder or add-on. Reads the filesystem only.
*/
std::string Problem(const Config& config);
/**
* The processor a make uses: `wanted` (UgcProcessOptions::PROCESSOR) when it can be, else native; `why` gets the
* reason when it falls back. Unknown names are native too.
*/
std::string_view Resolve(std::string_view wanted, const std::string& problem, std::string& why);
struct Result {
bool ok{};
std::string error;
std::string nif; // the .nif made
nlohmann::json ms; // the worker's steps' times: reset, import, process, bake, export
double blenderCpuSeconds{}; // Blender's CPU time for it (all its threads)
double waitedMs{}; // waiting for Blender (another model, or it starting)
bool started{}; // Blender was started for it
std::string blender; // "Blender 3.1.2, LU Toolbox 2.4.0, niftools 0.1.1, cpu"
};
/**
* The Blender worker. Make is called by the UGC workers (any thread; they take turns); Configure and Stop by the main
* thread.
*/
class Worker {
public:
Worker() = default;
~Worker();
Worker(const Worker&) = delete;
Worker& operator=(const Worker&) = delete;
// New settings: used from the next model on (Blender is started again when they changed)
void Configure(const Config& config);
Config GetConfig() const;
/**
* Makes a model's .nif from its LXFML with LU Toolbox, importing `lods`. `pace` is called about every 100 ms while
* Blender works, with the CPU seconds it used since the last call and a function that pauses (true) or resumes
* (false) it: the CPU budget (UgcThrottle) is kept through it, and it may throw to give up (the server stopping),
* which stops Blender. `id` names the files in the work folder.
*/
using Pause = std::function<void(bool)>;
Result Make(uint64_t id, const std::string& lxfml, const std::vector<uint32_t>& lods,
const std::function<void(double cpuSeconds, const Pause& pause)>& pace = {});
// Stops Blender (asks it to quit, then kills it); the next Make starts it again
void Stop();
// For /status: running, pid, models made by this Blender, starts, the versions it reported, the last error
nlohmann::json Status() const;
private:
struct Process;
bool StartLocked(std::string& error);
void StopLocked(bool kill);
std::string LogTail() const;
mutable std::mutex m_ConfigMutex;
Config m_Config;
std::mutex m_JobMutex; // one model at a time; guards everything below
Process* m_Process{};
Config m_Running; // the settings the running Blender was started with
uint64_t m_Jobs{}; // made by the running Blender
uint64_t m_Starts{};
uint64_t m_NextId{ 1 };
std::string m_Versions;
std::string m_LastError;
uint32_t m_FailedStarts{};
std::chrono::steady_clock::time_point m_RetryAfter{};
mutable std::mutex m_StatusMutex;
nlohmann::json m_Status = nlohmann::json::object();
void PublishStatus();
};
}

146
dUgcServer/Toolbox/UgcToolboxJob.cpp Normal file
View File

@@ -0,0 +1,146 @@
// UgcJobs::ProcessModelToolbox: a player model made by LU Toolbox in Blender (UgcToolbox), checked and turned into the
// files a native make writes
#include <algorithm>
#include <chrono>
#include <cmath>
#include "json.hpp"
#include "NifFile.h"
#include "UgcJobs.h"
#include "UgcModel.h"
#include "UgcThrottle.h"
#include "UgcToolbox.h"
namespace UgcJobs {
namespace {
double Since(std::chrono::steady_clock::time_point start) {
return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
}
size_t Triangles(const NifFile::Model& model) {
size_t triangles = 0;
for (const auto& mesh : model.meshes) triangles += mesh.indices.size() / 3;
return triangles;
}
}
Outcome ProcessModelToolbox(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, UgcToolbox::Worker& toolbox,
uint64_t id, const UgcIconParams::Values& iconValues) {
Outcome outcome;
const auto started = std::chrono::steady_clock::now();
const auto lxfml = LxfmlFromBlob(blob);
if (lxfml.empty()) {
outcome.error = "the stored LXFML can't be read";
return outcome;
}
std::string error;
const auto parts = UgcModel::ParseLxfml(lxfml, error);
if (parts.empty()) {
outcome.error = error;
outcome.empty = UgcModel::HasNoBricks(lxfml);
return outcome;
}
if (settings.maxBricks > 0 && parts.size() > settings.maxBricks) {
outcome.error = "the model has " + std::to_string(parts.size()) + " bricks, more than max_model_bricks (" + std::to_string(settings.maxBricks) + ")";
return outcome;
}
auto lods = settings.lods;
std::erase_if(lods, [](uint32_t lod) { return lod > 3; });
std::sort(lods.begin(), lods.end());
lods.erase(std::unique(lods.begin(), lods.end()), lods.end());
if (lods.empty()) lods.push_back(0);
// The triangles before: the bricks' own meshes, as LU Toolbox's importer reads them from the same files
nlohmann::json stats;
stats["version"] = 1;
stats["bricks"] = parts.size();
auto& lodStats = stats["lods"] = nlohmann::json::array();
auto step = std::chrono::steady_clock::now();
for (size_t i = 0; i < lods.size(); i++) {
auto options = settings.build;
options.seed = id;
options.lod = lods[i];
const auto model = UgcModel::Build(parts, library, options);
if (i == 0 && !model.missingDesigns.empty()) {
outcome.note = "no geometry for design(s)";
for (const auto design : model.missingDesigns) outcome.note += " " + std::to_string(design);
stats["missingDesigns"] = model.missingDesigns;
}
if (i == 0 && model.Empty()) {
outcome.error = "none of the model's bricks have geometry";
if (!outcome.note.empty()) outcome.error += " (" + outcome.note + ")";
return outcome;
}
lodStats.push_back({ { "lod", lods[i] }, { "opaqueBefore", model.opaque.TriangleCount() }, { "transparent", model.transparent.TriangleCount() } });
}
const double countMs = Since(step);
// LU Toolbox makes the .nif. Blender's CPU time is this thread's (the budget, the make's CPU time), and the
// budget is kept by pausing Blender.
step = std::chrono::steady_clock::now();
const auto made = toolbox.Make(id, lxfml, lods, [](double cpuSeconds, const UgcToolbox::Worker::Pause& pause) {
UgcThrottle::Charge(cpuSeconds);
UgcThrottle::Checkpoint(pause);
});
const double toolboxMs = Since(step);
if (!made.ok) {
outcome.error = made.error;
return outcome;
}
// Read back as the icon renderer and the dashboard read it: every level, for the triangles after
for (size_t i = 0; i < lods.size(); i++) {
std::string nifError;
const auto read = NifFile::Parse(made.nif, static_cast<uint32_t>(i), nifError);
if (!read) {
outcome.error = "LU Toolbox's .nif can't be read: " + nifError;
return outcome;
}
auto& entry = lodStats[i];
const auto triangles = Triangles(*read);
const auto transparent = entry.value("transparent", size_t{ 0 });
entry["opaqueAfter"] = triangles > transparent ? triangles - transparent : 0;
entry["trianglesInNif"] = triangles;
entry["shapes"] = read->meshes.size();
size_t vertices = 0;
for (const auto& mesh : read->meshes) vertices += mesh.positions.size() / 3;
entry["vertices"] = vertices;
if (i == 0 && read->meshes.empty()) {
outcome.error = "LU Toolbox's .nif has no meshes";
return outcome;
}
}
AddDownload(outcome.files, "model.nif", made.nif);
// The icon from the .nif, as a native model's (its baked light as it is: no denoising without a .nif before the bake)
const auto iconStart = std::chrono::steady_clock::now();
auto iconOptions = settings.icon;
UgcIconParams::Apply(iconOptions, iconValues);
iconOptions.denoise = UgcRender::eDenoise::OFF;
std::string nifError;
if (!IconFromNif(made.nif, iconOptions, outcome.files, nifError, settings.shaders.TagLooks(), settings.shaders.OverlayTags())) {
outcome.error = "LU Toolbox's .nif can't be read back for the icon: " + nifError;
return outcome;
}
const double iconMs = Since(iconStart);
// The steps as the native ones are named, so the comparison lines up: Process Model (hidden faces removed, and
// LU Toolbox's other preparations) as hidden surfaces, Bake Lighting as ambient occlusion
const auto part = [&made](const char* name) { return std::lround(made.ms.value(name, 0.0)); };
stats["ms"] = { { "build", part("import") }, { "hiddenSurfaces", part("process") }, { "ambientOcclusion", part("bake") }, { "export", part("export") },
{ "reset", part("reset") }, { "icon", std::lround(iconMs) }, { "count", std::lround(countMs) }, { "toolbox", std::lround(toolboxMs) },
{ "waited", std::lround(made.waitedMs) }, { "blenderCpu", std::lround(made.blenderCpuSeconds * 1000.0) }, { "total", std::lround(Since(started)) } };
const auto& config = toolbox.GetConfig();
stats["settings"] = { { "processor", std::string(UgcProcessOptions::TOOLBOX_BLENDER) }, { "toolbox", made.blender }, { "device", config.device },
{ "threads", config.threads } };
if (made.started) stats["blenderStarted"] = true;
outcome.options = std::string(UgcProcessOptions::TOOLBOX_BLENDER);
outcome.aoBaked = true;
outcome.stats = stats.dump();
outcome.files["stats.json"] = outcome.stats;
outcome.ok = true;
return outcome;
}
}

82
dUgcServer/Toolbox/UgcToolboxProtocol.cpp Normal file
View File

@@ -0,0 +1,82 @@
#include "UgcToolboxProtocol.h"
namespace UgcToolboxProtocol {
std::string Frame(nlohmann::json message) {
message["dlutb"] = VERSION;
// dump() escapes control characters, so a message never has a newline of its own
return message.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n";
}
nlohmann::json MakeRequest(uint64_t id, const std::string& input, const std::string& output, const std::vector<uint32_t>& lods) {
return { { "cmd", "make" }, { "id", id }, { "input", input }, { "output", output }, { "lods", lods } };
}
void LineReader::Feed(std::string_view data) {
// Drop what was read already before growing
if (m_Start > 0 && m_Start >= m_Buffer.size() / 2) {
m_Buffer.erase(0, m_Start);
m_Start = 0;
}
m_Buffer.append(data);
}
std::optional<std::string> LineReader::Next() {
while (true) {
const auto end = m_Buffer.find('\n', m_Start);
if (end == std::string::npos) {
// A line longer than a message can be: dropped up to its newline
if (Pending() > MAX_LINE) {
if (!m_Skipping) m_Overflows++;
m_Skipping = true;
m_Buffer.clear();
m_Start = 0;
}
return std::nullopt;
}
const auto start = m_Start;
m_Start = end + 1;
if (m_Skipping || end - start > MAX_LINE) {
if (!m_Skipping) m_Overflows++;
m_Skipping = false;
continue;
}
auto line = m_Buffer.substr(start, end - start);
if (!line.empty() && line.back() == '\r') line.pop_back();
return line;
}
}
std::optional<nlohmann::json> Parse(std::string_view line) {
auto parsed = nlohmann::json::parse(line, nullptr, false);
if (!parsed.is_object()) return std::nullopt;
const auto version = parsed.find("dlutb");
if (version == parsed.end() || !version->is_number_integer() || version->get<int>() != VERSION) return std::nullopt;
return parsed;
}
eType TypeOf(const nlohmann::json& message) {
const auto type = message.value("type", std::string());
if (type == "ready") return eType::READY;
if (type == "failed") return eType::FAILED;
if (type == "done") return eType::DONE;
if (type == "pong") return eType::PONG;
return eType::OTHER;
}
std::optional<Done> ParseDone(const nlohmann::json& message) {
if (TypeOf(message) != eType::DONE) return std::nullopt;
const auto id = message.find("id");
if (id == message.end() || !id->is_number_unsigned()) return std::nullopt;
Done done;
done.id = id->get<uint64_t>();
done.ok = message.value("ok", false);
done.error = message.contains("error") && message["error"].is_string() ? message["error"].get<std::string>() : std::string();
if (!done.ok && done.error.empty()) done.error = "the worker gave no reason";
if (const auto ms = message.find("ms"); ms != message.end() && ms->is_object()) {
for (const auto& [name, value] : ms->items()) {
if (value.is_number()) done.ms.emplace_back(name, value.get<double>());
}
}
return done;
}
}

61
dUgcServer/Toolbox/UgcToolboxProtocol.h Normal file
View File

@@ -0,0 +1,61 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
#include "json.hpp"
/**
* The messages between the UGC server and its LU Toolbox worker (dlu_toolbox_worker.py, running in Blender): one JSON
* object per line, each with "dlutb": VERSION. Pure, so the framing is tested without Blender.
*/
namespace UgcToolboxProtocol {
inline constexpr int VERSION = 1;
// A line longer than this is not a message (the worker's replies are a few hundred bytes)
inline constexpr size_t MAX_LINE = 1024 * 1024;
// A message as its line: compact JSON with "dlutb" set, and a newline
std::string Frame(nlohmann::json message);
// The request to make one model: `input` the LXFML file, `output` the .nif to write, `lods` the brickprimitives
// levels to import (LU Toolbox's importLOD0..3)
nlohmann::json MakeRequest(uint64_t id, const std::string& input, const std::string& output, const std::vector<uint32_t>& lods);
/**
* Splits what is read from the worker into lines, however it arrives (parts of lines, many at once). A line longer
* than MAX_LINE is dropped up to its newline and counted in Overflows.
*/
class LineReader {
public:
void Feed(std::string_view data);
// The next whole line without its newline (and a '\r' before it), or nullopt until one has arrived
std::optional<std::string> Next();
size_t Pending() const { return m_Buffer.size() - m_Start; }
size_t Overflows() const { return m_Overflows; }
private:
std::string m_Buffer;
size_t m_Start{};
bool m_Skipping{}; // inside a line that was too long
size_t m_Overflows{};
};
// A line as a message: a JSON object with this "dlutb" version; nullopt for anything else (blank lines too)
std::optional<nlohmann::json> Parse(std::string_view line);
enum class eType { READY, FAILED, DONE, PONG, OTHER };
eType TypeOf(const nlohmann::json& message);
// A "done" reply: whether it made the model, why not, and its steps' times in milliseconds
struct Done {
uint64_t id{};
bool ok{};
std::string error;
std::vector<std::pair<std::string, double>> ms; // reset, import, process, bake, export
};
std::optional<Done> ParseDone(const nlohmann::json& message);
}

244
dUgcServer/Toolbox/dlu_toolbox_worker.py Normal file
View File

@@ -0,0 +1,244 @@
# The UGC server's LU Toolbox worker (docs/UgcServer.md, "LU Toolbox in Blender"). Runs inside a headless Blender
# started by the UGC server and stays up, making one model after another:
#
# blender -b --factory-startup -t <threads> --python dlu_toolbox_worker.py -- \
# --standalone <LU-Toolbox-Standalone> --brickdb <folder> --res <client res> --device cpu
#
# Each model goes through LU Toolbox itself, as LU-Toolbox-Standalone's lu_batch_driver.py runs it (its functions are
# used, not copied): the LU Toolbox importer, Process Model, Bake Lighting and the niftools .nif export for LEGO
# Universe. Nothing of LU Toolbox is changed or re-implemented here; this file only keeps Blender up between models.
#
# The protocol: one JSON object per line. Requests come on stdin; replies go to the original stdout, which only this
# script writes to (everything Blender and the add-ons print is moved to stderr). Every message has "dlutb": 1.
# -> {"dlutb":1,"cmd":"make","id":7,"input":"/x/7.lxfml","output":"/x/7.nif","lods":[0,2]}
# <- {"dlutb":1,"type":"done","id":7,"ok":true,"ms":{"reset":..,"import":..,"process":..,"bake":..,"export":..}}
# <- {"dlutb":1,"type":"done","id":7,"ok":false,"error":"..."}
# -> {"dlutb":1,"cmd":"ping","id":8} <- {"dlutb":1,"type":"pong","id":8}
# -> {"dlutb":1,"cmd":"quit"} (the worker exits; it also exits when stdin closes)
# When it is ready for work it sends {"dlutb":1,"type":"ready","blender":..,"toolbox":..,"niftools":..,"device":..},
# or {"dlutb":1,"type":"failed","error":..} and exits when it can't be.
import argparse
import json
import os
import sys
import time
import traceback
import zipfile
PROTOCOL = 1
# The replies' channel is the original stdout; stdout itself now goes to stderr, so nothing else can write into it
_reply = os.fdopen(os.dup(1), "w", encoding="utf-8", newline="\n")
os.dup2(2, 1)
def send(message):
message = dict(message)
message["dlutb"] = PROTOCOL
_reply.write(json.dumps(message, separators=(",", ":")) + "\n")
_reply.flush()
def log(*parts):
print("[dlu-toolbox]", *parts, file=sys.stderr, flush=True)
def script_args():
argv = sys.argv
return argv[argv.index("--") + 1:] if "--" in argv else []
def prepare_brickdb(brickdb, res):
"""
LU Toolbox reads the brick data from a folder with the brick database unpacked (Assemblies/, Primitives/,
Materials.xml) and the client's brickprimitives/. Given the client's res folder it unpacks brickdb.zip there
itself, so a folder of its own is made instead: brickdb.zip unpacked into it and brickprimitives/ linked file by
file (LU Toolbox lists the folder without following linked folders).
"""
os.makedirs(brickdb, exist_ok=True)
if not os.path.isdir(os.path.join(brickdb, "Assemblies")):
with zipfile.ZipFile(os.path.join(res, "brickdb.zip")) as archive:
archive.extractall(brickdb)
log("unpacked brickdb.zip into", brickdb)
source = os.path.join(res, "brickprimitives")
target = os.path.join(brickdb, "brickprimitives")
if os.path.isdir(source) and not os.path.isdir(target):
linked = 0
for root, _, files in os.walk(source):
folder = os.path.join(target, os.path.relpath(root, source))
os.makedirs(folder, exist_ok=True)
for name in files:
src, dst = os.path.join(root, name), os.path.join(folder, name)
if os.path.lexists(dst):
continue
try:
os.symlink(src, dst)
except OSError:
import shutil
shutil.copyfile(src, dst)
linked += 1
log("linked", linked, "brickprimitives files into", target)
def set_device(driver, device):
import bpy
if device == "hip":
# lu_batch_driver's device choice knows CPU, CUDA and OptiX; AMD GPUs through HIP the same way
try:
bpy.ops.preferences.addon_enable(module="cycles")
except Exception:
pass
cp = bpy.context.preferences.addons["cycles"].preferences
try:
cp.compute_device_type = "HIP"
driver._refresh_cycles_devices(cp)
found = False
for d in cp.devices:
d.use = d.type in ("HIP", "CPU")
found = found or d.type == "HIP"
if found:
bpy.context.scene.cycles.device = "GPU"
return "hip"
except Exception as ex:
log("HIP unavailable:", ex)
bpy.context.scene.cycles.device = "CPU"
return "cpu"
if device == "auto":
return driver.set_cycles_device_auto()
return driver.set_cycles_device_forced(device)
def addon_version(name):
import addon_utils
for module in addon_utils.modules():
if module.__name__ == name:
return ".".join(str(part) for part in module.bl_info.get("version", ()))
return ""
class Worker:
def __init__(self, args):
self.args = args
sys.path.insert(0, args.standalone)
import lu_batch_driver # LU-Toolbox-Standalone's driver: its steps, run as it runs them
self.driver = lu_batch_driver
self.device = "cpu"
def setup(self):
"""What lu_batch_driver's main does before importing, for a scene just loaded"""
import bpy
for module in ("lu_toolbox", "io_scene_niftools"):
try:
bpy.ops.preferences.addon_enable(module=module)
except Exception as ex:
log("enabling", module, "failed:", ex)
if "lu_toolbox" not in bpy.context.preferences.addons:
raise RuntimeError("the lu_toolbox add-on can't be enabled (is it in the Blender scripts folder's addons?)")
if "io_scene_niftools" not in bpy.context.preferences.addons:
raise RuntimeError("the io_scene_niftools add-on can't be enabled (is it in the Blender scripts folder's addons?)")
bpy.context.preferences.addons["lu_toolbox"].preferences.brickdbpath = self.args.brickdb
self.driver._apply_headless_patches()
self.device = set_device(self.driver, self.args.device)
self.driver.set_lu_gpu_flags(use_gpu=(self.device != "cpu"))
def reset(self):
"""Every model starts from a fresh scene, as a Blender started for it would"""
import bpy
# niftools keeps the last export operator to report through, which is gone once it has finished; a second
# export (or enabling the add-on again) would report through it and fail. Its default reporter instead.
try:
from io_scene_niftools.utils import logging as nif_logging
nif_logging.NifLog.op = nif_logging._MockOperator()
except Exception:
pass
bpy.ops.wm.read_factory_settings(use_empty=False)
self.setup()
def make(self, request):
ms = {}
step = time.perf_counter()
def lap(name):
nonlocal step
now = time.perf_counter()
ms[name] = round((now - step) * 1000)
step = now
self.reset()
lap("reset")
lods = request.get("lods") or []
lod_kwargs = {f"importLOD{n}": n in lods for n in range(4)} if lods else None
self.driver.try_import_lxf(os.path.abspath(request["input"]), op_override=None, lod_kwargs=lod_kwargs)
lap("import")
self.driver.call_op("lutb.process_model", "Process Model")
lap("process")
self.driver.ensure_vertex_colors_exist()
self.driver.call_op("lutb.bake_lighting", "Bake Lighting")
lap("bake")
output = os.path.abspath(request["output"])
os.makedirs(os.path.dirname(output), exist_ok=True)
self.driver.set_niftools_game_to_lu()
self.driver.export_nif(output)
lap("export")
if not os.path.isfile(output):
raise RuntimeError("the .nif export wrote nothing")
return ms
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--standalone", required=True)
parser.add_argument("--brickdb", required=True)
parser.add_argument("--res", default="")
parser.add_argument("--device", default="cpu", choices=["auto", "cpu", "cuda", "optix", "hip"])
args = parser.parse_args(script_args())
try:
import bpy
if args.res:
prepare_brickdb(args.brickdb, args.res)
worker = Worker(args)
worker.setup()
send({"type": "ready", "blender": bpy.app.version_string, "toolbox": addon_version("lu_toolbox"),
"niftools": addon_version("io_scene_niftools"), "device": worker.device})
except Exception as ex:
traceback.print_exc()
send({"type": "failed", "error": f"{type(ex).__name__}: {ex}"})
return 2
for line in sys.stdin:
line = line.strip()
if not line:
continue
try:
request = json.loads(line)
except ValueError:
log("not JSON:", line[:200])
continue
if not isinstance(request, dict) or request.get("dlutb") != PROTOCOL:
log("unknown message:", line[:200])
continue
cmd = request.get("cmd")
if cmd == "quit":
break
if cmd == "ping":
send({"type": "pong", "id": request.get("id")})
continue
if cmd != "make":
send({"type": "done", "id": request.get("id"), "ok": False, "error": f"unknown command {cmd}"})
continue
log("making", request.get("input"))
try:
ms = worker.make(request)
send({"type": "done", "id": request.get("id"), "ok": True, "ms": ms})
except BaseException as ex: # SystemExit too: the driver's helpers exit on failures
traceback.print_exc()
send({"type": "done", "id": request.get("id"), "ok": False, "error": f"{type(ex).__name__}: {ex}"})
return 0
if __name__ == "__main__":
code = main()
_reply.close()
# Blender would go on to its own shutdown, which is slow with add-ons loaded and not needed
os._exit(code)