feat(ugc): models with no bricks are "empty", not failed

A model with no bricks has nothing to make: it gets its own state
(is_optimized = 3), isn't counted as a failure or retried, shows as Empty
on the UGC page (with its own filter and count), in the status and in
Prometheus, and its downloads answer 404 like HKX. State names come from
the enum (magic_enum). Migrations dlu/mysql/87 and dlu/sqlite/70 move the
rows that failed only because they had no bricks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-27 14:52:03 -05:00
parent 585839a539
commit 1309000460
17 changed files with 90 additions and 29 deletions

View File

@@ -223,10 +223,9 @@ namespace {
ok &= Timed(w, "ugc", [&] { ok &= Timed(w, "ugc", [&] {
w.Declare("darkflame_ugc_items", "Player models (model) and cars and rockets (modular) by UGC processing state", "gauge"); w.Declare("darkflame_ugc_items", "Player models (model) and cars and rockets (modular) by UGC processing state", "gauge");
const auto add = [&](const char* kind, const std::vector<std::pair<IUgc::eProcessState, uint64_t>>& rows) { const auto add = [&](const char* kind, const std::vector<std::pair<IUgc::eProcessState, uint64_t>>& rows) {
std::map<std::string, uint64_t> states{ {"pending", 0}, {"done", 0}, {"failed", 0} }; std::map<std::string, uint64_t> states;
for (const auto& [state, count] : rows) { for (const auto state : magic_enum::enum_values<IUgc::eProcessState>()) states[IUgc::ProcessStateName(state)] = 0;
states[state == IUgc::eProcessState::PENDING ? "pending" : state == IUgc::eProcessState::DONE ? "done" : "failed"] += count; for (const auto& [state, count] : rows) states[IUgc::ProcessStateName(state)] += count;
}
for (const auto& [state, count] : states) w.Add("darkflame_ugc_items", "", "gauge", { {"kind", kind}, {"state", state} }, static_cast<double>(count)); for (const auto& [state, count] : states) w.Add("darkflame_ugc_items", "", "gauge", { {"kind", kind}, {"state", state} }, static_cast<double>(count));
}; };
add("model", db->GetUgcProcessCounts()); add("model", db->GetUgcProcessCounts());

View File

@@ -65,9 +65,10 @@ namespace {
// The UGC server's queue in the database (its work list), from the model and modular build tables // The UGC server's queue in the database (its work list), from the model and modular build tables
nlohmann::json UgcCounts() { nlohmann::json UgcCounts() {
const auto counts = [](const std::vector<std::pair<IUgc::eProcessState, uint64_t>>& rows) { const auto counts = [](const std::vector<std::pair<IUgc::eProcessState, uint64_t>>& rows) {
nlohmann::json out = { {"pending", 0}, {"done", 0}, {"failed", 0} }; nlohmann::json out = nlohmann::json::object();
for (const auto state : magic_enum::enum_values<IUgc::eProcessState>()) out[IUgc::ProcessStateName(state)] = 0;
for (const auto& [state, count] : rows) { for (const auto& [state, count] : rows) {
out[state == IUgc::eProcessState::PENDING ? "pending" : state == IUgc::eProcessState::DONE ? "done" : "failed"] = count; out[IUgc::ProcessStateName(state)] = count;
} }
return out; return out;
}; };

View File

@@ -28,21 +28,10 @@ using namespace RouteUtils;
namespace { namespace {
constexpr uint32_t PAGE_SIZE = 50; constexpr uint32_t PAGE_SIZE = 50;
const char* StateName(IUgc::eProcessState state) { std::string StateName(IUgc::eProcessState state) { return IUgc::ProcessStateName(state); }
switch (state) {
case IUgc::eProcessState::PENDING: return "pending"; std::optional<IUgc::eProcessState> ParseState(const std::string& text) { return IUgc::ParseProcessState(text); }
case IUgc::eProcessState::DONE: return "done";
case IUgc::eProcessState::FAILED: return "failed";
}
return "unknown";
}
std::optional<IUgc::eProcessState> ParseState(const std::string& text) {
if (text == "pending") return IUgc::eProcessState::PENDING;
if (text == "done") return IUgc::eProcessState::DONE;
if (text == "failed") return IUgc::eProcessState::FAILED;
return std::nullopt;
}
/** /**
* The dashboard's own way to the UGC server (ugc_internal_url, normally the same machine): its status and the files * The dashboard's own way to the UGC server (ugc_internal_url, normally the same machine): its status and the files
@@ -235,7 +224,8 @@ namespace {
} }
nlohmann::json Counts(const std::vector<std::pair<IUgc::eProcessState, uint64_t>>& counts) { nlohmann::json Counts(const std::vector<std::pair<IUgc::eProcessState, uint64_t>>& counts) {
nlohmann::json out = { { "pending", 0 }, { "done", 0 }, { "failed", 0 } }; nlohmann::json out = nlohmann::json::object();
for (const auto state : magic_enum::enum_values<IUgc::eProcessState>()) out[IUgc::ProcessStateName(state)] = 0;
for (const auto& [state, count] : counts) out[StateName(state)] = count; for (const auto& [state, count] : counts) out[StateName(state)] = count;
return out; return out;
} }

View File

@@ -9,7 +9,7 @@
var kind = 'model', state = '', search = '', page = 0, view = 'gallery', canManage = false, publicUrl = ''; var kind = 'model', state = '', search = '', page = 0, view = 'gallery', canManage = false, publicUrl = '';
var nifViewer = null, lxfmlViewer = null, current = null; var nifViewer = null, lxfmlViewer = null, current = null;
var STATES = { pending: ['Waiting', 'secondary'], done: ['Made', 'success'], failed: ['Failed', 'danger'] }; var STATES = { pending: ['Waiting', 'secondary'], done: ['Made', 'success'], failed: ['Failed', 'danger'], empty: ['Empty', 'light'] };
var PAGE_SIZE = { gallery: 48, list: 50 }; var PAGE_SIZE = { gallery: 48, list: 50 };
try { view = localStorage.getItem('ugcView') === 'list' ? 'list' : 'gallery'; } catch (e) { /* storage blocked */ } try { view = localStorage.getItem('ugcView') === 'list' ? 'list' : 'gallery'; } catch (e) { /* storage blocked */ }
@@ -25,7 +25,7 @@
function countCard(title, c) { function countCard(title, c) {
return '<div class="col-md-6"><div class="card"><div class="card-body py-2"><div class="small text-body-secondary">' + esc(title) + '</div>' + return '<div class="col-md-6"><div class="card"><div class="card-body py-2"><div class="small text-body-secondary">' + esc(title) + '</div>' +
fmt.badge(c.done + ' made', 'success') + ' ' + fmt.badge(c.pending + ' waiting', 'secondary') + ' ' + fmt.badge(c.failed + ' failed', c.failed ? 'danger' : 'secondary') + fmt.badge(c.done + ' made', 'success') + ' ' + fmt.badge(c.pending + ' waiting', 'secondary') + ' ' + fmt.badge(c.failed + ' failed', c.failed ? 'danger' : 'secondary') +
'</div></div></div>'; (c.empty ? ' ' + fmt.badge(c.empty + ' empty', 'light') : '') + '</div></div></div>';
} }
function owner(i) { function owner(i) {

View File

@@ -41,7 +41,7 @@
<button type="button" class="btn btn-outline-primary" data-kind="modular">Cars and rockets</button> <button type="button" class="btn btn-outline-primary" data-kind="modular">Cars and rockets</button>
</div> </div>
<select class="form-select form-select-sm w-auto" id="stateFilter" aria-label="State"> <select class="form-select form-select-sm w-auto" id="stateFilter" aria-label="State">
<option value="">Every state</option><option value="pending">Waiting</option><option value="done">Made</option><option value="failed">Failed</option> <option value="">Every state</option><option value="pending">Waiting</option><option value="done">Made</option><option value="failed">Failed</option><option value="empty">Empty (no bricks)</option>
</select> </select>
<input type="search" class="form-control form-control-sm w-auto" id="search" placeholder="ID or owner" aria-label="Search by ID or owner"> <input type="search" class="form-control form-control-sm w-auto" id="search" placeholder="ID or owner" aria-label="Search by ID or owner">
<div class="btn-group btn-group-sm" role="group" id="viewButtons"> <div class="btn-group btn-group-sm" role="group" id="viewButtons">

View File

@@ -8,6 +8,8 @@
#include <string_view> #include <string_view>
#include <vector> #include <vector>
#include "magic_enum.hpp"
class IUgc { class IUgc {
public: public:
struct Model { struct Model {
@@ -40,8 +42,23 @@ public:
PENDING = 0, // not made yet (or to be made again) PENDING = 0, // not made yet (or to be made again)
DONE = 1, // the UGC server's files are made DONE = 1, // the UGC server's files are made
FAILED = 2, // gave up after the allowed attempts FAILED = 2, // gave up after the allowed attempts
EMPTY = 3, // nothing to make: the model has no bricks (not a failure; never retried)
}; };
// A state's name for the dashboard, the API and metrics ("pending", "done", "failed", "empty")
static std::string ProcessStateName(eProcessState state) {
std::string name(magic_enum::enum_name(state));
for (auto& c : name) c = static_cast<char>(c >= 'A' && c <= 'Z' ? c - 'A' + 'a' : c);
return name.empty() ? "unknown" : name;
}
static std::optional<eProcessState> ParseProcessState(std::string_view text) {
for (const auto state : magic_enum::enum_values<eProcessState>()) {
if (ProcessStateName(state) == text) return state;
}
return std::nullopt;
}
// A row's processing state, for the UGC server and the dashboard // A row's processing state, for the UGC server and the dashboard
struct ProcessInfo { struct ProcessInfo {
LWOOBJID id{}; LWOOBJID id{};

View File

@@ -86,6 +86,8 @@ namespace UgcJobs {
const auto parts = UgcModel::ParseLxfml(lxfml, error); const auto parts = UgcModel::ParseLxfml(lxfml, error);
if (parts.empty()) { if (parts.empty()) {
outcome.error = error; outcome.error = error;
// A model with no bricks has nothing to make; the LXFML itself is still served
outcome.empty = UgcModel::HasNoBricks(lxfml);
return outcome; return outcome;
} }
if (settings.maxBricks > 0 && parts.size() > settings.maxBricks) { if (settings.maxBricks > 0 && parts.size() > settings.maxBricks) {

View File

@@ -40,6 +40,7 @@ namespace UgcJobs {
std::string note; // what was odd but not fatal (missing bricks, ...) std::string note; // what was odd but not fatal (missing bricks, ...)
UgcStorage::Files files; // name -> bytes, when ok UgcStorage::Files files; // name -> bytes, when ok
bool aoBaked{}; bool aoBaked{};
bool empty{}; // nothing to make (no bricks): not a failure
std::string stats; // stats.json: bricks, triangles before and after per LOD, timings std::string stats; // stats.json: bricks, triangles before and after per LOD, timings
}; };

View File

@@ -142,6 +142,13 @@ namespace UgcModel {
return parts; return parts;
} }
bool HasNoBricks(std::string_view lxfml) {
tinyxml2::XMLDocument doc;
if (doc.Parse(lxfml.data(), lxfml.size()) != tinyxml2::XML_SUCCESS || !doc.FirstChildElement("LXFML")) return false;
std::string error;
return ParseLxfml(lxfml, error).empty() && lxfml.find("<Part") == std::string_view::npos;
}
void Mesh::Append(const Mesh& other) { void Mesh::Append(const Mesh& other) {
const auto base = static_cast<uint32_t>(positions.size()); const auto base = static_cast<uint32_t>(positions.size());
positions.insert(positions.end(), other.positions.begin(), other.positions.end()); positions.insert(positions.end(), other.positions.begin(), other.positions.end());

View File

@@ -29,6 +29,9 @@ namespace UgcModel {
// Empty with `error` set when it can't be read. // Empty with `error` set when it can't be read.
std::vector<Part> ParseLxfml(std::string_view lxfml, std::string& error); std::vector<Part> ParseLxfml(std::string_view lxfml, std::string& error);
// Whether an LXFML reads but has no bricks at all (nothing to make; not a failure)
bool HasNoBricks(std::string_view lxfml);
struct Mesh { struct Mesh {
std::vector<glm::vec3> positions; std::vector<glm::vec3> positions;
std::vector<glm::vec3> normals; std::vector<glm::vec3> normals;

View File

@@ -327,6 +327,7 @@ void UgcProcessor::Record(const Done& done) {
auto error = done.outcome.error.substr(0, MAX_ERROR_LENGTH); auto error = done.outcome.error.substr(0, MAX_ERROR_LENGTH);
const auto attempts = done.attempts + 1; const auto attempts = done.attempts + 1;
const auto state = done.outcome.ok ? IUgc::eProcessState::DONE const auto state = done.outcome.ok ? IUgc::eProcessState::DONE
: done.outcome.empty ? IUgc::eProcessState::EMPTY
: attempts >= m_Config.maxAttempts ? IUgc::eProcessState::FAILED : IUgc::eProcessState::PENDING; : attempts >= m_Config.maxAttempts ? IUgc::eProcessState::FAILED : IUgc::eProcessState::PENDING;
if (done.kind == Kind::MODEL) { if (done.kind == Kind::MODEL) {
Database::Get()->SetUgcModelProcessed(done.id, state, attempts, error, done.outcome.ok && done.outcome.aoBaked); Database::Get()->SetUgcModelProcessed(done.id, state, attempts, error, done.outcome.ok && done.outcome.aoBaked);
@@ -335,7 +336,10 @@ void UgcProcessor::Record(const Done& done) {
} }
m_Recent.erase({ done.kind, done.id }); m_Recent.erase({ done.kind, done.id });
if (done.outcome.ok) { if (done.outcome.empty) {
m_Empty++;
LOG_DEBUG("%s %llu has no bricks: nothing to make", KindName(done.kind), static_cast<unsigned long long>(done.id));
} else if (done.outcome.ok) {
m_Made++; m_Made++;
m_StoredBytes += done.bytes; m_StoredBytes += done.bytes;
LOG_DEBUG("Made %s %llu in %.0f ms%s%s", KindName(done.kind), static_cast<unsigned long long>(done.id), done.milliseconds, LOG_DEBUG("Made %s %llu in %.0f ms%s%s", KindName(done.kind), static_cast<unsigned long long>(done.id), done.milliseconds,
@@ -344,7 +348,7 @@ void UgcProcessor::Record(const Done& done) {
m_Failed++; m_Failed++;
LOG("Couldn't make %s %llu (attempt %u): %s", KindName(done.kind), static_cast<unsigned long long>(done.id), attempts, error.c_str()); LOG("Couldn't make %s %llu (attempt %u): %s", KindName(done.kind), static_cast<unsigned long long>(done.id), attempts, error.c_str());
} }
m_Log.push_back({ done.kind, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? done.outcome.note : error, UnixNow() }); m_Log.push_back({ done.kind, done.id, done.outcome.ok || done.outcome.empty, done.milliseconds, done.outcome.empty ? std::string("no bricks: nothing to make") : done.outcome.ok ? done.outcome.note : error, UnixNow() });
while (m_Log.size() > LOG_LENGTH) m_Log.pop_front(); while (m_Log.size() > LOG_LENGTH) m_Log.pop_front();
} }
@@ -432,7 +436,8 @@ UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) {
const auto info = kind == Kind::MODEL ? Database::Get()->GetUgcProcessInfo(id) : Database::Get()->GetModularBuildProcessInfo(id); const auto info = kind == Kind::MODEL ? Database::Get()->GetUgcProcessInfo(id) : Database::Get()->GetModularBuildProcessInfo(id);
Availability answer = Availability::UNKNOWN; Availability answer = Availability::UNKNOWN;
if (info && info->state != IUgc::eProcessState::FAILED) { // Failed, or nothing to make (no bricks): 404, as for HKX, so the client doesn't wait
if (info && info->state != IUgc::eProcessState::FAILED && info->state != IUgc::eProcessState::EMPTY) {
// Made before but the files are gone (deleted to save space): make them again, first // Made before but the files are gone (deleted to save space): make them again, first
if (info->state == IUgc::eProcessState::DONE) { if (info->state == IUgc::eProcessState::DONE) {
if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false); if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false);
@@ -480,6 +485,7 @@ nlohmann::json UgcProcessor::Status() const {
}; };
status["made"] = m_Made; status["made"] = m_Made;
status["failed"] = m_Failed; status["failed"] = m_Failed;
status["empty"] = m_Empty;
status["evicted"] = m_Evicted; status["evicted"] = m_Evicted;
status["storedBytes"] = m_StoredBytes; status["storedBytes"] = m_StoredBytes;
status["maxStorageBytes"] = m_Config.maxStorageBytes; status["maxStorageBytes"] = m_Config.maxStorageBytes;

View File

@@ -189,7 +189,8 @@ private:
std::set<LWOOBJID> m_ComboJobs; // combinations being made std::set<LWOOBJID> m_ComboJobs; // combinations being made
std::map<LWOOBJID, std::vector<std::pair<LWOOBJID, uint32_t>>> m_ComboRows; // combination -> builds (id, attempts) waiting for it std::map<LWOOBJID, std::vector<std::pair<LWOOBJID, uint32_t>>> m_ComboRows; // combination -> builds (id, attempts) waiting for it
std::map<LWOOBJID, LWOOBJID> m_ComboOf; // build -> combination (StorageId) std::map<LWOOBJID, LWOOBJID> m_ComboOf; // build -> combination (StorageId)
uint64_t m_Reused{}; // builds whose combination was made already uint64_t m_Reused{};
uint64_t m_Empty{}; // models with no bricks (nothing to make) // builds whose combination was made already
std::chrono::steady_clock::time_point m_NextPoll{}; std::chrono::steady_clock::time_point m_NextPoll{};
std::chrono::steady_clock::time_point m_NextEviction{}; std::chrono::steady_clock::time_point m_NextEviction{};
std::map<std::pair<Kind, LWOOBJID>, std::pair<std::chrono::steady_clock::time_point, Availability>> m_Recent; // answers for missing files std::map<std::pair<Kind, LWOOBJID>, std::pair<std::chrono::steady_clock::time_point, Availability>> m_Recent; // answers for missing files

View File

@@ -151,7 +151,9 @@ Migrations `dlu/mysql/81_ugc_processing.sql` and `dlu/sqlite/64_ugc_processing.s
that aren't there yet). that aren't there yet).
`ugc`: `is_optimized` (existing) is 0 until the model has been processed, 1 once its files are made, 2 when processing `ugc`: `is_optimized` (existing) is 0 until the model has been processed, 1 once its files are made, 2 when processing
failed. New: `processed_at` (Unix seconds of the last attempt), `process_attempts`, `process_error` (the last failure's failed, 3 when there was nothing to make (the model has no bricks; not a failure, never retried, counted and shown as
"empty", answered with 404 like HKX so the client doesn't wait; migrations 87/70 move the rows that had failed only for
that). The names come from `IUgc::eProcessState` (`IUgc::ProcessStateName`). New: `processed_at` (Unix seconds of the last attempt), `process_attempts`, `process_error` (the last failure's
reason, empty when none). `bake_ao` (existing) records whether lighting was baked into the model. Changing a model's reason, empty when none). `bake_ao` (existing) records whether lighting was baked into the model. Changing a model's
LXFML (`UpdateUgcModelData`) sets it back to unprocessed. LXFML (`UpdateUgcModelData`) sets it back to unprocessed.

View File

@@ -0,0 +1,3 @@
/* UGC server: models with no bricks have nothing to make. They are a state of their own (is_optimized = 3, "empty"),
not a failure: the ones marked failed only because they had no bricks move to it. */
UPDATE ugc SET is_optimized = 3, process_error = '' WHERE is_optimized = 2 AND process_error = 'the LXFML has no bricks';

View File

@@ -0,0 +1,2 @@
/* UGC server: models with no bricks are a state of their own. See the MySQL migration. */
UPDATE ugc SET is_optimized = 3, process_error = '' WHERE is_optimized = 2 AND process_error = 'the LXFML has no bricks';

View File

@@ -1,3 +1,4 @@
add_executable(dUgcTests "UgcTests.cpp") add_executable(dUgcTests "UgcTests.cpp")
target_include_directories(dUgcTests PRIVATE "${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase/ITables")
target_link_libraries(dUgcTests ${COMMON_LIBRARIES} dUgc GTest::gtest_main) target_link_libraries(dUgcTests ${COMMON_LIBRARIES} dUgc GTest::gtest_main)
gtest_discover_tests(dUgcTests) gtest_discover_tests(dUgcTests)

View File

@@ -13,6 +13,7 @@
#include "UgcFormats.h" #include "UgcFormats.h"
#include "UgcModel.h" #include "UgcModel.h"
#include "UgcJobs.h" #include "UgcJobs.h"
#include "IUgc.h"
#include "UgcIconParams.h" #include "UgcIconParams.h"
#include "UgcKeys.h" #include "UgcKeys.h"
#include "UgcModular.h" #include "UgcModular.h"
@@ -709,3 +710,28 @@ TEST(UgcIconParams, OneListDrivesEverything) {
shifted.offsetX = 0.25f; shifted.offsetX = 0.25f;
EXPECT_NEAR(centroid(UgcRender::RenderIcon(model, shifted)) - centroid(UgcRender::RenderIcon(model, plain)), 16.0, 1.0); EXPECT_NEAR(centroid(UgcRender::RenderIcon(model, shifted)) - centroid(UgcRender::RenderIcon(model, plain)), 16.0, 1.0);
} }
TEST(UgcJobs, ModelsWithoutBricksAreEmptyNotFailed) {
UgcBricks::BrickLibrary library(MakeRes(), 0);
UgcJobs::Settings settings;
const auto empty = UgcJobs::ProcessModel(R"(<?xml version="1.0"?><LXFML versionMajor="5"><Meta/><Bricks/></LXFML>)", library, settings);
EXPECT_FALSE(empty.ok);
EXPECT_TRUE(empty.empty);
EXPECT_TRUE(UgcModel::HasNoBricks(R"(<LXFML versionMajor="5"><Bricks/></LXFML>)"));
// Broken LXFML, or bricks without geometry, are failures
EXPECT_FALSE(UgcJobs::ProcessModel("<LXFML><nope", library, settings).empty);
const auto missing = UgcJobs::ProcessModel(R"(<LXFML versionMajor="5"><Bricks><Brick><Part designID="9999" materials="1">
<Bone transformation="1,0,0,0,1,0,0,0,1,0,0,0"/></Part></Brick></Bricks></LXFML>)", library, settings);
EXPECT_FALSE(missing.ok);
EXPECT_FALSE(missing.empty);
EXPECT_FALSE(UgcModel::HasNoBricks(R"(<LXFML versionMajor="5"><Bricks><Brick><Part designID="9999"/></Brick></Bricks></LXFML>)"));
}
TEST(UgcStates, NamesComeFromTheEnum) {
EXPECT_EQ(IUgc::ProcessStateName(IUgc::eProcessState::EMPTY), "empty");
EXPECT_EQ(IUgc::ProcessStateName(IUgc::eProcessState::FAILED), "failed");
EXPECT_EQ(IUgc::ParseProcessState("empty"), IUgc::eProcessState::EMPTY);
EXPECT_EQ(IUgc::ParseProcessState("pending"), IUgc::eProcessState::PENDING);
EXPECT_FALSE(IUgc::ParseProcessState("nonsense").has_value());
EXPECT_EQ(magic_enum::enum_count<IUgc::eProcessState>(), 4u);
}