From 5de59b99ef81454283dc51c7a5fe4a2d5cb7efd9 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sun, 27 Sep 2026 14:52:03 -0500 Subject: [PATCH] feat(ugc): models with no bricks are "empty", not failed A model with no bricks has nothing to make: it gets its own state (is_optimized = 3), isn't counted as a failure or retried, shows as Empty on the UGC page (with its own filter and count), in the status and in Prometheus, and its downloads answer 404 like HKX. State names come from the enum (magic_enum). Migrations dlu/mysql/87 and dlu/sqlite/70 move the rows that failed only because they had no bricks. Co-Authored-By: Claude Opus 5.5 --- dDashboardServer/routes/PrometheusMetrics.cpp | 7 +++-- dDashboardServer/routes/ServerRoutes.cpp | 5 ++-- dDashboardServer/routes/UgcRoutes.cpp | 20 ++++---------- dDashboardServer/static/js/ugc.js | 4 +-- dDashboardServer/templates/ugc.jinja2 | 2 +- dDatabase/GameDatabase/ITables/IUgc.h | 17 ++++++++++++ dUgcServer/UgcJobs.cpp | 2 ++ dUgcServer/UgcJobs.h | 1 + dUgcServer/UgcModel.cpp | 7 +++++ dUgcServer/UgcModel.h | 3 +++ dUgcServer/UgcProcessor.cpp | 12 ++++++--- dUgcServer/UgcProcessor.h | 3 ++- docs/UgcServer.md | 4 ++- migrations/dlu/mysql/87_ugc_empty_models.sql | 3 +++ migrations/dlu/sqlite/70_ugc_empty_models.sql | 2 ++ tests/dUgcTests/CMakeLists.txt | 1 + tests/dUgcTests/UgcTests.cpp | 26 +++++++++++++++++++ 17 files changed, 90 insertions(+), 29 deletions(-) create mode 100644 migrations/dlu/mysql/87_ugc_empty_models.sql create mode 100644 migrations/dlu/sqlite/70_ugc_empty_models.sql diff --git a/dDashboardServer/routes/PrometheusMetrics.cpp b/dDashboardServer/routes/PrometheusMetrics.cpp index b4ae33376..27ea651ae 100644 --- a/dDashboardServer/routes/PrometheusMetrics.cpp +++ b/dDashboardServer/routes/PrometheusMetrics.cpp @@ -223,10 +223,9 @@ namespace { ok &= Timed(w, "ugc", [&] { w.Declare("darkflame_ugc_items", "Player models (model) and cars and rockets (modular) by UGC processing state", "gauge"); const auto add = [&](const char* kind, const std::vector>& rows) { - std::map states{ {"pending", 0}, {"done", 0}, {"failed", 0} }; - for (const auto& [state, count] : rows) { - states[state == IUgc::eProcessState::PENDING ? "pending" : state == IUgc::eProcessState::DONE ? "done" : "failed"] += count; - } + std::map states; + for (const auto state : magic_enum::enum_values()) states[IUgc::ProcessStateName(state)] = 0; + for (const auto& [state, count] : rows) states[IUgc::ProcessStateName(state)] += count; for (const auto& [state, count] : states) w.Add("darkflame_ugc_items", "", "gauge", { {"kind", kind}, {"state", state} }, static_cast(count)); }; add("model", db->GetUgcProcessCounts()); diff --git a/dDashboardServer/routes/ServerRoutes.cpp b/dDashboardServer/routes/ServerRoutes.cpp index dec5c02db..91808f025 100644 --- a/dDashboardServer/routes/ServerRoutes.cpp +++ b/dDashboardServer/routes/ServerRoutes.cpp @@ -65,9 +65,10 @@ namespace { // The UGC server's queue in the database (its work list), from the model and modular build tables nlohmann::json UgcCounts() { const auto counts = [](const std::vector>& rows) { - nlohmann::json out = { {"pending", 0}, {"done", 0}, {"failed", 0} }; + nlohmann::json out = nlohmann::json::object(); + for (const auto state : magic_enum::enum_values()) out[IUgc::ProcessStateName(state)] = 0; for (const auto& [state, count] : rows) { - out[state == IUgc::eProcessState::PENDING ? "pending" : state == IUgc::eProcessState::DONE ? "done" : "failed"] = count; + out[IUgc::ProcessStateName(state)] = count; } return out; }; diff --git a/dDashboardServer/routes/UgcRoutes.cpp b/dDashboardServer/routes/UgcRoutes.cpp index b845ac52b..d50c30e7f 100644 --- a/dDashboardServer/routes/UgcRoutes.cpp +++ b/dDashboardServer/routes/UgcRoutes.cpp @@ -28,21 +28,10 @@ using namespace RouteUtils; namespace { constexpr uint32_t PAGE_SIZE = 50; - const char* StateName(IUgc::eProcessState state) { - switch (state) { - case IUgc::eProcessState::PENDING: return "pending"; - case IUgc::eProcessState::DONE: return "done"; - case IUgc::eProcessState::FAILED: return "failed"; - } - return "unknown"; - } + std::string StateName(IUgc::eProcessState state) { return IUgc::ProcessStateName(state); } + + std::optional ParseState(const std::string& text) { return IUgc::ParseProcessState(text); } - std::optional ParseState(const std::string& text) { - if (text == "pending") return IUgc::eProcessState::PENDING; - if (text == "done") return IUgc::eProcessState::DONE; - if (text == "failed") return IUgc::eProcessState::FAILED; - return std::nullopt; - } /** * The dashboard's own way to the UGC server (ugc_internal_url, normally the same machine): its status and the files @@ -235,7 +224,8 @@ namespace { } nlohmann::json Counts(const std::vector>& counts) { - nlohmann::json out = { { "pending", 0 }, { "done", 0 }, { "failed", 0 } }; + nlohmann::json out = nlohmann::json::object(); + for (const auto state : magic_enum::enum_values()) out[IUgc::ProcessStateName(state)] = 0; for (const auto& [state, count] : counts) out[StateName(state)] = count; return out; } diff --git a/dDashboardServer/static/js/ugc.js b/dDashboardServer/static/js/ugc.js index 22642fe9f..e7a148025 100644 --- a/dDashboardServer/static/js/ugc.js +++ b/dDashboardServer/static/js/ugc.js @@ -9,7 +9,7 @@ var kind = 'model', state = '', search = '', page = 0, view = 'gallery', canManage = false, publicUrl = ''; var nifViewer = null, lxfmlViewer = null, current = null; - var STATES = { pending: ['Waiting', 'secondary'], done: ['Made', 'success'], failed: ['Failed', 'danger'] }; + var STATES = { pending: ['Waiting', 'secondary'], done: ['Made', 'success'], failed: ['Failed', 'danger'], empty: ['Empty', 'light'] }; var PAGE_SIZE = { gallery: 48, list: 50 }; try { view = localStorage.getItem('ugcView') === 'list' ? 'list' : 'gallery'; } catch (e) { /* storage blocked */ } @@ -25,7 +25,7 @@ function countCard(title, c) { return '
' + esc(title) + '
' + fmt.badge(c.done + ' made', 'success') + ' ' + fmt.badge(c.pending + ' waiting', 'secondary') + ' ' + fmt.badge(c.failed + ' failed', c.failed ? 'danger' : 'secondary') + - '
'; + (c.empty ? ' ' + fmt.badge(c.empty + ' empty', 'light') : '') + ''; } function owner(i) { diff --git a/dDashboardServer/templates/ugc.jinja2 b/dDashboardServer/templates/ugc.jinja2 index 02e17e4b9..c8a511762 100644 --- a/dDashboardServer/templates/ugc.jinja2 +++ b/dDashboardServer/templates/ugc.jinja2 @@ -41,7 +41,7 @@
diff --git a/dDatabase/GameDatabase/ITables/IUgc.h b/dDatabase/GameDatabase/ITables/IUgc.h index 5e0e4c21c..4113b02bf 100644 --- a/dDatabase/GameDatabase/ITables/IUgc.h +++ b/dDatabase/GameDatabase/ITables/IUgc.h @@ -8,6 +8,8 @@ #include #include +#include "magic_enum.hpp" + class IUgc { public: struct Model { @@ -40,8 +42,23 @@ public: PENDING = 0, // not made yet (or to be made again) DONE = 1, // the UGC server's files are made FAILED = 2, // gave up after the allowed attempts + EMPTY = 3, // nothing to make: the model has no bricks (not a failure; never retried) }; + // A state's name for the dashboard, the API and metrics ("pending", "done", "failed", "empty") + static std::string ProcessStateName(eProcessState state) { + std::string name(magic_enum::enum_name(state)); + for (auto& c : name) c = static_cast(c >= 'A' && c <= 'Z' ? c - 'A' + 'a' : c); + return name.empty() ? "unknown" : name; + } + + static std::optional ParseProcessState(std::string_view text) { + for (const auto state : magic_enum::enum_values()) { + if (ProcessStateName(state) == text) return state; + } + return std::nullopt; + } + // A row's processing state, for the UGC server and the dashboard struct ProcessInfo { LWOOBJID id{}; diff --git a/dUgcServer/UgcJobs.cpp b/dUgcServer/UgcJobs.cpp index 3db9eb4e1..647e1acb3 100644 --- a/dUgcServer/UgcJobs.cpp +++ b/dUgcServer/UgcJobs.cpp @@ -86,6 +86,8 @@ namespace UgcJobs { const auto parts = UgcModel::ParseLxfml(lxfml, error); if (parts.empty()) { outcome.error = error; + // A model with no bricks has nothing to make; the LXFML itself is still served + outcome.empty = UgcModel::HasNoBricks(lxfml); return outcome; } if (settings.maxBricks > 0 && parts.size() > settings.maxBricks) { diff --git a/dUgcServer/UgcJobs.h b/dUgcServer/UgcJobs.h index edbcd75f2..8a9e2246e 100644 --- a/dUgcServer/UgcJobs.h +++ b/dUgcServer/UgcJobs.h @@ -40,6 +40,7 @@ namespace UgcJobs { std::string note; // what was odd but not fatal (missing bricks, ...) UgcStorage::Files files; // name -> bytes, when ok bool aoBaked{}; + bool empty{}; // nothing to make (no bricks): not a failure std::string stats; // stats.json: bricks, triangles before and after per LOD, timings }; diff --git a/dUgcServer/UgcModel.cpp b/dUgcServer/UgcModel.cpp index 01dcd444e..f52a6804f 100644 --- a/dUgcServer/UgcModel.cpp +++ b/dUgcServer/UgcModel.cpp @@ -142,6 +142,13 @@ namespace UgcModel { return parts; } + bool HasNoBricks(std::string_view lxfml) { + tinyxml2::XMLDocument doc; + if (doc.Parse(lxfml.data(), lxfml.size()) != tinyxml2::XML_SUCCESS || !doc.FirstChildElement("LXFML")) return false; + std::string error; + return ParseLxfml(lxfml, error).empty() && lxfml.find("(positions.size()); positions.insert(positions.end(), other.positions.begin(), other.positions.end()); diff --git a/dUgcServer/UgcModel.h b/dUgcServer/UgcModel.h index ef17e9837..dddce09ff 100644 --- a/dUgcServer/UgcModel.h +++ b/dUgcServer/UgcModel.h @@ -29,6 +29,9 @@ namespace UgcModel { // Empty with `error` set when it can't be read. std::vector ParseLxfml(std::string_view lxfml, std::string& error); + // Whether an LXFML reads but has no bricks at all (nothing to make; not a failure) + bool HasNoBricks(std::string_view lxfml); + struct Mesh { std::vector positions; std::vector normals; diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index fc8ae4873..aff1c1e49 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -327,6 +327,7 @@ void UgcProcessor::Record(const Done& done) { auto error = done.outcome.error.substr(0, MAX_ERROR_LENGTH); const auto attempts = done.attempts + 1; const auto state = done.outcome.ok ? IUgc::eProcessState::DONE + : done.outcome.empty ? IUgc::eProcessState::EMPTY : attempts >= m_Config.maxAttempts ? IUgc::eProcessState::FAILED : IUgc::eProcessState::PENDING; if (done.kind == Kind::MODEL) { Database::Get()->SetUgcModelProcessed(done.id, state, attempts, error, done.outcome.ok && done.outcome.aoBaked); @@ -335,7 +336,10 @@ void UgcProcessor::Record(const Done& done) { } m_Recent.erase({ done.kind, done.id }); - if (done.outcome.ok) { + if (done.outcome.empty) { + m_Empty++; + LOG_DEBUG("%s %llu has no bricks: nothing to make", KindName(done.kind), static_cast(done.id)); + } else if (done.outcome.ok) { m_Made++; m_StoredBytes += done.bytes; LOG_DEBUG("Made %s %llu in %.0f ms%s%s", KindName(done.kind), static_cast(done.id), done.milliseconds, @@ -344,7 +348,7 @@ void UgcProcessor::Record(const Done& done) { m_Failed++; LOG("Couldn't make %s %llu (attempt %u): %s", KindName(done.kind), static_cast(done.id), attempts, error.c_str()); } - m_Log.push_back({ done.kind, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? done.outcome.note : error, UnixNow() }); + m_Log.push_back({ done.kind, done.id, done.outcome.ok || done.outcome.empty, done.milliseconds, done.outcome.empty ? std::string("no bricks: nothing to make") : done.outcome.ok ? done.outcome.note : error, UnixNow() }); while (m_Log.size() > LOG_LENGTH) m_Log.pop_front(); } @@ -432,7 +436,8 @@ UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) { const auto info = kind == Kind::MODEL ? Database::Get()->GetUgcProcessInfo(id) : Database::Get()->GetModularBuildProcessInfo(id); Availability answer = Availability::UNKNOWN; - if (info && info->state != IUgc::eProcessState::FAILED) { + // Failed, or nothing to make (no bricks): 404, as for HKX, so the client doesn't wait + if (info && info->state != IUgc::eProcessState::FAILED && info->state != IUgc::eProcessState::EMPTY) { // Made before but the files are gone (deleted to save space): make them again, first if (info->state == IUgc::eProcessState::DONE) { if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false); @@ -480,6 +485,7 @@ nlohmann::json UgcProcessor::Status() const { }; status["made"] = m_Made; status["failed"] = m_Failed; + status["empty"] = m_Empty; status["evicted"] = m_Evicted; status["storedBytes"] = m_StoredBytes; status["maxStorageBytes"] = m_Config.maxStorageBytes; diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h index bcc3e54ce..a231ec4c2 100644 --- a/dUgcServer/UgcProcessor.h +++ b/dUgcServer/UgcProcessor.h @@ -189,7 +189,8 @@ private: std::set m_ComboJobs; // combinations being made std::map>> m_ComboRows; // combination -> builds (id, attempts) waiting for it std::map m_ComboOf; // build -> combination (StorageId) - uint64_t m_Reused{}; // builds whose combination was made already + uint64_t m_Reused{}; + uint64_t m_Empty{}; // models with no bricks (nothing to make) // builds whose combination was made already std::chrono::steady_clock::time_point m_NextPoll{}; std::chrono::steady_clock::time_point m_NextEviction{}; std::map, std::pair> m_Recent; // answers for missing files diff --git a/docs/UgcServer.md b/docs/UgcServer.md index 0b7750bb2..b4d5e4b9e 100644 --- a/docs/UgcServer.md +++ b/docs/UgcServer.md @@ -151,7 +151,9 @@ Migrations `dlu/mysql/81_ugc_processing.sql` and `dlu/sqlite/64_ugc_processing.s that aren't there yet). `ugc`: `is_optimized` (existing) is 0 until the model has been processed, 1 once its files are made, 2 when processing -failed. New: `processed_at` (Unix seconds of the last attempt), `process_attempts`, `process_error` (the last failure's +failed, 3 when there was nothing to make (the model has no bricks; not a failure, never retried, counted and shown as +"empty", answered with 404 like HKX so the client doesn't wait; migrations 87/70 move the rows that had failed only for +that). The names come from `IUgc::eProcessState` (`IUgc::ProcessStateName`). New: `processed_at` (Unix seconds of the last attempt), `process_attempts`, `process_error` (the last failure's reason, empty when none). `bake_ao` (existing) records whether lighting was baked into the model. Changing a model's LXFML (`UpdateUgcModelData`) sets it back to unprocessed. diff --git a/migrations/dlu/mysql/87_ugc_empty_models.sql b/migrations/dlu/mysql/87_ugc_empty_models.sql new file mode 100644 index 000000000..365546789 --- /dev/null +++ b/migrations/dlu/mysql/87_ugc_empty_models.sql @@ -0,0 +1,3 @@ +/* UGC server: models with no bricks have nothing to make. They are a state of their own (is_optimized = 3, "empty"), + not a failure: the ones marked failed only because they had no bricks move to it. */ +UPDATE ugc SET is_optimized = 3, process_error = '' WHERE is_optimized = 2 AND process_error = 'the LXFML has no bricks'; diff --git a/migrations/dlu/sqlite/70_ugc_empty_models.sql b/migrations/dlu/sqlite/70_ugc_empty_models.sql new file mode 100644 index 000000000..d2b00ece8 --- /dev/null +++ b/migrations/dlu/sqlite/70_ugc_empty_models.sql @@ -0,0 +1,2 @@ +/* UGC server: models with no bricks are a state of their own. See the MySQL migration. */ +UPDATE ugc SET is_optimized = 3, process_error = '' WHERE is_optimized = 2 AND process_error = 'the LXFML has no bricks'; diff --git a/tests/dUgcTests/CMakeLists.txt b/tests/dUgcTests/CMakeLists.txt index f7cd8925d..69c56d81b 100644 --- a/tests/dUgcTests/CMakeLists.txt +++ b/tests/dUgcTests/CMakeLists.txt @@ -1,3 +1,4 @@ add_executable(dUgcTests "UgcTests.cpp") +target_include_directories(dUgcTests PRIVATE "${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase/ITables") target_link_libraries(dUgcTests ${COMMON_LIBRARIES} dUgc GTest::gtest_main) gtest_discover_tests(dUgcTests) diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp index ac36af8b2..fb5e1b9a6 100644 --- a/tests/dUgcTests/UgcTests.cpp +++ b/tests/dUgcTests/UgcTests.cpp @@ -13,6 +13,7 @@ #include "UgcFormats.h" #include "UgcModel.h" #include "UgcJobs.h" +#include "IUgc.h" #include "UgcIconParams.h" #include "UgcKeys.h" #include "UgcModular.h" @@ -709,3 +710,28 @@ TEST(UgcIconParams, OneListDrivesEverything) { shifted.offsetX = 0.25f; EXPECT_NEAR(centroid(UgcRender::RenderIcon(model, shifted)) - centroid(UgcRender::RenderIcon(model, plain)), 16.0, 1.0); } + +TEST(UgcJobs, ModelsWithoutBricksAreEmptyNotFailed) { + UgcBricks::BrickLibrary library(MakeRes(), 0); + UgcJobs::Settings settings; + const auto empty = UgcJobs::ProcessModel(R"()", library, settings); + EXPECT_FALSE(empty.ok); + EXPECT_TRUE(empty.empty); + EXPECT_TRUE(UgcModel::HasNoBricks(R"()")); + // Broken LXFML, or bricks without geometry, are failures + EXPECT_FALSE(UgcJobs::ProcessModel(" + )", library, settings); + EXPECT_FALSE(missing.ok); + EXPECT_FALSE(missing.empty); + EXPECT_FALSE(UgcModel::HasNoBricks(R"()")); +} + +TEST(UgcStates, NamesComeFromTheEnum) { + EXPECT_EQ(IUgc::ProcessStateName(IUgc::eProcessState::EMPTY), "empty"); + EXPECT_EQ(IUgc::ProcessStateName(IUgc::eProcessState::FAILED), "failed"); + EXPECT_EQ(IUgc::ParseProcessState("empty"), IUgc::eProcessState::EMPTY); + EXPECT_EQ(IUgc::ParseProcessState("pending"), IUgc::eProcessState::PENDING); + EXPECT_FALSE(IUgc::ParseProcessState("nonsense").has_value()); + EXPECT_EQ(magic_enum::enum_count(), 4u); +}