diff --git a/dDashboardServer/routes/PrometheusMetrics.cpp b/dDashboardServer/routes/PrometheusMetrics.cpp index b4ae33376..27ea651ae 100644 --- a/dDashboardServer/routes/PrometheusMetrics.cpp +++ b/dDashboardServer/routes/PrometheusMetrics.cpp @@ -223,10 +223,9 @@ namespace { ok &= Timed(w, "ugc", [&] { w.Declare("darkflame_ugc_items", "Player models (model) and cars and rockets (modular) by UGC processing state", "gauge"); const auto add = [&](const char* kind, const std::vector>& rows) { - std::map states{ {"pending", 0}, {"done", 0}, {"failed", 0} }; - for (const auto& [state, count] : rows) { - states[state == IUgc::eProcessState::PENDING ? "pending" : state == IUgc::eProcessState::DONE ? "done" : "failed"] += count; - } + std::map states; + for (const auto state : magic_enum::enum_values()) states[IUgc::ProcessStateName(state)] = 0; + for (const auto& [state, count] : rows) states[IUgc::ProcessStateName(state)] += count; for (const auto& [state, count] : states) w.Add("darkflame_ugc_items", "", "gauge", { {"kind", kind}, {"state", state} }, static_cast(count)); }; add("model", db->GetUgcProcessCounts()); diff --git a/dDashboardServer/routes/ServerRoutes.cpp b/dDashboardServer/routes/ServerRoutes.cpp index dec5c02db..91808f025 100644 --- a/dDashboardServer/routes/ServerRoutes.cpp +++ b/dDashboardServer/routes/ServerRoutes.cpp @@ -65,9 +65,10 @@ namespace { // The UGC server's queue in the database (its work list), from the model and modular build tables nlohmann::json UgcCounts() { const auto counts = [](const std::vector>& rows) { - nlohmann::json out = { {"pending", 0}, {"done", 0}, {"failed", 0} }; + nlohmann::json out = nlohmann::json::object(); + for (const auto state : magic_enum::enum_values()) out[IUgc::ProcessStateName(state)] = 0; for (const auto& [state, count] : rows) { - out[state == IUgc::eProcessState::PENDING ? "pending" : state == IUgc::eProcessState::DONE ? "done" : "failed"] = count; + out[IUgc::ProcessStateName(state)] = count; } return out; }; diff --git a/dDashboardServer/routes/UgcRoutes.cpp b/dDashboardServer/routes/UgcRoutes.cpp index b845ac52b..d50c30e7f 100644 --- a/dDashboardServer/routes/UgcRoutes.cpp +++ b/dDashboardServer/routes/UgcRoutes.cpp @@ -28,21 +28,10 @@ using namespace RouteUtils; namespace { constexpr uint32_t PAGE_SIZE = 50; - const char* StateName(IUgc::eProcessState state) { - switch (state) { - case IUgc::eProcessState::PENDING: return "pending"; - case IUgc::eProcessState::DONE: return "done"; - case IUgc::eProcessState::FAILED: return "failed"; - } - return "unknown"; - } + std::string StateName(IUgc::eProcessState state) { return IUgc::ProcessStateName(state); } + + std::optional ParseState(const std::string& text) { return IUgc::ParseProcessState(text); } - std::optional ParseState(const std::string& text) { - if (text == "pending") return IUgc::eProcessState::PENDING; - if (text == "done") return IUgc::eProcessState::DONE; - if (text == "failed") return IUgc::eProcessState::FAILED; - return std::nullopt; - } /** * The dashboard's own way to the UGC server (ugc_internal_url, normally the same machine): its status and the files @@ -235,7 +224,8 @@ namespace { } nlohmann::json Counts(const std::vector>& counts) { - nlohmann::json out = { { "pending", 0 }, { "done", 0 }, { "failed", 0 } }; + nlohmann::json out = nlohmann::json::object(); + for (const auto state : magic_enum::enum_values()) out[IUgc::ProcessStateName(state)] = 0; for (const auto& [state, count] : counts) out[StateName(state)] = count; return out; } diff --git a/dDashboardServer/static/js/ugc.js b/dDashboardServer/static/js/ugc.js index 22642fe9f..e7a148025 100644 --- a/dDashboardServer/static/js/ugc.js +++ b/dDashboardServer/static/js/ugc.js @@ -9,7 +9,7 @@ var kind = 'model', state = '', search = '', page = 0, view = 'gallery', canManage = false, publicUrl = ''; var nifViewer = null, lxfmlViewer = null, current = null; - var STATES = { pending: ['Waiting', 'secondary'], done: ['Made', 'success'], failed: ['Failed', 'danger'] }; + var STATES = { pending: ['Waiting', 'secondary'], done: ['Made', 'success'], failed: ['Failed', 'danger'], empty: ['Empty', 'light'] }; var PAGE_SIZE = { gallery: 48, list: 50 }; try { view = localStorage.getItem('ugcView') === 'list' ? 'list' : 'gallery'; } catch (e) { /* storage blocked */ } @@ -25,7 +25,7 @@ function countCard(title, c) { return '
' + esc(title) + '
' + fmt.badge(c.done + ' made', 'success') + ' ' + fmt.badge(c.pending + ' waiting', 'secondary') + ' ' + fmt.badge(c.failed + ' failed', c.failed ? 'danger' : 'secondary') + - '
'; + (c.empty ? ' ' + fmt.badge(c.empty + ' empty', 'light') : '') + ''; } function owner(i) { diff --git a/dDashboardServer/templates/ugc.jinja2 b/dDashboardServer/templates/ugc.jinja2 index 02e17e4b9..c8a511762 100644 --- a/dDashboardServer/templates/ugc.jinja2 +++ b/dDashboardServer/templates/ugc.jinja2 @@ -41,7 +41,7 @@
diff --git a/dDatabase/GameDatabase/ITables/IUgc.h b/dDatabase/GameDatabase/ITables/IUgc.h index 5e0e4c21c..4113b02bf 100644 --- a/dDatabase/GameDatabase/ITables/IUgc.h +++ b/dDatabase/GameDatabase/ITables/IUgc.h @@ -8,6 +8,8 @@ #include #include +#include "magic_enum.hpp" + class IUgc { public: struct Model { @@ -40,8 +42,23 @@ public: PENDING = 0, // not made yet (or to be made again) DONE = 1, // the UGC server's files are made FAILED = 2, // gave up after the allowed attempts + EMPTY = 3, // nothing to make: the model has no bricks (not a failure; never retried) }; + // A state's name for the dashboard, the API and metrics ("pending", "done", "failed", "empty") + static std::string ProcessStateName(eProcessState state) { + std::string name(magic_enum::enum_name(state)); + for (auto& c : name) c = static_cast(c >= 'A' && c <= 'Z' ? c - 'A' + 'a' : c); + return name.empty() ? "unknown" : name; + } + + static std::optional ParseProcessState(std::string_view text) { + for (const auto state : magic_enum::enum_values()) { + if (ProcessStateName(state) == text) return state; + } + return std::nullopt; + } + // A row's processing state, for the UGC server and the dashboard struct ProcessInfo { LWOOBJID id{}; diff --git a/dUgcServer/UgcJobs.cpp b/dUgcServer/UgcJobs.cpp index 3db9eb4e1..647e1acb3 100644 --- a/dUgcServer/UgcJobs.cpp +++ b/dUgcServer/UgcJobs.cpp @@ -86,6 +86,8 @@ namespace UgcJobs { const auto parts = UgcModel::ParseLxfml(lxfml, error); if (parts.empty()) { outcome.error = error; + // A model with no bricks has nothing to make; the LXFML itself is still served + outcome.empty = UgcModel::HasNoBricks(lxfml); return outcome; } if (settings.maxBricks > 0 && parts.size() > settings.maxBricks) { diff --git a/dUgcServer/UgcJobs.h b/dUgcServer/UgcJobs.h index edbcd75f2..8a9e2246e 100644 --- a/dUgcServer/UgcJobs.h +++ b/dUgcServer/UgcJobs.h @@ -40,6 +40,7 @@ namespace UgcJobs { std::string note; // what was odd but not fatal (missing bricks, ...) UgcStorage::Files files; // name -> bytes, when ok bool aoBaked{}; + bool empty{}; // nothing to make (no bricks): not a failure std::string stats; // stats.json: bricks, triangles before and after per LOD, timings }; diff --git a/dUgcServer/UgcModel.cpp b/dUgcServer/UgcModel.cpp index 01dcd444e..f52a6804f 100644 --- a/dUgcServer/UgcModel.cpp +++ b/dUgcServer/UgcModel.cpp @@ -142,6 +142,13 @@ namespace UgcModel { return parts; } + bool HasNoBricks(std::string_view lxfml) { + tinyxml2::XMLDocument doc; + if (doc.Parse(lxfml.data(), lxfml.size()) != tinyxml2::XML_SUCCESS || !doc.FirstChildElement("LXFML")) return false; + std::string error; + return ParseLxfml(lxfml, error).empty() && lxfml.find("(positions.size()); positions.insert(positions.end(), other.positions.begin(), other.positions.end()); diff --git a/dUgcServer/UgcModel.h b/dUgcServer/UgcModel.h index ef17e9837..dddce09ff 100644 --- a/dUgcServer/UgcModel.h +++ b/dUgcServer/UgcModel.h @@ -29,6 +29,9 @@ namespace UgcModel { // Empty with `error` set when it can't be read. std::vector ParseLxfml(std::string_view lxfml, std::string& error); + // Whether an LXFML reads but has no bricks at all (nothing to make; not a failure) + bool HasNoBricks(std::string_view lxfml); + struct Mesh { std::vector positions; std::vector normals; diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index fc8ae4873..aff1c1e49 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -327,6 +327,7 @@ void UgcProcessor::Record(const Done& done) { auto error = done.outcome.error.substr(0, MAX_ERROR_LENGTH); const auto attempts = done.attempts + 1; const auto state = done.outcome.ok ? IUgc::eProcessState::DONE + : done.outcome.empty ? IUgc::eProcessState::EMPTY : attempts >= m_Config.maxAttempts ? IUgc::eProcessState::FAILED : IUgc::eProcessState::PENDING; if (done.kind == Kind::MODEL) { Database::Get()->SetUgcModelProcessed(done.id, state, attempts, error, done.outcome.ok && done.outcome.aoBaked); @@ -335,7 +336,10 @@ void UgcProcessor::Record(const Done& done) { } m_Recent.erase({ done.kind, done.id }); - if (done.outcome.ok) { + if (done.outcome.empty) { + m_Empty++; + LOG_DEBUG("%s %llu has no bricks: nothing to make", KindName(done.kind), static_cast(done.id)); + } else if (done.outcome.ok) { m_Made++; m_StoredBytes += done.bytes; LOG_DEBUG("Made %s %llu in %.0f ms%s%s", KindName(done.kind), static_cast(done.id), done.milliseconds, @@ -344,7 +348,7 @@ void UgcProcessor::Record(const Done& done) { m_Failed++; LOG("Couldn't make %s %llu (attempt %u): %s", KindName(done.kind), static_cast(done.id), attempts, error.c_str()); } - m_Log.push_back({ done.kind, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? done.outcome.note : error, UnixNow() }); + m_Log.push_back({ done.kind, done.id, done.outcome.ok || done.outcome.empty, done.milliseconds, done.outcome.empty ? std::string("no bricks: nothing to make") : done.outcome.ok ? done.outcome.note : error, UnixNow() }); while (m_Log.size() > LOG_LENGTH) m_Log.pop_front(); } @@ -432,7 +436,8 @@ UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) { const auto info = kind == Kind::MODEL ? Database::Get()->GetUgcProcessInfo(id) : Database::Get()->GetModularBuildProcessInfo(id); Availability answer = Availability::UNKNOWN; - if (info && info->state != IUgc::eProcessState::FAILED) { + // Failed, or nothing to make (no bricks): 404, as for HKX, so the client doesn't wait + if (info && info->state != IUgc::eProcessState::FAILED && info->state != IUgc::eProcessState::EMPTY) { // Made before but the files are gone (deleted to save space): make them again, first if (info->state == IUgc::eProcessState::DONE) { if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false); @@ -480,6 +485,7 @@ nlohmann::json UgcProcessor::Status() const { }; status["made"] = m_Made; status["failed"] = m_Failed; + status["empty"] = m_Empty; status["evicted"] = m_Evicted; status["storedBytes"] = m_StoredBytes; status["maxStorageBytes"] = m_Config.maxStorageBytes; diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h index bcc3e54ce..a231ec4c2 100644 --- a/dUgcServer/UgcProcessor.h +++ b/dUgcServer/UgcProcessor.h @@ -189,7 +189,8 @@ private: std::set m_ComboJobs; // combinations being made std::map>> m_ComboRows; // combination -> builds (id, attempts) waiting for it std::map m_ComboOf; // build -> combination (StorageId) - uint64_t m_Reused{}; // builds whose combination was made already + uint64_t m_Reused{}; + uint64_t m_Empty{}; // models with no bricks (nothing to make) // builds whose combination was made already std::chrono::steady_clock::time_point m_NextPoll{}; std::chrono::steady_clock::time_point m_NextEviction{}; std::map, std::pair> m_Recent; // answers for missing files diff --git a/docs/UgcServer.md b/docs/UgcServer.md index 0b7750bb2..b4d5e4b9e 100644 --- a/docs/UgcServer.md +++ b/docs/UgcServer.md @@ -151,7 +151,9 @@ Migrations `dlu/mysql/81_ugc_processing.sql` and `dlu/sqlite/64_ugc_processing.s that aren't there yet). `ugc`: `is_optimized` (existing) is 0 until the model has been processed, 1 once its files are made, 2 when processing -failed. New: `processed_at` (Unix seconds of the last attempt), `process_attempts`, `process_error` (the last failure's +failed, 3 when there was nothing to make (the model has no bricks; not a failure, never retried, counted and shown as +"empty", answered with 404 like HKX so the client doesn't wait; migrations 87/70 move the rows that had failed only for +that). The names come from `IUgc::eProcessState` (`IUgc::ProcessStateName`). New: `processed_at` (Unix seconds of the last attempt), `process_attempts`, `process_error` (the last failure's reason, empty when none). `bake_ao` (existing) records whether lighting was baked into the model. Changing a model's LXFML (`UpdateUgcModelData`) sets it back to unprocessed. diff --git a/migrations/dlu/mysql/87_ugc_empty_models.sql b/migrations/dlu/mysql/87_ugc_empty_models.sql new file mode 100644 index 000000000..365546789 --- /dev/null +++ b/migrations/dlu/mysql/87_ugc_empty_models.sql @@ -0,0 +1,3 @@ +/* UGC server: models with no bricks have nothing to make. They are a state of their own (is_optimized = 3, "empty"), + not a failure: the ones marked failed only because they had no bricks move to it. */ +UPDATE ugc SET is_optimized = 3, process_error = '' WHERE is_optimized = 2 AND process_error = 'the LXFML has no bricks'; diff --git a/migrations/dlu/sqlite/70_ugc_empty_models.sql b/migrations/dlu/sqlite/70_ugc_empty_models.sql new file mode 100644 index 000000000..d2b00ece8 --- /dev/null +++ b/migrations/dlu/sqlite/70_ugc_empty_models.sql @@ -0,0 +1,2 @@ +/* UGC server: models with no bricks are a state of their own. See the MySQL migration. */ +UPDATE ugc SET is_optimized = 3, process_error = '' WHERE is_optimized = 2 AND process_error = 'the LXFML has no bricks'; diff --git a/tests/dUgcTests/CMakeLists.txt b/tests/dUgcTests/CMakeLists.txt index f7cd8925d..69c56d81b 100644 --- a/tests/dUgcTests/CMakeLists.txt +++ b/tests/dUgcTests/CMakeLists.txt @@ -1,3 +1,4 @@ add_executable(dUgcTests "UgcTests.cpp") +target_include_directories(dUgcTests PRIVATE "${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase/ITables") target_link_libraries(dUgcTests ${COMMON_LIBRARIES} dUgc GTest::gtest_main) gtest_discover_tests(dUgcTests) diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp index ac36af8b2..fb5e1b9a6 100644 --- a/tests/dUgcTests/UgcTests.cpp +++ b/tests/dUgcTests/UgcTests.cpp @@ -13,6 +13,7 @@ #include "UgcFormats.h" #include "UgcModel.h" #include "UgcJobs.h" +#include "IUgc.h" #include "UgcIconParams.h" #include "UgcKeys.h" #include "UgcModular.h" @@ -709,3 +710,28 @@ TEST(UgcIconParams, OneListDrivesEverything) { shifted.offsetX = 0.25f; EXPECT_NEAR(centroid(UgcRender::RenderIcon(model, shifted)) - centroid(UgcRender::RenderIcon(model, plain)), 16.0, 1.0); } + +TEST(UgcJobs, ModelsWithoutBricksAreEmptyNotFailed) { + UgcBricks::BrickLibrary library(MakeRes(), 0); + UgcJobs::Settings settings; + const auto empty = UgcJobs::ProcessModel(R"()", library, settings); + EXPECT_FALSE(empty.ok); + EXPECT_TRUE(empty.empty); + EXPECT_TRUE(UgcModel::HasNoBricks(R"()")); + // Broken LXFML, or bricks without geometry, are failures + EXPECT_FALSE(UgcJobs::ProcessModel(" + )", library, settings); + EXPECT_FALSE(missing.ok); + EXPECT_FALSE(missing.empty); + EXPECT_FALSE(UgcModel::HasNoBricks(R"()")); +} + +TEST(UgcStates, NamesComeFromTheEnum) { + EXPECT_EQ(IUgc::ProcessStateName(IUgc::eProcessState::EMPTY), "empty"); + EXPECT_EQ(IUgc::ProcessStateName(IUgc::eProcessState::FAILED), "failed"); + EXPECT_EQ(IUgc::ParseProcessState("empty"), IUgc::eProcessState::EMPTY); + EXPECT_EQ(IUgc::ParseProcessState("pending"), IUgc::eProcessState::PENDING); + EXPECT_FALSE(IUgc::ParseProcessState("nonsense").has_value()); + EXPECT_EQ(magic_enum::enum_count(), 4u); +}