From f252c6ff26d246dce7d76229b2f1b5d9d8d10359 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sun, 27 Sep 2026 21:36:20 -0500 Subject: [PATCH] feat(ugc): how many triangles each model saved, on the UGC page The UGC server stores a model's most detailed level's triangles before it removed the faces that can't be seen (ugc.triangle_count_before; migrations sqlite 75, mysql 92), from stats.json when it makes a model and, once, for the models made before. The UGC page's models list has a Saved column (the share and number of triangles removed, before/after in its tooltip), sortable by the share (sort=savings), and the gallery can sort by most triangles saved. Co-Authored-By: Claude Opus 5.5 --- dDashboardServer/routes/UgcRoutes.cpp | 6 +-- dDashboardServer/static/js/ugc.js | 8 +++- dDashboardServer/templates/ugc.jinja2 | 2 +- dDatabase/GameDatabase/ITables/IUgc.h | 5 ++- dDatabase/GameDatabase/ITables/IUgcLookup.h | 2 + dDatabase/GameDatabase/ITables/UgcLookupSql.h | 6 ++- dDatabase/GameDatabase/MySQL/MySQLDatabase.h | 2 +- dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp | 4 +- .../GameDatabase/MySQL/Tables/UgcLookup.cpp | 1 + .../GameDatabase/SQLite/SQLiteDatabase.h | 2 +- dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp | 4 +- .../GameDatabase/SQLite/Tables/UgcLookup.cpp | 1 + .../GameDatabase/TestSQL/TestSQLDatabase.h | 2 +- dUgcServer/UgcProcessor.cpp | 43 ++++++++++++++++--- dUgcServer/UgcProcessor.h | 2 + .../dlu/mysql/92_ugc_triangles_before.sql | 7 +++ .../dlu/sqlite/75_ugc_triangles_before.sql | 2 + tests/dWebTests/UgcLookupTests.cpp | 8 ++-- 18 files changed, 83 insertions(+), 24 deletions(-) create mode 100644 migrations/dlu/mysql/92_ugc_triangles_before.sql create mode 100644 migrations/dlu/sqlite/75_ugc_triangles_before.sql diff --git a/dDashboardServer/routes/UgcRoutes.cpp b/dDashboardServer/routes/UgcRoutes.cpp index 0e9d81c3e..c2b30911f 100644 --- a/dDashboardServer/routes/UgcRoutes.cpp +++ b/dDashboardServer/routes/UgcRoutes.cpp @@ -198,7 +198,7 @@ namespace { return { { "id", std::to_string(entry.id) }, { "characterId", std::to_string(entry.characterId) }, { "characterName", entry.characterName }, { "accountId", entry.accountId }, { "accountName", entry.accountName }, { "state", IUgc::ProcessStateName(entry.state) }, { "attempts", entry.attempts }, { "processedAt", entry.processedAt }, { "error", entry.error }, { "bakeAo", entry.bakeAo }, { "processAfter", entry.processAfter }, - { "detail", entry.detail }, { "bricks", entry.bricks }, { "triangles", entry.triangles }, { "processMs", entry.processMs }, { "processCpuMs", entry.processCpuMs }, { "processMemoryKb", entry.processMemoryKb }, { "modelName", entry.modelName } }; + { "detail", entry.detail }, { "bricks", entry.bricks }, { "triangles", entry.triangles }, { "processMs", entry.processMs }, { "processCpuMs", entry.processCpuMs }, { "processMemoryKb", entry.processMemoryKb }, { "modelName", entry.modelName }, { "trianglesBefore", entry.trianglesBefore } }; } // Web thread: the cars and rockets grouped into assemblies (one per combination of modules), from every build. @@ -271,7 +271,7 @@ namespace UgcRoutes { "A page of player models (kind=model) or of car and rocket assemblies (kind=modular: one per combination of modules, however many builds use " "it). Query: q= (\"state:\", \"kind:\"/\"type:\" (a build type, e.g. build6), \"owner:\", \"account:\", \"property:\", \"name:\", " "\"lot:\"/\"module:\" (a LOT or a module's name), \"id:\", or plain text across names, owners and ids), state=, type=, sort=newest|oldest|owner|name|" - "bricks|triangles|slowest|made|cpu|memory (models) or newest|oldest|references|name (assemblies), reverse=1 (the sort's other direction), page= (from 0), size= (1-200). {items, total, page, size, counts, " + "bricks|triangles|slowest|made|cpu|memory|savings (models) or newest|oldest|references|name (assemblies), reverse=1 (the sort's other direction), page= (from 0), size= (1-200). {items, total, page, size, counts, " "kinds, ugcPublicUrl, canManage}", [](HTTPReply& reply, const HTTPContext& context) { const bool modular = QueryValue(context.queryString, "kind") == "modular"; @@ -290,7 +290,7 @@ namespace UgcRoutes { query.search = parsed.search; query.state = parsed.state; static const std::map SORTS = { { "newest", IUgcLookup::eSort::NEWEST }, { "oldest", IUgcLookup::eSort::OLDEST }, - { "owner", IUgcLookup::eSort::OWNER }, { "name", IUgcLookup::eSort::NAME }, { "bricks", IUgcLookup::eSort::BRICKS }, { "triangles", IUgcLookup::eSort::TRIANGLES }, { "slowest", IUgcLookup::eSort::SLOWEST }, { "made", IUgcLookup::eSort::MADE }, { "cpu", IUgcLookup::eSort::CPU }, { "memory", IUgcLookup::eSort::MEMORY } }; + { "owner", IUgcLookup::eSort::OWNER }, { "name", IUgcLookup::eSort::NAME }, { "bricks", IUgcLookup::eSort::BRICKS }, { "triangles", IUgcLookup::eSort::TRIANGLES }, { "slowest", IUgcLookup::eSort::SLOWEST }, { "made", IUgcLookup::eSort::MADE }, { "cpu", IUgcLookup::eSort::CPU }, { "memory", IUgcLookup::eSort::MEMORY }, { "savings", IUgcLookup::eSort::SAVINGS } }; if (const auto it = SORTS.find(sortText); it != SORTS.end()) query.sort = it->second; query.reverse = reverse; query.offset = page * size; diff --git a/dDashboardServer/static/js/ugc.js b/dDashboardServer/static/js/ugc.js index d9be9bc3e..aa90f9b53 100644 --- a/dDashboardServer/static/js/ugc.js +++ b/dDashboardServer/static/js/ugc.js @@ -11,7 +11,7 @@ var STATES = { pending: ['Waiting', 'secondary'], done: ['Made', 'success'], failed: ['Failed', 'danger'], empty: ['Empty', 'light'] }; var SORTS = { - model: [['newest', 'Newest'], ['oldest', 'Oldest'], ['bricks', 'Most bricks'], ['triangles', 'Most triangles'], ['made', 'Recently made'], ['slowest', 'Slowest to make'], ['cpu', 'Most CPU'], ['memory', 'Most RAM'], ['owner', 'Owner'], ['name', 'File name']], + model: [['newest', 'Newest'], ['oldest', 'Oldest'], ['bricks', 'Most bricks'], ['triangles', 'Most triangles'], ['made', 'Recently made'], ['slowest', 'Slowest to make'], ['cpu', 'Most CPU'], ['memory', 'Most RAM'], ['savings', 'Most triangles saved'], ['owner', 'Owner'], ['name', 'File name']], modular: [['newest', 'Newest'], ['oldest', 'Oldest'], ['references', 'Most builds'], ['name', 'Name']] }; // The list's state, as in the address: ?kind=&q=&state=&type=&sort=&page= (from 1)&view=, and the open item (&item=, &build=) @@ -152,6 +152,12 @@ column('Took', function (i) { return '' + (i.processMs ? esc(duration(i.processMs)) : '') + ''; }, 'slowest', true), column('CPU', function (i) { return '' + (i.processCpuMs ? esc(duration(i.processCpuMs)) : '') + ''; }, 'cpu', true), column('RAM (est.)', function (i) { return '' + (i.processMemoryKb ? '~' + esc(megabytes(i.processMemoryKb)) : '') + ''; }, 'memory', true), + column('Saved', function (i) { + if (!i.trianglesBefore) return ''; + var removed = i.trianglesBefore - i.triangles, share = removed / i.trianglesBefore * 100; + return '' + + esc(share.toFixed(share < 10 ? 1 : 0)) + '%
' + esc(removed.toLocaleString()) + ' tris'; + }, 'savings', true), column('Size', function (i) { return '' + (i.bricks ? esc(i.bricks) + ' bricks
' + esc(i.triangles.toLocaleString()) + ' triangles' : '') + ''; }, 'bricks', true), column('File', function (i) { return '' + esc(fileName(i)) + '' + errorText(i); }, 'name'), column('', function (i) { return '
' + actions(i, i.id) + '
'; }) diff --git a/dDashboardServer/templates/ugc.jinja2 b/dDashboardServer/templates/ugc.jinja2 index 32cd46b9b..b58d0f1b2 100644 --- a/dDashboardServer/templates/ugc.jinja2 +++ b/dDashboardServer/templates/ugc.jinja2 @@ -74,7 +74,7 @@
-
IconIDOwnerStateMadeTookCPURAM (est.)SizeFile
+ IconIDOwnerStateMadeTookCPURAM (est.)SavedSizeFile
IconNewest buildTypeModulesStateBuildsOwners
diff --git a/dDatabase/GameDatabase/ITables/IUgc.h b/dDatabase/GameDatabase/ITables/IUgc.h index 1fb236250..0c2510804 100644 --- a/dDatabase/GameDatabase/ITables/IUgc.h +++ b/dDatabase/GameDatabase/ITables/IUgc.h @@ -92,8 +92,9 @@ public: // whether lighting was baked in; processed_at becomes now virtual void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) = 0; - // What the UGC server counted when it made a model: its bricks and the most detailed mesh's triangles - virtual void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) = 0; + // What the UGC server counted when it made a model: its bricks and the most detailed mesh's triangles, after and + // before it removed the faces that can't be seen + virtual void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles, const uint32_t trianglesBefore) = 0; // The last successful make of a model: how long it took, the worker's CPU time and the estimated memory struct ProcessStats { diff --git a/dDatabase/GameDatabase/ITables/IUgcLookup.h b/dDatabase/GameDatabase/ITables/IUgcLookup.h index 9c467322d..bbc351d68 100644 --- a/dDatabase/GameDatabase/ITables/IUgcLookup.h +++ b/dDatabase/GameDatabase/ITables/IUgcLookup.h @@ -40,6 +40,7 @@ public: uint32_t processCpuMs{}; // the worker thread's CPU time for it uint32_t processMemoryKb{}; // the memory the UGC server estimated for it (not measured) std::string modelName; // models: the name a player gave it where it is placed (empty: none) + uint32_t trianglesBefore{}; // models: the most detailed level's triangles before hidden faces were removed (0: unknown) }; // What SearchUgc matches. A number (when set) is matched against ids; text against names @@ -68,6 +69,7 @@ public: MADE, // the most recently made (or attempted) first (processed_at) CPU, // the most CPU time in the last make first (process_cpu_ms) MEMORY, // the most estimated memory in the last make first (process_memory_kb) + SAVINGS, // models: the biggest share of triangles removed first }; // A page of one kind: all of them or those matching the search (the same matching as SearchUgc), in a state or any diff --git a/dDatabase/GameDatabase/ITables/UgcLookupSql.h b/dDatabase/GameDatabase/ITables/UgcLookupSql.h index b39010b77..949d1a914 100644 --- a/dDatabase/GameDatabase/ITables/UgcLookupSql.h +++ b/dDatabase/GameDatabase/ITables/UgcLookupSql.h @@ -31,10 +31,10 @@ namespace UgcLookupSql { return (modular ? "SELECT b.ugc_id AS id, b.character_id, c.name AS character_name, COALESCE(c.account_id, 0) AS account_id, a.name AS account_name, " "b.is_optimized, b.process_error, b.ldf_config AS detail, b.process_attempts, b.processed_at, 0 AS process_after, 0 AS bake_ao, " - "0 AS brick_count, 0 AS triangle_count, b.process_ms, b.process_cpu_ms, b.process_memory_kb, '' AS model_name " + "0 AS brick_count, 0 AS triangle_count, b.process_ms, b.process_cpu_ms, b.process_memory_kb, '' AS model_name, 0 AS triangle_count_before " : "SELECT u.id, u.character_id, c.name AS character_name, u.account_id, a.name AS account_name, " "u.is_optimized, u.process_error, u.filename AS detail, u.process_attempts, u.processed_at, u.process_after, u.bake_ao, " - "u.brick_count, u.triangle_count, u.process_ms, u.process_cpu_ms, u.process_memory_kb, " + "u.brick_count, u.triangle_count, u.process_ms, u.process_cpu_ms, u.process_memory_kb, u.triangle_count_before, " // The name a player gave the model where it is placed (empty: none) "COALESCE((SELECT pc.model_name FROM properties_contents AS pc WHERE pc.ugc_id = u.id AND pc.model_name <> '' LIMIT 1), '') AS model_name ") + From(modular); } @@ -61,6 +61,8 @@ namespace UgcLookupSql { case eSort::NAME: return std::string("ORDER BY ") + (modular ? "b.ldf_config" : "LOWER(COALESCE(NULLIF(model_name, ''), u.filename))") + dir(false) + ", " + id + " DESC "; case eSort::BRICKS: return modular ? "ORDER BY " + id + dir(true) + " " : "ORDER BY u.brick_count" + std::string(dir(true)) + ", u.id DESC "; case eSort::TRIANGLES: return modular ? "ORDER BY " + id + dir(true) + " " : "ORDER BY u.triangle_count" + std::string(dir(true)) + ", u.id DESC "; + // The biggest share of triangles removed first; models without a before count last + case eSort::SAVINGS: return modular ? "ORDER BY " + id + dir(true) + " " : "ORDER BY CASE WHEN u.triangle_count_before > 0 THEN (u.triangle_count_before - u.triangle_count) * 1.0 / u.triangle_count_before ELSE -1 END" + std::string(dir(true)) + ", u.id DESC "; case eSort::CPU: return "ORDER BY " + std::string(modular ? "b" : "u") + ".process_cpu_ms" + dir(true) + ", " + id + " DESC "; case eSort::MEMORY: return "ORDER BY " + std::string(modular ? "b" : "u") + ".process_memory_kb" + dir(true) + ", " + id + " DESC "; case eSort::MADE: return "ORDER BY " + std::string(modular ? "b" : "u") + ".processed_at" + dir(true) + ", " + id + " DESC "; diff --git a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h index d22bf8467..ef40b7938 100644 --- a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h +++ b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h @@ -428,7 +428,7 @@ public: void SetUgcIconSettings(const std::string_view target, const std::string_view params) override; void DeleteUgcIconSettings(const std::string_view target) override; void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override; - void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override; + void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles, const uint32_t trianglesBefore) override; void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override; std::optional GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override; void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override; diff --git a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp index 38e8a9b65..7db4a49f7 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp @@ -178,8 +178,8 @@ void MySQLDatabase::ExpediteUgcModels(const LWOOBJID characterId) { ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE character_id = ? AND is_optimized = 0 AND process_after > 0;", characterId); } -void MySQLDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) { - ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ? WHERE id = ?;", bricks, triangles, id); +void MySQLDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles, const uint32_t trianglesBefore) { + ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ?, triangle_count_before = ? WHERE id = ?;", bricks, triangles, trianglesBefore, id); } void MySQLDatabase::SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) { diff --git a/dDatabase/GameDatabase/MySQL/Tables/UgcLookup.cpp b/dDatabase/GameDatabase/MySQL/Tables/UgcLookup.cpp index 97a6cfd94..9e7d82393 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/UgcLookup.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/UgcLookup.cpp @@ -24,6 +24,7 @@ namespace { entry.processCpuMs = static_cast(result->getInt64("process_cpu_ms")); entry.processMemoryKb = static_cast(result->getInt64("process_memory_kb")); entry.modelName = result->getString("model_name").c_str(); + entry.trianglesBefore = static_cast(result->getInt64("triangle_count_before")); return entry; } } diff --git a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h index a7e609532..eac981d29 100644 --- a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h +++ b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h @@ -412,7 +412,7 @@ public: void SetUgcIconSettings(const std::string_view target, const std::string_view params) override; void DeleteUgcIconSettings(const std::string_view target) override; void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override; - void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override; + void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles, const uint32_t trianglesBefore) override; void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override; std::optional GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override; void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override; diff --git a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp index c3de4f3ec..3655871ef 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp @@ -181,8 +181,8 @@ void SQLiteDatabase::ExpediteUgcModels(const LWOOBJID characterId) { ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE character_id = ? AND is_optimized = 0 AND process_after > 0;", characterId); } -void SQLiteDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) { - ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ? WHERE id = ?;", bricks, triangles, id); +void SQLiteDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles, const uint32_t trianglesBefore) { + ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ?, triangle_count_before = ? WHERE id = ?;", bricks, triangles, trianglesBefore, id); } void SQLiteDatabase::SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) { diff --git a/dDatabase/GameDatabase/SQLite/Tables/UgcLookup.cpp b/dDatabase/GameDatabase/SQLite/Tables/UgcLookup.cpp index 0921ec8f3..cae79d6f4 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/UgcLookup.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/UgcLookup.cpp @@ -24,6 +24,7 @@ namespace { entry.processCpuMs = static_cast(result.getInt64Field("process_cpu_ms")); entry.processMemoryKb = static_cast(result.getInt64Field("process_memory_kb")); entry.modelName = result.getStringField("model_name", ""); + entry.trianglesBefore = static_cast(result.getInt64Field("triangle_count_before")); return entry; } } diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h index a4a3c1b55..e77c331f5 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h @@ -410,7 +410,7 @@ class TestSQLDatabase : public GameDatabase { void SetUgcIconSettings(const std::string_view target, const std::string_view params) override {} void DeleteUgcIconSettings(const std::string_view target) override {} void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override {} - void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override {} + void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles, const uint32_t trianglesBefore) override {} void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override {} std::optional GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override { return std::nullopt; } void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override {} diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index aab21f790..fb628273c 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -31,6 +31,22 @@ namespace { constexpr auto RECENT_ANSWER_TIME = std::chrono::seconds(10); // In the storage folder once every item stored has its sd0 icon and checksums (Backfill) constexpr auto BACKFILL_MARKER = ".checksums-stored"; + constexpr auto STATS_BACKFILL_MARKER = ".stats-stored"; + + // What a model's stats.json counted: bricks, and the most detailed level's triangles after and before the faces that + // can't be seen were removed (the transparent ones are kept as they are) + struct ModelCounts { + uint32_t bricks{}; + uint32_t triangles{}; + uint32_t trianglesBefore{}; + }; + std::optional CountsOf(const nlohmann::json& stats) { + if (!stats.is_object()) return std::nullopt; + const auto& lods = stats.value("lods", nlohmann::json::array()); + const auto& first = lods.is_array() && !lods.empty() ? lods.front() : nlohmann::json::object(); + const auto transparent = first.value("transparent", 0u); + return ModelCounts{ stats.value("bricks", 0u), first.value("opaqueAfter", 0u) + transparent, first.value("opaqueBefore", 0u) + transparent }; + } constexpr size_t BACKFILL_ITEMS_PER_UPDATE = 16; constexpr uint32_t BACKFILL_BUILDS_PER_UPDATE = 200; // Combination id of a build whose modules can't be told (so it isn't looked at again) @@ -110,6 +126,11 @@ void UgcProcessor::Start() { std::error_code error; m_BackfillItemsDone = std::filesystem::exists(m_Storage.GetRoot() / BACKFILL_MARKER, error); if (!m_BackfillItemsDone) m_BackfillItems.assign(stored.begin(), stored.end()); + // Models made before the triangles before hidden-face removal were stored get them from their stats.json once + m_StatsBackfillDone = std::filesystem::exists(m_Storage.GetRoot() / STATS_BACKFILL_MARKER, error); + if (!m_StatsBackfillDone) { + for (const auto& entry : stored) if (entry.kind == Kind::MODEL) m_StatsBackfill.push_back(entry.id); + } m_NextEviction = std::chrono::steady_clock::now(); m_Stopping = false; for (size_t i = 0; i < std::max(m_Config.threads, 1); i++) m_Threads.emplace_back(&UgcProcessor::Worker, this); @@ -382,11 +403,7 @@ void UgcProcessor::Record(const Done& done) { if (done.outcome.ok) Database::Get()->SetUgcModelProcessStats(done.id, cost); // What it counted (stats.json), for sorting on the dashboard const auto stats = done.outcome.ok && !done.outcome.stats.empty() ? nlohmann::json::parse(done.outcome.stats, nullptr, false) : nlohmann::json(); - if (stats.is_object()) { - const auto& lods = stats.value("lods", nlohmann::json::array()); - const auto& first = lods.is_array() && !lods.empty() ? lods.front() : nlohmann::json::object(); - Database::Get()->SetUgcModelStats(done.id, stats.value("bricks", 0u), first.value("opaqueAfter", 0u) + first.value("transparent", 0u)); - } + if (const auto counts = CountsOf(stats)) Database::Get()->SetUgcModelStats(done.id, counts->bricks, counts->triangles, counts->trianglesBefore); } else { Database::Get()->SetModularBuildProcessed(done.id, state, attempts, error); if (done.outcome.ok) Database::Get()->SetModularBuildProcessStats(done.id, cost); @@ -475,6 +492,22 @@ void UgcProcessor::Backfill() { } if (builds.size() < BACKFILL_BUILDS_PER_UPDATE) m_BackfillBuildsDone = true; } + if (!m_StatsBackfillDone) { + for (size_t i = 0; i < BACKFILL_ITEMS_PER_UPDATE && !m_StatsBackfill.empty(); i++) { + const auto id = m_StatsBackfill.front(); + m_StatsBackfill.pop_front(); + const auto file = m_Storage.File(Kind::MODEL, id, "stats.json"); + if (!file) continue; + std::ifstream in(*file, std::ios::binary); + const auto counts = CountsOf(nlohmann::json::parse(std::string(std::istreambuf_iterator(in), {}), nullptr, false)); + if (counts) Database::Get()->SetUgcModelStats(id, counts->bricks, counts->triangles, counts->trianglesBefore); + } + if (m_StatsBackfill.empty()) { + m_StatsBackfillDone = true; + std::ofstream(m_Storage.GetRoot() / STATS_BACKFILL_MARKER) << "1\n"; + LOG("Stored the triangle counts of the models made before"); + } + } if (m_BackfillItemsDone) return; const auto read = [](const std::filesystem::path& path) { std::ifstream in(path, std::ios::binary); diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h index e60a1ae7e..58b541882 100644 --- a/dUgcServer/UgcProcessor.h +++ b/dUgcServer/UgcProcessor.h @@ -187,6 +187,8 @@ private: void Backfill(); std::deque m_BackfillItems; bool m_BackfillItemsDone{ true }; + std::deque m_StatsBackfill; // models whose triangle counts are read from their stats.json once + bool m_StatsBackfillDone{ true }; bool m_BackfillBuildsDone{ false }; void Poll(); diff --git a/migrations/dlu/mysql/92_ugc_triangles_before.sql b/migrations/dlu/mysql/92_ugc_triangles_before.sql new file mode 100644 index 000000000..2039d9eab --- /dev/null +++ b/migrations/dlu/mysql/92_ugc_triangles_before.sql @@ -0,0 +1,7 @@ +/* UGC server: ugc.triangle_count_before is the most detailed level's triangles before the UGC server removed the faces + that can't be seen (0 until known), so the dashboard can show how much each model saved (with triangle_count). */ +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN triangle_count_before INT UNSIGNED NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'triangle_count_before'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; diff --git a/migrations/dlu/sqlite/75_ugc_triangles_before.sql b/migrations/dlu/sqlite/75_ugc_triangles_before.sql new file mode 100644 index 000000000..ca0c45f12 --- /dev/null +++ b/migrations/dlu/sqlite/75_ugc_triangles_before.sql @@ -0,0 +1,2 @@ +/* UGC server: a model's triangles before hidden faces were removed. See the MySQL migration. */ +ALTER TABLE ugc ADD COLUMN triangle_count_before INTEGER NOT NULL DEFAULT 0; diff --git a/tests/dWebTests/UgcLookupTests.cpp b/tests/dWebTests/UgcLookupTests.cpp index dc0096f75..e2e92420e 100644 --- a/tests/dWebTests/UgcLookupTests.cpp +++ b/tests/dWebTests/UgcLookupTests.cpp @@ -105,15 +105,15 @@ namespace { "CREATE TABLE charinfo (id BIGINT PRIMARY KEY, account_id INTEGER, name TEXT);" "CREATE TABLE properties (id BIGINT PRIMARY KEY, owner_id BIGINT, name TEXT, zone_id INTEGER);" "CREATE TABLE ugc (id INTEGER PRIMARY KEY, account_id INTEGER, character_id BIGINT, is_optimized INTEGER, filename TEXT, process_error TEXT DEFAULT '', " - "process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_after BIGINT DEFAULT 0, bake_ao INTEGER DEFAULT 0, brick_count INTEGER DEFAULT 0, triangle_count INTEGER DEFAULT 0, process_ms INTEGER DEFAULT 0, process_cpu_ms INTEGER DEFAULT 0, process_memory_kb INTEGER DEFAULT 0);" + "process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_after BIGINT DEFAULT 0, bake_ao INTEGER DEFAULT 0, brick_count INTEGER DEFAULT 0, triangle_count INTEGER DEFAULT 0, process_ms INTEGER DEFAULT 0, process_cpu_ms INTEGER DEFAULT 0, process_memory_kb INTEGER DEFAULT 0, triangle_count_before INTEGER DEFAULT 0);" "CREATE TABLE ugc_modular_build (ugc_id BIGINT PRIMARY KEY, character_id BIGINT, ldf_config TEXT, is_optimized INTEGER DEFAULT 0, process_error TEXT DEFAULT '', " "process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_ms INTEGER DEFAULT 0, process_cpu_ms INTEGER DEFAULT 0, process_memory_kb INTEGER DEFAULT 0);" "CREATE TABLE properties_contents (id BIGINT PRIMARY KEY, property_id BIGINT, ugc_id BIGINT, lot INTEGER, model_name TEXT DEFAULT '', model_description TEXT DEFAULT '');" "INSERT INTO accounts VALUES (1, 'builder'), (2, 'racer');" "INSERT INTO charinfo VALUES (10, 1, 'Bricky'), (20, 2, 'Speedy');" "INSERT INTO properties VALUES (500, 10, 'Castle Hill', 1150);" - "INSERT INTO ugc (id, account_id, character_id, is_optimized, filename, process_error, brick_count, triangle_count, process_ms) VALUES " - "(1000, 1, 10, 1, 'tower.lxfml', '', 40, 900, 5200), (1001, 1, 10, 0, 'boat.lxfml', '', 0, 0, 0);" + "INSERT INTO ugc (id, account_id, character_id, is_optimized, filename, process_error, brick_count, triangle_count, process_ms, triangle_count_before) VALUES " + "(1000, 1, 10, 1, 'tower.lxfml', '', 40, 900, 5200, 1500), (1001, 1, 10, 0, 'boat.lxfml', '', 0, 0, 0, 0);" "INSERT INTO ugc_modular_build (ugc_id, character_id, ldf_config, is_optimized, process_error) VALUES (2000, 20, '1:8129+1:8130+1:9330', 1, '');" "INSERT INTO properties_contents VALUES (7000, 500, 1000, 14, 'Big tower', 'the tallest');"); } @@ -182,6 +182,8 @@ TEST_F(UgcLookupSqlTests, ListsPagesWithCounts) { query.reverse = true; // the oldest first EXPECT_EQ(List(query, false).first, (std::vector{ 1000, 1001 })); query.reverse = false; + query.sort = IUgcLookup::eSort::SAVINGS; // the tower lost 40% of its triangles; the boat isn't known + EXPECT_EQ(List(query, false).first, (std::vector{ 1000, 1001 })); query.sort = IUgcLookup::eSort::CPU; // none recorded: ties go newest first EXPECT_EQ(List(query, false).first, (std::vector{ 1001, 1000 })); query.sort = IUgcLookup::eSort::MEMORY;