From cd81cf4c74090530e387f18f2a73320ba1cdad8f Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sun, 27 Sep 2026 20:33:09 -0500 Subject: [PATCH] feat(ugc): each make's CPU time and memory; durations in readable units The UGC server records, with each successful make's time, the worker thread's CPU time for it and the memory it estimated the job needs (the figure its memory budget counts; not a measurement) (process_cpu_ms and process_memory_kb on ugc and ugc_modular_build; migrations sqlite 74, mysql 91). The UGC page shows them with the time and duration ("took 12.4 s, CPU 11.9 s, ~96 MB RAM (est.)"), and durations use the largest units that fit, up to days. Co-Authored-By: Claude Opus 5.5 --- dDashboardServer/routes/UgcRoutes.cpp | 2 +- dDashboardServer/static/js/ugc.js | 22 +++++++++++----- dDatabase/GameDatabase/ITables/IUgc.h | 9 +++++-- dDatabase/GameDatabase/ITables/IUgcLookup.h | 2 ++ .../GameDatabase/ITables/IUgcModularBuild.h | 4 +-- dDatabase/GameDatabase/ITables/UgcLookupSql.h | 4 +-- dDatabase/GameDatabase/MySQL/MySQLDatabase.h | 4 +-- dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp | 4 +-- .../GameDatabase/MySQL/Tables/UgcLookup.cpp | 2 ++ .../MySQL/Tables/UgcModularBuild.cpp | 4 +-- .../GameDatabase/SQLite/SQLiteDatabase.h | 4 +-- dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp | 4 +-- .../GameDatabase/SQLite/Tables/UgcLookup.cpp | 2 ++ .../SQLite/Tables/UgcModularBuild.cpp | 4 +-- .../GameDatabase/TestSQL/TestSQLDatabase.h | 4 +-- dUgcServer/UgcProcessor.cpp | 9 +++++-- dUgcServer/UgcProcessor.h | 2 ++ .../dlu/mysql/91_ugc_process_diagnostics.sql | 25 +++++++++++++++++++ .../dlu/sqlite/74_ugc_process_diagnostics.sql | 5 ++++ tests/dWebTests/UgcLookupTests.cpp | 4 +-- 20 files changed, 89 insertions(+), 31 deletions(-) create mode 100644 migrations/dlu/mysql/91_ugc_process_diagnostics.sql create mode 100644 migrations/dlu/sqlite/74_ugc_process_diagnostics.sql diff --git a/dDashboardServer/routes/UgcRoutes.cpp b/dDashboardServer/routes/UgcRoutes.cpp index 54f5578b8..6754a7269 100644 --- a/dDashboardServer/routes/UgcRoutes.cpp +++ b/dDashboardServer/routes/UgcRoutes.cpp @@ -191,7 +191,7 @@ namespace { return { { "id", std::to_string(entry.id) }, { "characterId", std::to_string(entry.characterId) }, { "characterName", entry.characterName }, { "accountId", entry.accountId }, { "accountName", entry.accountName }, { "state", IUgc::ProcessStateName(entry.state) }, { "attempts", entry.attempts }, { "processedAt", entry.processedAt }, { "error", entry.error }, { "bakeAo", entry.bakeAo }, { "processAfter", entry.processAfter }, - { "detail", entry.detail }, { "bricks", entry.bricks }, { "triangles", entry.triangles }, { "processMs", entry.processMs } }; + { "detail", entry.detail }, { "bricks", entry.bricks }, { "triangles", entry.triangles }, { "processMs", entry.processMs }, { "processCpuMs", entry.processCpuMs }, { "processMemoryKb", entry.processMemoryKb } }; } // Web thread: every car and rocket build (the assemblies are made from them) diff --git a/dDashboardServer/static/js/ugc.js b/dDashboardServer/static/js/ugc.js index 57395f42a..ecb758770 100644 --- a/dDashboardServer/static/js/ugc.js +++ b/dDashboardServer/static/js/ugc.js @@ -89,16 +89,26 @@ return ''; } function moduleNames(i) { return (i.moduleList || []).map(function (m) { return m.name || m.lot; }).join(', '); } - // How long a make took: "850 ms", "12.4 s", "2 min 5 s" + // A duration in the largest units that fit: "850 ms", "12.4 s", "2 min 5 s", "1 h 4 min", "2 d 3 h" function duration(ms) { + ms = Math.round(ms); if (ms < 1000) return ms + ' ms'; - if (ms < 60000) return (ms / 1000).toFixed(1) + ' s'; - return Math.floor(ms / 60000) + ' min ' + Math.round((ms % 60000) / 1000) + ' s'; + if (ms < 60000) return (ms / 1000).toFixed(ms < 10000 ? 2 : 1) + ' s'; + var s = Math.round(ms / 1000), m = Math.floor(s / 60), h = Math.floor(m / 60), d = Math.floor(h / 24); + if (d) return d + ' d' + (h % 24 ? ' ' + (h % 24) + ' h' : ''); + if (h) return h + ' h' + (m % 60 ? ' ' + (m % 60) + ' min' : ''); + return m + ' min' + (s % 60 ? ' ' + (s % 60) + ' s' : ''); } - // When it was made and how long that took (the time is only known for makes since it was recorded) + function megabytes(kb) { return kb >= 1048576 ? (kb / 1048576).toFixed(1) + ' GB' : kb >= 1024 ? Math.round(kb / 1024) + ' MB' : kb + ' KB'; } + // What the last make cost: its time, the worker's CPU time and the memory the UGC server estimated for it + function costText(i) { + if (!i.processMs) return ''; + return 'took ' + duration(i.processMs) + (i.processCpuMs ? ', CPU ' + duration(i.processCpuMs) : '') + (i.processMemoryKb ? ', ~' + megabytes(i.processMemoryKb) + ' RAM (est.)' : ''); + } + // When it was made and what that cost (the cost is only known for makes since it was recorded) function madeText(i) { if (!i.processedAt) return ''; - return fmt.unix(i.processedAt) + (i.processMs ? ' \u00b7 took ' + duration(i.processMs) : ''); + return fmt.unix(i.processedAt) + (costText(i) ? ' \u00b7 ' + costText(i) : ''); } function waitBadge(i) { return i.state === 'pending' && i.processAfter > Date.now() / 1000 ? ' after ' + esc(fmt.unix(i.processAfter)) + '' : ''; @@ -288,7 +298,7 @@ meter('Jobs\' memory', s.jobMemoryBytes || 0, l.maxMemoryBytes || 0, esc(mb(s.jobMemoryBytes || 0)) + ' estimated' + (l.maxMemoryBytes ? ' of ' + esc(mb(l.maxMemoryBytes)) : ', no limit') + (s.memoryWaits ? ', ' + esc(s.memoryWaits) + ' waits' : '')) + '' + (l.nice ? '
Workers run at priority ' + esc(l.nice) + '.
' : '') + - (last ? '
Last: ' + esc(last.kind) + ' ' + esc(last.id) + (last.ok ? ' made in ' + esc(last.ms) + ' ms' : ' failed') + + (last ? '
Last: ' + esc(last.kind) + ' ' + esc(last.id) + (last.ok ? ' made in ' + esc(duration(last.ms)) : ' failed') + (last.message ? ' (' + esc(last.message) + ')' : '') + '
' : ''); }).catch(function () { box.innerHTML = fmt.badge('Unknown', 'secondary') + ' Couldn\'t ask the dashboard for the UGC server\'s status.'; }); } diff --git a/dDatabase/GameDatabase/ITables/IUgc.h b/dDatabase/GameDatabase/ITables/IUgc.h index dd11b333f..1fb236250 100644 --- a/dDatabase/GameDatabase/ITables/IUgc.h +++ b/dDatabase/GameDatabase/ITables/IUgc.h @@ -95,8 +95,13 @@ public: // What the UGC server counted when it made a model: its bricks and the most detailed mesh's triangles virtual void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) = 0; - // How long the last successful make of a model took (process_ms), for the dashboard - virtual void SetUgcModelProcessMs(const LWOOBJID id, const uint32_t milliseconds) = 0; + // The last successful make of a model: how long it took, the worker's CPU time and the estimated memory + struct ProcessStats { + uint32_t milliseconds{}; + uint32_t cpuMilliseconds{}; + uint32_t memoryKb{}; + }; + virtual void SetUgcModelProcessStats(const LWOOBJID id, const ProcessStats& stats) = 0; virtual std::optional GetUgcProcessInfo(const LWOOBJID id) = 0; diff --git a/dDatabase/GameDatabase/ITables/IUgcLookup.h b/dDatabase/GameDatabase/ITables/IUgcLookup.h index e1a3117fa..f0228a01e 100644 --- a/dDatabase/GameDatabase/ITables/IUgcLookup.h +++ b/dDatabase/GameDatabase/ITables/IUgcLookup.h @@ -37,6 +37,8 @@ public: uint32_t bricks{}; // models: counted by the UGC server when it made them (0: not yet) uint32_t triangles{}; // models: of the made mesh's most detailed level (0: not yet) uint32_t processMs{}; // how long the last successful make took (0: not made, or made before it was timed) + uint32_t processCpuMs{}; // the worker thread's CPU time for it + uint32_t processMemoryKb{}; // the memory the UGC server estimated for it (not measured) }; // What SearchUgc matches. A number (when set) is matched against ids; text against names diff --git a/dDatabase/GameDatabase/ITables/IUgcModularBuild.h b/dDatabase/GameDatabase/ITables/IUgcModularBuild.h index b04c3ff10..937b28f4e 100644 --- a/dDatabase/GameDatabase/ITables/IUgcModularBuild.h +++ b/dDatabase/GameDatabase/ITables/IUgcModularBuild.h @@ -32,8 +32,8 @@ public: // The combination of modules a build was made as (UgcModularKey::StorageId of its modules), whose files it shares virtual void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) = 0; - // How long the last successful make of a build's icon took (process_ms), for the dashboard - virtual void SetModularBuildProcessMs(const LWOOBJID id, const uint32_t milliseconds) = 0; + // The last successful make of a build's icon (see IUgc::ProcessStats) + virtual void SetModularBuildProcessStats(const LWOOBJID id, const IUgc::ProcessStats& stats) = 0; // Up to `limit` builds whose combination isn't recorded yet (combination_id 0): their ids and modules virtual std::vector GetModularBuildsWithoutCombination(const uint32_t limit) = 0; diff --git a/dDatabase/GameDatabase/ITables/UgcLookupSql.h b/dDatabase/GameDatabase/ITables/UgcLookupSql.h index 06084d68e..d259db041 100644 --- a/dDatabase/GameDatabase/ITables/UgcLookupSql.h +++ b/dDatabase/GameDatabase/ITables/UgcLookupSql.h @@ -31,10 +31,10 @@ namespace UgcLookupSql { return (modular ? "SELECT b.ugc_id AS id, b.character_id, c.name AS character_name, COALESCE(c.account_id, 0) AS account_id, a.name AS account_name, " "b.is_optimized, b.process_error, b.ldf_config AS detail, b.process_attempts, b.processed_at, 0 AS process_after, 0 AS bake_ao, " - "0 AS brick_count, 0 AS triangle_count, b.process_ms " + "0 AS brick_count, 0 AS triangle_count, b.process_ms, b.process_cpu_ms, b.process_memory_kb " : "SELECT u.id, u.character_id, c.name AS character_name, u.account_id, a.name AS account_name, " "u.is_optimized, u.process_error, u.filename AS detail, u.process_attempts, u.processed_at, u.process_after, u.bake_ao, " - "u.brick_count, u.triangle_count, u.process_ms ") + From(modular); + "u.brick_count, u.triangle_count, u.process_ms, u.process_cpu_ms, u.process_memory_kb ") + From(modular); } // Whether a search has anything to match (else a list is of everything) diff --git a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h index fea0b6bc8..d22bf8467 100644 --- a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h +++ b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h @@ -432,8 +432,8 @@ public: void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override; std::optional GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override; void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override; - void SetUgcModelProcessMs(const LWOOBJID id, const uint32_t milliseconds) override; - void SetModularBuildProcessMs(const LWOOBJID id, const uint32_t milliseconds) override; + void SetUgcModelProcessStats(const LWOOBJID id, const ProcessStats& stats) override; + void SetModularBuildProcessStats(const LWOOBJID id, const IUgc::ProcessStats& stats) override; std::vector GetModularBuildsWithoutCombination(const uint32_t limit) override; std::optional GetUgcProcessInfo(const LWOOBJID id) override; uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override; diff --git a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp index cbe17cc43..38e8a9b65 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp @@ -195,6 +195,6 @@ std::optional MySQLDatabase::GetUgcFileChecksum(const LWOOBJ return IUgc::FileChecksum{ std::string(result->getString("md5").c_str()), static_cast(result->getUInt("size")) }; } -void MySQLDatabase::SetUgcModelProcessMs(const LWOOBJID id, const uint32_t milliseconds) { - ExecuteUpdate("UPDATE ugc SET process_ms = ? WHERE id = ?;", milliseconds, id); +void MySQLDatabase::SetUgcModelProcessStats(const LWOOBJID id, const ProcessStats& stats) { + ExecuteUpdate("UPDATE ugc SET process_ms = ?, process_cpu_ms = ?, process_memory_kb = ? WHERE id = ?;", stats.milliseconds, stats.cpuMilliseconds, stats.memoryKb, id); } diff --git a/dDatabase/GameDatabase/MySQL/Tables/UgcLookup.cpp b/dDatabase/GameDatabase/MySQL/Tables/UgcLookup.cpp index 2a5286d21..99403b4a3 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/UgcLookup.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/UgcLookup.cpp @@ -21,6 +21,8 @@ namespace { entry.bricks = static_cast(result->getInt64("brick_count")); entry.triangles = static_cast(result->getInt64("triangle_count")); entry.processMs = static_cast(result->getInt64("process_ms")); + entry.processCpuMs = static_cast(result->getInt64("process_cpu_ms")); + entry.processMemoryKb = static_cast(result->getInt64("process_memory_kb")); return entry; } } diff --git a/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp b/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp index 403bdde00..ed427d366 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp @@ -120,6 +120,6 @@ std::vector MySQLDatabase::GetModularBuildsWitho return builds; } -void MySQLDatabase::SetModularBuildProcessMs(const LWOOBJID id, const uint32_t milliseconds) { - ExecuteUpdate("UPDATE ugc_modular_build SET process_ms = ? WHERE ugc_id = ?;", milliseconds, id); +void MySQLDatabase::SetModularBuildProcessStats(const LWOOBJID id, const IUgc::ProcessStats& stats) { + ExecuteUpdate("UPDATE ugc_modular_build SET process_ms = ?, process_cpu_ms = ?, process_memory_kb = ? WHERE ugc_id = ?;", stats.milliseconds, stats.cpuMilliseconds, stats.memoryKb, id); } diff --git a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h index 981075b0f..a7e609532 100644 --- a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h +++ b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h @@ -416,8 +416,8 @@ public: void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override; std::optional GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override; void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override; - void SetUgcModelProcessMs(const LWOOBJID id, const uint32_t milliseconds) override; - void SetModularBuildProcessMs(const LWOOBJID id, const uint32_t milliseconds) override; + void SetUgcModelProcessStats(const LWOOBJID id, const ProcessStats& stats) override; + void SetModularBuildProcessStats(const LWOOBJID id, const IUgc::ProcessStats& stats) override; std::vector GetModularBuildsWithoutCombination(const uint32_t limit) override; std::optional GetUgcProcessInfo(const LWOOBJID id) override; uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override; diff --git a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp index 72ec262af..c3de4f3ec 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp @@ -198,6 +198,6 @@ std::optional SQLiteDatabase::GetUgcFileChecksum(const LWOOB return IUgc::FileChecksum{ std::string(result.getStringField("md5", "")), static_cast(result.getInt64Field("size")) }; } -void SQLiteDatabase::SetUgcModelProcessMs(const LWOOBJID id, const uint32_t milliseconds) { - ExecuteUpdate("UPDATE ugc SET process_ms = ? WHERE id = ?;", milliseconds, id); +void SQLiteDatabase::SetUgcModelProcessStats(const LWOOBJID id, const ProcessStats& stats) { + ExecuteUpdate("UPDATE ugc SET process_ms = ?, process_cpu_ms = ?, process_memory_kb = ? WHERE id = ?;", stats.milliseconds, stats.cpuMilliseconds, stats.memoryKb, id); } diff --git a/dDatabase/GameDatabase/SQLite/Tables/UgcLookup.cpp b/dDatabase/GameDatabase/SQLite/Tables/UgcLookup.cpp index a9209767a..397a7644e 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/UgcLookup.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/UgcLookup.cpp @@ -21,6 +21,8 @@ namespace { entry.bricks = static_cast(result.getInt64Field("brick_count")); entry.triangles = static_cast(result.getInt64Field("triangle_count")); entry.processMs = static_cast(result.getInt64Field("process_ms")); + entry.processCpuMs = static_cast(result.getInt64Field("process_cpu_ms")); + entry.processMemoryKb = static_cast(result.getInt64Field("process_memory_kb")); return entry; } } diff --git a/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp b/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp index 076816270..9e70d662e 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp @@ -127,6 +127,6 @@ std::vector SQLiteDatabase::GetModularBuildsWith return builds; } -void SQLiteDatabase::SetModularBuildProcessMs(const LWOOBJID id, const uint32_t milliseconds) { - ExecuteUpdate("UPDATE ugc_modular_build SET process_ms = ? WHERE ugc_id = ?;", milliseconds, id); +void SQLiteDatabase::SetModularBuildProcessStats(const LWOOBJID id, const IUgc::ProcessStats& stats) { + ExecuteUpdate("UPDATE ugc_modular_build SET process_ms = ?, process_cpu_ms = ?, process_memory_kb = ? WHERE ugc_id = ?;", stats.milliseconds, stats.cpuMilliseconds, stats.memoryKb, id); } diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h index 5ff358949..a4a3c1b55 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h @@ -414,8 +414,8 @@ class TestSQLDatabase : public GameDatabase { void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override {} std::optional GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override { return std::nullopt; } void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override {} - void SetUgcModelProcessMs(const LWOOBJID id, const uint32_t milliseconds) override {} - void SetModularBuildProcessMs(const LWOOBJID id, const uint32_t milliseconds) override {} + void SetUgcModelProcessStats(const LWOOBJID id, const ProcessStats& stats) override {} + void SetModularBuildProcessStats(const LWOOBJID id, const IUgc::ProcessStats& stats) override {} std::vector GetModularBuildsWithoutCombination(const uint32_t limit) override { return {}; } std::optional GetUgcProcessInfo(const LWOOBJID id) override { return {}; } uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override { return 0; } diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index feac3a83b..aab21f790 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -221,7 +221,9 @@ void UgcProcessor::Worker() { continue; } const auto start = std::chrono::steady_clock::now(); + const double cpuStart = UgcThrottle::ThreadCpuSeconds(); Done done{ job.kind, job.id, job.attempts }; + done.memoryEstimate = job.memory; done.iconOnly = job.iconOnly; try { if (job.iconOnly) { @@ -262,6 +264,7 @@ void UgcProcessor::Worker() { } done.outcome.files.clear(); done.milliseconds = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + done.cpuMilliseconds = std::max(0.0, UgcThrottle::ThreadCpuSeconds() - cpuStart) * 1000.0; UgcThrottle::Checkpoint(); { @@ -367,6 +370,8 @@ void UgcProcessor::Poll() { } void UgcProcessor::Record(const Done& done) { + // What the make cost (wall time, the worker's CPU time, the estimated memory), for the dashboard + const IUgc::ProcessStats cost{ static_cast(done.milliseconds), static_cast(done.cpuMilliseconds), static_cast(done.memoryEstimate / 1024) }; auto error = done.outcome.error.substr(0, MAX_ERROR_LENGTH); const auto attempts = done.attempts + 1; const auto state = done.outcome.ok ? IUgc::eProcessState::DONE @@ -374,7 +379,7 @@ void UgcProcessor::Record(const Done& done) { : attempts >= m_Config.maxAttempts ? IUgc::eProcessState::FAILED : IUgc::eProcessState::PENDING; if (done.kind == Kind::MODEL) { Database::Get()->SetUgcModelProcessed(done.id, state, attempts, error, done.outcome.ok && done.outcome.aoBaked); - if (done.outcome.ok) Database::Get()->SetUgcModelProcessMs(done.id, static_cast(done.milliseconds)); + if (done.outcome.ok) Database::Get()->SetUgcModelProcessStats(done.id, cost); // What it counted (stats.json), for sorting on the dashboard const auto stats = done.outcome.ok && !done.outcome.stats.empty() ? nlohmann::json::parse(done.outcome.stats, nullptr, false) : nlohmann::json(); if (stats.is_object()) { @@ -384,7 +389,7 @@ void UgcProcessor::Record(const Done& done) { } } else { Database::Get()->SetModularBuildProcessed(done.id, state, attempts, error); - if (done.outcome.ok) Database::Get()->SetModularBuildProcessMs(done.id, static_cast(done.milliseconds)); + if (done.outcome.ok) Database::Get()->SetModularBuildProcessStats(done.id, cost); // Which combination's files it shares, for the worlds' manifest answers if (const auto combo = m_ComboOf.find(done.id); combo != m_ComboOf.end()) Database::Get()->SetModularBuildCombination(done.id, combo->second); } diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h index 23582c2e2..e60a1ae7e 100644 --- a/dUgcServer/UgcProcessor.h +++ b/dUgcServer/UgcProcessor.h @@ -173,6 +173,8 @@ private: UgcJobs::Outcome outcome; uint64_t bytes{}; double milliseconds{}; + double cpuMilliseconds{}; // the worker thread's CPU time for it + uint64_t memoryEstimate{}; // the bytes it was estimated to need (the memory budget's figure) bool iconOnly{}; std::vector checksums; // of the files written that the client downloads as sd0 }; diff --git a/migrations/dlu/mysql/91_ugc_process_diagnostics.sql b/migrations/dlu/mysql/91_ugc_process_diagnostics.sql new file mode 100644 index 000000000..bc45bd814 --- /dev/null +++ b/migrations/dlu/mysql/91_ugc_process_diagnostics.sql @@ -0,0 +1,25 @@ +/* UGC server: process_cpu_ms is the CPU time the worker thread spent on the last successful make, process_memory_kb + the memory the UGC server estimated for it (what its memory budget counts; not measured), for the dashboard. */ +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN process_cpu_ms INT UNSIGNED NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'process_cpu_ms'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; + +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN process_memory_kb INT UNSIGNED NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'process_memory_kb'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; + +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN process_cpu_ms INT UNSIGNED NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'process_cpu_ms'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; + +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN process_memory_kb INT UNSIGNED NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'process_memory_kb'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; diff --git a/migrations/dlu/sqlite/74_ugc_process_diagnostics.sql b/migrations/dlu/sqlite/74_ugc_process_diagnostics.sql new file mode 100644 index 000000000..d04e1a1b8 --- /dev/null +++ b/migrations/dlu/sqlite/74_ugc_process_diagnostics.sql @@ -0,0 +1,5 @@ +/* UGC server: the CPU time and estimated memory of each model's and build's last make. See the MySQL migration. */ +ALTER TABLE ugc ADD COLUMN process_cpu_ms INTEGER NOT NULL DEFAULT 0; +ALTER TABLE ugc ADD COLUMN process_memory_kb INTEGER NOT NULL DEFAULT 0; +ALTER TABLE ugc_modular_build ADD COLUMN process_cpu_ms INTEGER NOT NULL DEFAULT 0; +ALTER TABLE ugc_modular_build ADD COLUMN process_memory_kb INTEGER NOT NULL DEFAULT 0; diff --git a/tests/dWebTests/UgcLookupTests.cpp b/tests/dWebTests/UgcLookupTests.cpp index 6c584d6ac..0f0bbe07d 100644 --- a/tests/dWebTests/UgcLookupTests.cpp +++ b/tests/dWebTests/UgcLookupTests.cpp @@ -105,9 +105,9 @@ namespace { "CREATE TABLE charinfo (id BIGINT PRIMARY KEY, account_id INTEGER, name TEXT);" "CREATE TABLE properties (id BIGINT PRIMARY KEY, owner_id BIGINT, name TEXT, zone_id INTEGER);" "CREATE TABLE ugc (id INTEGER PRIMARY KEY, account_id INTEGER, character_id BIGINT, is_optimized INTEGER, filename TEXT, process_error TEXT DEFAULT '', " - "process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_after BIGINT DEFAULT 0, bake_ao INTEGER DEFAULT 0, brick_count INTEGER DEFAULT 0, triangle_count INTEGER DEFAULT 0, process_ms INTEGER DEFAULT 0);" + "process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_after BIGINT DEFAULT 0, bake_ao INTEGER DEFAULT 0, brick_count INTEGER DEFAULT 0, triangle_count INTEGER DEFAULT 0, process_ms INTEGER DEFAULT 0, process_cpu_ms INTEGER DEFAULT 0, process_memory_kb INTEGER DEFAULT 0);" "CREATE TABLE ugc_modular_build (ugc_id BIGINT PRIMARY KEY, character_id BIGINT, ldf_config TEXT, is_optimized INTEGER DEFAULT 0, process_error TEXT DEFAULT '', " - "process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_ms INTEGER DEFAULT 0);" + "process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_ms INTEGER DEFAULT 0, process_cpu_ms INTEGER DEFAULT 0, process_memory_kb INTEGER DEFAULT 0);" "CREATE TABLE properties_contents (id BIGINT PRIMARY KEY, property_id BIGINT, ugc_id BIGINT, lot INTEGER, model_name TEXT DEFAULT '', model_description TEXT DEFAULT '');" "INSERT INTO accounts VALUES (1, 'builder'), (2, 'racer');" "INSERT INTO charinfo VALUES (10, 1, 'Bricky'), (20, 2, 'Speedy');"