diff --git a/dDashboardServer/routes/UgcRoutes.cpp b/dDashboardServer/routes/UgcRoutes.cpp index 67be52882..c5b97ec2f 100644 --- a/dDashboardServer/routes/UgcRoutes.cpp +++ b/dDashboardServer/routes/UgcRoutes.cpp @@ -458,9 +458,10 @@ namespace UgcRoutes { online = ServerState::g_UgcStatus.online; } const auto base = InternalUrl(); - Workers::Reply(reply, context, false, [base, enabled, online](HTTPReply& out) { - nlohmann::json body = { { "success", true }, { "enabled", enabled }, { "online", online }, - { "counts", { { "model", Counts(Database::Get()->GetUgcProcessCounts()) }, { "modular", Counts(Database::Get()->GetModularBuildProcessCounts()) } } } }; + // The database only on this thread; the worker only fetches + nlohmann::json body = { { "success", true }, { "enabled", enabled }, { "online", online }, + { "counts", { { "model", Counts(Database::Get()->GetUgcProcessCounts()) }, { "modular", Counts(Database::Get()->GetModularBuildProcessCounts()) } } } }; + Workers::Reply(reply, context, false, [base, enabled, body = std::move(body)](HTTPReply& out) mutable { if (enabled) { const auto fetched = CachedGet(base + "/status", eCache::STATUS); if (fetched->status == 200) body["status"] = nlohmann::json::parse(fetched->body, nullptr, false); diff --git a/dDashboardServer/templates/diagnostics.jinja2 b/dDashboardServer/templates/diagnostics.jinja2 index 83d236285..abafdb3d7 100644 --- a/dDashboardServer/templates/diagnostics.jinja2 +++ b/dDashboardServer/templates/diagnostics.jinja2 @@ -83,37 +83,29 @@ var direction = 'in'; var data = null; - // The UGC server (when it's enabled): its work, what it uses, and what it has made + // The UGC server (when it's enabled): whether it's up, what it's doing and uses, and what waits or failed. What it + // made, and how long each took, is on the UGC page. function loadUgc() { api.get('/api/diagnostics/ugc').then(function (d) { var card = document.getElementById('ugcCard'), body = document.getElementById('ugcBody'); card.classList.toggle('d-none', !d.enabled); if (!d.enabled) return; - var counts = function (title, c) { - return '
' + esc(title) + ': ' + fmt.badge(nf.format(c.done || 0) + ' made', 'success') + ' ' + fmt.badge(nf.format(c.pending || 0) + ' waiting', 'secondary') + ' ' + - fmt.badge(nf.format(c.failed || 0) + ' failed', c.failed ? 'danger' : 'secondary') + (c.empty ? ' ' + fmt.badge(nf.format(c.empty) + ' empty', 'light') : '') + '
'; - }; - var html = counts('Models', d.counts.model) + counts('Cars and rockets', d.counts.modular); + var waiting = (d.counts.model.pending || 0) + (d.counts.modular.pending || 0), failed = (d.counts.model.failed || 0) + (d.counts.modular.failed || 0); + var row = function (k, v) { return '' + esc(k) + '' + v + ''; }; + var queue = row('Waiting', esc(nf.format(waiting)) + (failed ? ', ' + fmt.link('/ugc?view=list&state=failed', nf.format(failed) + ' failed') : '')); var s = d.status; if (!s) { - body.innerHTML = fmt.badge('Unreachable', 'warning') + ' ' + esc(d.error || '') + '
' + html; + body.innerHTML = '
' + fmt.badge('Unreachable', 'warning') + ' ' + esc(d.error || '') + '
' + queue + '
'; return; } var u = s.usage || {}, l = s.limits || {}, cores = u.cores || 1; - var row = function (k, v) { return '' + esc(k) + '' + v + ''; }; - html = '
' + fmt.badge('Online', 'success') + (u.throttled ? ' ' + fmt.badge('Throttled', 'warning') : '') + (u.paused ? ' ' + fmt.badge('Paused', 'info') : '') + '
' + - '' + - row('Work', esc(s.active) + ' of ' + esc(s.workers) + ' workers busy, ' + esc(nf.format(s.queued)) + ' queued') + - row('CPU', esc(((u.cpuPercent || 0) / cores).toFixed(0)) + '% of ' + esc(cores) + ' cores' + (l.maxCpus ? ' (limit ' + esc((l.maxCpus / cores * 100).toFixed(0)) + '%)' : '')) + - row('Memory', esc(bytes(u.residentBytes || 0)) + ' in use; jobs ~' + esc(bytes(s.jobMemoryBytes || 0)) + (l.maxMemoryBytes ? ' of ' + esc(bytes(l.maxMemoryBytes)) : '') + (s.memoryWaits ? ', ' + esc(s.memoryWaits) + ' waits' : '')) + - row('Files', esc(bytes(s.storedBytes || 0)) + (s.maxStorageBytes ? ' of ' + esc(bytes(s.maxStorageBytes)) : '') + (s.evicted ? ', ' + esc(s.evicted) + ' evicted' : '')) + - row('Since start', esc(nf.format(s.made || 0)) + ' made, ' + esc(nf.format(s.failed || 0)) + ' failed attempts') + - '
' + html; - var recent = (s.recent || []).slice(0, 5); - if (recent.length) html += '
Last: ' + recent.map(function (r) { - return esc(r.kind) + ' ' + esc(r.id) + (r.ok ? ' ' + esc(ms(r.ms)) : ' failed'); - }).join(' ยท ') + '
'; - body.innerHTML = html; + body.innerHTML = '
' + fmt.badge('Online', 'success') + (u.throttled ? ' ' + fmt.badge('Throttled', 'warning') : '') + (u.paused ? ' ' + fmt.badge('Paused', 'info') : '') + '
' + + '' + queue + + row('Workers', esc(s.active) + ' of ' + esc(s.workers) + ' busy') + + row('CPU', esc(((u.cpuPercent || 0) / cores).toFixed(0)) + '%' + (l.maxCpus ? ' of ' + esc((l.maxCpus / cores * 100).toFixed(0)) + '% allowed' : '')) + + row('Memory', esc(bytes(u.residentBytes || 0))) + + row('Files', esc(bytes(s.storedBytes || 0)) + (s.maxStorageBytes ? ' of ' + esc(bytes(s.maxStorageBytes)) : '')) + + '
'; }).catch(function () {}); } loadUgc(); diff --git a/docs/Dashboard.md b/docs/Dashboard.md index 4ca0b9209..1f5ac74fb 100644 --- a/docs/Dashboard.md +++ b/docs/Dashboard.md @@ -698,7 +698,9 @@ second of each server, HTTP requests per second of the dashboard and the UGC ser their request counts, errors and latency. **Live** shows the last 5 minutes at one second and updates as reports arrive (the `traffic` WebSocket topic); **1 hour** is at 10 seconds; **24 hours** (5 minutes) and **7 days** (30 minutes) come from the database. The servers table also shows each server's RakNet connections, average ping and resent -messages, and the worker threads of the dashboard and the UGC server. +messages, and the worker threads of the dashboard and the UGC server. While the UGC server is enabled, its card says +whether it is up (throttled, paused), how many items wait and failed (a link to the failed ones), its busy workers, CPU, +memory and stored files; what it made and how long each took is on the UGC page. How it is counted: