mirror of
https://github.com/DarkflameUniverse/DarkflameServer.git
synced 2026-10-29 16:07:14 +00:00
feat(ugc): how many triangles each model saved, on the UGC page
The UGC server stores a model's most detailed level's triangles before it removed the faces that can't be seen (ugc.triangle_count_before; migrations sqlite 75, mysql 92), from stats.json when it makes a model and, once, for the models made before. The UGC page's models list has a Saved column (the share and number of triangles removed, before/after in its tooltip), sortable by the share (sort=savings), and the gallery can sort by most triangles saved. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -198,7 +198,7 @@ namespace {
|
|||||||
return { { "id", std::to_string(entry.id) }, { "characterId", std::to_string(entry.characterId) }, { "characterName", entry.characterName },
|
return { { "id", std::to_string(entry.id) }, { "characterId", std::to_string(entry.characterId) }, { "characterName", entry.characterName },
|
||||||
{ "accountId", entry.accountId }, { "accountName", entry.accountName }, { "state", IUgc::ProcessStateName(entry.state) }, { "attempts", entry.attempts },
|
{ "accountId", entry.accountId }, { "accountName", entry.accountName }, { "state", IUgc::ProcessStateName(entry.state) }, { "attempts", entry.attempts },
|
||||||
{ "processedAt", entry.processedAt }, { "error", entry.error }, { "bakeAo", entry.bakeAo }, { "processAfter", entry.processAfter },
|
{ "processedAt", entry.processedAt }, { "error", entry.error }, { "bakeAo", entry.bakeAo }, { "processAfter", entry.processAfter },
|
||||||
{ "detail", entry.detail }, { "bricks", entry.bricks }, { "triangles", entry.triangles }, { "processMs", entry.processMs }, { "processCpuMs", entry.processCpuMs }, { "processMemoryKb", entry.processMemoryKb }, { "modelName", entry.modelName } };
|
{ "detail", entry.detail }, { "bricks", entry.bricks }, { "triangles", entry.triangles }, { "processMs", entry.processMs }, { "processCpuMs", entry.processCpuMs }, { "processMemoryKb", entry.processMemoryKb }, { "modelName", entry.modelName }, { "trianglesBefore", entry.trianglesBefore } };
|
||||||
}
|
}
|
||||||
|
|
||||||
// Web thread: the cars and rockets grouped into assemblies (one per combination of modules), from every build.
|
// Web thread: the cars and rockets grouped into assemblies (one per combination of modules), from every build.
|
||||||
@@ -271,7 +271,7 @@ namespace UgcRoutes {
|
|||||||
"A page of player models (kind=model) or of car and rocket assemblies (kind=modular: one per combination of modules, however many builds use "
|
"A page of player models (kind=model) or of car and rocket assemblies (kind=modular: one per combination of modules, however many builds use "
|
||||||
"it). Query: q= (\"state:\", \"kind:\"/\"type:\" (a build type, e.g. build6), \"owner:\", \"account:\", \"property:\", \"name:\", "
|
"it). Query: q= (\"state:\", \"kind:\"/\"type:\" (a build type, e.g. build6), \"owner:\", \"account:\", \"property:\", \"name:\", "
|
||||||
"\"lot:\"/\"module:\" (a LOT or a module's name), \"id:\", or plain text across names, owners and ids), state=, type=, sort=newest|oldest|owner|name|"
|
"\"lot:\"/\"module:\" (a LOT or a module's name), \"id:\", or plain text across names, owners and ids), state=, type=, sort=newest|oldest|owner|name|"
|
||||||
"bricks|triangles|slowest|made|cpu|memory (models) or newest|oldest|references|name (assemblies), reverse=1 (the sort's other direction), page= (from 0), size= (1-200). {items, total, page, size, counts, "
|
"bricks|triangles|slowest|made|cpu|memory|savings (models) or newest|oldest|references|name (assemblies), reverse=1 (the sort's other direction), page= (from 0), size= (1-200). {items, total, page, size, counts, "
|
||||||
"kinds, ugcPublicUrl, canManage}",
|
"kinds, ugcPublicUrl, canManage}",
|
||||||
[](HTTPReply& reply, const HTTPContext& context) {
|
[](HTTPReply& reply, const HTTPContext& context) {
|
||||||
const bool modular = QueryValue(context.queryString, "kind") == "modular";
|
const bool modular = QueryValue(context.queryString, "kind") == "modular";
|
||||||
@@ -290,7 +290,7 @@ namespace UgcRoutes {
|
|||||||
query.search = parsed.search;
|
query.search = parsed.search;
|
||||||
query.state = parsed.state;
|
query.state = parsed.state;
|
||||||
static const std::map<std::string, IUgcLookup::eSort> SORTS = { { "newest", IUgcLookup::eSort::NEWEST }, { "oldest", IUgcLookup::eSort::OLDEST },
|
static const std::map<std::string, IUgcLookup::eSort> SORTS = { { "newest", IUgcLookup::eSort::NEWEST }, { "oldest", IUgcLookup::eSort::OLDEST },
|
||||||
{ "owner", IUgcLookup::eSort::OWNER }, { "name", IUgcLookup::eSort::NAME }, { "bricks", IUgcLookup::eSort::BRICKS }, { "triangles", IUgcLookup::eSort::TRIANGLES }, { "slowest", IUgcLookup::eSort::SLOWEST }, { "made", IUgcLookup::eSort::MADE }, { "cpu", IUgcLookup::eSort::CPU }, { "memory", IUgcLookup::eSort::MEMORY } };
|
{ "owner", IUgcLookup::eSort::OWNER }, { "name", IUgcLookup::eSort::NAME }, { "bricks", IUgcLookup::eSort::BRICKS }, { "triangles", IUgcLookup::eSort::TRIANGLES }, { "slowest", IUgcLookup::eSort::SLOWEST }, { "made", IUgcLookup::eSort::MADE }, { "cpu", IUgcLookup::eSort::CPU }, { "memory", IUgcLookup::eSort::MEMORY }, { "savings", IUgcLookup::eSort::SAVINGS } };
|
||||||
if (const auto it = SORTS.find(sortText); it != SORTS.end()) query.sort = it->second;
|
if (const auto it = SORTS.find(sortText); it != SORTS.end()) query.sort = it->second;
|
||||||
query.reverse = reverse;
|
query.reverse = reverse;
|
||||||
query.offset = page * size;
|
query.offset = page * size;
|
||||||
|
|||||||
@@ -11,7 +11,7 @@
|
|||||||
|
|
||||||
var STATES = { pending: ['Waiting', 'secondary'], done: ['Made', 'success'], failed: ['Failed', 'danger'], empty: ['Empty', 'light'] };
|
var STATES = { pending: ['Waiting', 'secondary'], done: ['Made', 'success'], failed: ['Failed', 'danger'], empty: ['Empty', 'light'] };
|
||||||
var SORTS = {
|
var SORTS = {
|
||||||
model: [['newest', 'Newest'], ['oldest', 'Oldest'], ['bricks', 'Most bricks'], ['triangles', 'Most triangles'], ['made', 'Recently made'], ['slowest', 'Slowest to make'], ['cpu', 'Most CPU'], ['memory', 'Most RAM'], ['owner', 'Owner'], ['name', 'File name']],
|
model: [['newest', 'Newest'], ['oldest', 'Oldest'], ['bricks', 'Most bricks'], ['triangles', 'Most triangles'], ['made', 'Recently made'], ['slowest', 'Slowest to make'], ['cpu', 'Most CPU'], ['memory', 'Most RAM'], ['savings', 'Most triangles saved'], ['owner', 'Owner'], ['name', 'File name']],
|
||||||
modular: [['newest', 'Newest'], ['oldest', 'Oldest'], ['references', 'Most builds'], ['name', 'Name']]
|
modular: [['newest', 'Newest'], ['oldest', 'Oldest'], ['references', 'Most builds'], ['name', 'Name']]
|
||||||
};
|
};
|
||||||
// The list's state, as in the address: ?kind=&q=&state=&type=&sort=&page= (from 1)&view=, and the open item (&item=, &build=)
|
// The list's state, as in the address: ?kind=&q=&state=&type=&sort=&page= (from 1)&view=, and the open item (&item=, &build=)
|
||||||
@@ -152,6 +152,12 @@
|
|||||||
column('Took', function (i) { return '<span class="small">' + (i.processMs ? esc(duration(i.processMs)) : '') + '</span>'; }, 'slowest', true),
|
column('Took', function (i) { return '<span class="small">' + (i.processMs ? esc(duration(i.processMs)) : '') + '</span>'; }, 'slowest', true),
|
||||||
column('CPU', function (i) { return '<span class="small">' + (i.processCpuMs ? esc(duration(i.processCpuMs)) : '') + '</span>'; }, 'cpu', true),
|
column('CPU', function (i) { return '<span class="small">' + (i.processCpuMs ? esc(duration(i.processCpuMs)) : '') + '</span>'; }, 'cpu', true),
|
||||||
column('RAM (est.)', function (i) { return '<span class="small">' + (i.processMemoryKb ? '~' + esc(megabytes(i.processMemoryKb)) : '') + '</span>'; }, 'memory', true),
|
column('RAM (est.)', function (i) { return '<span class="small">' + (i.processMemoryKb ? '~' + esc(megabytes(i.processMemoryKb)) : '') + '</span>'; }, 'memory', true),
|
||||||
|
column('Saved', function (i) {
|
||||||
|
if (!i.trianglesBefore) return '';
|
||||||
|
var removed = i.trianglesBefore - i.triangles, share = removed / i.trianglesBefore * 100;
|
||||||
|
return '<span class="small" title="' + esc(i.trianglesBefore.toLocaleString()) + ' triangles before, ' + esc(i.triangles.toLocaleString()) + ' after">' +
|
||||||
|
esc(share.toFixed(share < 10 ? 1 : 0)) + '%<br><span class="text-body-secondary">' + esc(removed.toLocaleString()) + ' tris</span></span>';
|
||||||
|
}, 'savings', true),
|
||||||
column('Size', function (i) { return '<span class="small">' + (i.bricks ? esc(i.bricks) + ' bricks<br>' + esc(i.triangles.toLocaleString()) + ' triangles' : '') + '</span>'; }, 'bricks', true),
|
column('Size', function (i) { return '<span class="small">' + (i.bricks ? esc(i.bricks) + ' bricks<br>' + esc(i.triangles.toLocaleString()) + ' triangles' : '') + '</span>'; }, 'bricks', true),
|
||||||
column('File', function (i) { return '<span class="small" title="' + esc(i.detail || '') + '">' + esc(fileName(i)) + '</span>' + errorText(i); }, 'name'),
|
column('File', function (i) { return '<span class="small" title="' + esc(i.detail || '') + '">' + esc(fileName(i)) + '</span>' + errorText(i); }, 'name'),
|
||||||
column('', function (i) { return '<div class="text-end text-nowrap">' + actions(i, i.id) + '</div>'; })
|
column('', function (i) { return '<div class="text-end text-nowrap">' + actions(i, i.id) + '</div>'; })
|
||||||
|
|||||||
@@ -74,7 +74,7 @@
|
|||||||
<div id="gallery" class="ugc-gallery mb-2"></div>
|
<div id="gallery" class="ugc-gallery mb-2"></div>
|
||||||
<div class="card d-none" id="listCard"><div class="card-body p-2">
|
<div class="card d-none" id="listCard"><div class="card-body p-2">
|
||||||
<div id="modelTableBox"><table class="table table-sm align-middle mb-0 w-100 table-stack" id="ugcModelTable">
|
<div id="modelTableBox"><table class="table table-sm align-middle mb-0 w-100 table-stack" id="ugcModelTable">
|
||||||
<thead><tr><th>Icon</th><th>ID</th><th>Owner</th><th>State</th><th>Made</th><th>Took</th><th>CPU</th><th>RAM (est.)</th><th>Size</th><th>File</th><th></th></tr></thead></table></div>
|
<thead><tr><th>Icon</th><th>ID</th><th>Owner</th><th>State</th><th>Made</th><th>Took</th><th>CPU</th><th>RAM (est.)</th><th>Saved</th><th>Size</th><th>File</th><th></th></tr></thead></table></div>
|
||||||
<div id="assemblyTableBox" class="d-none"><table class="table table-sm align-middle mb-0 w-100 table-stack" id="ugcAssemblyTable">
|
<div id="assemblyTableBox" class="d-none"><table class="table table-sm align-middle mb-0 w-100 table-stack" id="ugcAssemblyTable">
|
||||||
<thead><tr><th>Icon</th><th>Newest build</th><th>Type</th><th>Modules</th><th>State</th><th>Builds</th><th>Owners</th><th></th></tr></thead></table></div>
|
<thead><tr><th>Icon</th><th>Newest build</th><th>Type</th><th>Modules</th><th>State</th><th>Builds</th><th>Owners</th><th></th></tr></thead></table></div>
|
||||||
</div></div>
|
</div></div>
|
||||||
|
|||||||
@@ -92,8 +92,9 @@ public:
|
|||||||
// whether lighting was baked in; processed_at becomes now
|
// whether lighting was baked in; processed_at becomes now
|
||||||
virtual void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) = 0;
|
virtual void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) = 0;
|
||||||
|
|
||||||
// What the UGC server counted when it made a model: its bricks and the most detailed mesh's triangles
|
// What the UGC server counted when it made a model: its bricks and the most detailed mesh's triangles, after and
|
||||||
virtual void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) = 0;
|
// before it removed the faces that can't be seen
|
||||||
|
virtual void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles, const uint32_t trianglesBefore) = 0;
|
||||||
|
|
||||||
// The last successful make of a model: how long it took, the worker's CPU time and the estimated memory
|
// The last successful make of a model: how long it took, the worker's CPU time and the estimated memory
|
||||||
struct ProcessStats {
|
struct ProcessStats {
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ public:
|
|||||||
uint32_t processCpuMs{}; // the worker thread's CPU time for it
|
uint32_t processCpuMs{}; // the worker thread's CPU time for it
|
||||||
uint32_t processMemoryKb{}; // the memory the UGC server estimated for it (not measured)
|
uint32_t processMemoryKb{}; // the memory the UGC server estimated for it (not measured)
|
||||||
std::string modelName; // models: the name a player gave it where it is placed (empty: none)
|
std::string modelName; // models: the name a player gave it where it is placed (empty: none)
|
||||||
|
uint32_t trianglesBefore{}; // models: the most detailed level's triangles before hidden faces were removed (0: unknown)
|
||||||
};
|
};
|
||||||
|
|
||||||
// What SearchUgc matches. A number (when set) is matched against ids; text against names
|
// What SearchUgc matches. A number (when set) is matched against ids; text against names
|
||||||
@@ -68,6 +69,7 @@ public:
|
|||||||
MADE, // the most recently made (or attempted) first (processed_at)
|
MADE, // the most recently made (or attempted) first (processed_at)
|
||||||
CPU, // the most CPU time in the last make first (process_cpu_ms)
|
CPU, // the most CPU time in the last make first (process_cpu_ms)
|
||||||
MEMORY, // the most estimated memory in the last make first (process_memory_kb)
|
MEMORY, // the most estimated memory in the last make first (process_memory_kb)
|
||||||
|
SAVINGS, // models: the biggest share of triangles removed first
|
||||||
};
|
};
|
||||||
|
|
||||||
// A page of one kind: all of them or those matching the search (the same matching as SearchUgc), in a state or any
|
// A page of one kind: all of them or those matching the search (the same matching as SearchUgc), in a state or any
|
||||||
|
|||||||
@@ -31,10 +31,10 @@ namespace UgcLookupSql {
|
|||||||
return (modular
|
return (modular
|
||||||
? "SELECT b.ugc_id AS id, b.character_id, c.name AS character_name, COALESCE(c.account_id, 0) AS account_id, a.name AS account_name, "
|
? "SELECT b.ugc_id AS id, b.character_id, c.name AS character_name, COALESCE(c.account_id, 0) AS account_id, a.name AS account_name, "
|
||||||
"b.is_optimized, b.process_error, b.ldf_config AS detail, b.process_attempts, b.processed_at, 0 AS process_after, 0 AS bake_ao, "
|
"b.is_optimized, b.process_error, b.ldf_config AS detail, b.process_attempts, b.processed_at, 0 AS process_after, 0 AS bake_ao, "
|
||||||
"0 AS brick_count, 0 AS triangle_count, b.process_ms, b.process_cpu_ms, b.process_memory_kb, '' AS model_name "
|
"0 AS brick_count, 0 AS triangle_count, b.process_ms, b.process_cpu_ms, b.process_memory_kb, '' AS model_name, 0 AS triangle_count_before "
|
||||||
: "SELECT u.id, u.character_id, c.name AS character_name, u.account_id, a.name AS account_name, "
|
: "SELECT u.id, u.character_id, c.name AS character_name, u.account_id, a.name AS account_name, "
|
||||||
"u.is_optimized, u.process_error, u.filename AS detail, u.process_attempts, u.processed_at, u.process_after, u.bake_ao, "
|
"u.is_optimized, u.process_error, u.filename AS detail, u.process_attempts, u.processed_at, u.process_after, u.bake_ao, "
|
||||||
"u.brick_count, u.triangle_count, u.process_ms, u.process_cpu_ms, u.process_memory_kb, "
|
"u.brick_count, u.triangle_count, u.process_ms, u.process_cpu_ms, u.process_memory_kb, u.triangle_count_before, "
|
||||||
// The name a player gave the model where it is placed (empty: none)
|
// The name a player gave the model where it is placed (empty: none)
|
||||||
"COALESCE((SELECT pc.model_name FROM properties_contents AS pc WHERE pc.ugc_id = u.id AND pc.model_name <> '' LIMIT 1), '') AS model_name ") + From(modular);
|
"COALESCE((SELECT pc.model_name FROM properties_contents AS pc WHERE pc.ugc_id = u.id AND pc.model_name <> '' LIMIT 1), '') AS model_name ") + From(modular);
|
||||||
}
|
}
|
||||||
@@ -61,6 +61,8 @@ namespace UgcLookupSql {
|
|||||||
case eSort::NAME: return std::string("ORDER BY ") + (modular ? "b.ldf_config" : "LOWER(COALESCE(NULLIF(model_name, ''), u.filename))") + dir(false) + ", " + id + " DESC ";
|
case eSort::NAME: return std::string("ORDER BY ") + (modular ? "b.ldf_config" : "LOWER(COALESCE(NULLIF(model_name, ''), u.filename))") + dir(false) + ", " + id + " DESC ";
|
||||||
case eSort::BRICKS: return modular ? "ORDER BY " + id + dir(true) + " " : "ORDER BY u.brick_count" + std::string(dir(true)) + ", u.id DESC ";
|
case eSort::BRICKS: return modular ? "ORDER BY " + id + dir(true) + " " : "ORDER BY u.brick_count" + std::string(dir(true)) + ", u.id DESC ";
|
||||||
case eSort::TRIANGLES: return modular ? "ORDER BY " + id + dir(true) + " " : "ORDER BY u.triangle_count" + std::string(dir(true)) + ", u.id DESC ";
|
case eSort::TRIANGLES: return modular ? "ORDER BY " + id + dir(true) + " " : "ORDER BY u.triangle_count" + std::string(dir(true)) + ", u.id DESC ";
|
||||||
|
// The biggest share of triangles removed first; models without a before count last
|
||||||
|
case eSort::SAVINGS: return modular ? "ORDER BY " + id + dir(true) + " " : "ORDER BY CASE WHEN u.triangle_count_before > 0 THEN (u.triangle_count_before - u.triangle_count) * 1.0 / u.triangle_count_before ELSE -1 END" + std::string(dir(true)) + ", u.id DESC ";
|
||||||
case eSort::CPU: return "ORDER BY " + std::string(modular ? "b" : "u") + ".process_cpu_ms" + dir(true) + ", " + id + " DESC ";
|
case eSort::CPU: return "ORDER BY " + std::string(modular ? "b" : "u") + ".process_cpu_ms" + dir(true) + ", " + id + " DESC ";
|
||||||
case eSort::MEMORY: return "ORDER BY " + std::string(modular ? "b" : "u") + ".process_memory_kb" + dir(true) + ", " + id + " DESC ";
|
case eSort::MEMORY: return "ORDER BY " + std::string(modular ? "b" : "u") + ".process_memory_kb" + dir(true) + ", " + id + " DESC ";
|
||||||
case eSort::MADE: return "ORDER BY " + std::string(modular ? "b" : "u") + ".processed_at" + dir(true) + ", " + id + " DESC ";
|
case eSort::MADE: return "ORDER BY " + std::string(modular ? "b" : "u") + ".processed_at" + dir(true) + ", " + id + " DESC ";
|
||||||
|
|||||||
@@ -428,7 +428,7 @@ public:
|
|||||||
void SetUgcIconSettings(const std::string_view target, const std::string_view params) override;
|
void SetUgcIconSettings(const std::string_view target, const std::string_view params) override;
|
||||||
void DeleteUgcIconSettings(const std::string_view target) override;
|
void DeleteUgcIconSettings(const std::string_view target) override;
|
||||||
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override;
|
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override;
|
||||||
void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override;
|
void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles, const uint32_t trianglesBefore) override;
|
||||||
void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override;
|
void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override;
|
||||||
std::optional<IUgc::FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override;
|
std::optional<IUgc::FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override;
|
||||||
void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override;
|
void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override;
|
||||||
|
|||||||
@@ -178,8 +178,8 @@ void MySQLDatabase::ExpediteUgcModels(const LWOOBJID characterId) {
|
|||||||
ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE character_id = ? AND is_optimized = 0 AND process_after > 0;", characterId);
|
ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE character_id = ? AND is_optimized = 0 AND process_after > 0;", characterId);
|
||||||
}
|
}
|
||||||
|
|
||||||
void MySQLDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) {
|
void MySQLDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles, const uint32_t trianglesBefore) {
|
||||||
ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ? WHERE id = ?;", bricks, triangles, id);
|
ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ?, triangle_count_before = ? WHERE id = ?;", bricks, triangles, trianglesBefore, id);
|
||||||
}
|
}
|
||||||
|
|
||||||
void MySQLDatabase::SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) {
|
void MySQLDatabase::SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) {
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ namespace {
|
|||||||
entry.processCpuMs = static_cast<uint32_t>(result->getInt64("process_cpu_ms"));
|
entry.processCpuMs = static_cast<uint32_t>(result->getInt64("process_cpu_ms"));
|
||||||
entry.processMemoryKb = static_cast<uint32_t>(result->getInt64("process_memory_kb"));
|
entry.processMemoryKb = static_cast<uint32_t>(result->getInt64("process_memory_kb"));
|
||||||
entry.modelName = result->getString("model_name").c_str();
|
entry.modelName = result->getString("model_name").c_str();
|
||||||
|
entry.trianglesBefore = static_cast<uint32_t>(result->getInt64("triangle_count_before"));
|
||||||
return entry;
|
return entry;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -412,7 +412,7 @@ public:
|
|||||||
void SetUgcIconSettings(const std::string_view target, const std::string_view params) override;
|
void SetUgcIconSettings(const std::string_view target, const std::string_view params) override;
|
||||||
void DeleteUgcIconSettings(const std::string_view target) override;
|
void DeleteUgcIconSettings(const std::string_view target) override;
|
||||||
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override;
|
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override;
|
||||||
void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override;
|
void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles, const uint32_t trianglesBefore) override;
|
||||||
void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override;
|
void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override;
|
||||||
std::optional<IUgc::FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override;
|
std::optional<IUgc::FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override;
|
||||||
void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override;
|
void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override;
|
||||||
|
|||||||
@@ -181,8 +181,8 @@ void SQLiteDatabase::ExpediteUgcModels(const LWOOBJID characterId) {
|
|||||||
ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE character_id = ? AND is_optimized = 0 AND process_after > 0;", characterId);
|
ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE character_id = ? AND is_optimized = 0 AND process_after > 0;", characterId);
|
||||||
}
|
}
|
||||||
|
|
||||||
void SQLiteDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) {
|
void SQLiteDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles, const uint32_t trianglesBefore) {
|
||||||
ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ? WHERE id = ?;", bricks, triangles, id);
|
ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ?, triangle_count_before = ? WHERE id = ?;", bricks, triangles, trianglesBefore, id);
|
||||||
}
|
}
|
||||||
|
|
||||||
void SQLiteDatabase::SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) {
|
void SQLiteDatabase::SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) {
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ namespace {
|
|||||||
entry.processCpuMs = static_cast<uint32_t>(result.getInt64Field("process_cpu_ms"));
|
entry.processCpuMs = static_cast<uint32_t>(result.getInt64Field("process_cpu_ms"));
|
||||||
entry.processMemoryKb = static_cast<uint32_t>(result.getInt64Field("process_memory_kb"));
|
entry.processMemoryKb = static_cast<uint32_t>(result.getInt64Field("process_memory_kb"));
|
||||||
entry.modelName = result.getStringField("model_name", "");
|
entry.modelName = result.getStringField("model_name", "");
|
||||||
|
entry.trianglesBefore = static_cast<uint32_t>(result.getInt64Field("triangle_count_before"));
|
||||||
return entry;
|
return entry;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -410,7 +410,7 @@ class TestSQLDatabase : public GameDatabase {
|
|||||||
void SetUgcIconSettings(const std::string_view target, const std::string_view params) override {}
|
void SetUgcIconSettings(const std::string_view target, const std::string_view params) override {}
|
||||||
void DeleteUgcIconSettings(const std::string_view target) override {}
|
void DeleteUgcIconSettings(const std::string_view target) override {}
|
||||||
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override {}
|
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override {}
|
||||||
void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override {}
|
void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles, const uint32_t trianglesBefore) override {}
|
||||||
void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override {}
|
void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override {}
|
||||||
std::optional<IUgc::FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override { return std::nullopt; }
|
std::optional<IUgc::FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override { return std::nullopt; }
|
||||||
void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override {}
|
void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override {}
|
||||||
|
|||||||
@@ -31,6 +31,22 @@ namespace {
|
|||||||
constexpr auto RECENT_ANSWER_TIME = std::chrono::seconds(10);
|
constexpr auto RECENT_ANSWER_TIME = std::chrono::seconds(10);
|
||||||
// In the storage folder once every item stored has its sd0 icon and checksums (Backfill)
|
// In the storage folder once every item stored has its sd0 icon and checksums (Backfill)
|
||||||
constexpr auto BACKFILL_MARKER = ".checksums-stored";
|
constexpr auto BACKFILL_MARKER = ".checksums-stored";
|
||||||
|
constexpr auto STATS_BACKFILL_MARKER = ".stats-stored";
|
||||||
|
|
||||||
|
// What a model's stats.json counted: bricks, and the most detailed level's triangles after and before the faces that
|
||||||
|
// can't be seen were removed (the transparent ones are kept as they are)
|
||||||
|
struct ModelCounts {
|
||||||
|
uint32_t bricks{};
|
||||||
|
uint32_t triangles{};
|
||||||
|
uint32_t trianglesBefore{};
|
||||||
|
};
|
||||||
|
std::optional<ModelCounts> CountsOf(const nlohmann::json& stats) {
|
||||||
|
if (!stats.is_object()) return std::nullopt;
|
||||||
|
const auto& lods = stats.value("lods", nlohmann::json::array());
|
||||||
|
const auto& first = lods.is_array() && !lods.empty() ? lods.front() : nlohmann::json::object();
|
||||||
|
const auto transparent = first.value("transparent", 0u);
|
||||||
|
return ModelCounts{ stats.value("bricks", 0u), first.value("opaqueAfter", 0u) + transparent, first.value("opaqueBefore", 0u) + transparent };
|
||||||
|
}
|
||||||
constexpr size_t BACKFILL_ITEMS_PER_UPDATE = 16;
|
constexpr size_t BACKFILL_ITEMS_PER_UPDATE = 16;
|
||||||
constexpr uint32_t BACKFILL_BUILDS_PER_UPDATE = 200;
|
constexpr uint32_t BACKFILL_BUILDS_PER_UPDATE = 200;
|
||||||
// Combination id of a build whose modules can't be told (so it isn't looked at again)
|
// Combination id of a build whose modules can't be told (so it isn't looked at again)
|
||||||
@@ -110,6 +126,11 @@ void UgcProcessor::Start() {
|
|||||||
std::error_code error;
|
std::error_code error;
|
||||||
m_BackfillItemsDone = std::filesystem::exists(m_Storage.GetRoot() / BACKFILL_MARKER, error);
|
m_BackfillItemsDone = std::filesystem::exists(m_Storage.GetRoot() / BACKFILL_MARKER, error);
|
||||||
if (!m_BackfillItemsDone) m_BackfillItems.assign(stored.begin(), stored.end());
|
if (!m_BackfillItemsDone) m_BackfillItems.assign(stored.begin(), stored.end());
|
||||||
|
// Models made before the triangles before hidden-face removal were stored get them from their stats.json once
|
||||||
|
m_StatsBackfillDone = std::filesystem::exists(m_Storage.GetRoot() / STATS_BACKFILL_MARKER, error);
|
||||||
|
if (!m_StatsBackfillDone) {
|
||||||
|
for (const auto& entry : stored) if (entry.kind == Kind::MODEL) m_StatsBackfill.push_back(entry.id);
|
||||||
|
}
|
||||||
m_NextEviction = std::chrono::steady_clock::now();
|
m_NextEviction = std::chrono::steady_clock::now();
|
||||||
m_Stopping = false;
|
m_Stopping = false;
|
||||||
for (size_t i = 0; i < std::max<size_t>(m_Config.threads, 1); i++) m_Threads.emplace_back(&UgcProcessor::Worker, this);
|
for (size_t i = 0; i < std::max<size_t>(m_Config.threads, 1); i++) m_Threads.emplace_back(&UgcProcessor::Worker, this);
|
||||||
@@ -382,11 +403,7 @@ void UgcProcessor::Record(const Done& done) {
|
|||||||
if (done.outcome.ok) Database::Get()->SetUgcModelProcessStats(done.id, cost);
|
if (done.outcome.ok) Database::Get()->SetUgcModelProcessStats(done.id, cost);
|
||||||
// What it counted (stats.json), for sorting on the dashboard
|
// What it counted (stats.json), for sorting on the dashboard
|
||||||
const auto stats = done.outcome.ok && !done.outcome.stats.empty() ? nlohmann::json::parse(done.outcome.stats, nullptr, false) : nlohmann::json();
|
const auto stats = done.outcome.ok && !done.outcome.stats.empty() ? nlohmann::json::parse(done.outcome.stats, nullptr, false) : nlohmann::json();
|
||||||
if (stats.is_object()) {
|
if (const auto counts = CountsOf(stats)) Database::Get()->SetUgcModelStats(done.id, counts->bricks, counts->triangles, counts->trianglesBefore);
|
||||||
const auto& lods = stats.value("lods", nlohmann::json::array());
|
|
||||||
const auto& first = lods.is_array() && !lods.empty() ? lods.front() : nlohmann::json::object();
|
|
||||||
Database::Get()->SetUgcModelStats(done.id, stats.value("bricks", 0u), first.value("opaqueAfter", 0u) + first.value("transparent", 0u));
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
Database::Get()->SetModularBuildProcessed(done.id, state, attempts, error);
|
Database::Get()->SetModularBuildProcessed(done.id, state, attempts, error);
|
||||||
if (done.outcome.ok) Database::Get()->SetModularBuildProcessStats(done.id, cost);
|
if (done.outcome.ok) Database::Get()->SetModularBuildProcessStats(done.id, cost);
|
||||||
@@ -475,6 +492,22 @@ void UgcProcessor::Backfill() {
|
|||||||
}
|
}
|
||||||
if (builds.size() < BACKFILL_BUILDS_PER_UPDATE) m_BackfillBuildsDone = true;
|
if (builds.size() < BACKFILL_BUILDS_PER_UPDATE) m_BackfillBuildsDone = true;
|
||||||
}
|
}
|
||||||
|
if (!m_StatsBackfillDone) {
|
||||||
|
for (size_t i = 0; i < BACKFILL_ITEMS_PER_UPDATE && !m_StatsBackfill.empty(); i++) {
|
||||||
|
const auto id = m_StatsBackfill.front();
|
||||||
|
m_StatsBackfill.pop_front();
|
||||||
|
const auto file = m_Storage.File(Kind::MODEL, id, "stats.json");
|
||||||
|
if (!file) continue;
|
||||||
|
std::ifstream in(*file, std::ios::binary);
|
||||||
|
const auto counts = CountsOf(nlohmann::json::parse(std::string(std::istreambuf_iterator<char>(in), {}), nullptr, false));
|
||||||
|
if (counts) Database::Get()->SetUgcModelStats(id, counts->bricks, counts->triangles, counts->trianglesBefore);
|
||||||
|
}
|
||||||
|
if (m_StatsBackfill.empty()) {
|
||||||
|
m_StatsBackfillDone = true;
|
||||||
|
std::ofstream(m_Storage.GetRoot() / STATS_BACKFILL_MARKER) << "1\n";
|
||||||
|
LOG("Stored the triangle counts of the models made before");
|
||||||
|
}
|
||||||
|
}
|
||||||
if (m_BackfillItemsDone) return;
|
if (m_BackfillItemsDone) return;
|
||||||
const auto read = [](const std::filesystem::path& path) {
|
const auto read = [](const std::filesystem::path& path) {
|
||||||
std::ifstream in(path, std::ios::binary);
|
std::ifstream in(path, std::ios::binary);
|
||||||
|
|||||||
@@ -187,6 +187,8 @@ private:
|
|||||||
void Backfill();
|
void Backfill();
|
||||||
std::deque<UgcStorage::Entry> m_BackfillItems;
|
std::deque<UgcStorage::Entry> m_BackfillItems;
|
||||||
bool m_BackfillItemsDone{ true };
|
bool m_BackfillItemsDone{ true };
|
||||||
|
std::deque<LWOOBJID> m_StatsBackfill; // models whose triangle counts are read from their stats.json once
|
||||||
|
bool m_StatsBackfillDone{ true };
|
||||||
bool m_BackfillBuildsDone{ false };
|
bool m_BackfillBuildsDone{ false };
|
||||||
|
|
||||||
void Poll();
|
void Poll();
|
||||||
|
|||||||
7
migrations/dlu/mysql/92_ugc_triangles_before.sql
Normal file
7
migrations/dlu/mysql/92_ugc_triangles_before.sql
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
/* UGC server: ugc.triangle_count_before is the most detailed level's triangles before the UGC server removed the faces
|
||||||
|
that can't be seen (0 until known), so the dashboard can show how much each model saved (with triangle_count). */
|
||||||
|
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN triangle_count_before INT UNSIGNED NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
|
||||||
|
WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'triangle_count_before');
|
||||||
|
PREPARE dlu_column_stmt FROM @dlu_column;
|
||||||
|
EXECUTE dlu_column_stmt;
|
||||||
|
DEALLOCATE PREPARE dlu_column_stmt;
|
||||||
2
migrations/dlu/sqlite/75_ugc_triangles_before.sql
Normal file
2
migrations/dlu/sqlite/75_ugc_triangles_before.sql
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
/* UGC server: a model's triangles before hidden faces were removed. See the MySQL migration. */
|
||||||
|
ALTER TABLE ugc ADD COLUMN triangle_count_before INTEGER NOT NULL DEFAULT 0;
|
||||||
@@ -105,15 +105,15 @@ namespace {
|
|||||||
"CREATE TABLE charinfo (id BIGINT PRIMARY KEY, account_id INTEGER, name TEXT);"
|
"CREATE TABLE charinfo (id BIGINT PRIMARY KEY, account_id INTEGER, name TEXT);"
|
||||||
"CREATE TABLE properties (id BIGINT PRIMARY KEY, owner_id BIGINT, name TEXT, zone_id INTEGER);"
|
"CREATE TABLE properties (id BIGINT PRIMARY KEY, owner_id BIGINT, name TEXT, zone_id INTEGER);"
|
||||||
"CREATE TABLE ugc (id INTEGER PRIMARY KEY, account_id INTEGER, character_id BIGINT, is_optimized INTEGER, filename TEXT, process_error TEXT DEFAULT '', "
|
"CREATE TABLE ugc (id INTEGER PRIMARY KEY, account_id INTEGER, character_id BIGINT, is_optimized INTEGER, filename TEXT, process_error TEXT DEFAULT '', "
|
||||||
"process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_after BIGINT DEFAULT 0, bake_ao INTEGER DEFAULT 0, brick_count INTEGER DEFAULT 0, triangle_count INTEGER DEFAULT 0, process_ms INTEGER DEFAULT 0, process_cpu_ms INTEGER DEFAULT 0, process_memory_kb INTEGER DEFAULT 0);"
|
"process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_after BIGINT DEFAULT 0, bake_ao INTEGER DEFAULT 0, brick_count INTEGER DEFAULT 0, triangle_count INTEGER DEFAULT 0, process_ms INTEGER DEFAULT 0, process_cpu_ms INTEGER DEFAULT 0, process_memory_kb INTEGER DEFAULT 0, triangle_count_before INTEGER DEFAULT 0);"
|
||||||
"CREATE TABLE ugc_modular_build (ugc_id BIGINT PRIMARY KEY, character_id BIGINT, ldf_config TEXT, is_optimized INTEGER DEFAULT 0, process_error TEXT DEFAULT '', "
|
"CREATE TABLE ugc_modular_build (ugc_id BIGINT PRIMARY KEY, character_id BIGINT, ldf_config TEXT, is_optimized INTEGER DEFAULT 0, process_error TEXT DEFAULT '', "
|
||||||
"process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_ms INTEGER DEFAULT 0, process_cpu_ms INTEGER DEFAULT 0, process_memory_kb INTEGER DEFAULT 0);"
|
"process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_ms INTEGER DEFAULT 0, process_cpu_ms INTEGER DEFAULT 0, process_memory_kb INTEGER DEFAULT 0);"
|
||||||
"CREATE TABLE properties_contents (id BIGINT PRIMARY KEY, property_id BIGINT, ugc_id BIGINT, lot INTEGER, model_name TEXT DEFAULT '', model_description TEXT DEFAULT '');"
|
"CREATE TABLE properties_contents (id BIGINT PRIMARY KEY, property_id BIGINT, ugc_id BIGINT, lot INTEGER, model_name TEXT DEFAULT '', model_description TEXT DEFAULT '');"
|
||||||
"INSERT INTO accounts VALUES (1, 'builder'), (2, 'racer');"
|
"INSERT INTO accounts VALUES (1, 'builder'), (2, 'racer');"
|
||||||
"INSERT INTO charinfo VALUES (10, 1, 'Bricky'), (20, 2, 'Speedy');"
|
"INSERT INTO charinfo VALUES (10, 1, 'Bricky'), (20, 2, 'Speedy');"
|
||||||
"INSERT INTO properties VALUES (500, 10, 'Castle Hill', 1150);"
|
"INSERT INTO properties VALUES (500, 10, 'Castle Hill', 1150);"
|
||||||
"INSERT INTO ugc (id, account_id, character_id, is_optimized, filename, process_error, brick_count, triangle_count, process_ms) VALUES "
|
"INSERT INTO ugc (id, account_id, character_id, is_optimized, filename, process_error, brick_count, triangle_count, process_ms, triangle_count_before) VALUES "
|
||||||
"(1000, 1, 10, 1, 'tower.lxfml', '', 40, 900, 5200), (1001, 1, 10, 0, 'boat.lxfml', '', 0, 0, 0);"
|
"(1000, 1, 10, 1, 'tower.lxfml', '', 40, 900, 5200, 1500), (1001, 1, 10, 0, 'boat.lxfml', '', 0, 0, 0, 0);"
|
||||||
"INSERT INTO ugc_modular_build (ugc_id, character_id, ldf_config, is_optimized, process_error) VALUES (2000, 20, '1:8129+1:8130+1:9330', 1, '');"
|
"INSERT INTO ugc_modular_build (ugc_id, character_id, ldf_config, is_optimized, process_error) VALUES (2000, 20, '1:8129+1:8130+1:9330', 1, '');"
|
||||||
"INSERT INTO properties_contents VALUES (7000, 500, 1000, 14, 'Big tower', 'the tallest');");
|
"INSERT INTO properties_contents VALUES (7000, 500, 1000, 14, 'Big tower', 'the tallest');");
|
||||||
}
|
}
|
||||||
@@ -182,6 +182,8 @@ TEST_F(UgcLookupSqlTests, ListsPagesWithCounts) {
|
|||||||
query.reverse = true; // the oldest first
|
query.reverse = true; // the oldest first
|
||||||
EXPECT_EQ(List(query, false).first, (std::vector<int64_t>{ 1000, 1001 }));
|
EXPECT_EQ(List(query, false).first, (std::vector<int64_t>{ 1000, 1001 }));
|
||||||
query.reverse = false;
|
query.reverse = false;
|
||||||
|
query.sort = IUgcLookup::eSort::SAVINGS; // the tower lost 40% of its triangles; the boat isn't known
|
||||||
|
EXPECT_EQ(List(query, false).first, (std::vector<int64_t>{ 1000, 1001 }));
|
||||||
query.sort = IUgcLookup::eSort::CPU; // none recorded: ties go newest first
|
query.sort = IUgcLookup::eSort::CPU; // none recorded: ties go newest first
|
||||||
EXPECT_EQ(List(query, false).first, (std::vector<int64_t>{ 1001, 1000 }));
|
EXPECT_EQ(List(query, false).first, (std::vector<int64_t>{ 1001, 1000 }));
|
||||||
query.sort = IUgcLookup::eSort::MEMORY;
|
query.sort = IUgcLookup::eSort::MEMORY;
|
||||||
|
|||||||
Reference in New Issue
Block a user