Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-02 02:43:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ugc): processing options per make, recorded per model with their times

Browse Source 
Staff can make models again with other processing options than the UGC
settings' (ray backend, hidden-face method, denoising) and every make records
what made it, for comparing the options.

- migration mysql 99 / sqlite 82 (ugc_process_options): ugc.process_options
  (picked for the next make, cleared once made), ugc.made_options (what made
  the current files) and ugc_process_runs (every successful make: options,
  wall and CPU time, hidden faces', occlusion's and icon's time, bricks,
  triangles before and after)
- IUgc: ResetUgcModelProcessing and ResetPropertyUgcModelProcessing take the
  options; PendingModel carries them; RecordUgcModelRun, GetUgcRunSummaries;
  list entries have madeOptions and processOptions
- the UGC server applies a job's options over its settings and records the run
- /api/ugc/reprocess takes options ("embree fast oidn"); /api/ugc/options
  lists the choices, the settings' defaults and averages per combination
- /reprocessproperty [builtin|embree|hiprt] [toolbox|fast] [off|oidn], any
  order, all optional

Check: run the migration on MySQL and SQLite; /reprocessproperty embree fast
on a property, then the models' made_options and ugc_process_runs rows;
/api/ugc/options.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-29 11:28:52 -05:00
parent 06be40471b
commit ef3c74f715
22 changed files with 290 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
dDashboardServer/routes/UgcRoutes.cpp
View File

@@ -199,7 +199,8 @@ namespace {
return { { "id", std::to_string(entry.id) }, { "characterId", std::to_string(entry.characterId) }, { "characterName", entry.characterName },
{ "accountId", entry.accountId }, { "accountName", entry.accountName }, { "state", IUgc::ProcessStateName(entry.state) }, { "attempts", entry.attempts },
{ "processedAt", entry.processedAt }, { "error", entry.error }, { "bakeAo", entry.bakeAo }, { "processAfter", entry.processAfter },
{ "detail", entry.detail }, { "bricks", entry.bricks }, { "triangles", entry.triangles }, { "processMs", entry.processMs }, { "processCpuMs", entry.processCpuMs }, { "processMemoryKb", entry.processMemoryKb }, { "modelName", entry.modelName }, { "trianglesBefore", entry.trianglesBefore } };
{ "detail", entry.detail }, { "bricks", entry.bricks }, { "triangles", entry.triangles }, { "processMs", entry.processMs }, { "processCpuMs", entry.processCpuMs }, { "processMemoryKb", entry.processMemoryKb }, { "modelName", entry.modelName }, { "trianglesBefore", entry.trianglesBefore },
{ "madeOptions", entry.madeOptions }, { "processOptions", entry.processOptions } };
}
// Web thread: the cars and rockets grouped into assemblies (one per combination of modules), from every build.
@@ -456,6 +457,32 @@ namespace UgcRoutes {
reply.message = sd0.GetAsStringUncompressed();
});
Route(eHTTPMethod::GET, "/api/ugc/options", Perm("properties_view"),
"The processing options (docs/UgcServer.md, \"Processing options\"): the choices ({rays, hsr, denoise}: names), the settings' defaults "
"({rays, hsr, denoise}) and every combination the made models used, with its makes (runs), models and averages per make "
"(ms, cpuMs, hsrMs, aoMs, iconMs, bricks, trianglesBefore, triangles, removed: the share of triangles removed)",
[](HTTPReply& reply, const HTTPContext& context) {
const auto names = [](const auto& list) {
nlohmann::json out = nlohmann::json::array();
for (const auto name : list) out.push_back(std::string(name));
return out;
};
nlohmann::json combos = nlohmann::json::array();
for (const auto& summary : Database::Get()->GetUgcRunSummaries()) {
const double runs = static_cast<double>(std::max<uint64_t>(summary.runs, 1));
const auto average = [runs](uint64_t total) { return std::lround(static_cast<double>(total) / runs); };
combos.push_back({ { "options", summary.options }, { "runs", summary.runs }, { "models", summary.models }, { "ms", average(summary.milliseconds) },
{ "cpuMs", average(summary.cpuMilliseconds) }, { "hsrMs", average(summary.hsrMilliseconds) }, { "aoMs", average(summary.aoMilliseconds) },
{ "iconMs", average(summary.iconMilliseconds) }, { "bricks", average(summary.bricks) }, { "trianglesBefore", average(summary.trianglesBefore) },
{ "triangles", average(summary.triangles) },
{ "removed", summary.trianglesBefore > 0 ? 1.0 - static_cast<double>(summary.triangles) / static_cast<double>(summary.trianglesBefore) : 0.0 } });
}
JsonSuccess(reply, { { "choices", { { "rays", names(UgcProcessOptions::RAYS) }, { "hsr", names(UgcProcessOptions::HSR) }, { "denoise", names(UgcProcessOptions::DENOISE) } } },
{ "defaults", { { "rays", UgcSetting("ray_backend").value_or("builtin") }, { "hsr", UgcSetting("hsr_method").value_or("toolbox") },
{ "denoise", UgcSetting("denoise").value_or("off") } } },
{ "combinations", combos }, { "canManage", Can(context, "ugc_manage") } });
});
Route(eHTTPMethod::GET, "/api/ugc/server/status", Perm("properties_view"),
"The UGC server's live status (its /status, fetched by the dashboard from ugc_internal_url): queue, workers, CPU and memory use, "
"limits and throttling. {success: false, error, url} when it doesn't answer",
@@ -744,16 +771,23 @@ namespace UgcRoutes {
});
Route(eHTTPMethod::POST, "/api/ugc/reprocess", Perm("ugc_manage"),
"Have the UGC server make items again. Body: {kind: model|modular, id} for one, {kind: model, property} for every model placed on a property, {kind, failedOnly: true} for the failed ones, {kind} for all",
"Have the UGC server make items again. Body: {kind: model|modular, id} for one, {kind: model, property} for every model placed on a property, {kind, failedOnly: true} for the failed ones, {kind} for all; "
"models: options (processing options for this make, e.g. \"embree fast oidn\": ray backend, hidden-face method, denoising; left out: the UGC settings')",
[](HTTPReply& reply, const HTTPContext& context) {
const auto body = ParseBody(context);
if (!body) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid JSON");
const bool modular = body->value("kind", "model") == "modular";
UgcProcessOptions::Choice choice;
if (!UgcProcessOptions::Parse(body->contains("options") && (*body)["options"].is_string() ? (*body)["options"].get<std::string>() : "", choice)) {
return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Unknown processing options");
}
const auto options = UgcProcessOptions::ToString(choice);
const auto with = options.empty() ? std::string() : " with " + options;
if (!modular && body->contains("property")) {
const auto property = GeneralUtils::TryParse<LWOOBJID>((*body)["property"].is_string() ? (*body)["property"].get<std::string>() : (*body)["property"].dump());
if (!property) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid property");
const auto changed = Database::Get()->ResetPropertyUgcModelProcessing(*property);
Audit(context, "ugc_reprocess", "Queued " + std::to_string(changed) + " model(s) on property " + std::to_string(*property) + " to be made again");
const auto changed = Database::Get()->ResetPropertyUgcModelProcessing(*property, options);
Audit(context, "ugc_reprocess", "Queued " + std::to_string(changed) + " model(s) on property " + std::to_string(*property) + " to be made again" + with);
BroadcastTableChanged("ugc");
return JsonSuccess(reply, { { "message", std::to_string(changed) + " model" + (changed == 1 ? "" : "s") + " will be made again" } });
}
@@ -763,10 +797,10 @@ namespace UgcRoutes {
if (!id) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid id");
}
const bool failedOnly = body->value("failedOnly", false);
const auto changed = modular ? Database::Get()->ResetModularBuildProcessing(id, failedOnly) : Database::Get()->ResetUgcModelProcessing(id, failedOnly);
const auto changed = modular ? Database::Get()->ResetModularBuildProcessing(id, failedOnly) : Database::Get()->ResetUgcModelProcessing(id, failedOnly, options);
const std::string what = modular ? "modular build" : "model";
Audit(context, "ugc_reprocess", "Queued " + std::to_string(changed) + " " + what + "(s) to be made again" +
(id ? " (" + std::to_string(*id) + ")" : failedOnly ? " (the failed ones)" : " (all)"));
(id ? " (" + std::to_string(*id) + ")" : failedOnly ? " (the failed ones)" : " (all)") + (modular ? "" : with));
BroadcastTableChanged("ugc");
JsonSuccess(reply, { { "message", std::to_string(changed) + " " + what + (changed == 1 ? "" : "s") + " will be made again" } });
});

49
dDatabase/GameDatabase/ITables/IUgc.h
View File

@@ -79,6 +79,7 @@ public:
std::string lxfml;
uint32_t attempts{};
bool priority{}; // staff asked for it to be made again: before any other model
std::string options; // the processing options staff picked for this make (UgcProcessOptions), empty: the settings'
};
// Up to `limit` pending models whose quiet period is over (process_after): the priority ones first, then the least
@@ -91,7 +92,8 @@ public:
virtual void ExpediteUgcModels(const LWOOBJID characterId) = 0;
// Records an attempt: the new state, how many attempts there have been, why it failed (empty when it didn't) and
// whether lighting was baked in; processed_at becomes now
// whether lighting was baked in; processed_at becomes now. Once it isn't pending any more, the priority and the
// processing options staff picked for it are cleared.
virtual void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) = 0;
// What the UGC server counted when it made a model: its bricks and the most detailed mesh's triangles, after and
@@ -106,15 +108,50 @@ public:
};
virtual void SetUgcModelProcessStats(const LWOOBJID id, const ProcessStats& stats) = 0;
// A successful make of a model with the processing options that made it (UgcProcessOptions, every choice filled)
// and what it took (stats.json's times), for comparing the options
struct ProcessRun {
LWOOBJID id{};
std::string options;
uint32_t milliseconds{};
uint32_t cpuMilliseconds{};
uint32_t hsrMilliseconds{}; // removing hidden faces, every level
uint32_t aoMilliseconds{}; // the occlusion bake, every level
uint32_t iconMilliseconds{};
uint32_t bricks{};
uint32_t trianglesBefore{}; // the most detailed level's, before and after hidden faces were removed
uint32_t triangles{};
};
// Records a run (ugc_process_runs) and that the options made the model's files (made_options)
virtual void RecordUgcModelRun(const ProcessRun& run) = 0;
// The runs summed per options, the most used first
struct RunSummary {
std::string options;
uint64_t runs{};
uint64_t models{}; // different models
uint64_t milliseconds{}; // summed, as the rest
uint64_t cpuMilliseconds{};
uint64_t hsrMilliseconds{};
uint64_t aoMilliseconds{};
uint64_t iconMilliseconds{};
uint64_t bricks{};
uint64_t trianglesBefore{};
uint64_t triangles{};
};
virtual std::vector<RunSummary> GetUgcRunSummaries() = 0;
virtual std::optional<ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) = 0;
// Sets models back to pending with no attempts: one (`id`), or all of them (`id` nullopt; only the failed ones
// with `failedOnly`). Returns how many rows changed.
virtual uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) = 0;
// with `failedOnly`). `options`: the processing options for the next make (UgcProcessOptions), empty: the
// settings'. Returns how many rows changed.
virtual uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly, const std::string_view options) = 0;
// Sets every model placed on a property back to pending with no attempts, as priority (staff asked for it). Returns
// how many rows changed.
virtual uint64_t ResetPropertyUgcModelProcessing(const LWOOBJID propertyId) = 0;
// Sets every model placed on a property back to pending with no attempts, as priority (staff asked for it), with
// `options` for the next make as above. Returns how many rows changed.
virtual uint64_t ResetPropertyUgcModelProcessing(const LWOOBJID propertyId, const std::string_view options) = 0;
// A page of models (all, or those in `state`), the newest first; `search` (when not empty) matches the model's id
// exactly or part of its owner's name

2
dDatabase/GameDatabase/ITables/IUgcLookup.h
View File

@@ -41,6 +41,8 @@ public:
uint32_t processMemoryKb{}; // the memory the UGC server estimated for it (not measured)
std::string modelName; // models: the name a player gave it where it is placed (empty: none)
uint32_t trianglesBefore{}; // models: the most detailed level's triangles before hidden faces were removed (0: unknown)
std::string madeOptions; // models: the processing options that made its files (UgcProcessOptions; empty: unknown)
std::string processOptions; // models: the options staff picked for its next make (empty: the settings')
};
// What SearchUgc matches. A number (when set) is matched against ids; text against names

5
dDatabase/GameDatabase/ITables/UgcLookupSql.h
View File

@@ -31,10 +31,11 @@ namespace UgcLookupSql {
return (modular
? "SELECT b.ugc_id AS id, b.character_id, c.name AS character_name, COALESCE(c.account_id, 0) AS account_id, a.name AS account_name, "
"b.is_optimized, b.process_error, b.ldf_config AS detail, b.process_attempts, b.processed_at, 0 AS process_after, 0 AS bake_ao, "
"0 AS brick_count, 0 AS triangle_count, b.process_ms, b.process_cpu_ms, b.process_memory_kb, '' AS model_name, 0 AS triangle_count_before "
"0 AS brick_count, 0 AS triangle_count, b.process_ms, b.process_cpu_ms, b.process_memory_kb, '' AS model_name, 0 AS triangle_count_before, "
"'' AS made_options, '' AS process_options "
: "SELECT u.id, u.character_id, c.name AS character_name, u.account_id, a.name AS account_name, "
"u.is_optimized, u.process_error, u.filename AS detail, u.process_attempts, u.processed_at, u.process_after, u.bake_ao, "
"u.brick_count, u.triangle_count, u.process_ms, u.process_cpu_ms, u.process_memory_kb, u.triangle_count_before, "
"u.brick_count, u.triangle_count, u.process_ms, u.process_cpu_ms, u.process_memory_kb, u.triangle_count_before, u.made_options, u.process_options, "
// The name a player gave the model where it is placed (empty: none)
"COALESCE((SELECT pc.model_name FROM properties_contents AS pc WHERE pc.ugc_id = u.id AND pc.model_name <> '' LIMIT 1), '') AS model_name ") + From(modular);
}

6
dDatabase/GameDatabase/MySQL/MySQLDatabase.h
View File

@@ -460,8 +460,10 @@ public:
void SetModularBuildProcessStats(const LWOOBJID id, const IUgc::ProcessStats& stats) override;
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsWithoutCombination(const uint32_t limit) override;
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override;
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
uint64_t ResetPropertyUgcModelProcessing(const LWOOBJID propertyId) override;
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly, const std::string_view options) override;
uint64_t ResetPropertyUgcModelProcessing(const LWOOBJID propertyId, const std::string_view options) override;
void RecordUgcModelRun(const ProcessRun& run) override;
std::vector<IUgc::RunSummary> GetUgcRunSummaries() override;
IUgc::ProcessTotals GetUgcProcessTotals(const bool modular) override;
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) override;
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetUgcProcessCounts() override;

42
dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp
View File

@@ -112,7 +112,7 @@ std::optional<IUgc::Model> MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) {
}
std::vector<IUgc::PendingModel> MySQLDatabase::GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) {
auto result = ExecuteSelect("SELECT id, lxfml, process_attempts, priority FROM ugc WHERE is_optimized = 0 AND process_after <= ? AND (? = 0 OR priority = 1) "
auto result = ExecuteSelect("SELECT id, lxfml, process_attempts, priority, process_options FROM ugc WHERE is_optimized = 0 AND process_after <= ? AND (? = 0 OR priority = 1) "
"ORDER BY priority DESC, process_attempts ASC, id DESC LIMIT ?;", UnixNow(), priorityOnly ? 1 : 0, limit);
std::vector<IUgc::PendingModel> models;
while (result->next()) {
@@ -124,13 +124,15 @@ std::vector<IUgc::PendingModel> MySQLDatabase::GetUgcModelsToProcess(const uint3
model.lxfml = contents.str();
model.attempts = static_cast<uint32_t>(result->getInt("process_attempts"));
model.priority = result->getInt("priority") != 0;
model.options = std::string(result->getString("process_options").c_str());
}
return models;
}
void MySQLDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) {
ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ?, priority = CASE WHEN ? = 0 THEN priority ELSE 0 END WHERE id = ?;",
static_cast<int32_t>(state), attempts, error, bakeAo, UnixNow(), static_cast<int32_t>(state), id);
ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ?, priority = CASE WHEN ? = 0 THEN priority ELSE 0 END, "
"process_options = CASE WHEN ? = 0 THEN process_options ELSE '' END WHERE id = ?;",
static_cast<int32_t>(state), attempts, error, bakeAo, UnixNow(), static_cast<int32_t>(state), static_cast<int32_t>(state), id);
}
std::optional<IUgc::ProcessInfo> MySQLDatabase::GetUgcProcessInfo(const LWOOBJID id) {
@@ -141,10 +143,10 @@ std::optional<IUgc::ProcessInfo> MySQLDatabase::GetUgcProcessInfo(const LWOOBJID
return ReadUgcProcessInfo(result, false);
}
uint64_t MySQLDatabase::ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) {
if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 WHERE id = ?;", *id);
if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 WHERE is_optimized = 2;");
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0;");
uint64_t MySQLDatabase::ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly, const std::string_view options) {
if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, process_options = ? WHERE id = ?;", options, *id);
if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, process_options = ? WHERE is_optimized = 2;", options);
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, process_options = ?;", options);
}
IUgc::ProcessTotals MySQLDatabase::GetUgcProcessTotals(const bool modular) {
@@ -165,9 +167,29 @@ IUgc::ProcessTotals MySQLDatabase::GetUgcProcessTotals(const bool modular) {
return totals;
}
uint64_t MySQLDatabase::ResetPropertyUgcModelProcessing(const LWOOBJID propertyId) {
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, priority = 1 "
"WHERE id IN (SELECT ugc_id FROM properties_contents WHERE property_id = ? AND ugc_id IS NOT NULL);", propertyId);
uint64_t MySQLDatabase::ResetPropertyUgcModelProcessing(const LWOOBJID propertyId, const std::string_view options) {
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, priority = 1, process_options = ? "
"WHERE id IN (SELECT ugc_id FROM properties_contents WHERE property_id = ? AND ugc_id IS NOT NULL);", options, propertyId);
}
void MySQLDatabase::RecordUgcModelRun(const ProcessRun& run) {
ExecuteInsert("INSERT INTO ugc_process_runs (ugc_id, options, made_at, process_ms, process_cpu_ms, hsr_ms, ao_ms, icon_ms, bricks, triangles_before, triangles) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", run.id, run.options, UnixNow(), run.milliseconds, run.cpuMilliseconds, run.hsrMilliseconds,
run.aoMilliseconds, run.iconMilliseconds, run.bricks, run.trianglesBefore, run.triangles);
ExecuteUpdate("UPDATE ugc SET made_options = ? WHERE id = ?;", run.options, run.id);
}
std::vector<IUgc::RunSummary> MySQLDatabase::GetUgcRunSummaries() {
auto result = ExecuteSelect("SELECT options, COUNT(*) AS runs, COUNT(DISTINCT ugc_id) AS models, SUM(process_ms) AS ms, SUM(process_cpu_ms) AS cpu, "
"SUM(hsr_ms) AS hsr, SUM(ao_ms) AS ao, SUM(icon_ms) AS icon, SUM(bricks) AS bricks, SUM(triangles_before) AS before_tris, SUM(triangles) AS tris "
"FROM ugc_process_runs GROUP BY options ORDER BY runs DESC, options;");
std::vector<RunSummary> summaries;
while (result->next()) {
const auto field = [&](const char* name) { return static_cast<uint64_t>(std::llround(result->getDouble(name))); };
summaries.push_back({ std::string(result->getString("options").c_str()), field("runs"), field("models"), field("ms"), field("cpu"), field("hsr"), field("ao"),
field("icon"), field("bricks"), field("before_tris"), field("tris") });
}
return summaries;
}
std::vector<IUgc::ProcessInfo> MySQLDatabase::GetUgcProcessList(const std::optional<eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) {

2
dDatabase/GameDatabase/MySQL/Tables/UgcLookup.cpp
View File

@@ -25,6 +25,8 @@ namespace {
entry.processMemoryKb = static_cast<uint32_t>(result->getInt64("process_memory_kb"));
entry.modelName = result->getString("model_name").c_str();
entry.trianglesBefore = static_cast<uint32_t>(result->getInt64("triangle_count_before"));
entry.madeOptions = std::string(result->getString("made_options").c_str());
entry.processOptions = std::string(result->getString("process_options").c_str());
return entry;
}
}

6
dDatabase/GameDatabase/SQLite/SQLiteDatabase.h
View File

@@ -444,8 +444,10 @@ public:
void SetModularBuildProcessStats(const LWOOBJID id, const IUgc::ProcessStats& stats) override;
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsWithoutCombination(const uint32_t limit) override;
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override;
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
uint64_t ResetPropertyUgcModelProcessing(const LWOOBJID propertyId) override;
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly, const std::string_view options) override;
uint64_t ResetPropertyUgcModelProcessing(const LWOOBJID propertyId, const std::string_view options) override;
void RecordUgcModelRun(const ProcessRun& run) override;
std::vector<IUgc::RunSummary> GetUgcRunSummaries() override;
IUgc::ProcessTotals GetUgcProcessTotals(const bool modular) override;
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) override;
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetUgcProcessCounts() override;

43
dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp
View File

@@ -113,7 +113,7 @@ std::optional<IUgc::Model> SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) {
}
std::vector<IUgc::PendingModel> SQLiteDatabase::GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) {
auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts, priority FROM ugc WHERE is_optimized = 0 AND process_after <= ? AND (? = 0 OR priority = 1) "
auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts, priority, process_options FROM ugc WHERE is_optimized = 0 AND process_after <= ? AND (? = 0 OR priority = 1) "
"ORDER BY priority DESC, process_attempts ASC, id DESC LIMIT ?;", UnixNow(), priorityOnly ? 1 : 0, limit);
std::vector<IUgc::PendingModel> models;
while (!result.eof()) {
@@ -124,14 +124,16 @@ std::vector<IUgc::PendingModel> SQLiteDatabase::GetUgcModelsToProcess(const uint
model.lxfml.assign(reinterpret_cast<const char*>(blob), blobSize);
model.attempts = static_cast<uint32_t>(result.getIntField("process_attempts"));
model.priority = result.getIntField("priority") != 0;
model.options = result.getStringField("process_options", "");
result.nextRow();
}
return models;
}
void SQLiteDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) {
ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ?, priority = CASE WHEN ? = 0 THEN priority ELSE 0 END WHERE id = ?;",
static_cast<int32_t>(state), attempts, error, bakeAo, UnixNow(), static_cast<int32_t>(state), id);
ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ?, priority = CASE WHEN ? = 0 THEN priority ELSE 0 END, "
"process_options = CASE WHEN ? = 0 THEN process_options ELSE '' END WHERE id = ?;",
static_cast<int32_t>(state), attempts, error, bakeAo, UnixNow(), static_cast<int32_t>(state), static_cast<int32_t>(state), id);
}
std::optional<IUgc::ProcessInfo> SQLiteDatabase::GetUgcProcessInfo(const LWOOBJID id) {
@@ -142,10 +144,10 @@ std::optional<IUgc::ProcessInfo> SQLiteDatabase::GetUgcProcessInfo(const LWOOBJI
return ReadUgcProcessInfo(result, false);
}
uint64_t SQLiteDatabase::ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) {
if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 WHERE id = ?;", *id);
if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 WHERE is_optimized = 2;");
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0;");
uint64_t SQLiteDatabase::ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly, const std::string_view options) {
if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, process_options = ? WHERE id = ?;", options, *id);
if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, process_options = ? WHERE is_optimized = 2;", options);
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, process_options = ?;", options);
}
IUgc::ProcessTotals SQLiteDatabase::GetUgcProcessTotals(const bool modular) {
@@ -166,9 +168,30 @@ IUgc::ProcessTotals SQLiteDatabase::GetUgcProcessTotals(const bool modular) {
return totals;
}
uint64_t SQLiteDatabase::ResetPropertyUgcModelProcessing(const LWOOBJID propertyId) {
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, priority = 1 "
"WHERE id IN (SELECT ugc_id FROM properties_contents WHERE property_id = ? AND ugc_id IS NOT NULL);", propertyId);
uint64_t SQLiteDatabase::ResetPropertyUgcModelProcessing(const LWOOBJID propertyId, const std::string_view options) {
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, priority = 1, process_options = ? "
"WHERE id IN (SELECT ugc_id FROM properties_contents WHERE property_id = ? AND ugc_id IS NOT NULL);", options, propertyId);
}
void SQLiteDatabase::RecordUgcModelRun(const ProcessRun& run) {
ExecuteInsert("INSERT INTO ugc_process_runs (ugc_id, options, made_at, process_ms, process_cpu_ms, hsr_ms, ao_ms, icon_ms, bricks, triangles_before, triangles) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", run.id, run.options, UnixNow(), run.milliseconds, run.cpuMilliseconds, run.hsrMilliseconds,
run.aoMilliseconds, run.iconMilliseconds, run.bricks, run.trianglesBefore, run.triangles);
ExecuteUpdate("UPDATE ugc SET made_options = ? WHERE id = ?;", run.options, run.id);
}
std::vector<IUgc::RunSummary> SQLiteDatabase::GetUgcRunSummaries() {
auto [_, result] = ExecuteSelect("SELECT options, COUNT(*) AS runs, COUNT(DISTINCT ugc_id) AS models, SUM(process_ms) AS ms, SUM(process_cpu_ms) AS cpu, "
"SUM(hsr_ms) AS hsr, SUM(ao_ms) AS ao, SUM(icon_ms) AS icon, SUM(bricks) AS bricks, SUM(triangles_before) AS before_tris, SUM(triangles) AS tris "
"FROM ugc_process_runs GROUP BY options ORDER BY runs DESC, options;");
std::vector<RunSummary> summaries;
while (!result.eof()) {
const auto field = [&](const char* name) { return static_cast<uint64_t>(std::llround(result.getFloatField(name, 0.0))); };
summaries.push_back({ result.getStringField("options", ""), field("runs"), field("models"), field("ms"), field("cpu"), field("hsr"), field("ao"),
field("icon"), field("bricks"), field("before_tris"), field("tris") });
result.nextRow();
}
return summaries;
}
std::vector<IUgc::ProcessInfo> SQLiteDatabase::GetUgcProcessList(const std::optional<eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) {

2
dDatabase/GameDatabase/SQLite/Tables/UgcLookup.cpp
View File

@@ -25,6 +25,8 @@ namespace {
entry.processMemoryKb = static_cast<uint32_t>(result.getInt64Field("process_memory_kb"));
entry.modelName = result.getStringField("model_name", "");
entry.trianglesBefore = static_cast<uint32_t>(result.getInt64Field("triangle_count_before"));
entry.madeOptions = result.getStringField("made_options", "");
entry.processOptions = result.getStringField("process_options", "");
return entry;
}
}

6
dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h
View File

@@ -440,8 +440,10 @@ class TestSQLDatabase : public GameDatabase {
void SetModularBuildProcessStats(const LWOOBJID id, const IUgc::ProcessStats& stats) override {}
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsWithoutCombination(const uint32_t limit) override { return {}; }
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override { return {}; }
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override { return 0; }
uint64_t ResetPropertyUgcModelProcessing(const LWOOBJID propertyId) override { return 0; }
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly, const std::string_view options) override { return 0; }
uint64_t ResetPropertyUgcModelProcessing(const LWOOBJID propertyId, const std::string_view options) override { return 0; }
void RecordUgcModelRun(const ProcessRun& run) override {}
std::vector<IUgc::RunSummary> GetUgcRunSummaries() override { return {}; }
IUgc::ProcessTotals GetUgcProcessTotals(const bool modular) override { return {}; }
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) override { return {}; }
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetUgcProcessCounts() override { return {}; }

2
dGame/dUtilities/SlashCommandHandler.cpp
View File

@@ -497,7 +497,7 @@ void SlashCommandHandler::Startup() {
Command ReprocessPropertyCommand{
.help = "Make this property's models again and reload it",
.info = "The UGC server makes every model placed on the property you are on again (with the current UGC settings). Once they are made, everyone on the property is sent back into it, so their game loads the new meshes",
.info = "The UGC server makes every model placed on the property you are on again, with the current UGC settings or the processing options given (in any order: builtin, embree or hiprt; toolbox or fast; off or oidn). Once they are made, everyone on the property is sent back into it, so their game loads the new meshes",
.aliases = { "reprocessproperty", "reloadpropertymodels" },
.handle = DEVGMCommands::ReprocessProperty,
.requiredLevel = eGameMasterLevel::DEVELOPER

14
dGame/dUtilities/SlashCommands/DEVGMCommands.cpp
View File

@@ -1,5 +1,6 @@
#include "DEVGMCommands.h"
#include "UgcKeys.h"
#include "UgcManifest.h"
#include "PropertyManagementComponent.h"
#include "MasterPackets.h"
@@ -1989,12 +1990,19 @@ namespace DEVGMCommands {
ChatPackets::SendSystemMessage(sysAddr, u"You aren't on a property.");
return;
}
const auto queued = UgcManifest::ReprocessProperty(property->GetId());
// Optional processing options for this make: ray backend, hidden-face method, denoising, in any order
UgcProcessOptions::Choice choice;
if (!UgcProcessOptions::Parse(args, choice)) {
ChatPackets::SendSystemMessage(sysAddr, u"Usage: /reprocessproperty [builtin|embree|hiprt] [toolbox|fast] [off|oidn] (left out: the UGC settings')");
return;
}
const auto options = UgcProcessOptions::ToString(choice);
const auto queued = UgcManifest::ReprocessProperty(property->GetId(), options);
if (queued == 0) {
ChatPackets::SendSystemMessage(sysAddr, u"This property has no brick built models.");
return;
}
ChatPackets::SendSystemMessage(sysAddr, GeneralUtils::UTF8ToUTF16("Making " + std::to_string(queued) +
" model(s) again. Everyone on the property is reloaded when they're made."));
ChatPackets::SendSystemMessage(sysAddr, GeneralUtils::UTF8ToUTF16("Making " + std::to_string(queued) + " model(s) again" +
(options.empty() ? "" : " with " + options) + ". Everyone on the property is reloaded when they're made."));
}
};

7
dGame/dUtilities/UgcManifest.cpp
View File

@@ -346,16 +346,17 @@ namespace {
}
}
size_t UgcManifest::ReprocessProperty(const LWOOBJID propertyId) {
size_t UgcManifest::ReprocessProperty(const LWOOBJID propertyId, const std::string& options) {
std::set<LWOOBJID> blueprintIds;
for (const auto& model : Database::Get()->GetPropertyModels(propertyId)) {
if (model.ugcId != 0) blueprintIds.insert(model.ugcId);
}
if (blueprintIds.empty()) return 0;
Database::Get()->ResetPropertyUgcModelProcessing(propertyId);
Database::Get()->ResetPropertyUgcModelProcessing(propertyId, options);
const auto now = std::chrono::steady_clock::now();
g_PropertyReload = PropertyReload{ std::move(blueprintIds), now, now + RETRY_INTERVAL };
LOG("Making the %zu models of property %llu again; players are reloaded when they're made", g_PropertyReload->blueprintIds.size(), static_cast<unsigned long long>(propertyId));
LOG("Making the %zu models of property %llu again%s; players are reloaded when they're made", g_PropertyReload->blueprintIds.size(), static_cast<unsigned long long>(propertyId),
options.empty() ? "" : (" with " + options).c_str());
return g_PropertyReload->blueprintIds.size();
}

3
dGame/dUtilities/UgcManifest.h
View File

@@ -101,7 +101,8 @@ namespace UgcManifest {
// /reprocessproperty: every model placed on the property is made again by the UGC server; once none is waiting
// (or after 15 minutes), every player in this world is sent the new mesh checksums and transferred back into it,
// so their client loads the property again with the new meshes. Returns how many models were queued.
size_t ReprocessProperty(LWOOBJID propertyId);
// `options`: the processing options for this make (UgcProcessOptions, e.g. "embree fast"), empty: the UGC settings'
size_t ReprocessProperty(LWOOBJID propertyId, const std::string& options = "");
// A client left (or went back to character select): its waiting requests and pending switches are dropped
void OnDisconnect(const SystemAddress& sysAddr);

25
dUgcServer/Processing/UgcProcessor.cpp
View File

@@ -297,6 +297,9 @@ void UgcProcessor::Worker() {
const double iconMs = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - iconStart).count();
if (const auto updated = stats ? UgcJobs::WithIconTime(*stats, iconMs, done.iconChangeMs) : std::nullopt) done.outcome.files["stats.json"] = *updated;
} else {
// The processing options staff picked for this make, over the settings (unknown ones: the settings')
UgcProcessOptions::Choice choice;
if (job.kind == Kind::MODEL && UgcProcessOptions::Parse(job.options, choice)) UgcJobs::ApplyOptions(settings, choice);
done.outcome = job.kind == Kind::MODEL
? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast<uint64_t>(job.id), job.iconValues)
: UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings);
@@ -393,6 +396,7 @@ void UgcProcessor::Poll() {
if (m_InFlight.contains({ Kind::MODEL, model.id })) continue;
Job job{ Kind::MODEL, model.id, model.attempts, UgcJobs::LxfmlFromBlob(model.lxfml) };
job.priority = model.priority;
job.options = std::move(model.options);
if (job.blob.empty()) job.blob = std::move(model.lxfml); // the worker reports it can't be read
job.parts = UgcJobs::CountParts(job.blob);
job.iconValues = IconValues(UgcIconParams::ModelKind(), UgcIconParams::ModelTarget(model.id));
@@ -469,7 +473,20 @@ void UgcProcessor::Record(const Done& done) {
if (done.outcome.ok) Database::Get()->SetUgcModelProcessStats(done.id, cost);
// What it counted (stats.json), for sorting on the dashboard
const auto stats = done.outcome.ok && !done.outcome.stats.empty() ? nlohmann::json::parse(done.outcome.stats, nullptr, false) : nlohmann::json();
if (const auto counts = CountsOf(stats)) Database::Get()->SetUgcModelStats(done.id, counts->bricks, counts->triangles, counts->trianglesBefore);
const auto counts = CountsOf(stats);
if (counts) Database::Get()->SetUgcModelStats(done.id, counts->bricks, counts->triangles, counts->trianglesBefore);
// The make with its options and times, for comparing the options on the dashboard
if (done.outcome.ok && !done.outcome.options.empty()) {
const auto& ms = stats.is_object() ? stats.value("ms", nlohmann::json::object()) : nlohmann::json::object();
const auto part = [&ms](const char* name) { return ms.is_object() ? ms.value(name, 0u) : 0u; };
IUgc::ProcessRun run{ done.id, done.outcome.options, cost.milliseconds, cost.cpuMilliseconds, part("hiddenSurfaces"), part("ambientOcclusion"), part("icon") };
if (counts) {
run.bricks = counts->bricks;
run.trianglesBefore = counts->trianglesBefore;
run.triangles = counts->triangles;
}
Database::Get()->RecordUgcModelRun(run);
}
} else {
Database::Get()->SetModularBuildProcessed(done.id, state, attempts, error);
if (done.outcome.ok) Database::Get()->SetModularBuildProcessStats(done.id, cost);
@@ -656,7 +673,7 @@ UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) {
if (info && info->state != IUgc::eProcessState::FAILED && info->state != IUgc::eProcessState::EMPTY) {
// Made before but the files are gone (deleted to save space): make them again, first
if (info->state == IUgc::eProcessState::DONE) {
if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false);
if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false, "");
else Database::Get()->ResetModularBuildProcessing(id, false);
}
// Someone wants it now: no need to wait out the quiet period after its save
@@ -1007,7 +1024,7 @@ void UgcProcessor::CollectFileTask() {
constexpr size_t ROWS_PER_TICK = 250;
if (m_Purge.after == eAfterDelete::NOW && m_Purge.all) {
if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(std::nullopt, false);
if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(std::nullopt, false, "");
else Database::Get()->ResetModularBuildProcessing(std::nullopt, false);
m_PurgeRowsDone = m_PurgeRows.size();
} else if (m_Purge.after != eAfterDelete::ON_DEMAND) {
@@ -1017,7 +1034,7 @@ void UgcProcessor::CollectFileTask() {
for (; m_PurgeRowsDone < end; m_PurgeRowsDone++) {
const auto row = m_PurgeRows[m_PurgeRowsDone];
if (m_Purge.after == eAfterDelete::NOW) {
if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(row, false);
if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(row, false, "");
else Database::Get()->ResetModularBuildProcessing(row, false);
} else {
if (kind == Kind::MODEL) Database::Get()->SetUgcModelProcessed(row, IUgc::eProcessState::FAILED, m_Config.maxAttempts, "Deleted from the dashboard", false);

1
dUgcServer/Processing/UgcProcessor.h
View File

@@ -178,6 +178,7 @@ private:
bool assembly{}; // with `preview`: answered with the assembled .nif instead of an icon
UgcIconParams::Values iconValues; // models: the preset and override (UgcIconParams)
bool priority{}; // models staff asked to be made again: queued at the front
std::string options; // models: the processing options staff picked (UgcProcessOptions), over the settings
};
// A written file's MD5 and size as the client has it after inflating it (from its .checksum)

2
docs/Commands.md
View File

@@ -137,7 +137,7 @@ These commands are primarily for development and testing. The usage of many of t
|resurrect|`/resurrect`|Resurrects the player.|8|
|setminifig|`/setminifig <body part> <minifig item id>`|Alters your player's minifig. Body part can be one of "Eyebrows", "Eyes", "HairColor", "HairStyle", "Pants", "LeftHand", "Mouth", "RightHand", "Shirt", or "Hands". Changing minifig parts could break the character so this command is limited to GMs.|1|
|testmap|`/testmap <zone> (clone-id) (instance-id) (spawn-point)`|Transfers you to the given zone by id and clone id and then spawns you at the specified spawn point if one was specified. Ignores instance-id for now. Aliases: `/tm`.|1|
|reprocessproperty|`/reprocessproperty`|Has the UGC server make every brick built model on the property you are on again (with the current UGC settings). When they are all made (at most 15 minutes), everyone on the property is sent back into it, so their game downloads the new meshes. Aliases: `/reloadpropertymodels`.|8|
|reprocessproperty|`/reprocessproperty [builtin\|embree\|hiprt] [toolbox\|fast] [off\|oidn]`|Has the UGC server make every brick built model on the property you are on again, with the current UGC settings or the processing options given, in any order (ray backend, hidden-face method, denoising; see docs/UgcServer.md, "Processing options"). When they are all made (at most 15 minutes), everyone on the property is sent back into it, so their game downloads the new meshes. Aliases: `/reloadpropertymodels`.|8|
|reportproxphys|`/reportproxphys`|Prints to console the position and radius of proximity sensors.|9|
|spawnphysicsverts|`/spawnphysicsverts`|Spawns a 1x1 brick at all vertices of phantom physics objects|8|
|teleport|`/teleport <x/source player> (y) <z/target player>`|Teleports you. If no Y is given, you are teleported to the height of the terrain or physics object at (x, z). Any of the coordinates can use the syntax of an exact position (10.0), or a relative position (~+10.0). A ~ means use the current value of that axis as the base value. Addition or subtraction is supported (~+10) (~-10). If source player and target player are players that exist in the world, then the source player will be teleported to target player. Aliases: `/tele`, `/tp`.|6|

33
migrations/dlu/mysql/99_ugc_process_options.sql Normal file
View File

@@ -0,0 +1,33 @@
/* UGC server processing options (docs/UgcServer.md, "Processing options"): ugc.process_options holds the options staff
picked for a model's next make (the dashboard's make again, /reprocessproperty), as "embree fast oidn"; empty: the
UGC settings'. Cleared once the model is made. ugc.made_options is what made the model's current files ("embree
toolbox off"; empty for models made before). ugc_process_runs has every successful make of a model with its options,
times and triangles, for the dashboard's comparison of the options. */
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN process_options VARCHAR(64) NOT NULL DEFAULT ''''', 'DO 0') FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'process_options');
PREPARE dlu_column_stmt FROM @dlu_column;
EXECUTE dlu_column_stmt;
DEALLOCATE PREPARE dlu_column_stmt;
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN made_options VARCHAR(64) NOT NULL DEFAULT ''''', 'DO 0') FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'made_options');
PREPARE dlu_column_stmt FROM @dlu_column;
EXECUTE dlu_column_stmt;
DEALLOCATE PREPARE dlu_column_stmt;
CREATE TABLE IF NOT EXISTS ugc_process_runs (
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
ugc_id BIGINT NOT NULL,
options VARCHAR(64) NOT NULL,
made_at BIGINT NOT NULL,
process_ms INT UNSIGNED NOT NULL,
process_cpu_ms INT UNSIGNED NOT NULL,
hsr_ms INT UNSIGNED NOT NULL,
ao_ms INT UNSIGNED NOT NULL,
icon_ms INT UNSIGNED NOT NULL,
bricks INT UNSIGNED NOT NULL,
triangles_before INT UNSIGNED NOT NULL,
triangles INT UNSIGNED NOT NULL,
INDEX ugc_process_runs_options (options),
INDEX ugc_process_runs_ugc_id (ugc_id)
);

21
migrations/dlu/sqlite/82_ugc_process_options.sql Normal file
View File

@@ -0,0 +1,21 @@
/* UGC server processing options: what staff picked for a model's next make, what made it, and every make's options and
times. See the MySQL migration. */
ALTER TABLE ugc ADD COLUMN process_options TEXT NOT NULL DEFAULT '';
ALTER TABLE ugc ADD COLUMN made_options TEXT NOT NULL DEFAULT '';
CREATE TABLE IF NOT EXISTS ugc_process_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ugc_id BIGINT NOT NULL,
options TEXT NOT NULL,
made_at BIGINT NOT NULL,
process_ms INTEGER NOT NULL,
process_cpu_ms INTEGER NOT NULL,
hsr_ms INTEGER NOT NULL,
ao_ms INTEGER NOT NULL,
icon_ms INTEGER NOT NULL,
bricks INTEGER NOT NULL,
triangles_before INTEGER NOT NULL,
triangles INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS ugc_process_runs_options ON ugc_process_runs (options);
CREATE INDEX IF NOT EXISTS ugc_process_runs_ugc_id ON ugc_process_runs (ugc_id);

29
tests/dDatabaseTests/DatabaseParityTests.cpp
View File

@@ -774,7 +774,7 @@ TEST_F(ParitySeeded, UgcModel) {
return out;
});
Both("ResetUgcModelProcessing", [&](GameDatabase& db) {
const auto changed = db.ResetUgcModelProcessing(std::nullopt, true);
const auto changed = db.ResetUgcModelProcessing(std::nullopt, true, "");
db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::DONE, 1, "", true);
return json{ changed, infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), infoJson(db.GetUgcProcessInfo(1)) };
});
@@ -793,17 +793,38 @@ TEST_F(ParitySeeded, UgcModel) {
IPropertyContents::Model model;
model.id = 1152921510000400009LL; model.lot = 14; model.ugcId = 1152921510000500001LL;
db.InsertNewPropertyModel(PROP1, model, "Objects_14_name");
const auto changed = db.ResetPropertyUgcModelProcessing(PROP1);
const auto changed = db.ResetPropertyUgcModelProcessing(PROP1, "");
// Reset as priority: first in line, and the only one when asking for priority models; made, it's cleared
json priority = json::array();
for (const auto& model : db.GetUgcModelsToProcess(10, true)) priority.push_back({ model.id, model.priority });
const auto first = db.GetUgcModelsToProcess(10);
db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::DONE, 1, "", true);
db.ResetUgcModelProcessing(1152921510000500001LL, false);
db.ResetUgcModelProcessing(1152921510000500001LL, false, "");
const auto afterMade = db.GetUgcModelsToProcess(10, true).size();
return json{ changed, infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), db.ResetPropertyUgcModelProcessing(1), priority,
return json{ changed, infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), db.ResetPropertyUgcModelProcessing(1, ""), priority,
first.empty() ? json() : json{ first.front().id, first.front().priority }, afterMade };
});
Both("Processing options and runs", [&](GameDatabase& db) {
// Options picked for the next make reach the UGC server and are cleared once it's made; a failed attempt keeps them
json out = json::array();
out.push_back(db.ResetUgcModelProcessing(1152921510000500001LL, false, "embree fast"));
for (const auto& model : db.GetUgcModelsToProcess(10)) out.push_back({ model.id, model.options });
db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::PENDING, 1, "crashed", false);
for (const auto& model : db.GetUgcModelsToProcess(10)) out.push_back({ model.id, model.options });
db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::DONE, 2, "", true);
out.push_back(db.ResetPropertyUgcModelProcessing(PROP1, "hiprt oidn"));
for (const auto& model : db.GetUgcModelsToProcess(10)) out.push_back({ model.id, model.options, model.priority });
db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::DONE, 1, "", true);
// Runs: the made options on the model, and the sums per options
db.RecordUgcModelRun({ 1152921510000500001LL, "embree fast off", 1000, 900, 100, 200, 50, 12, 500, 300 });
db.RecordUgcModelRun({ 1152921510000500001LL, "embree fast off", 3000, 2900, 300, 400, 70, 12, 500, 280 });
db.RecordUgcModelRun({ 1152921510000500001LL, "builtin toolbox off", 9000, 8900, 7000, 900, 60, 12, 500, 350 });
for (const auto& entry : db.GetUgcEntries({ 1152921510000500001LL })) out.push_back({ entry.madeOptions, entry.processOptions });
for (const auto& s : db.GetUgcRunSummaries()) {
out.push_back({ s.options, s.runs, s.models, s.milliseconds, s.cpuMilliseconds, s.hsrMilliseconds, s.aoMilliseconds, s.iconMilliseconds, s.bricks, s.trianglesBefore, s.triangles });
}
return out;
});
Both("Modular build processing", [&](GameDatabase& db) {
db.InsertUgcBuild("1:4713+1:4714+1:4715", 1152921510000500002LL, CHAR_ALICE);
json out = json::array();

2
tests/dWebTests/UgcLookupTests.cpp
View File

@@ -105,7 +105,7 @@ namespace {
"CREATE TABLE charinfo (id BIGINT PRIMARY KEY, account_id INTEGER, name TEXT);"
"CREATE TABLE properties (id BIGINT PRIMARY KEY, owner_id BIGINT, name TEXT, zone_id INTEGER);"
"CREATE TABLE ugc (id INTEGER PRIMARY KEY, account_id INTEGER, character_id BIGINT, is_optimized INTEGER, filename TEXT, process_error TEXT DEFAULT '', "
"process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_after BIGINT DEFAULT 0, bake_ao INTEGER DEFAULT 0, brick_count INTEGER DEFAULT 0, triangle_count INTEGER DEFAULT 0, process_ms INTEGER DEFAULT 0, process_cpu_ms INTEGER DEFAULT 0, process_memory_kb INTEGER DEFAULT 0, triangle_count_before INTEGER DEFAULT 0);"
"process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_after BIGINT DEFAULT 0, bake_ao INTEGER DEFAULT 0, brick_count INTEGER DEFAULT 0, triangle_count INTEGER DEFAULT 0, process_ms INTEGER DEFAULT 0, process_cpu_ms INTEGER DEFAULT 0, process_memory_kb INTEGER DEFAULT 0, triangle_count_before INTEGER DEFAULT 0, made_options TEXT DEFAULT '', process_options TEXT DEFAULT '');"
"CREATE TABLE ugc_modular_build (ugc_id BIGINT PRIMARY KEY, character_id BIGINT, ldf_config TEXT, is_optimized INTEGER DEFAULT 0, process_error TEXT DEFAULT '', "
"process_attempts INTEGER DEFAULT 0, processed_at BIGINT DEFAULT 0, process_ms INTEGER DEFAULT 0, process_cpu_ms INTEGER DEFAULT 0, process_memory_kb INTEGER DEFAULT 0);"
"CREATE TABLE properties_contents (id BIGINT PRIMARY KEY, property_id BIGINT, ugc_id BIGINT, lot INTEGER, model_name TEXT DEFAULT '', model_description TEXT DEFAULT '');"