From 9e4f2818c3cbd79641f9ff58f6c5e1a4136cd0f4 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Mon, 28 Sep 2026 12:26:56 -0500 Subject: [PATCH] feat(ugc): staff reprocessing of a property goes to the front of the queue /reprocessproperty and the dashboard's Reprocess all models mark the property's models as priority (ugc.priority, migrations mysql 94 and sqlite 77). The UGC server takes priority models first, polls them even when its queue is full, and puts them at its front, as it does cars and rockets. The flag clears once a model is made. Co-Authored-By: Claude Opus 5.5 --- dDatabase/GameDatabase/ITables/IUgc.h | 9 ++++++--- dDatabase/GameDatabase/MySQL/MySQLDatabase.h | 2 +- dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp | 12 +++++++----- dDatabase/GameDatabase/SQLite/SQLiteDatabase.h | 2 +- dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp | 12 +++++++----- dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h | 2 +- dUgcServer/UgcProcessor.cpp | 5 ++++- dUgcServer/UgcProcessor.h | 1 + docs/UgcServer.md | 3 +++ migrations/dlu/mysql/94_ugc_priority.sql | 7 +++++++ migrations/dlu/sqlite/77_ugc_priority.sql | 2 ++ tests/dDatabaseTests/DatabaseParityTests.cpp | 10 +++++++++- 12 files changed, 49 insertions(+), 18 deletions(-) create mode 100644 migrations/dlu/mysql/94_ugc_priority.sql create mode 100644 migrations/dlu/sqlite/77_ugc_priority.sql diff --git a/dDatabase/GameDatabase/ITables/IUgc.h b/dDatabase/GameDatabase/ITables/IUgc.h index f61d9c957..7a1b09c02 100644 --- a/dDatabase/GameDatabase/ITables/IUgc.h +++ b/dDatabase/GameDatabase/ITables/IUgc.h @@ -78,10 +78,12 @@ public: LWOOBJID id{}; std::string lxfml; uint32_t attempts{}; + bool priority{}; // staff asked for it to be made again: before any other model }; - // Up to `limit` pending models whose quiet period is over (process_after), the least tried and then the newest first - virtual std::vector GetUgcModelsToProcess(const uint32_t limit) = 0; + // Up to `limit` pending models whose quiet period is over (process_after): the priority ones first, then the least + // tried and then the newest. `priorityOnly`: only the priority ones. + virtual std::vector GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly = false) = 0; // Ends the quiet period of a waiting model (a client asked for it), or of all a character's waiting models (they // left the property) @@ -110,7 +112,8 @@ public: // with `failedOnly`). Returns how many rows changed. virtual uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) = 0; - // Sets every model placed on a property back to pending with no attempts. Returns how many rows changed. + // Sets every model placed on a property back to pending with no attempts, as priority (staff asked for it). Returns + // how many rows changed. virtual uint64_t ResetPropertyUgcModelProcessing(const LWOOBJID propertyId) = 0; // A page of models (all, or those in `state`), the newest first; `search` (when not empty) matches the model's id diff --git a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h index ac536a47f..a9ff83b91 100644 --- a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h +++ b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h @@ -420,7 +420,7 @@ public: nlohmann::json GetCharacterById(const LWOOBJID charId) override; std::optional GetModel(const LWOOBJID modelID) override; std::optional GetUgcModel(const LWOOBJID ugcId) override; - std::vector GetUgcModelsToProcess(const uint32_t limit) override; + std::vector GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) override; void ExpediteUgcModel(const LWOOBJID id) override; void ExpediteUgcModels(const LWOOBJID characterId) override; std::vector> GetModularBuildConfigCounts() override; diff --git a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp index 286876da6..065d57564 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp @@ -110,8 +110,9 @@ std::optional MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) { return toReturn; } -std::vector MySQLDatabase::GetUgcModelsToProcess(const uint32_t limit) { - auto result = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 AND process_after <= ? ORDER BY process_attempts ASC, id DESC LIMIT ?;", UnixNow(), limit); +std::vector MySQLDatabase::GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) { + auto result = ExecuteSelect("SELECT id, lxfml, process_attempts, priority FROM ugc WHERE is_optimized = 0 AND process_after <= ? AND (? = 0 OR priority = 1) " + "ORDER BY priority DESC, process_attempts ASC, id DESC LIMIT ?;", UnixNow(), priorityOnly ? 1 : 0, limit); std::vector models; while (result->next()) { auto& model = models.emplace_back(); @@ -121,13 +122,14 @@ std::vector MySQLDatabase::GetUgcModelsToProcess(const uint3 contents << blob->rdbuf(); model.lxfml = contents.str(); model.attempts = static_cast(result->getInt("process_attempts")); + model.priority = result->getInt("priority") != 0; } return models; } void MySQLDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) { - ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ? WHERE id = ?;", - static_cast(state), attempts, error, bakeAo, UnixNow(), id); + ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ?, priority = CASE WHEN ? = 0 THEN priority ELSE 0 END WHERE id = ?;", + static_cast(state), attempts, error, bakeAo, UnixNow(), static_cast(state), id); } std::optional MySQLDatabase::GetUgcProcessInfo(const LWOOBJID id) { @@ -145,7 +147,7 @@ uint64_t MySQLDatabase::ResetUgcModelProcessing(const std::optional id } uint64_t MySQLDatabase::ResetPropertyUgcModelProcessing(const LWOOBJID propertyId) { - return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 " + return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, priority = 1 " "WHERE id IN (SELECT ugc_id FROM properties_contents WHERE property_id = ? AND ugc_id IS NOT NULL);", propertyId); } diff --git a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h index 24379524c..45ab42157 100644 --- a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h +++ b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h @@ -404,7 +404,7 @@ public: nlohmann::json GetCharacterById(const LWOOBJID charId) override; std::optional GetModel(const LWOOBJID modelID) override; std::optional GetUgcModel(const LWOOBJID ugcId) override; - std::vector GetUgcModelsToProcess(const uint32_t limit) override; + std::vector GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) override; void ExpediteUgcModel(const LWOOBJID id) override; void ExpediteUgcModels(const LWOOBJID characterId) override; std::vector> GetModularBuildConfigCounts() override; diff --git a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp index b893d7fb7..c60532e5f 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp @@ -111,8 +111,9 @@ std::optional SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) { return toReturn; } -std::vector SQLiteDatabase::GetUgcModelsToProcess(const uint32_t limit) { - auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 AND process_after <= ? ORDER BY process_attempts ASC, id DESC LIMIT ?;", UnixNow(), limit); +std::vector SQLiteDatabase::GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) { + auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts, priority FROM ugc WHERE is_optimized = 0 AND process_after <= ? AND (? = 0 OR priority = 1) " + "ORDER BY priority DESC, process_attempts ASC, id DESC LIMIT ?;", UnixNow(), priorityOnly ? 1 : 0, limit); std::vector models; while (!result.eof()) { auto& model = models.emplace_back(); @@ -121,14 +122,15 @@ std::vector SQLiteDatabase::GetUgcModelsToProcess(const uint const auto* blob = result.getBlobField("lxfml", blobSize); model.lxfml.assign(reinterpret_cast(blob), blobSize); model.attempts = static_cast(result.getIntField("process_attempts")); + model.priority = result.getIntField("priority") != 0; result.nextRow(); } return models; } void SQLiteDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) { - ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ? WHERE id = ?;", - static_cast(state), attempts, error, bakeAo, UnixNow(), id); + ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ?, priority = CASE WHEN ? = 0 THEN priority ELSE 0 END WHERE id = ?;", + static_cast(state), attempts, error, bakeAo, UnixNow(), static_cast(state), id); } std::optional SQLiteDatabase::GetUgcProcessInfo(const LWOOBJID id) { @@ -146,7 +148,7 @@ uint64_t SQLiteDatabase::ResetUgcModelProcessing(const std::optional i } uint64_t SQLiteDatabase::ResetPropertyUgcModelProcessing(const LWOOBJID propertyId) { - return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 " + return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, priority = 1 " "WHERE id IN (SELECT ugc_id FROM properties_contents WHERE property_id = ? AND ugc_id IS NOT NULL);", propertyId); } diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h index 5a93c7581..0d658380b 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h @@ -402,7 +402,7 @@ class TestSQLDatabase : public GameDatabase { std::optional GetModel(const LWOOBJID modelID) override { return {}; } std::optional GetPropertyInfo(const LWOOBJID id) override { return {}; } std::optional GetUgcModel(const LWOOBJID ugcId) override { return {}; } - std::vector GetUgcModelsToProcess(const uint32_t limit) override { return {}; } + std::vector GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) override { return {}; } void ExpediteUgcModel(const LWOOBJID id) override {} void ExpediteUgcModels(const LWOOBJID characterId) override {} std::vector> GetModularBuildConfigCounts() override { return {}; } diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index fb689f6ce..b1433cc6c 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -332,10 +332,13 @@ void UgcProcessor::Poll() { std::vector jobs; std::vector models; + // Models staff asked to be made again come first too: polled even when the queue is full, to its front if (limit > 0) models = Database::Get()->GetUgcModelsToProcess(limit); + else models = Database::Get()->GetUgcModelsToProcess(buildLimit, true); for (auto& model : models) { if (m_InFlight.contains({ Kind::MODEL, model.id })) continue; Job job{ Kind::MODEL, model.id, model.attempts, UgcJobs::LxfmlFromBlob(model.lxfml) }; + job.priority = model.priority; if (job.blob.empty()) job.blob = std::move(model.lxfml); // the worker reports it can't be read job.parts = UgcJobs::CountParts(job.blob); job.iconValues = IconValues(UgcIconParams::ModelKind(), UgcIconParams::ModelTarget(model.id)); @@ -383,7 +386,7 @@ void UgcProcessor::Poll() { std::lock_guard lock(m_Mutex); for (auto& job : jobs) { if (job.kind == Kind::MODEL) m_InFlight.insert({ job.kind, job.id }); - if (job.kind == Kind::MODULAR) m_Jobs.push_front(std::move(job)); + if (job.kind == Kind::MODULAR || job.priority) m_Jobs.push_front(std::move(job)); else m_Jobs.push_back(std::move(job)); } } diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h index 58b541882..fd51a97ed 100644 --- a/dUgcServer/UgcProcessor.h +++ b/dUgcServer/UgcProcessor.h @@ -157,6 +157,7 @@ private: bool iconOnly{}; // a model's icon drawn again from its stored .nif bool assembly{}; // with `preview`: answered with the assembled .nif instead of an icon UgcIconParams::Values iconValues; // models: the preset and override (UgcIconParams) + bool priority{}; // models staff asked to be made again: queued at the front }; // A written file's MD5 and size as the client has it after inflating it (from its .checksum) diff --git a/docs/UgcServer.md b/docs/UgcServer.md index b3987a038..54c4b669d 100644 --- a/docs/UgcServer.md +++ b/docs/UgcServer.md @@ -431,6 +431,9 @@ serving), the worlds: minutes) it sends every player in the world the new `model.nif` checksums and transfers them back into the same zone and clone. The client loads the property again and downloads the new meshes (its manifest cache has the new checksums, which its files don't match). Models of a reprocess skip the "made again" switch. + The models are queued as priority (`ugc.priority`, also set by the dashboard's Reprocess all models): the UGC + server takes them before any other model, polls them even when its queue is full and puts them at its front (as + cars and rockets); the flag clears once a model is made. ### Waiting while the owner is still building diff --git a/migrations/dlu/mysql/94_ugc_priority.sql b/migrations/dlu/mysql/94_ugc_priority.sql new file mode 100644 index 000000000..4e31b4a77 --- /dev/null +++ b/migrations/dlu/mysql/94_ugc_priority.sql @@ -0,0 +1,7 @@ +/* ugc.priority: models staff asked to be made again (/reprocessproperty, the dashboard's Reprocess all models). The + UGC server makes them before any other model; cleared once made. */ +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN priority TINYINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'priority'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; diff --git a/migrations/dlu/sqlite/77_ugc_priority.sql b/migrations/dlu/sqlite/77_ugc_priority.sql new file mode 100644 index 000000000..e6321ff8b --- /dev/null +++ b/migrations/dlu/sqlite/77_ugc_priority.sql @@ -0,0 +1,2 @@ +/* ugc.priority: models staff asked to be made again. See the MySQL migration. */ +ALTER TABLE ugc ADD COLUMN priority INTEGER NOT NULL DEFAULT 0; diff --git a/tests/dDatabaseTests/DatabaseParityTests.cpp b/tests/dDatabaseTests/DatabaseParityTests.cpp index dc11c1374..c4d399c71 100644 --- a/tests/dDatabaseTests/DatabaseParityTests.cpp +++ b/tests/dDatabaseTests/DatabaseParityTests.cpp @@ -777,7 +777,15 @@ TEST_F(ParitySeeded, UgcModel) { model.id = 1152921510000400009LL; model.lot = 14; model.ugcId = 1152921510000500001LL; db.InsertNewPropertyModel(PROP1, model, "Objects_14_name"); const auto changed = db.ResetPropertyUgcModelProcessing(PROP1); - return json{ changed, infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), db.ResetPropertyUgcModelProcessing(1) }; + // Reset as priority: first in line, and the only one when asking for priority models; made, it's cleared + json priority = json::array(); + for (const auto& model : db.GetUgcModelsToProcess(10, true)) priority.push_back({ model.id, model.priority }); + const auto first = db.GetUgcModelsToProcess(10); + db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::DONE, 1, "", true); + db.ResetUgcModelProcessing(1152921510000500001LL, false); + const auto afterMade = db.GetUgcModelsToProcess(10, true).size(); + return json{ changed, infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), db.ResetPropertyUgcModelProcessing(1), priority, + first.empty() ? json() : json{ first.front().id, first.front().priority }, afterMade }; }); Both("Modular build processing", [&](GameDatabase& db) { db.InsertUgcBuild("1:4713+1:4714+1:4715", 1152921510000500002LL, CHAR_ALICE);