Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-02 10:53:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ugc): staff reprocessing of a property goes to the front of the queue

Browse Source 
/reprocessproperty and the dashboard's Reprocess all models mark the
property's models as priority (ugc.priority, migrations mysql 94 and sqlite
77). The UGC server takes priority models first, polls them even when its
queue is full, and puts them at its front, as it does cars and rockets. The
flag clears once a model is made.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-28 12:26:56 -05:00
parent 8d57cc82cf
commit 9e4f2818c3
12 changed files with 49 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
dDatabase/GameDatabase/ITables/IUgc.h
View File

@@ -78,10 +78,12 @@ public:
LWOOBJID id{};
std::string lxfml;
uint32_t attempts{};
bool priority{}; // staff asked for it to be made again: before any other model
};
// Up to `limit` pending models whose quiet period is over (process_after), the least tried and then the newest first
virtual std::vector<PendingModel> GetUgcModelsToProcess(const uint32_t limit) = 0;
// Up to `limit` pending models whose quiet period is over (process_after): the priority ones first, then the least
// tried and then the newest. `priorityOnly`: only the priority ones.
virtual std::vector<PendingModel> GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly = false) = 0;
// Ends the quiet period of a waiting model (a client asked for it), or of all a character's waiting models (they
// left the property)
@@ -110,7 +112,8 @@ public:
// with `failedOnly`). Returns how many rows changed.
virtual uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) = 0;
// Sets every model placed on a property back to pending with no attempts. Returns how many rows changed.
// Sets every model placed on a property back to pending with no attempts, as priority (staff asked for it). Returns
// how many rows changed.
virtual uint64_t ResetPropertyUgcModelProcessing(const LWOOBJID propertyId) = 0;
// A page of models (all, or those in `state`), the newest first; `search` (when not empty) matches the model's id

2
dDatabase/GameDatabase/MySQL/MySQLDatabase.h
View File

@@ -420,7 +420,7 @@ public:
nlohmann::json GetCharacterById(const LWOOBJID charId) override;
std::optional<IPropertyContents::Model> GetModel(const LWOOBJID modelID) override;
std::optional<IUgc::Model> GetUgcModel(const LWOOBJID ugcId) override;
std::vector<IUgc::PendingModel> GetUgcModelsToProcess(const uint32_t limit) override;
std::vector<IUgc::PendingModel> GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) override;
void ExpediteUgcModel(const LWOOBJID id) override;
void ExpediteUgcModels(const LWOOBJID characterId) override;
std::vector<std::pair<std::string, uint64_t>> GetModularBuildConfigCounts() override;

12
dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp
View File

@@ -110,8 +110,9 @@ std::optional<IUgc::Model> MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) {
return toReturn;
}
std::vector<IUgc::PendingModel> MySQLDatabase::GetUgcModelsToProcess(const uint32_t limit) {
auto result = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 AND process_after <= ? ORDER BY process_attempts ASC, id DESC LIMIT ?;", UnixNow(), limit);
std::vector<IUgc::PendingModel> MySQLDatabase::GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) {
auto result = ExecuteSelect("SELECT id, lxfml, process_attempts, priority FROM ugc WHERE is_optimized = 0 AND process_after <= ? AND (? = 0 OR priority = 1) "
"ORDER BY priority DESC, process_attempts ASC, id DESC LIMIT ?;", UnixNow(), priorityOnly ? 1 : 0, limit);
std::vector<IUgc::PendingModel> models;
while (result->next()) {
auto& model = models.emplace_back();
@@ -121,13 +122,14 @@ std::vector<IUgc::PendingModel> MySQLDatabase::GetUgcModelsToProcess(const uint3
contents << blob->rdbuf();
model.lxfml = contents.str();
model.attempts = static_cast<uint32_t>(result->getInt("process_attempts"));
model.priority = result->getInt("priority") != 0;
}
return models;
}
void MySQLDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) {
ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ? WHERE id = ?;",
static_cast<int32_t>(state), attempts, error, bakeAo, UnixNow(), id);
ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ?, priority = CASE WHEN ? = 0 THEN priority ELSE 0 END WHERE id = ?;",
static_cast<int32_t>(state), attempts, error, bakeAo, UnixNow(), static_cast<int32_t>(state), id);
}
std::optional<IUgc::ProcessInfo> MySQLDatabase::GetUgcProcessInfo(const LWOOBJID id) {
@@ -145,7 +147,7 @@ uint64_t MySQLDatabase::ResetUgcModelProcessing(const std::optional<LWOOBJID> id
}
uint64_t MySQLDatabase::ResetPropertyUgcModelProcessing(const LWOOBJID propertyId) {
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 "
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, priority = 1 "
"WHERE id IN (SELECT ugc_id FROM properties_contents WHERE property_id = ? AND ugc_id IS NOT NULL);", propertyId);
}

2
dDatabase/GameDatabase/SQLite/SQLiteDatabase.h
View File

@@ -404,7 +404,7 @@ public:
nlohmann::json GetCharacterById(const LWOOBJID charId) override;
std::optional<IPropertyContents::Model> GetModel(const LWOOBJID modelID) override;
std::optional<IUgc::Model> GetUgcModel(const LWOOBJID ugcId) override;
std::vector<IUgc::PendingModel> GetUgcModelsToProcess(const uint32_t limit) override;
std::vector<IUgc::PendingModel> GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) override;
void ExpediteUgcModel(const LWOOBJID id) override;
void ExpediteUgcModels(const LWOOBJID characterId) override;
std::vector<std::pair<std::string, uint64_t>> GetModularBuildConfigCounts() override;

12
dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp
View File

@@ -111,8 +111,9 @@ std::optional<IUgc::Model> SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) {
return toReturn;
}
std::vector<IUgc::PendingModel> SQLiteDatabase::GetUgcModelsToProcess(const uint32_t limit) {
auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 AND process_after <= ? ORDER BY process_attempts ASC, id DESC LIMIT ?;", UnixNow(), limit);
std::vector<IUgc::PendingModel> SQLiteDatabase::GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) {
auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts, priority FROM ugc WHERE is_optimized = 0 AND process_after <= ? AND (? = 0 OR priority = 1) "
"ORDER BY priority DESC, process_attempts ASC, id DESC LIMIT ?;", UnixNow(), priorityOnly ? 1 : 0, limit);
std::vector<IUgc::PendingModel> models;
while (!result.eof()) {
auto& model = models.emplace_back();
@@ -121,14 +122,15 @@ std::vector<IUgc::PendingModel> SQLiteDatabase::GetUgcModelsToProcess(const uint
const auto* blob = result.getBlobField("lxfml", blobSize);
model.lxfml.assign(reinterpret_cast<const char*>(blob), blobSize);
model.attempts = static_cast<uint32_t>(result.getIntField("process_attempts"));
model.priority = result.getIntField("priority") != 0;
result.nextRow();
}
return models;
}
void SQLiteDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) {
ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ? WHERE id = ?;",
static_cast<int32_t>(state), attempts, error, bakeAo, UnixNow(), id);
ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ?, priority = CASE WHEN ? = 0 THEN priority ELSE 0 END WHERE id = ?;",
static_cast<int32_t>(state), attempts, error, bakeAo, UnixNow(), static_cast<int32_t>(state), id);
}
std::optional<IUgc::ProcessInfo> SQLiteDatabase::GetUgcProcessInfo(const LWOOBJID id) {
@@ -146,7 +148,7 @@ uint64_t SQLiteDatabase::ResetUgcModelProcessing(const std::optional<LWOOBJID> i
}
uint64_t SQLiteDatabase::ResetPropertyUgcModelProcessing(const LWOOBJID propertyId) {
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 "
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, priority = 1 "
"WHERE id IN (SELECT ugc_id FROM properties_contents WHERE property_id = ? AND ugc_id IS NOT NULL);", propertyId);
}

2
dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h
View File

@@ -402,7 +402,7 @@ class TestSQLDatabase : public GameDatabase {
std::optional<IPropertyContents::Model> GetModel(const LWOOBJID modelID) override { return {}; }
std::optional<IProperty::Info> GetPropertyInfo(const LWOOBJID id) override { return {}; }
std::optional<IUgc::Model> GetUgcModel(const LWOOBJID ugcId) override { return {}; }
std::vector<IUgc::PendingModel> GetUgcModelsToProcess(const uint32_t limit) override { return {}; }
std::vector<IUgc::PendingModel> GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) override { return {}; }
void ExpediteUgcModel(const LWOOBJID id) override {}
void ExpediteUgcModels(const LWOOBJID characterId) override {}
std::vector<std::pair<std::string, uint64_t>> GetModularBuildConfigCounts() override { return {}; }

5
dUgcServer/UgcProcessor.cpp
View File

@@ -332,10 +332,13 @@ void UgcProcessor::Poll() {
std::vector<Job> jobs;
std::vector<IUgc::PendingModel> models;
// Models staff asked to be made again come first too: polled even when the queue is full, to its front
if (limit > 0) models = Database::Get()->GetUgcModelsToProcess(limit);
else models = Database::Get()->GetUgcModelsToProcess(buildLimit, true);
for (auto& model : models) {
if (m_InFlight.contains({ Kind::MODEL, model.id })) continue;
Job job{ Kind::MODEL, model.id, model.attempts, UgcJobs::LxfmlFromBlob(model.lxfml) };
job.priority = model.priority;
if (job.blob.empty()) job.blob = std::move(model.lxfml); // the worker reports it can't be read
job.parts = UgcJobs::CountParts(job.blob);
job.iconValues = IconValues(UgcIconParams::ModelKind(), UgcIconParams::ModelTarget(model.id));
@@ -383,7 +386,7 @@ void UgcProcessor::Poll() {
std::lock_guard lock(m_Mutex);
for (auto& job : jobs) {
if (job.kind == Kind::MODEL) m_InFlight.insert({ job.kind, job.id });
if (job.kind == Kind::MODULAR) m_Jobs.push_front(std::move(job));
if (job.kind == Kind::MODULAR || job.priority) m_Jobs.push_front(std::move(job));
else m_Jobs.push_back(std::move(job));
}
}

1
dUgcServer/UgcProcessor.h
View File

@@ -157,6 +157,7 @@ private:
bool iconOnly{}; // a model's icon drawn again from its stored .nif
bool assembly{}; // with `preview`: answered with the assembled .nif instead of an icon
UgcIconParams::Values iconValues; // models: the preset and override (UgcIconParams)
bool priority{}; // models staff asked to be made again: queued at the front
};
// A written file's MD5 and size as the client has it after inflating it (from its .checksum)

3
docs/UgcServer.md
View File

@@ -431,6 +431,9 @@ serving), the worlds:
minutes) it sends every player in the world the new `model.nif` checksums and transfers them back into the same
zone and clone. The client loads the property again and downloads the new meshes (its manifest cache has the new
checksums, which its files don't match). Models of a reprocess skip the "made again" switch.
The models are queued as priority (`ugc.priority`, also set by the dashboard's Reprocess all models): the UGC
server takes them before any other model, polls them even when its queue is full and puts them at its front (as
cars and rockets); the flag clears once a model is made.
### Waiting while the owner is still building

7
migrations/dlu/mysql/94_ugc_priority.sql Normal file
View File

@@ -0,0 +1,7 @@
/* ugc.priority: models staff asked to be made again (/reprocessproperty, the dashboard's Reprocess all models). The
UGC server makes them before any other model; cleared once made. */
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN priority TINYINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'priority');
PREPARE dlu_column_stmt FROM @dlu_column;
EXECUTE dlu_column_stmt;
DEALLOCATE PREPARE dlu_column_stmt;

2
migrations/dlu/sqlite/77_ugc_priority.sql Normal file
View File

@@ -0,0 +1,2 @@
/* ugc.priority: models staff asked to be made again. See the MySQL migration. */
ALTER TABLE ugc ADD COLUMN priority INTEGER NOT NULL DEFAULT 0;

10
tests/dDatabaseTests/DatabaseParityTests.cpp
View File

@@ -777,7 +777,15 @@ TEST_F(ParitySeeded, UgcModel) {
model.id = 1152921510000400009LL; model.lot = 14; model.ugcId = 1152921510000500001LL;
db.InsertNewPropertyModel(PROP1, model, "Objects_14_name");
const auto changed = db.ResetPropertyUgcModelProcessing(PROP1);
return json{ changed, infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), db.ResetPropertyUgcModelProcessing(1) };
// Reset as priority: first in line, and the only one when asking for priority models; made, it's cleared
json priority = json::array();
for (const auto& model : db.GetUgcModelsToProcess(10, true)) priority.push_back({ model.id, model.priority });
const auto first = db.GetUgcModelsToProcess(10);
db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::DONE, 1, "", true);
db.ResetUgcModelProcessing(1152921510000500001LL, false);
const auto afterMade = db.GetUgcModelsToProcess(10, true).size();
return json{ changed, infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), db.ResetPropertyUgcModelProcessing(1), priority,
first.empty() ? json() : json{ first.front().id, first.front().priority }, afterMade };
});
Both("Modular build processing", [&](GameDatabase& db) {
db.InsertUgcBuild("1:4713+1:4714+1:4715", 1152921510000500002LL, CHAR_ALICE);