diff --git a/dDatabase/GameDatabase/ITables/IUgc.h b/dDatabase/GameDatabase/ITables/IUgc.h index 7e1397ed9..338bf2bf6 100644 --- a/dDatabase/GameDatabase/ITables/IUgc.h +++ b/dDatabase/GameDatabase/ITables/IUgc.h @@ -83,8 +83,16 @@ public: }; // Up to `limit` pending models whose quiet period is over (process_after): the priority ones first, then the least - // tried and then the newest. `priorityOnly`: only the priority ones. - virtual std::vector GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly = false) = 0; + // tried and then the newest. `priorityOnly`: only the priority ones. `queuedOnly`: only the queued ones (the UGC + // setting queue_on_property_save). + virtual std::vector GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly = false, const bool queuedOnly = false) = 0; + + // Queues every model placed on a property (its owner saved it): with queue_on_property_save only queued models are + // made. Returns how many were newly queued. + virtual uint64_t QueuePropertyUgcModels(const LWOOBJID propertyId) = 0; + + // Pending models not queued: waiting for their property to be saved while queue_on_property_save is on + virtual uint64_t CountUnqueuedUgcModels() = 0; // Ends the quiet period of a waiting model (a client asked for it), or of all a character's waiting models (they // left the property) diff --git a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h index 00efae042..be6336acd 100644 --- a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h +++ b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h @@ -462,7 +462,9 @@ public: nlohmann::json GetCharacterById(const LWOOBJID charId) override; std::optional GetModel(const LWOOBJID modelID) override; std::optional GetUgcModel(const LWOOBJID ugcId) override; - std::vector GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) override; + std::vector GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly, const bool queuedOnly) override; + uint64_t QueuePropertyUgcModels(const LWOOBJID propertyId) override; + uint64_t CountUnqueuedUgcModels() override; void ExpediteUgcModel(const LWOOBJID id) override; void ExpediteUgcModels(const LWOOBJID characterId) override; std::vector> GetModularBuildConfigCounts() override; diff --git a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp index b3dfb4694..2b1f42122 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp @@ -74,7 +74,7 @@ void MySQLDatabase::InsertNewUgcModel( const int64_t processAfter) { const std::istream stream(sd0Data.rdbuf()); ExecuteInsert( - "INSERT INTO `ugc`(`id`, `account_id`, `character_id`, `is_optimized`, `lxfml`, `bake_ao`, `filename`, `process_after`) VALUES (?,?,?,?,?,?,?,?)", + "INSERT INTO `ugc`(`id`, `account_id`, `character_id`, `is_optimized`, `lxfml`, `bake_ao`, `filename`, `process_after`, `queued`) VALUES (?,?,?,?,?,?,?,?,1)", blueprintId, accountId, characterId, @@ -96,7 +96,7 @@ void MySQLDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { void MySQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) { const std::istream stream(lxfml.rdbuf()); // The UGC server makes the model's files again - ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '', process_after = ? WHERE id = ?;", &stream, processAfter, modelId); + ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '', process_after = ?, queued = 1 WHERE id = ?;", &stream, processAfter, modelId); } std::optional MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) { @@ -111,9 +111,9 @@ std::optional MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) { return toReturn; } -std::vector MySQLDatabase::GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) { - auto result = ExecuteSelect("SELECT id, lxfml, process_attempts, priority, process_options FROM ugc WHERE is_optimized = 0 AND process_after <= ? AND (? = 0 OR priority = 1) " - "ORDER BY priority DESC, process_attempts ASC, id DESC LIMIT ?;", UnixNow(), priorityOnly ? 1 : 0, limit); +std::vector MySQLDatabase::GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly, const bool queuedOnly) { + auto result = ExecuteSelect("SELECT id, lxfml, process_attempts, priority, process_options FROM ugc WHERE is_optimized = 0 AND process_after <= ? AND (? = 0 OR priority = 1) AND (? = 0 OR queued = 1) " + "ORDER BY priority DESC, process_attempts ASC, id DESC LIMIT ?;", UnixNow(), priorityOnly ? 1 : 0, queuedOnly ? 1 : 0, limit); std::vector models; while (result->next()) { auto& model = models.emplace_back(); @@ -144,9 +144,9 @@ std::optional MySQLDatabase::GetUgcProcessInfo(const LWOOBJID } uint64_t MySQLDatabase::ResetUgcModelProcessing(const std::optional id, const bool failedOnly, const std::string_view options) { - if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, process_options = ? WHERE id = ?;", options, *id); - if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, process_options = ? WHERE is_optimized = 2;", options); - return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, process_options = ?;", options); + if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, queued = 1, process_options = ? WHERE id = ?;", options, *id); + if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, queued = 1, process_options = ? WHERE is_optimized = 2;", options); + return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, queued = 1, process_options = ?;", options); } IUgc::ProcessTotals MySQLDatabase::GetUgcProcessTotals(const bool modular) { @@ -168,7 +168,7 @@ IUgc::ProcessTotals MySQLDatabase::GetUgcProcessTotals(const bool modular) { } uint64_t MySQLDatabase::ResetPropertyUgcModelProcessing(const LWOOBJID propertyId, const std::string_view options) { - return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, priority = 1, process_options = ? " + return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, priority = 1, queued = 1, process_options = ? " "WHERE id IN (SELECT ugc_id FROM properties_contents WHERE property_id = ? AND ugc_id IS NOT NULL);", options, propertyId); } @@ -258,3 +258,12 @@ std::optional MySQLDatabase::GetUgcFileChecksum(const LWOOBJ void MySQLDatabase::SetUgcModelProcessStats(const LWOOBJID id, const ProcessStats& stats) { ExecuteUpdate("UPDATE ugc SET process_ms = ?, process_cpu_ms = ?, process_memory_kb = ? WHERE id = ?;", stats.milliseconds, stats.cpuMilliseconds, stats.memoryKb, id); } + +uint64_t MySQLDatabase::QueuePropertyUgcModels(const LWOOBJID propertyId) { + return ExecuteUpdate("UPDATE ugc SET queued = 1 WHERE queued = 0 AND id IN (SELECT ugc_id FROM properties_contents WHERE property_id = ? AND ugc_id IS NOT NULL);", propertyId); +} + +uint64_t MySQLDatabase::CountUnqueuedUgcModels() { + auto result = ExecuteSelect("SELECT COUNT(*) AS count FROM ugc WHERE is_optimized = 0 AND queued = 0;"); + return result->next() ? static_cast(result->getUInt64("count")) : 0; +} diff --git a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h index 9ae481132..e010a2994 100644 --- a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h +++ b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h @@ -446,7 +446,9 @@ public: nlohmann::json GetCharacterById(const LWOOBJID charId) override; std::optional GetModel(const LWOOBJID modelID) override; std::optional GetUgcModel(const LWOOBJID ugcId) override; - std::vector GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) override; + std::vector GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly, const bool queuedOnly) override; + uint64_t QueuePropertyUgcModels(const LWOOBJID propertyId) override; + uint64_t CountUnqueuedUgcModels() override; void ExpediteUgcModel(const LWOOBJID id) override; void ExpediteUgcModels(const LWOOBJID characterId) override; std::vector> GetModularBuildConfigCounts() override; diff --git a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp index 2a55b49c6..fde02c6d6 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp @@ -75,7 +75,7 @@ void SQLiteDatabase::InsertNewUgcModel( const int64_t processAfter) { const std::istream stream(sd0Data.rdbuf()); ExecuteInsert( - "INSERT INTO `ugc`(`id`, `account_id`, `character_id`, `is_optimized`, `lxfml`, `bake_ao`, `filename`, `process_after`) VALUES (?,?,?,?,?,?,?,?)", + "INSERT INTO `ugc`(`id`, `account_id`, `character_id`, `is_optimized`, `lxfml`, `bake_ao`, `filename`, `process_after`, `queued`) VALUES (?,?,?,?,?,?,?,?,1)", blueprintId, accountId, characterId, @@ -97,7 +97,7 @@ void SQLiteDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { void SQLiteDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) { const std::istream stream(lxfml.rdbuf()); // The UGC server makes the model's files again - ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '', process_after = ? WHERE id = ?;", &stream, processAfter, modelId); + ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '', process_after = ?, queued = 1 WHERE id = ?;", &stream, processAfter, modelId); } std::optional SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) { @@ -112,9 +112,9 @@ std::optional SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) { return toReturn; } -std::vector SQLiteDatabase::GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) { - auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts, priority, process_options FROM ugc WHERE is_optimized = 0 AND process_after <= ? AND (? = 0 OR priority = 1) " - "ORDER BY priority DESC, process_attempts ASC, id DESC LIMIT ?;", UnixNow(), priorityOnly ? 1 : 0, limit); +std::vector SQLiteDatabase::GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly, const bool queuedOnly) { + auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts, priority, process_options FROM ugc WHERE is_optimized = 0 AND process_after <= ? AND (? = 0 OR priority = 1) AND (? = 0 OR queued = 1) " + "ORDER BY priority DESC, process_attempts ASC, id DESC LIMIT ?;", UnixNow(), priorityOnly ? 1 : 0, queuedOnly ? 1 : 0, limit); std::vector models; while (!result.eof()) { auto& model = models.emplace_back(); @@ -145,9 +145,9 @@ std::optional SQLiteDatabase::GetUgcProcessInfo(const LWOOBJI } uint64_t SQLiteDatabase::ResetUgcModelProcessing(const std::optional id, const bool failedOnly, const std::string_view options) { - if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, process_options = ? WHERE id = ?;", options, *id); - if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, process_options = ? WHERE is_optimized = 2;", options); - return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, process_options = ?;", options); + if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, queued = 1, process_options = ? WHERE id = ?;", options, *id); + if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, queued = 1, process_options = ? WHERE is_optimized = 2;", options); + return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, queued = 1, process_options = ?;", options); } IUgc::ProcessTotals SQLiteDatabase::GetUgcProcessTotals(const bool modular) { @@ -169,7 +169,7 @@ IUgc::ProcessTotals SQLiteDatabase::GetUgcProcessTotals(const bool modular) { } uint64_t SQLiteDatabase::ResetPropertyUgcModelProcessing(const LWOOBJID propertyId, const std::string_view options) { - return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, priority = 1, process_options = ? " + return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0, priority = 1, queued = 1, process_options = ? " "WHERE id IN (SELECT ugc_id FROM properties_contents WHERE property_id = ? AND ugc_id IS NOT NULL);", options, propertyId); } @@ -263,3 +263,12 @@ std::optional SQLiteDatabase::GetUgcFileChecksum(const LWOOB void SQLiteDatabase::SetUgcModelProcessStats(const LWOOBJID id, const ProcessStats& stats) { ExecuteUpdate("UPDATE ugc SET process_ms = ?, process_cpu_ms = ?, process_memory_kb = ? WHERE id = ?;", stats.milliseconds, stats.cpuMilliseconds, stats.memoryKb, id); } + +uint64_t SQLiteDatabase::QueuePropertyUgcModels(const LWOOBJID propertyId) { + return ExecuteUpdate("UPDATE ugc SET queued = 1 WHERE queued = 0 AND id IN (SELECT ugc_id FROM properties_contents WHERE property_id = ? AND ugc_id IS NOT NULL);", propertyId); +} + +uint64_t SQLiteDatabase::CountUnqueuedUgcModels() { + auto [_, result] = ExecuteSelect("SELECT COUNT(*) AS count FROM ugc WHERE is_optimized = 0 AND queued = 0;"); + return result.eof() ? 0 : static_cast(result.getInt64Field("count", 0)); +} diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h index d0a595247..2a97cbeca 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h @@ -441,7 +441,9 @@ class TestSQLDatabase : public GameDatabase { std::optional GetModel(const LWOOBJID modelID) override { return {}; } std::optional GetPropertyInfo(const LWOOBJID id) override { return {}; } std::optional GetUgcModel(const LWOOBJID ugcId) override { return {}; } - std::vector GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly) override { return {}; } + std::vector GetUgcModelsToProcess(const uint32_t limit, const bool priorityOnly, const bool queuedOnly) override { return {}; } + uint64_t QueuePropertyUgcModels(const LWOOBJID propertyId) override { return 0; } + uint64_t CountUnqueuedUgcModels() override { return 0; } void ExpediteUgcModel(const LWOOBJID id) override {} void ExpediteUgcModels(const LWOOBJID characterId) override {} std::vector> GetModularBuildConfigCounts() override { return {}; } diff --git a/migrations/dlu/mysql/104_ugc_queued.sql b/migrations/dlu/mysql/104_ugc_queued.sql new file mode 100644 index 000000000..f716895c7 --- /dev/null +++ b/migrations/dlu/mysql/104_ugc_queued.sql @@ -0,0 +1,8 @@ +/* ugc.queued: whether a model may be made while the UGC setting queue_on_property_save is on (docs/UgcServer.md, + "Queue on property save"). Set when a model is saved from Brick By Brick, made again by staff, or its property is + saved. Models from before stay 0, so turning the setting on doesn't queue a whole server's backlog at once. */ +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN queued TINYINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'queued'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; diff --git a/migrations/dlu/sqlite/87_ugc_queued.sql b/migrations/dlu/sqlite/87_ugc_queued.sql new file mode 100644 index 000000000..55aba2c5a --- /dev/null +++ b/migrations/dlu/sqlite/87_ugc_queued.sql @@ -0,0 +1,2 @@ +/* ugc.queued: see the MySQL migration. */ +ALTER TABLE ugc ADD COLUMN queued INTEGER NOT NULL DEFAULT 0; diff --git a/tests/dDatabaseTests/DatabaseParityTests.cpp b/tests/dDatabaseTests/DatabaseParityTests.cpp index d5e5564f5..57eeec1d0 100644 --- a/tests/dDatabaseTests/DatabaseParityTests.cpp +++ b/tests/dDatabaseTests/DatabaseParityTests.cpp @@ -760,6 +760,17 @@ TEST_F(ParitySeeded, UgcModel) { db.DeleteUgcModelData(1152921510000500004LL); return out; }); + Both("Queue on property save", [&](GameDatabase& db) { + // The seeded model is from before queued: only taken by queue_on_property_save once its property is saved + json out = json::array(); + out.push_back(db.CountUnqueuedUgcModels()); + out.push_back(db.GetUgcModelsToProcess(10, false, true).size()); + out.push_back(db.QueuePropertyUgcModels(PROP1)); + out.push_back(db.GetUgcModelsToProcess(10, false, true).size()); + out.push_back(db.CountUnqueuedUgcModels()); + out.push_back(db.QueuePropertyUgcModels(PROP1)); + return out; + }); Both("SetUgcModelProcessed failed", [&](GameDatabase& db) { db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::FAILED, 3, "no bricks", false); return json{ infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), countsJson(db.GetUgcProcessCounts()), db.GetUgcModelsToProcess(10).size() };