diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index bba8e24c9..4f5edb3bc 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -473,6 +473,7 @@ namespace { c.Add(Int(UGC, "worker_nice", "Worker priority", "Linux nice value of the workers: 0 normal, 19 only when nothing else wants the CPU.", "0", 0, 19)); c.Add(Unit(Int(UGC, "max_memory_mb", "Most memory for jobs", "Jobs whose estimated memory would go past this together wait for running ones; a bigger one runs alone. 0: no limit.", "0", 0, 1000000), "MB")); c.Add(Unit(Int(UGC, "max_model_bricks", "Biggest model", "Models with more bricks fail with that reason. 0: no limit.", "0", 0, 1000000), "bricks")); + c.Add(Bool(UGC, "queue_on_property_save", "Only make models once their property is saved", "On: models already waiting (say, a server turning the UGC server on) aren't all made at once; a model is made once it is saved from Brick By Brick, made again by staff, or its property is saved. Off: every waiting model is made.", false)); c.Add(Text(UGC, "pause_hours", "Pause hours", "Local hours in which no new jobs start, e.g. 18-23 or 22-6. Empty: never.", "")); c.Add(Unit(Int(UGC, "poll_interval_ms", "Look for new models every", "", "2000", 100, 600000, true), "ms")); c.Add(Int(UGC, "poll_batch", "Models taken at once", "", "32", 1, 1000, true)); diff --git a/dDashboardServer/routes/UgcRoutes.cpp b/dDashboardServer/routes/UgcRoutes.cpp index 7b6b28543..87127c037 100644 --- a/dDashboardServer/routes/UgcRoutes.cpp +++ b/dDashboardServer/routes/UgcRoutes.cpp @@ -264,6 +264,14 @@ namespace { for (const auto& [state, count] : counts) out[StateName(state)] = count; return out; } + + // The models' counts, with how many of the waiting ones wait for their property to be saved while + // queue_on_property_save is on (notQueued) + nlohmann::json ModelCounts() { + auto out = Counts(Database::Get()->GetUgcProcessCounts()); + if (UgcSetting("queue_on_property_save").value_or("0") == "1") out["notQueued"] = Database::Get()->CountUnqueuedUgcModels(); + return out; + } } namespace UgcRoutes { @@ -385,7 +393,7 @@ namespace UgcRoutes { } if (searching) matches = { { "model", models(0, 1).second }, { "modular", total } }; } - JsonSuccess(reply, { { "counts", { { "model", Counts(Database::Get()->GetUgcProcessCounts()) }, { "modular", Counts(Database::Get()->GetModularBuildProcessCounts()) } } }, + JsonSuccess(reply, { { "counts", { { "model", ModelCounts() }, { "modular", Counts(Database::Get()->GetModularBuildProcessCounts()) } } }, { "totals", { { "model", Totals(Database::Get()->GetUgcProcessTotals(false)) }, { "modular", Totals(Database::Get()->GetUgcProcessTotals(true)) } } }, { "items", items }, { "total", total }, { "page", page }, { "size", size }, { "more", static_cast(page + 1) * size < total }, { "kinds", kinds }, { "matches", matches }, { "ugcPublicUrl", Game::config->GetValue("ugc_public_url") }, { "canManage", Can(context, "ugc_manage") } }); @@ -530,7 +538,7 @@ namespace UgcRoutes { const auto base = InternalUrl(); // The database only on this thread; the worker only fetches nlohmann::json body = { { "success", true }, { "enabled", enabled }, { "online", online }, - { "counts", { { "model", Counts(Database::Get()->GetUgcProcessCounts()) }, { "modular", Counts(Database::Get()->GetModularBuildProcessCounts()) } } } }; + { "counts", { { "model", ModelCounts() }, { "modular", Counts(Database::Get()->GetModularBuildProcessCounts()) } } } }; Workers::Reply(reply, context, false, [base, enabled, body = std::move(body)](HTTPReply& out) mutable { if (enabled) { const auto fetched = CachedGet(base + "/status", eCache::STATUS); diff --git a/dDashboardServer/static/js/ugc.js b/dDashboardServer/static/js/ugc.js index 8a76f2a39..753fc583a 100644 --- a/dDashboardServer/static/js/ugc.js +++ b/dDashboardServer/static/js/ugc.js @@ -76,7 +76,9 @@ function countCard(title, c) { return '
' + esc(title) + '
' + fmt.badge(c.done + ' made', 'success') + ' ' + fmt.badge(c.pending + ' waiting', 'secondary') + ' ' + fmt.badge(c.failed + ' failed', c.failed ? 'danger' : 'secondary') + - (c.empty ? ' ' + fmt.badge(c.empty + ' empty', 'light') : '') + '
'; + (c.empty ? ' ' + fmt.badge(c.empty + ' empty', 'light') : '') + + (c.notQueued ? '
' + + esc(Number(c.notQueued).toLocaleString()) + ' of the waiting ones wait for their property to be saved
' : '') + ''; } // What making them all took, and what it saved (the rows' recorded stats; ones made before a stat existed don't count) function totalsCard(title, t) { diff --git a/docs/UgcServer.md b/docs/UgcServer.md index d8c0a265d..3e2317de6 100644 --- a/docs/UgcServer.md +++ b/docs/UgcServer.md @@ -842,10 +842,21 @@ need no change to have new models processed. Rows that failed are tried again up `max_attempts` (default 3) times. Reprocessing (dashboard) sets rows back to `is_optimized = 0, process_attempts = 0`. No master messages are needed for any of it. +### Queue on property save (`queue_on_property_save`) + +Off by default. On, the UGC server only takes pending models with `ugc.queued = 1` (mysql 104 / sqlite 87), so a +server that turns the UGC server on with years of never-made models doesn't queue them all at once. `queued` is set by +`InsertNewUgcModel` and `UpdateUgcModelData` (a Brick By Brick save), by every reset (`ResetUgcModelProcessing`, +`ResetPropertyUgcModelProcessing`: the dashboard's Make again, `/reprocessproperty`), and for every model placed on a +property when the property is saved (`PropertyManagementComponent::Save` calls `QueuePropertyUgcModels`: finishing +building, placing, moving or picking up a model, a hand-off). Rows from before the column are 0. The UGC page counts +the waiting models that aren't queued apart. Turning it off makes every waiting model eligible again. + ### Marking models for processing (for code that writes `ugc` rows) -* A new row needs nothing: `is_optimized` defaults to 0 (`InsertNewUgcModel` writes 0, and its `process_after`). -* When a model's LXFML changes, `UpdateUgcModelData` sets `is_optimized = 0, process_attempts = 0, process_error = ''` +* A new row needs nothing: `is_optimized` defaults to 0 (`InsertNewUgcModel` writes 0, its `process_after`, and + `queued = 1`). +* When a model's LXFML changes, `UpdateUgcModelData` sets `is_optimized = 0, process_attempts = 0, process_error = '', queued = 1` in the same statement. Anything that writes `ugc.lxfml` another way must do the same, or call `ResetUgcModelProcessing(id, false)`. * Deleting the row is enough when a model is deleted; its files are left until the storage cap removes them. diff --git a/resources/ugcconfig.ini b/resources/ugcconfig.ini index 1c40b6a2b..48a5bbcd2 100644 --- a/resources/ugcconfig.ini +++ b/resources/ugcconfig.ini @@ -30,11 +30,15 @@ worker_threads=0 # bigger than it alone runs when nothing else does (0: no limit). # max_model_bricks: models with more bricks fail with that reason instead of being made (0: no limit). # pause_hours: local hours in which no new jobs start, e.g. 18-23 or 22-6 (empty: never). +# queue_on_property_save: 1: only models that were saved from Brick By Brick, made again by staff, or whose property +# was saved are made, so the models already waiting (a server turning the UGC server on) aren't all made at once. +# 0: every waiting model is made. max_cpu_percent=0 worker_nice=0 max_memory_mb=0 max_model_bricks=0 pause_hours= +queue_on_property_save=0 # How often to look for models to make (milliseconds), how many to take at once, and how often a model may fail # before it's given up on (the dashboard can try again)