feat(ugc): icon light matched to the game, debounce, shared car icons, purge, icon editor

Brighter icons: a world light, a fill from the camera, a highlight, exposure and contrast; with the defaults the icons' mean luminance matches the game's own model icons (118 against 120 on a scratch set). Every icon parameter is listed once (UgcIconParams: key, setting, range, default); the settings, the dashboard's settings entries and the icon editor come from that list. Presets per kind (player models, each car or rocket build type from ModularBuildComponent) and overrides per model or module combination are in ugc_icon_settings. Saved models wait ugc_debounce_seconds (sharedconfig) after the owner's last save before they're made (ugc.process_after); a client asking for one, the owner leaving the world or a reset ends the wait. Cars and rockets: one icon per combination of modules (sorted LOTs), shared by every build of it; builds of a combination made already are marked made right away, the client's per-blueprint downloads serve the shared files. The dashboard can delete one item's files, purge by filter or all, preview icons with any values on the UGC server (/admin routes, master password), save presets and overrides, and draw a kind's icons again (icons only). Migrations dlu/mysql/86 and dlu/sqlite/69. Not done yet: the dashboard editor's lighting controls in the page script (routes are there), docs for it, the empty-model state, /ugc?item= links, the shared fetch helper; the storage size and property loading bugs are next. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 17:14:07 +00:00 · 2026-09-27 13:51:04 -05:00
parent 0c1253cd16
commit 9d2ad053fd
42 changed files with 1576 additions and 136 deletions
--- a/dUgcServer/UgcProcessor.cpp
+++ b/dUgcServer/UgcProcessor.cpp
@@ -3,11 +3,15 @@
 #include "CDClientDatabase.h"
 #include "Database.h"
 #include "Logger.h"
+#include "UgcBricks.h"
 #include "UgcCdClient.h"
+#include "UgcKeys.h"
 #include "UgcThrottle.h"
 #include "json.hpp"

+#include <algorithm>
 #include <ctime>
+#include <filesystem>
 #include <fstream>
 #include <thread>
 #if defined(__linux__)
@@ -148,12 +152,63 @@ void UgcProcessor::Worker() {
 		}

 		UgcThrottle::Begin();
+		if (job.preview) {
+			// A preview: the icon goes back to whoever asked, nothing is stored or recorded
+			HTTPReply reply;
+			try {
+				UgcJobs::Outcome outcome{ false, "cancelled" };
+				if (!job.preview.Cancelled() && job.kind == Kind::MODULAR) {
+					outcome = UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings);
+				} else if (!job.preview.Cancelled()) {
+					// A player model's icon from its stored .nif
+					const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif");
+					const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt;
+					auto options = settings.icon;
+					UgcIconParams::Apply(options, job.iconValues);
+					outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, outcome.files, outcome.error);
+					if (!nif) outcome.error = "the model has no stored .nif yet";
+				}
+				if (outcome.ok && outcome.files.contains("icon.png")) {
+					reply.status = eHTTPStatusCode::OK;
+					reply.contentType = eContentType::IMAGE_PNG;
+					reply.message = std::move(outcome.files["icon.png"]);
+					reply.headers.push_back("Cache-Control: no-store");
+				} else {
+					reply.status = eHTTPStatusCode::UNPROCESSABLE_ENTITY;
+					reply.contentType = eContentType::TEXT_PLAIN;
+					reply.message = outcome.error;
+				}
+			} catch (const std::exception& ex) {
+				reply.status = eHTTPStatusCode::INTERNAL_SERVER_ERROR;
+				reply.contentType = eContentType::TEXT_PLAIN;
+				reply.message = ex.what();
+			}
+			job.preview.Send(std::move(reply));
+			{
+				std::lock_guard lock(m_Mutex);
+				m_Active--;
+				m_MemoryInUse -= std::min(m_MemoryInUse, job.memory);
+			}
+			m_Wake.notify_all();
+			continue;
+		}
 		const auto start = std::chrono::steady_clock::now();
 		Done done{ job.kind, job.id, job.attempts };
+		done.iconOnly = job.iconOnly;
 		try {
-			done.outcome = job.kind == Kind::MODEL
-				? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast<uint64_t>(job.id))
-				: UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings);
+			if (job.iconOnly) {
+				// Only the icon, from the .nif made before
+				const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif");
+				const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt;
+				auto options = settings.icon;
+				UgcIconParams::Apply(options, job.iconValues);
+				done.outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, done.outcome.files, done.outcome.error);
+				if (!nif) done.outcome.error = "no stored .nif";
+			} else {
+				done.outcome = job.kind == Kind::MODEL
+					? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast<uint64_t>(job.id), job.iconValues)
+					: UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings);
+			}
 		} catch (const std::exception& ex) {
 			done.outcome.ok = false;
 			done.outcome.error = std::string("crashed: ") + ex.what();
@@ -162,7 +217,7 @@ void UgcProcessor::Worker() {
 		job.blob.shrink_to_fit();
 		if (done.outcome.ok) {
 			std::string error;
-			const auto bytes = m_Storage.Write(job.kind, job.id, done.outcome.files, error);
+			const auto bytes = job.iconOnly ? m_Storage.Update(job.kind, job.id, done.outcome.files, error) : m_Storage.Write(job.kind, job.id, done.outcome.files, error);
 			if (bytes) {
 				done.bytes = *bytes;
 			} else {
@@ -219,26 +274,51 @@ void UgcProcessor::Poll() {
 		Job job{ Kind::MODEL, model.id, model.attempts, UgcJobs::LxfmlFromBlob(model.lxfml) };
 		if (job.blob.empty()) job.blob = std::move(model.lxfml); // the worker reports it can't be read
 		job.parts = UgcJobs::CountParts(job.blob);
+		job.iconValues = IconValues(UgcIconParams::ModelKind(), UgcIconParams::ModelTarget(model.id));
 		job.memory = UgcJobs::EstimateMemory(job.parts, settings);
 		jobs.push_back(std::move(job));
 	}
+	// Cars and rockets: one icon per combination of modules, shared by every build of it
 	for (auto& build : Database::Get()->GetModularBuildsToProcess(limit)) {
 		if (m_InFlight.contains({ Kind::MODULAR, build.id })) continue;
-		Job job{ Kind::MODULAR, build.id, build.attempts };
+		const auto key = UgcModularKey::Normalize(build.modules);
+		if (key.empty()) {
+			Record(Done{ Kind::MODULAR, build.id, build.attempts, UgcJobs::Outcome{ false, "no modules in \"" + build.modules + "\"" } });
+			continue;
+		}
+		const auto combo = UgcModularKey::StorageId(key);
+		m_ComboOf[build.id] = combo;
+		// Made already for another build of the same modules
+		if (!m_ComboJobs.contains(combo) && m_Storage.File(Kind::MODULAR, combo, "icon.png")) {
+			UgcJobs::Outcome reused{ true };
+			reused.note = "the same modules as a build made before (" + key + ")";
+			m_Reused++;
+			Record(Done{ Kind::MODULAR, build.id, build.attempts, std::move(reused) });
+			continue;
+		}
+		m_InFlight.insert({ Kind::MODULAR, build.id });
+		m_ComboRows[combo].emplace_back(build.id, build.attempts);
+		if (m_ComboJobs.contains(combo)) continue;
+		Job job{ Kind::MODULAR, combo, build.attempts };
 		job.memory = UgcJobs::EstimateMemory(64, settings);
 		std::string error;
 		if (!UgcCdClient::GatherModular(build.modules, job.modular, error)) {
 			// Nothing a worker could do: record it right away
+			m_InFlight.erase({ Kind::MODULAR, build.id });
+			m_ComboRows.erase(combo);
 			Record(Done{ Kind::MODULAR, build.id, build.attempts, UgcJobs::Outcome{ false, error } });
 			continue;
 		}
+		job.modular.key = key;
+		job.modular.iconValues = IconValues(UgcIconParams::BuildKind(job.modular.buildType), UgcIconParams::CombinationTarget(key));
+		m_ComboJobs.insert(combo);
 		jobs.push_back(std::move(job));
 	}
 	if (jobs.empty()) return;
 	{
 		std::lock_guard lock(m_Mutex);
 		for (auto& job : jobs) {
-			m_InFlight.insert({ job.kind, job.id });
+			if (job.kind == Kind::MODEL) m_InFlight.insert({ job.kind, job.id });
 			m_Jobs.push_back(std::move(job));
 		}
 	}
@@ -277,8 +357,33 @@ void UgcProcessor::Collect() {
 		finished.swap(m_Done);
 	}
 	for (const auto& done : finished) {
-		m_InFlight.erase({ done.kind, done.id });
-		Record(done);
+		if (done.kind == Kind::MODEL) {
+			m_InFlight.erase({ done.kind, done.id });
+			if (!done.iconOnly) {
+				Record(done);
+				continue;
+			}
+			m_Log.push_back({ Kind::MODEL, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? "icon drawn again" : done.outcome.error, UnixNow() });
+			while (m_Log.size() > LOG_LENGTH) m_Log.pop_front();
+			continue;
+		}
+		// A combination: every build waiting for it gets its outcome (none when only its icon was drawn again)
+		m_ComboJobs.erase(done.id);
+		if (done.outcome.ok) m_StoredBytes += done.bytes;
+		auto rows = std::move(m_ComboRows[done.id]);
+		m_ComboRows.erase(done.id);
+		for (const auto& [row, attempts] : rows) {
+			m_InFlight.erase({ Kind::MODULAR, row });
+			Done forRow = done;
+			forRow.id = row;
+			forRow.attempts = attempts;
+			forRow.bytes = 0;
+			Record(forRow);
+		}
+		if (rows.empty()) {
+			m_Log.push_back({ Kind::MODULAR, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? "icon drawn again" : done.outcome.error, UnixNow() });
+			while (m_Log.size() > LOG_LENGTH) m_Log.pop_front();
+		}
 	}
 	// Something finished: look for more now rather than at the next interval
 	if (!finished.empty()) m_NextPoll = std::min(m_NextPoll, std::chrono::steady_clock::now() + std::chrono::milliseconds(100));
@@ -307,10 +412,22 @@ void UgcProcessor::Update() {
 	std::erase_if(m_Recent, [now](const auto& item) { return now - item.second.first > RECENT_ANSWER_TIME; });
 }

+LWOOBJID UgcProcessor::StorageId(Kind kind, LWOOBJID id) {
+	if (kind == Kind::MODEL) return id;
+	if (const auto it = m_ComboOf.find(id); it != m_ComboOf.end()) return it->second;
+	const auto info = Database::Get()->GetModularBuildProcessInfo(id);
+	const auto key = info ? UgcModularKey::Normalize(info->details) : std::string();
+	const LWOOBJID combo = key.empty() ? 0 : UgcModularKey::StorageId(key);
+	if (combo != 0) m_ComboOf[id] = combo;
+	return combo;
+}
+
 UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) {
 	if (m_InFlight.contains({ kind, id })) return Availability::QUEUED;
-	if (m_Storage.File(kind, id, "icon.png")) {
-		m_Storage.Touch(kind, id);
+	const auto storageId = StorageId(kind, id);
+	if (kind == Kind::MODULAR && m_ComboJobs.contains(storageId)) return Availability::QUEUED;
+	if (storageId != 0 && m_Storage.File(kind, storageId, "icon.png")) {
+		m_Storage.Touch(kind, storageId);
 		return Availability::READY;
 	}
 	if (const auto it = m_Recent.find({ kind, id }); it != m_Recent.end()) return it->second.second;
@@ -323,6 +440,8 @@ UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) {
 			if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false);
 			else Database::Get()->ResetModularBuildProcessing(id, false);
 		}
+		// Someone wants it now: no need to wait out the quiet period after its save
+		if (kind == Kind::MODEL) Database::Get()->ExpediteUgcModel(id);
 		m_NextPoll = std::chrono::steady_clock::now();
 		answer = Availability::QUEUED;
 	}
@@ -345,6 +464,7 @@ nlohmann::json UgcProcessor::Status() const {
 		status["memoryWaits"] = m_Waiting;
 	}
 	status["workers"] = m_Threads.size();
+	status["reused"] = m_Reused;
 	const auto throttle = UgcThrottle::GetStats();
 	status["usage"] = {
 		{ "cpuPercent", std::lround(m_CpuPercent) }, // of one core
@@ -372,3 +492,161 @@ nlohmann::json UgcProcessor::Status() const {
 	}
 	return status;
 }
+
+UgcIconParams::Values UgcProcessor::IconValues(const std::string& kind, const std::string& itemTarget) {
+	UgcIconParams::Values values;
+	if (const auto preset = Database::Get()->GetUgcIconSettings(UgcIconParams::KindTarget(kind))) values = UgcIconParams::Parse(*preset);
+	if (const auto own = Database::Get()->GetUgcIconSettings(itemTarget)) {
+		for (const auto& [key, value] : UgcIconParams::Parse(*own)) values[key] = value;
+	}
+	return values;
+}
+
+bool UgcProcessor::QueuePreview(Kind kind, LWOOBJID id, const std::string& modules, const UgcIconParams::Values& values, DeferredReply reply, std::string& error) {
+	Job job{ kind, id, 0 };
+	if (kind == Kind::MODULAR) {
+		if (!UgcCdClient::GatherModular(modules, job.modular, error)) return false;
+		job.modular.key = UgcModularKey::Normalize(modules);
+		job.modular.iconValues = values;
+	} else {
+		if (!m_Storage.File(Kind::MODEL, id, "model.nif")) {
+			error = "the model has no stored .nif yet";
+			return false;
+		}
+		job.iconValues = values;
+	}
+	job.preview = std::move(reply);
+	{
+		std::lock_guard lock(m_Mutex);
+		job.memory = UgcJobs::EstimateMemory(64, m_Settings);
+		m_Jobs.push_front(std::move(job));
+	}
+	m_Wake.notify_all();
+	return true;
+}
+
+size_t UgcProcessor::RegenerateIcons(const std::string& kind) {
+	std::vector<Job> jobs;
+	UgcJobs::Settings settings;
+	{
+		std::lock_guard lock(m_Mutex);
+		settings = m_Settings;
+	}
+	const bool models = kind == UgcIconParams::ModelKind();
+	for (const auto& entry : m_Storage.List()) {
+		if (models) {
+			if (entry.kind != Kind::MODEL || m_InFlight.contains({ Kind::MODEL, entry.id }) || !m_Storage.File(Kind::MODEL, entry.id, "model.nif")) continue;
+			Job job{ Kind::MODEL, entry.id, 0 };
+			job.iconOnly = true;
+			job.iconValues = IconValues(kind, UgcIconParams::ModelTarget(entry.id));
+			job.memory = UgcJobs::EstimateMemory(64, settings);
+			m_InFlight.insert({ Kind::MODEL, entry.id });
+			jobs.push_back(std::move(job));
+			continue;
+		}
+		if (entry.kind != Kind::MODULAR || m_ComboJobs.contains(entry.id)) continue;
+		const auto path = m_Storage.File(Kind::MODULAR, entry.id, "combo.json");
+		const auto text = path ? UgcBricks::ReadFile(*path) : std::nullopt;
+		const auto combo = text ? nlohmann::json::parse(*text, nullptr, false) : nlohmann::json();
+		if (!combo.is_object() || UgcIconParams::BuildKind(combo.value("buildType", -1)) != kind) continue;
+		const auto key = combo.value("key", std::string());
+		Job job{ Kind::MODULAR, entry.id, 0 };
+		std::string error;
+		// The key's LOTs written as an ldf_config ("4713-4714" -> "4713+4714")
+		std::string modules = key;
+		std::replace(modules.begin(), modules.end(), '-', '+');
+		if (key.empty() || !UgcCdClient::GatherModular(modules, job.modular, error)) continue;
+		job.modular.key = key;
+		job.modular.iconValues = IconValues(kind, UgcIconParams::CombinationTarget(key));
+		job.memory = UgcJobs::EstimateMemory(64, settings);
+		m_ComboJobs.insert(entry.id);
+		jobs.push_back(std::move(job));
+	}
+	const auto count = jobs.size();
+	{
+		std::lock_guard lock(m_Mutex);
+		for (auto& job : jobs) m_Jobs.push_back(std::move(job));
+	}
+	m_Wake.notify_all();
+	return count;
+}
+
+UgcProcessor::DeleteResult UgcProcessor::Delete(const DeleteRequest& request) {
+	DeleteResult result;
+	const auto now = std::filesystem::file_time_type::clock::now();
+	const auto days = [](int64_t count) { return std::chrono::duration_cast<std::filesystem::file_time_type::duration>(std::chrono::hours(24 * count)); };
+
+	// What to delete: the files' ids, and the rows they came from (when they're known)
+	std::map<LWOOBJID, std::vector<LWOOBJID>> targets; // storage id -> rows
+	if (request.all) {
+		for (const auto& entry : m_Storage.List()) {
+			if (entry.kind == request.kind) targets[entry.id];
+		}
+	}
+	for (const auto id : request.ids) {
+		const auto storageId = StorageId(request.kind, id);
+		if (storageId != 0) targets[storageId].push_back(id);
+	}
+
+	std::vector<LWOOBJID> deletedRows;
+	for (const auto& [storageId, rows] : targets) {
+		const bool busy = request.kind == Kind::MODEL ? m_InFlight.contains({ Kind::MODEL, storageId }) : m_ComboJobs.contains(storageId);
+		if (busy) {
+			result.busy++;
+			continue;
+		}
+		const auto folder = m_Storage.Folder(request.kind, storageId);
+		std::error_code error;
+		if (!std::filesystem::exists(folder, error)) {
+			deletedRows.insert(deletedRows.end(), rows.begin(), rows.end());
+			continue;
+		}
+		if (request.unusedDays > 0) {
+			const auto used = std::filesystem::last_write_time(folder, error);
+			if (!error && now - used < days(request.unusedDays)) continue;
+		}
+		if (request.olderThanDays > 0) {
+			const auto made = std::filesystem::last_write_time(folder / "icon.png", error);
+			if (!error && now - made < days(request.olderThanDays)) continue;
+		}
+		uint64_t bytes = 0;
+		for (const auto& file : std::filesystem::directory_iterator(folder, error)) bytes += file.is_regular_file(error) ? file.file_size(error) : 0;
+		m_Storage.Remove(request.kind, storageId);
+		result.deleted++;
+		result.bytes += bytes;
+		m_StoredBytes -= std::min(m_StoredBytes, bytes);
+		m_Recent.clear();
+		deletedRows.insert(deletedRows.end(), rows.begin(), rows.end());
+		if (request.kind == Kind::MODEL) deletedRows.push_back(storageId);
+	}
+	std::sort(deletedRows.begin(), deletedRows.end());
+	deletedRows.erase(std::unique(deletedRows.begin(), deletedRows.end()), deletedRows.end());
+
+	// What happens to the rows
+	if (request.after == eAfterDelete::NOW) {
+		if (request.all) {
+			if (request.kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(std::nullopt, false);
+			else Database::Get()->ResetModularBuildProcessing(std::nullopt, false);
+		} else {
+			for (const auto row : deletedRows) {
+				if (request.kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(row, false);
+				else Database::Get()->ResetModularBuildProcessing(row, false);
+			}
+		}
+		m_NextPoll = std::chrono::steady_clock::now();
+		result.notes.push_back("They're queued to be made again now.");
+	} else if (request.after == eAfterDelete::GONE) {
+		for (const auto row : deletedRows) {
+			if (request.kind == Kind::MODEL) Database::Get()->SetUgcModelProcessed(row, IUgc::eProcessState::FAILED, m_Config.maxAttempts, "Deleted from the dashboard", false);
+			else Database::Get()->SetModularBuildProcessed(row, IUgc::eProcessState::FAILED, m_Config.maxAttempts, "Deleted from the dashboard");
+		}
+		result.notes.push_back("They're marked deleted and won't be made again unless made again from the dashboard.");
+	} else {
+		result.notes.push_back("They'll be made again when a game client asks for them.");
+	}
+	if (request.kind == Kind::MODULAR && result.deleted > 0) {
+		result.notes.push_back("Car and rocket icons are shared by every build of the same modules: the other builds' icons are made again when asked for.");
+	}
+	if (result.busy > 0) result.notes.push_back(std::to_string(result.busy) + " being made right now were left alone.");
+	return result;
+}