feat(dashboard): UGC gallery and viewer, reached through the dashboard

The /ugc page no longer needs the browser to reach the UGC server: the dashboard fetches its status (/api/ugc/server/status), the files it made (/api/ugc/files/<kind>/<id>/<file>) and its NIFs converted for the 3D view (/api/ugc/mesh/<id>, NifFile like the scenery) from ugc_internal_url (default http://127.0.0.1:2008) with libcurl on the worker threads, keeping small answers briefly. ugc_public_url is only an "open on the UGC server" link now. The page gets an icon gallery beside the list, filtered by kind, state and a search by id or owner (GetUgcProcessList/GetModularBuildProcessList take a search), and a viewer: the generated NIF in 3D at any LOD, now or before it was made again, with wireframe, vertex color and baked lighting switches, the LXFML beside it, the icon now and before, and the stats with triangles before and after hidden faces were removed. The status box shows CPU, memory, the jobs' memory estimate with their limits, and throttling. The settings page lists the UGC server's new settings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 07:57:14 +00:00 · 2026-09-27 09:38:51 -05:00
parent 175db95c98
commit 95158f69b6
16 changed files with 671 additions and 141 deletions
--- a/dDashboardServer/routes/SettingsCatalog.cpp
+++ b/dDashboardServer/routes/SettingsCatalog.cpp
@@ -296,7 +296,8 @@ namespace {
 		c.Add(Port(DASHBOARD, "port", "Web port", "Where the dashboard listens for browsers.", "2006"));
 		c.Add(Format(Text(DASHBOARD, "listen_ip", "Listen address", "127.0.0.1 for this machine only (put a reverse proxy in front); 0.0.0.0 for everyone.", "127.0.0.1", true), eFormat::HOST));
 		c.Add(Port(DASHBOARD, "net_port", "Server link port", "UDP port for the connection to master (the next one is used too). Keep clear of other servers' ports.", "2010"));
-		c.Add(Format(Text(DASHBOARD, "ugc_public_url", "UGC server address", "Where the browser loads previews from the UGC server, e.g. https://ugc.example.com. Empty: this host, port 2008.", ""), eFormat::URL));
+		c.Add(Format(Text(DASHBOARD, "ugc_internal_url", "UGC server address (internal)", "Where the dashboard itself reaches the UGC server for its status and files. Empty: http://127.0.0.1:2008.", ""), eFormat::URL));
+		c.Add(Format(Text(DASHBOARD, "ugc_public_url", "UGC server address (public)", "Only for the \"open on the UGC server\" links, e.g. https://ugc.example.com. The dashboard's own pages don't need the browser to reach it.", ""), eFormat::URL));
 		c.Add(Format(Text(DASHBOARD, "dashboard_url", "Public address", "For links in emails and alerts, e.g. https://dashboard.example.com.", ""), eFormat::URL));
 		c.Add(Bool(DASHBOARD, "secure_cookies", "HTTPS only cookies", "Turn on when the dashboard is served over HTTPS.", false));
 		c.Add(Bool(DASHBOARD, "behind_proxy", "Behind a reverse proxy", "Use the proxy's X-Forwarded-For for rate limits. Only when the dashboard can't be reached directly.", false));
@@ -415,20 +416,48 @@ namespace {
 		c.Add(Format(Text(UGC, "ugc_output_dir", "Files folder", "Relative to the server binaries.", "ugc", true), eFormat::PATH));
 		c.Add(Unit(Int(UGC, "ugc_max_storage_mb", "Most space for files", "The files used longest ago are deleted past this and made again when asked for. 0: no limit.", "2048", 0, std::nullopt, true), "MB"));
 		c.Add(Int(UGC, "worker_threads", "Worker threads", "0: half the CPU cores.", "0", 0, 64, true));
+		c.Add(Unit(Float(UGC, "max_cpu_percent", "Most CPU", "The workers together average at most this share of all CPU cores; they pause between bits of work to stay under it. 0: no limit.", "0", 0, 100), "%"));
+		c.Add(Int(UGC, "worker_nice", "Worker priority", "Linux nice value of the workers: 0 normal, 19 only when nothing else wants the CPU.", "0", 0, 19));
+		c.Add(Unit(Int(UGC, "max_memory_mb", "Most memory for jobs", "Jobs whose estimated memory would go past this together wait for running ones; a bigger one runs alone. 0: no limit.", "0", 0, 1000000), "MB"));
+		c.Add(Unit(Int(UGC, "max_model_bricks", "Biggest model", "Models with more bricks fail with that reason. 0: no limit.", "0", 0, 1000000), "bricks"));
+		c.Add(Text(UGC, "pause_hours", "Pause hours", "Local hours in which no new jobs start, e.g. 18-23 or 22-6. Empty: never.", ""));
 		c.Add(Unit(Int(UGC, "poll_interval_ms", "Look for new models every", "", "2000", 100, 600000, true), "ms"));
 		c.Add(Int(UGC, "poll_batch", "Models taken at once", "", "32", 1, 1000, true));
 		c.Add(Int(UGC, "max_attempts", "Attempts before giving up", "", "3", 1, 100, true));
-		c.Add(Int(UGC, "brick_lod", "Brick detail", "0 is the most detailed, 2 the least.", "0", 0, 2, true));
-		c.Add(Bool(UGC, "remove_hidden_faces", "Remove faces nobody can see", "", true, true));
-		c.Add(Bool(UGC, "bake_ao", "Darken hidden corners", "Ambient occlusion baked into the vertex colors.", true, true));
-		c.Add(Float(UGC, "ao_strength", "Darkening strength", "0 to 1.", "0.6", 0, 1));
-		c.Add(Unit(Int(UGC, "optimize_resolution", "Detail of the visibility renders", "", "1024", 64, 4096, true), "pixels"));
-		c.Add(Unit(Int(UGC, "icon_size", "Icon size", "", "128", 16, 1024, true), "pixels"));
-		c.Add(Unit(Float(UGC, "icon_yaw", "Icon angle around", "", "35", -360, 360), "degrees"));
-		c.Add(Unit(Float(UGC, "icon_pitch", "Icon angle above", "", "25", -90, 90), "degrees"));
+		c.Add(Labels(Choice(UGC, "color_palette", "Colors", "LU Toolbox's LU palette (unknown colors black), or the client's Materials.xml.", "lu_toolbox", { "lu_toolbox", "brickdb" }), { "LU Toolbox", "Brick database" }));
+		c.Add(Unit(Float(UGC, "color_variation", "Color variation", "Each brick's brightness is shifted by up to this much (the same every time the model is made). 0: none.", "5", 0, 100), "%"));
+		c.Add(Unit(Float(UGC, "transparent_opacity", "Transparent opacity", "", "58.82", 0, 100), "%"));
+		c.Add(Text(UGC, "lods", "Levels of detail", "brickprimitives levels made, 0 (most detailed) to 2, e.g. 0,2.", "0,2"));
+		c.Add(Float(UGC, "lod_distance_0", "LOD 0 distance", "", "0", 0, 100000));
+		c.Add(Float(UGC, "lod_distance_1", "LOD 1 distance", "", "50", 0, 100000));
+		c.Add(Float(UGC, "lod_distance_2", "LOD 2 distance", "", "100", 0, 100000));
+		c.Add(Float(UGC, "lod_distance_3", "LOD 3 distance", "", "280", 0, 100000));
+		c.Add(Float(UGC, "lod_cull", "Drawn up to", "", "10000", 0, 1000000));
+		c.Add(Text(UGC, "shader_opaque", "Opaque shader", "S<shader>_Opaque_Model.", "01"));
+		c.Add(Bool(UGC, "combine_transparent", "One shape for all transparent bricks", "Off: each transparent brick is its own shape, so the client can sort them.", false));
+		c.Add(Bool(UGC, "remove_hidden_faces", "Remove faces nobody can see", "", true));
+		c.Add(Bool(UGC, "hsr_ground_plane", "Nothing seen from below", "Also removes what can only be seen from under the model.", false));
+		c.Add(Unit(Int(UGC, "optimize_resolution", "Detail of the visibility renders", "", "1024", 64, 4096), "pixels"));
+		c.Add(Bool(UGC, "bake_ao", "Darken hidden corners", "Ambient occlusion baked into the vertex colors.", true));
+		c.Add(Int(UGC, "ao_samples", "Occlusion rays per vertex", "", "64", 1, 1024));
+		c.Add(Float(UGC, "ao_distance", "Occlusion distance", "", "5", 0, 1000));
+		c.Add(Float(UGC, "ao_strength", "Darkening strength", "0 to 1.", "1", 0, 1));
+		c.Add(Float(UGC, "glow_strength", "Glow strength", "What glowing colors add to the baked light.", "6", 0, 100));
+		c.Add(Unit(Int(UGC, "icon_size", "Icon size", "", "128", 16, 1024), "pixels"));
+		c.Add(Unit(Float(UGC, "icon_yaw", "Icon angle around", "", "53.36", -360, 360), "degrees"));
+		c.Add(Unit(Float(UGC, "icon_pitch", "Icon angle above", "", "19.54", -90, 90), "degrees"));
+		c.Add(Unit(Float(UGC, "icon_fov", "Icon field of view", "", "39.6", 1, 120), "degrees"));
 		c.Add(Float(UGC, "icon_margin", "Icon border", "1 fills the icon.", "1.03", 0.1, 10));
-		c.Add(Unit(Float(UGC, "modular_icon_yaw", "Car and rocket icon angle around", "", "35", -360, 360), "degrees"));
-		c.Add(Unit(Float(UGC, "modular_icon_pitch", "Car and rocket icon angle above", "", "20", -90, 90), "degrees"));
+		c.Add(Unit(Float(UGC, "icon_sun_yaw", "Icon sun angle around", "", "21", -360, 360), "degrees"));
+		c.Add(Unit(Float(UGC, "icon_sun_pitch", "Icon sun angle above", "", "50.3", -90, 90), "degrees"));
+		c.Add(Float(UGC, "icon_sun_strength", "Icon sun strength", "", "2.5", 0, 100));
+		c.Add(Float(UGC, "icon_ambient", "Icon world light", "", "0.192", 0, 10));
+		c.Add(Bool(UGC, "icon_shadows", "Icon shadows", "", true));
+		c.Add(Bool(UGC, "icon_ao", "Icon ambient occlusion", "", true));
+		c.Add(Bool(UGC, "icon_correct_colors", "Icon color corrections", "White and black toned down, as LU Toolbox's icon renderer does.", true));
+		c.Add(Unit(Float(UGC, "icon_color_variation", "Icon color variation", "", "0", 0, 100), "%"));
+		c.Add(Unit(Float(UGC, "modular_icon_yaw", "Car and rocket icon angle around", "", "53.36", -360, 360), "degrees"));
+		c.Add(Unit(Float(UGC, "modular_icon_pitch", "Car and rocket icon angle above", "", "19.54", -90, 90), "degrees"));

 		return c;
 	}
--- a/dDashboardServer/routes/UgcRoutes.cpp
+++ b/dDashboardServer/routes/UgcRoutes.cpp
@@ -1,6 +1,15 @@
 #include "UgcRoutes.h"

+#include <memory>
+#include <mutex>
+#include <set>
+
+#include <curl/curl.h>
+
 #include "Database.h"
+#include "NifFile.h"
+#include "TtlCache.h"
+#include "Workers.h"
 #include "Game.h"
 #include "GeneralUtils.h"
 #include "RouteUtils.h"
@@ -30,6 +39,91 @@ namespace {
 		return std::nullopt;
 	}

+	/**
+	 * The dashboard's own way to the UGC server (ugc_internal_url, normally the same machine): its status and the files
+	 * it made are fetched here and handed to the browser, which may not be able to reach the UGC server at all. Fetches
+	 * run on worker threads (Workers::Reply) with the URL worked out on the web thread; small answers are kept briefly.
+	 */
+	struct Fetched {
+		long status{};      // HTTP status, 0 when the UGC server didn't answer
+		std::string body;
+		std::string error;
+	};
+
+	std::mutex g_CacheMutex;
+	TtlCache<std::string, std::shared_ptr<const Fetched>> g_Cache(std::chrono::seconds(15), 32 * 1024 * 1024);
+	TtlCache<std::string, std::shared_ptr<const Fetched>> g_StatusCache(std::chrono::seconds(2), 1024 * 1024);
+
+	constexpr size_t MAX_FETCH_BYTES = 128 * 1024 * 1024;
+
+	size_t Collect(char* data, size_t size, size_t count, void* userData) {
+		auto* out = static_cast<std::string*>(userData);
+		if (out->size() + size * count > MAX_FETCH_BYTES) return 0;
+		out->append(data, size * count);
+		return size * count;
+	}
+
+	// Web thread: where the dashboard reaches the UGC server
+	std::string InternalUrl() {
+		auto url = Game::config->GetValue("ugc_internal_url");
+		if (url.empty()) url = "http://127.0.0.1:2008";
+		while (url.ends_with('/')) url.pop_back();
+		return url;
+	}
+
+	// Any thread: a GET to the UGC server
+	std::shared_ptr<const Fetched> Get(const std::string& url) {
+		auto out = std::make_shared<Fetched>();
+		CURL* curl = curl_easy_init();
+		if (!curl) {
+			out->error = "could not start a request";
+			return out;
+		}
+		curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+		curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https");
+		curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L);
+		curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
+		curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3L);
+		curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L);
+		curl_easy_setopt(curl, CURLOPT_USERAGENT, "DarkflameServer-Dashboard");
+		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Collect);
+		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out->body);
+		const auto code = curl_easy_perform(curl);
+		if (code == CURLE_OK) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &out->status);
+		else out->error = curl_easy_strerror(code);
+		curl_easy_cleanup(curl);
+		return out;
+	}
+
+	// Any thread: Get, through the cache (answers that aren't a file being made are kept)
+	std::shared_ptr<const Fetched> CachedGet(const std::string& url, bool status) {
+		{
+			std::lock_guard lock(g_CacheMutex);
+			if (auto hit = (status ? g_StatusCache : g_Cache).Get(url)) return *hit;
+		}
+		auto fetched = Get(url);
+		if (fetched->status == 200 || fetched->status == 404) {
+			std::lock_guard lock(g_CacheMutex);
+			(status ? g_StatusCache : g_Cache).Put(url, fetched, fetched->body.size() + 64);
+		}
+		return fetched;
+	}
+
+	void FetchError(HTTPReply& reply, const Fetched& fetched, const std::string& url) {
+		if (fetched.status == 408) return JsonError(reply, eHTTPStatusCode::REQUEST_TIMEOUT, "The UGC server is making it; try again in a moment");
+		if (fetched.status == 404) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "The UGC server has no such file");
+		if (fetched.status == 0) return JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server doesn't answer at " + url + " (" + fetched.error + ")");
+		JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server answered " + std::to_string(fetched.status));
+	}
+
+	const std::set<std::string> FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json",
+		"previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json" };
+
+	std::optional<std::string> KindOf(const std::string& text) {
+		if (text == "model" || text == "modular") return text;
+		return std::nullopt;
+	}
+
 	nlohmann::json Counts(const std::vector<std::pair<IUgc::eProcessState, uint64_t>>& counts) {
 		nlohmann::json out = { { "pending", 0 }, { "done", 0 }, { "failed", 0 } };
 		for (const auto& [state, count] : counts) out[StateName(state)] = count;
@@ -44,22 +138,24 @@ namespace UgcRoutes {

 		Route(eHTTPMethod::GET, "/api/ugc", Perm("properties_view"),
 			"Processing state: {counts: {model, modular: {pending, done, failed}}, items: [{id, characterId, characterName, state, attempts, processedAt, "
-			"error, bakeAo, modules}], ugcUrl, canManage}. Query: ?kind=model|modular&state=pending|done|failed&page=",
+			"error, bakeAo, modules}], ugcPublicUrl, canManage}. Query: ?kind=model|modular&state=pending|done|failed&search=(id or owner name)&page=&size=",
 			[](HTTPReply& reply, const HTTPContext& context) {
 				const bool modular = QueryValue(context.queryString, "kind") == "modular";
 				const auto state = ParseState(QueryValue(context.queryString, "state"));
 				const auto page = GeneralUtils::TryParse<uint32_t>(QueryValue(context.queryString, "page")).value_or(0);
-				const auto list = modular ? Database::Get()->GetModularBuildProcessList(state, page * PAGE_SIZE, PAGE_SIZE + 1)
-					: Database::Get()->GetUgcProcessList(state, page * PAGE_SIZE, PAGE_SIZE + 1);
+				const auto search = QueryValue(context.queryString, "search").substr(0, 64);
+				const auto size = std::clamp(GeneralUtils::TryParse<uint32_t>(QueryValue(context.queryString, "size")).value_or(PAGE_SIZE), 1u, 200u);
+				const auto list = modular ? Database::Get()->GetModularBuildProcessList(state, search, page * size, size + 1)
+					: Database::Get()->GetUgcProcessList(state, search, page * size, size + 1);
 				nlohmann::json items = nlohmann::json::array();
-				for (size_t i = 0; i < list.size() && i < PAGE_SIZE; i++) {
+				for (size_t i = 0; i < list.size() && i < size; i++) {
 					const auto& info = list[i];
 					items.push_back({ { "id", std::to_string(info.id) }, { "characterId", std::to_string(info.characterId) }, { "characterName", info.characterName },
 						{ "state", StateName(info.state) }, { "attempts", info.attempts }, { "processedAt", info.processedAt }, { "error", info.error },
 						{ "bakeAo", info.bakeAo }, { "modules", info.details } });
 				}
 				JsonSuccess(reply, { { "counts", { { "model", Counts(Database::Get()->GetUgcProcessCounts()) }, { "modular", Counts(Database::Get()->GetModularBuildProcessCounts()) } } },
-					{ "items", items }, { "more", list.size() > PAGE_SIZE }, { "ugcUrl", Game::config->GetValue("ugc_public_url") }, { "canManage", Can(context, "ugc_manage") } });
+					{ "items", items }, { "more", list.size() > size }, { "ugcPublicUrl", Game::config->GetValue("ugc_public_url") }, { "canManage", Can(context, "ugc_manage") } });
 			});

 		Route(eHTTPMethod::GET, "/api/ugc/:id/lxfml", Perm("properties_view"), "A player model's LXFML",
@@ -74,6 +170,67 @@ namespace UgcRoutes {
 				reply.message = sd0.GetAsStringUncompressed();
 			});

+		Route(eHTTPMethod::GET, "/api/ugc/server/status", Perm("properties_view"),
+			"The UGC server's live status (its /status, fetched by the dashboard from ugc_internal_url): queue, workers, CPU and memory use, "
+			"limits and throttling. {success: false, error, url} when it doesn't answer",
+			[](HTTPReply& reply, const HTTPContext& context) {
+				const auto base = InternalUrl();
+				Workers::Reply(reply, context, false, [base](HTTPReply& out) {
+					const auto fetched = CachedGet(base + "/status", true);
+					if (fetched->status != 200) {
+						JsonReply(out, eHTTPStatusCode::OK, { { "success", false }, { "url", base }, { "error", fetched->status == 0 ? fetched->error : "answered " + std::to_string(fetched->status) } });
+						return;
+					}
+					out.status = eHTTPStatusCode::OK;
+					out.contentType = eContentType::APPLICATION_JSON;
+					out.message = R"({"success":true,"status":)" + fetched->body + "}";
+					out.headers.push_back("Cache-Control: no-store");
+				}, WorkerPool::ePriority::URGENT);
+			});
+
+		Route(eHTTPMethod::GET, "/api/ugc/files/:kind/:id/:file", Perm("properties_view"),
+			"A file the UGC server made (fetched from ugc_internal_url): icon.png, model.nif, model.noao.nif (before the lighting bake), "
+			"stats.json, or previous.<one of those> (the version before it was made again). 408 while it's being made",
+			[](HTTPReply& reply, const HTTPContext& context) {
+				const auto kind = KindOf(std::string(PathSegment(context.path, 3)));
+				const auto id = PathId<LWOOBJID>(context.path, 4);
+				const auto file = std::string(PathSegment(context.path, 5));
+				if (!kind || !id || !FILES.contains(file)) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid kind, id or file");
+				const auto url = InternalUrl() + "/files/" + *kind + "/" + std::to_string(*id) + "/" + file;
+				Workers::Reply(reply, context, false, [url, file](HTTPReply& out) {
+					const auto fetched = CachedGet(url, false);
+					if (fetched->status != 200) return FetchError(out, *fetched, url);
+					out.status = eHTTPStatusCode::OK;
+					out.contentType = file.ends_with(".png") ? eContentType::IMAGE_PNG : file.ends_with(".json") ? eContentType::APPLICATION_JSON : eContentType::APPLICATION_OCTET_STREAM;
+					out.message = fetched->body;
+					out.headers.push_back("Cache-Control: private, no-cache");
+				}, file.ends_with(".png") ? WorkerPool::ePriority::URGENT : WorkerPool::ePriority::NORMAL);
+			});
+
+		Route(eHTTPMethod::GET, "/api/ugc/mesh/:id", Perm("properties_view"),
+			"A player model's generated .nif converted for the 3D view (NifFile::Encode, as the scenery meshes). Query: ?lod=0 (most detailed) "
+			"to 3, &version=current|previous, &ao=0 for the mesh before the lighting bake. The header adds triangles and vertices",
+			[](HTTPReply& reply, const HTTPContext& context) {
+				const auto id = PathId<LWOOBJID>(context.path, 3);
+				if (!id) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid id");
+				const auto lod = std::min(GeneralUtils::TryParse<uint32_t>(QueryValue(context.queryString, "lod")).value_or(0), 3u);
+				const bool previous = QueryValue(context.queryString, "version") == "previous";
+				const bool baked = QueryValue(context.queryString, "ao") != "0";
+				const std::string file = std::string(previous ? "previous." : "") + (baked ? "model.nif" : "model.noao.nif");
+				const auto url = InternalUrl() + "/files/model/" + std::to_string(*id) + "/" + file;
+				Workers::Reply(reply, context, false, [url, lod](HTTPReply& out) {
+					const auto fetched = CachedGet(url, false);
+					if (fetched->status != 200) return FetchError(out, *fetched, url);
+					std::string error;
+					const auto model = NifFile::Parse(fetched->body, lod, error);
+					if (!model) return JsonError(out, eHTTPStatusCode::UNPROCESSABLE_ENTITY, "The .nif can't be read: " + error);
+					out.status = eHTTPStatusCode::OK;
+					out.contentType = eContentType::APPLICATION_OCTET_STREAM;
+					out.message = NifFile::Encode(*model, std::vector<std::string>(model->meshes.size()));
+					out.headers.push_back("Cache-Control: private, no-cache");
+				});
+			});
+
 		Route(eHTTPMethod::POST, "/api/ugc/reprocess", Perm("ugc_manage"),
 			"Have the UGC server make items again. Body: {kind: model|modular, id} for one, {kind, failedOnly: true} for the failed ones, {kind} for all",
 			[](HTTPReply& reply, const HTTPContext& context) {