diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index 91f5facbf..1aa543755 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -296,7 +296,8 @@ namespace { c.Add(Port(DASHBOARD, "port", "Web port", "Where the dashboard listens for browsers.", "2006")); c.Add(Format(Text(DASHBOARD, "listen_ip", "Listen address", "127.0.0.1 for this machine only (put a reverse proxy in front); 0.0.0.0 for everyone.", "127.0.0.1", true), eFormat::HOST)); c.Add(Port(DASHBOARD, "net_port", "Server link port", "UDP port for the connection to master (the next one is used too). Keep clear of other servers' ports.", "2010")); - c.Add(Format(Text(DASHBOARD, "ugc_public_url", "UGC server address", "Where the browser loads previews from the UGC server, e.g. https://ugc.example.com. Empty: this host, port 2008.", ""), eFormat::URL)); + c.Add(Format(Text(DASHBOARD, "ugc_internal_url", "UGC server address (internal)", "Where the dashboard itself reaches the UGC server for its status and files. Empty: http://127.0.0.1:2008.", ""), eFormat::URL)); + c.Add(Format(Text(DASHBOARD, "ugc_public_url", "UGC server address (public)", "Only for the \"open on the UGC server\" links, e.g. https://ugc.example.com. The dashboard's own pages don't need the browser to reach it.", ""), eFormat::URL)); c.Add(Format(Text(DASHBOARD, "dashboard_url", "Public address", "For links in emails and alerts, e.g. https://dashboard.example.com.", ""), eFormat::URL)); c.Add(Bool(DASHBOARD, "secure_cookies", "HTTPS only cookies", "Turn on when the dashboard is served over HTTPS.", false)); c.Add(Bool(DASHBOARD, "behind_proxy", "Behind a reverse proxy", "Use the proxy's X-Forwarded-For for rate limits. Only when the dashboard can't be reached directly.", false)); @@ -415,20 +416,48 @@ namespace { c.Add(Format(Text(UGC, "ugc_output_dir", "Files folder", "Relative to the server binaries.", "ugc", true), eFormat::PATH)); c.Add(Unit(Int(UGC, "ugc_max_storage_mb", "Most space for files", "The files used longest ago are deleted past this and made again when asked for. 0: no limit.", "2048", 0, std::nullopt, true), "MB")); c.Add(Int(UGC, "worker_threads", "Worker threads", "0: half the CPU cores.", "0", 0, 64, true)); + c.Add(Unit(Float(UGC, "max_cpu_percent", "Most CPU", "The workers together average at most this share of all CPU cores; they pause between bits of work to stay under it. 0: no limit.", "0", 0, 100), "%")); + c.Add(Int(UGC, "worker_nice", "Worker priority", "Linux nice value of the workers: 0 normal, 19 only when nothing else wants the CPU.", "0", 0, 19)); + c.Add(Unit(Int(UGC, "max_memory_mb", "Most memory for jobs", "Jobs whose estimated memory would go past this together wait for running ones; a bigger one runs alone. 0: no limit.", "0", 0, 1000000), "MB")); + c.Add(Unit(Int(UGC, "max_model_bricks", "Biggest model", "Models with more bricks fail with that reason. 0: no limit.", "0", 0, 1000000), "bricks")); + c.Add(Text(UGC, "pause_hours", "Pause hours", "Local hours in which no new jobs start, e.g. 18-23 or 22-6. Empty: never.", "")); c.Add(Unit(Int(UGC, "poll_interval_ms", "Look for new models every", "", "2000", 100, 600000, true), "ms")); c.Add(Int(UGC, "poll_batch", "Models taken at once", "", "32", 1, 1000, true)); c.Add(Int(UGC, "max_attempts", "Attempts before giving up", "", "3", 1, 100, true)); - c.Add(Int(UGC, "brick_lod", "Brick detail", "0 is the most detailed, 2 the least.", "0", 0, 2, true)); - c.Add(Bool(UGC, "remove_hidden_faces", "Remove faces nobody can see", "", true, true)); - c.Add(Bool(UGC, "bake_ao", "Darken hidden corners", "Ambient occlusion baked into the vertex colors.", true, true)); - c.Add(Float(UGC, "ao_strength", "Darkening strength", "0 to 1.", "0.6", 0, 1)); - c.Add(Unit(Int(UGC, "optimize_resolution", "Detail of the visibility renders", "", "1024", 64, 4096, true), "pixels")); - c.Add(Unit(Int(UGC, "icon_size", "Icon size", "", "128", 16, 1024, true), "pixels")); - c.Add(Unit(Float(UGC, "icon_yaw", "Icon angle around", "", "35", -360, 360), "degrees")); - c.Add(Unit(Float(UGC, "icon_pitch", "Icon angle above", "", "25", -90, 90), "degrees")); + c.Add(Labels(Choice(UGC, "color_palette", "Colors", "LU Toolbox's LU palette (unknown colors black), or the client's Materials.xml.", "lu_toolbox", { "lu_toolbox", "brickdb" }), { "LU Toolbox", "Brick database" })); + c.Add(Unit(Float(UGC, "color_variation", "Color variation", "Each brick's brightness is shifted by up to this much (the same every time the model is made). 0: none.", "5", 0, 100), "%")); + c.Add(Unit(Float(UGC, "transparent_opacity", "Transparent opacity", "", "58.82", 0, 100), "%")); + c.Add(Text(UGC, "lods", "Levels of detail", "brickprimitives levels made, 0 (most detailed) to 2, e.g. 0,2.", "0,2")); + c.Add(Float(UGC, "lod_distance_0", "LOD 0 distance", "", "0", 0, 100000)); + c.Add(Float(UGC, "lod_distance_1", "LOD 1 distance", "", "50", 0, 100000)); + c.Add(Float(UGC, "lod_distance_2", "LOD 2 distance", "", "100", 0, 100000)); + c.Add(Float(UGC, "lod_distance_3", "LOD 3 distance", "", "280", 0, 100000)); + c.Add(Float(UGC, "lod_cull", "Drawn up to", "", "10000", 0, 1000000)); + c.Add(Text(UGC, "shader_opaque", "Opaque shader", "S_Opaque_Model.", "01")); + c.Add(Bool(UGC, "combine_transparent", "One shape for all transparent bricks", "Off: each transparent brick is its own shape, so the client can sort them.", false)); + c.Add(Bool(UGC, "remove_hidden_faces", "Remove faces nobody can see", "", true)); + c.Add(Bool(UGC, "hsr_ground_plane", "Nothing seen from below", "Also removes what can only be seen from under the model.", false)); + c.Add(Unit(Int(UGC, "optimize_resolution", "Detail of the visibility renders", "", "1024", 64, 4096), "pixels")); + c.Add(Bool(UGC, "bake_ao", "Darken hidden corners", "Ambient occlusion baked into the vertex colors.", true)); + c.Add(Int(UGC, "ao_samples", "Occlusion rays per vertex", "", "64", 1, 1024)); + c.Add(Float(UGC, "ao_distance", "Occlusion distance", "", "5", 0, 1000)); + c.Add(Float(UGC, "ao_strength", "Darkening strength", "0 to 1.", "1", 0, 1)); + c.Add(Float(UGC, "glow_strength", "Glow strength", "What glowing colors add to the baked light.", "6", 0, 100)); + c.Add(Unit(Int(UGC, "icon_size", "Icon size", "", "128", 16, 1024), "pixels")); + c.Add(Unit(Float(UGC, "icon_yaw", "Icon angle around", "", "53.36", -360, 360), "degrees")); + c.Add(Unit(Float(UGC, "icon_pitch", "Icon angle above", "", "19.54", -90, 90), "degrees")); + c.Add(Unit(Float(UGC, "icon_fov", "Icon field of view", "", "39.6", 1, 120), "degrees")); c.Add(Float(UGC, "icon_margin", "Icon border", "1 fills the icon.", "1.03", 0.1, 10)); - c.Add(Unit(Float(UGC, "modular_icon_yaw", "Car and rocket icon angle around", "", "35", -360, 360), "degrees")); - c.Add(Unit(Float(UGC, "modular_icon_pitch", "Car and rocket icon angle above", "", "20", -90, 90), "degrees")); + c.Add(Unit(Float(UGC, "icon_sun_yaw", "Icon sun angle around", "", "21", -360, 360), "degrees")); + c.Add(Unit(Float(UGC, "icon_sun_pitch", "Icon sun angle above", "", "50.3", -90, 90), "degrees")); + c.Add(Float(UGC, "icon_sun_strength", "Icon sun strength", "", "2.5", 0, 100)); + c.Add(Float(UGC, "icon_ambient", "Icon world light", "", "0.192", 0, 10)); + c.Add(Bool(UGC, "icon_shadows", "Icon shadows", "", true)); + c.Add(Bool(UGC, "icon_ao", "Icon ambient occlusion", "", true)); + c.Add(Bool(UGC, "icon_correct_colors", "Icon color corrections", "White and black toned down, as LU Toolbox's icon renderer does.", true)); + c.Add(Unit(Float(UGC, "icon_color_variation", "Icon color variation", "", "0", 0, 100), "%")); + c.Add(Unit(Float(UGC, "modular_icon_yaw", "Car and rocket icon angle around", "", "53.36", -360, 360), "degrees")); + c.Add(Unit(Float(UGC, "modular_icon_pitch", "Car and rocket icon angle above", "", "19.54", -90, 90), "degrees")); return c; } diff --git a/dDashboardServer/routes/UgcRoutes.cpp b/dDashboardServer/routes/UgcRoutes.cpp index c94998950..28f370311 100644 --- a/dDashboardServer/routes/UgcRoutes.cpp +++ b/dDashboardServer/routes/UgcRoutes.cpp @@ -1,6 +1,15 @@ #include "UgcRoutes.h" +#include +#include +#include + +#include + #include "Database.h" +#include "NifFile.h" +#include "TtlCache.h" +#include "Workers.h" #include "Game.h" #include "GeneralUtils.h" #include "RouteUtils.h" @@ -30,6 +39,91 @@ namespace { return std::nullopt; } + /** + * The dashboard's own way to the UGC server (ugc_internal_url, normally the same machine): its status and the files + * it made are fetched here and handed to the browser, which may not be able to reach the UGC server at all. Fetches + * run on worker threads (Workers::Reply) with the URL worked out on the web thread; small answers are kept briefly. + */ + struct Fetched { + long status{}; // HTTP status, 0 when the UGC server didn't answer + std::string body; + std::string error; + }; + + std::mutex g_CacheMutex; + TtlCache> g_Cache(std::chrono::seconds(15), 32 * 1024 * 1024); + TtlCache> g_StatusCache(std::chrono::seconds(2), 1024 * 1024); + + constexpr size_t MAX_FETCH_BYTES = 128 * 1024 * 1024; + + size_t Collect(char* data, size_t size, size_t count, void* userData) { + auto* out = static_cast(userData); + if (out->size() + size * count > MAX_FETCH_BYTES) return 0; + out->append(data, size * count); + return size * count; + } + + // Web thread: where the dashboard reaches the UGC server + std::string InternalUrl() { + auto url = Game::config->GetValue("ugc_internal_url"); + if (url.empty()) url = "http://127.0.0.1:2008"; + while (url.ends_with('/')) url.pop_back(); + return url; + } + + // Any thread: a GET to the UGC server + std::shared_ptr Get(const std::string& url) { + auto out = std::make_shared(); + CURL* curl = curl_easy_init(); + if (!curl) { + out->error = "could not start a request"; + return out; + } + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https"); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "DarkflameServer-Dashboard"); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Collect); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out->body); + const auto code = curl_easy_perform(curl); + if (code == CURLE_OK) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &out->status); + else out->error = curl_easy_strerror(code); + curl_easy_cleanup(curl); + return out; + } + + // Any thread: Get, through the cache (answers that aren't a file being made are kept) + std::shared_ptr CachedGet(const std::string& url, bool status) { + { + std::lock_guard lock(g_CacheMutex); + if (auto hit = (status ? g_StatusCache : g_Cache).Get(url)) return *hit; + } + auto fetched = Get(url); + if (fetched->status == 200 || fetched->status == 404) { + std::lock_guard lock(g_CacheMutex); + (status ? g_StatusCache : g_Cache).Put(url, fetched, fetched->body.size() + 64); + } + return fetched; + } + + void FetchError(HTTPReply& reply, const Fetched& fetched, const std::string& url) { + if (fetched.status == 408) return JsonError(reply, eHTTPStatusCode::REQUEST_TIMEOUT, "The UGC server is making it; try again in a moment"); + if (fetched.status == 404) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "The UGC server has no such file"); + if (fetched.status == 0) return JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server doesn't answer at " + url + " (" + fetched.error + ")"); + JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server answered " + std::to_string(fetched.status)); + } + + const std::set FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json", + "previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json" }; + + std::optional KindOf(const std::string& text) { + if (text == "model" || text == "modular") return text; + return std::nullopt; + } + nlohmann::json Counts(const std::vector>& counts) { nlohmann::json out = { { "pending", 0 }, { "done", 0 }, { "failed", 0 } }; for (const auto& [state, count] : counts) out[StateName(state)] = count; @@ -44,22 +138,24 @@ namespace UgcRoutes { Route(eHTTPMethod::GET, "/api/ugc", Perm("properties_view"), "Processing state: {counts: {model, modular: {pending, done, failed}}, items: [{id, characterId, characterName, state, attempts, processedAt, " - "error, bakeAo, modules}], ugcUrl, canManage}. Query: ?kind=model|modular&state=pending|done|failed&page=", + "error, bakeAo, modules}], ugcPublicUrl, canManage}. Query: ?kind=model|modular&state=pending|done|failed&search=(id or owner name)&page=&size=", [](HTTPReply& reply, const HTTPContext& context) { const bool modular = QueryValue(context.queryString, "kind") == "modular"; const auto state = ParseState(QueryValue(context.queryString, "state")); const auto page = GeneralUtils::TryParse(QueryValue(context.queryString, "page")).value_or(0); - const auto list = modular ? Database::Get()->GetModularBuildProcessList(state, page * PAGE_SIZE, PAGE_SIZE + 1) - : Database::Get()->GetUgcProcessList(state, page * PAGE_SIZE, PAGE_SIZE + 1); + const auto search = QueryValue(context.queryString, "search").substr(0, 64); + const auto size = std::clamp(GeneralUtils::TryParse(QueryValue(context.queryString, "size")).value_or(PAGE_SIZE), 1u, 200u); + const auto list = modular ? Database::Get()->GetModularBuildProcessList(state, search, page * size, size + 1) + : Database::Get()->GetUgcProcessList(state, search, page * size, size + 1); nlohmann::json items = nlohmann::json::array(); - for (size_t i = 0; i < list.size() && i < PAGE_SIZE; i++) { + for (size_t i = 0; i < list.size() && i < size; i++) { const auto& info = list[i]; items.push_back({ { "id", std::to_string(info.id) }, { "characterId", std::to_string(info.characterId) }, { "characterName", info.characterName }, { "state", StateName(info.state) }, { "attempts", info.attempts }, { "processedAt", info.processedAt }, { "error", info.error }, { "bakeAo", info.bakeAo }, { "modules", info.details } }); } JsonSuccess(reply, { { "counts", { { "model", Counts(Database::Get()->GetUgcProcessCounts()) }, { "modular", Counts(Database::Get()->GetModularBuildProcessCounts()) } } }, - { "items", items }, { "more", list.size() > PAGE_SIZE }, { "ugcUrl", Game::config->GetValue("ugc_public_url") }, { "canManage", Can(context, "ugc_manage") } }); + { "items", items }, { "more", list.size() > size }, { "ugcPublicUrl", Game::config->GetValue("ugc_public_url") }, { "canManage", Can(context, "ugc_manage") } }); }); Route(eHTTPMethod::GET, "/api/ugc/:id/lxfml", Perm("properties_view"), "A player model's LXFML", @@ -74,6 +170,67 @@ namespace UgcRoutes { reply.message = sd0.GetAsStringUncompressed(); }); + Route(eHTTPMethod::GET, "/api/ugc/server/status", Perm("properties_view"), + "The UGC server's live status (its /status, fetched by the dashboard from ugc_internal_url): queue, workers, CPU and memory use, " + "limits and throttling. {success: false, error, url} when it doesn't answer", + [](HTTPReply& reply, const HTTPContext& context) { + const auto base = InternalUrl(); + Workers::Reply(reply, context, false, [base](HTTPReply& out) { + const auto fetched = CachedGet(base + "/status", true); + if (fetched->status != 200) { + JsonReply(out, eHTTPStatusCode::OK, { { "success", false }, { "url", base }, { "error", fetched->status == 0 ? fetched->error : "answered " + std::to_string(fetched->status) } }); + return; + } + out.status = eHTTPStatusCode::OK; + out.contentType = eContentType::APPLICATION_JSON; + out.message = R"({"success":true,"status":)" + fetched->body + "}"; + out.headers.push_back("Cache-Control: no-store"); + }, WorkerPool::ePriority::URGENT); + }); + + Route(eHTTPMethod::GET, "/api/ugc/files/:kind/:id/:file", Perm("properties_view"), + "A file the UGC server made (fetched from ugc_internal_url): icon.png, model.nif, model.noao.nif (before the lighting bake), " + "stats.json, or previous. (the version before it was made again). 408 while it's being made", + [](HTTPReply& reply, const HTTPContext& context) { + const auto kind = KindOf(std::string(PathSegment(context.path, 3))); + const auto id = PathId(context.path, 4); + const auto file = std::string(PathSegment(context.path, 5)); + if (!kind || !id || !FILES.contains(file)) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid kind, id or file"); + const auto url = InternalUrl() + "/files/" + *kind + "/" + std::to_string(*id) + "/" + file; + Workers::Reply(reply, context, false, [url, file](HTTPReply& out) { + const auto fetched = CachedGet(url, false); + if (fetched->status != 200) return FetchError(out, *fetched, url); + out.status = eHTTPStatusCode::OK; + out.contentType = file.ends_with(".png") ? eContentType::IMAGE_PNG : file.ends_with(".json") ? eContentType::APPLICATION_JSON : eContentType::APPLICATION_OCTET_STREAM; + out.message = fetched->body; + out.headers.push_back("Cache-Control: private, no-cache"); + }, file.ends_with(".png") ? WorkerPool::ePriority::URGENT : WorkerPool::ePriority::NORMAL); + }); + + Route(eHTTPMethod::GET, "/api/ugc/mesh/:id", Perm("properties_view"), + "A player model's generated .nif converted for the 3D view (NifFile::Encode, as the scenery meshes). Query: ?lod=0 (most detailed) " + "to 3, &version=current|previous, &ao=0 for the mesh before the lighting bake. The header adds triangles and vertices", + [](HTTPReply& reply, const HTTPContext& context) { + const auto id = PathId(context.path, 3); + if (!id) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid id"); + const auto lod = std::min(GeneralUtils::TryParse(QueryValue(context.queryString, "lod")).value_or(0), 3u); + const bool previous = QueryValue(context.queryString, "version") == "previous"; + const bool baked = QueryValue(context.queryString, "ao") != "0"; + const std::string file = std::string(previous ? "previous." : "") + (baked ? "model.nif" : "model.noao.nif"); + const auto url = InternalUrl() + "/files/model/" + std::to_string(*id) + "/" + file; + Workers::Reply(reply, context, false, [url, lod](HTTPReply& out) { + const auto fetched = CachedGet(url, false); + if (fetched->status != 200) return FetchError(out, *fetched, url); + std::string error; + const auto model = NifFile::Parse(fetched->body, lod, error); + if (!model) return JsonError(out, eHTTPStatusCode::UNPROCESSABLE_ENTITY, "The .nif can't be read: " + error); + out.status = eHTTPStatusCode::OK; + out.contentType = eContentType::APPLICATION_OCTET_STREAM; + out.message = NifFile::Encode(*model, std::vector(model->meshes.size())); + out.headers.push_back("Cache-Control: private, no-cache"); + }); + }); + Route(eHTTPMethod::POST, "/api/ugc/reprocess", Perm("ugc_manage"), "Have the UGC server make items again. Body: {kind: model|modular, id} for one, {kind, failedOnly: true} for the failed ones, {kind} for all", [](HTTPReply& reply, const HTTPContext& context) { diff --git a/dDashboardServer/static/js/ugc-viewer.js b/dDashboardServer/static/js/ugc-viewer.js new file mode 100644 index 000000000..3678f5b1f --- /dev/null +++ b/dDashboardServer/static/js/ugc-viewer.js @@ -0,0 +1,141 @@ +/** + * A 3D view of a .nif the UGC server made, from /api/ugc/mesh/:id (the dashboard converts it with NifFile::Encode, + * as it does the scenery's models), with wireframe and vertex color switches and triangle counts. + */ +import * as THREE from 'three'; +import { OrbitControls } from 'three/addons/controls/OrbitControls.js'; +import { RoomEnvironment } from 'three/addons/environments/RoomEnvironment.js'; +import { parseModel, mergeMeshes, linearColors } from '/js/scenery-core.js'; + +export function createNifViewer(container) { + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2)); + container.textContent = ''; + container.style.position = 'relative'; + container.appendChild(renderer.domElement); + + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x1e2126); + const pmrem = new THREE.PMREMGenerator(renderer); + scene.environment = pmrem.fromScene(new RoomEnvironment(), 0.04).texture; + scene.environmentIntensity = 0.5; + const sun = new THREE.DirectionalLight(0xffffff, 1.4); + scene.add(sun, sun.target); + + const camera = new THREE.PerspectiveCamera(40, 1, 0.05, 20000); + const controls = new OrbitControls(camera, renderer.domElement); + controls.enableDamping = true; + const root = new THREE.Group(); + scene.add(root); + + let parts = []; + let wireframe = false, vertexColors = true, disposed = false, framed = false; + const grey = new THREE.Color(0xbbbbbb); + + function resize() { + const width = container.clientWidth || 300, height = container.clientHeight || 300; + renderer.setSize(width, height, false); + renderer.domElement.style.width = '100%'; + renderer.domElement.style.height = '100%'; + camera.aspect = width / height; + camera.updateProjectionMatrix(); + } + const observer = new ResizeObserver(resize); + observer.observe(container); + resize(); + + function loop() { + if (disposed) return; + controls.update(); + renderer.render(scene, camera); + requestAnimationFrame(loop); + } + requestAnimationFrame(loop); + + function clear() { + for (const part of parts) { + root.remove(part.mesh); + part.mesh.geometry.dispose(); + part.mesh.material.dispose(); + } + parts = []; + } + + function applyLook() { + for (const part of parts) { + const material = part.mesh.material; + material.wireframe = wireframe; + material.vertexColors = vertexColors && part.hasColors; + material.color.copy(vertexColors && part.hasColors ? part.baseColor : grey); + material.needsUpdate = true; + } + } + + function frame() { + const box = new THREE.Box3().setFromObject(root); + if (box.isEmpty()) return; + const center = box.getCenter(new THREE.Vector3()); + const radius = Math.max(box.getSize(new THREE.Vector3()).length() / 2, 0.1); + // From the icon's side: in front and to the side, a little above + const direction = new THREE.Vector3(Math.sin(0.93) * Math.cos(0.34), Math.sin(0.34), Math.cos(0.93) * Math.cos(0.34)); + camera.position.copy(center).addScaledVector(direction, radius / Math.sin(THREE.MathUtils.degToRad(camera.fov / 2)) * 1.05); + camera.near = radius / 100; + camera.far = radius * 100; + camera.updateProjectionMatrix(); + controls.target.copy(center); + sun.position.copy(center).add(new THREE.Vector3(0.23, 0.77, 0.6).multiplyScalar(radius * 4)); + sun.target.position.copy(center); + framed = true; + } + + return { + /** + * Loads a mesh; resolves {triangles, vertices, shapes}. The view keeps its angle between loads of the same model + * (LOD, version) so they can be compared; `reframe` points it at the model again. + */ + async load(url, reframe) { + const response = await fetch(url, { credentials: 'same-origin' }); + if (!response.ok) { + let message = response.status === 408 ? 'Being made, try again in a moment' : 'HTTP ' + response.status; + try { message = (await response.json()).error || message; } catch (e) { /* not JSON */ } + throw new Error(message); + } + const model = parseModel(await response.arrayBuffer()); + clear(); + let triangles = 0, vertices = 0; + for (const mesh of mergeMeshes(model.meshes)) { + if (!mesh.vertices || !mesh.indices.length) continue; + const geometry = new THREE.BufferGeometry(); + geometry.setAttribute('position', new THREE.BufferAttribute(mesh.positions, 3)); + if (mesh.normals) geometry.setAttribute('normal', new THREE.BufferAttribute(mesh.normals, 3, true)); + const hasColors = !!(mesh.colors && mesh.vertexColors !== 0); + if (hasColors) geometry.setAttribute('color', new THREE.BufferAttribute(linearColors(mesh.colors), 4, true)); + geometry.setIndex(new THREE.BufferAttribute(mesh.indices, 1)); + if (!mesh.normals) geometry.computeVertexNormals(); + const baseColor = new THREE.Color().setRGB(mesh.diffuse[0], mesh.diffuse[1], mesh.diffuse[2], THREE.SRGBColorSpace); + const material = new THREE.MeshStandardMaterial({ color: baseColor.clone(), vertexColors: hasColors, transparent: !!mesh.blend, roughness: 0.6, metalness: 0 }); + const object = new THREE.Mesh(geometry, material); + if (mesh.blend) object.renderOrder = 1; + root.add(object); + parts.push({ mesh: object, hasColors, baseColor }); + triangles += mesh.indices.length / 3; + vertices += mesh.vertices; + } + applyLook(); + if (reframe || !framed) frame(); + return { triangles, vertices, shapes: model.meshes.length }; + }, + setWireframe(on) { wireframe = on; applyLook(); }, + setVertexColors(on) { vertexColors = on; applyLook(); }, + frame, + dispose() { + disposed = true; + observer.disconnect(); + clear(); + controls.dispose(); + pmrem.dispose(); + renderer.dispose(); + renderer.domElement.remove(); + } + }; +} diff --git a/dDashboardServer/static/js/ugc.js b/dDashboardServer/static/js/ugc.js index 60b918042..c1d6df288 100644 --- a/dDashboardServer/static/js/ugc.js +++ b/dDashboardServer/static/js/ugc.js @@ -1,20 +1,26 @@ /** - * The UGC Server page: counts and a list of what the UGC server made of players' models and modular builds (from the - * database), its live status and previews (from the UGC server itself, at ugc_public_url), and making things again. + * The UGC Server page: counts, a gallery and a list of what the UGC server made of players' models and modular builds + * (from the database), its live status, and a viewer comparing a model's generated mesh, its LXFML and its icon, now + * and before it was made again. Everything from the UGC server comes through the dashboard (/api/ugc/server/status, + * /api/ugc/files/..., /api/ugc/mesh/...), so the browser never has to reach the UGC server itself. */ (function () { 'use strict'; - var kind = 'model', state = '', page = 0, ugcUrl = '', canManage = false, viewer = null; + var kind = 'model', state = '', search = '', page = 0, view = 'gallery', canManage = false, publicUrl = ''; + var nifViewer = null, lxfmlViewer = null, current = null; var STATES = { pending: ['Waiting', 'secondary'], done: ['Made', 'success'], failed: ['Failed', 'danger'] }; + var PAGE_SIZE = { gallery: 48, list: 50 }; - function serverUrl() { - return (ugcUrl || (location.protocol + '//' + location.hostname + ':2008')).replace(/\/+$/, ''); + try { view = localStorage.getItem('ugcView') === 'list' ? 'list' : 'gallery'; } catch (e) { /* storage blocked */ } + + function $(id) { return document.getElementById(id); } + + function fileUrl(itemKind, id, name) { + return '/api/ugc/files/' + itemKind + '/' + encodeURIComponent(id) + '/' + name; } - function fileUrl(id, name) { - return serverUrl() + '/files/' + kind + '/' + encodeURIComponent(id) + '/' + name; - } + function mb(bytes) { return (bytes / 1048576).toFixed(0) + ' MB'; } function countCard(title, c) { return '
' + esc(title) + '
' + @@ -22,47 +28,84 @@ '
'; } + function owner(i) { + return i.characterName ? '' + esc(i.characterName) + '' : '' + esc(i.characterId) + ''; + } + + function iconImg(i, size) { + if (i.state !== 'done') return '' + esc((STATES[i.state] || [i.state])[0]) + ''; + return ''; + } + + function tile(i) { + var s = STATES[i.state] || [i.state, 'secondary']; + return '
' + iconImg(i, 128) + + '
' + (i.characterName ? esc(i.characterName) : '' + esc(i.characterId) + '') + '
' + + '
' + esc(i.id) + '
' + fmt.badge(s[0], s[1]) + '
'; + } + function row(i) { var s = STATES[i.state] || [i.state, 'secondary']; - var icon = i.state === 'done' ? '' : ''; - var owner = i.characterName ? '' + esc(i.characterName) + '' : '' + esc(i.characterId) + ''; var details = kind === 'modular' ? '' + esc(i.modules) + '' : ''; if (i.error) details += '
' + esc(i.error) + '
'; - var buttons = (kind === 'model' ? '' : '') + + var buttons = '' + (canManage ? '' : ''); - return '' + icon + '' + esc(i.id) + '' + owner + '' + fmt.badge(s[0], s[1]) + + return '' + iconImg(i, 48) + '' + esc(i.id) + '' + owner(i) + '' + fmt.badge(s[0], s[1]) + (i.attempts ? ' ' + esc(i.attempts) + ' attempt' + (i.attempts === 1 ? '' : 's') + '' : '') + '' + '' + (i.processedAt ? esc(fmt.unix(i.processedAt)) : '') + '' + details + '' + buttons + ''; } + var items = []; function load() { - api.get('/api/ugc?kind=' + kind + '&state=' + state + '&page=' + page).then(function (d) { + var size = PAGE_SIZE[view]; + api.get('/api/ugc?kind=' + kind + '&state=' + state + '&page=' + page + '&size=' + size + '&search=' + encodeURIComponent(search)).then(function (d) { if (!d.success) return; - ugcUrl = d.ugcUrl; + items = d.items; canManage = d.canManage; - document.getElementById('manageButtons').classList.toggle('d-none', !canManage); - document.getElementById('counts').innerHTML = countCard('Models', d.counts.model) + countCard('Cars and rockets', d.counts.modular); - document.getElementById('rows').innerHTML = d.items.map(row).join('') || 'Nothing here.'; - document.getElementById('prevPage').disabled = page === 0; - document.getElementById('nextPage').disabled = !d.more; - loadStatus(); + publicUrl = (d.ugcPublicUrl || '').replace(/\/+$/, ''); + $('manageButtons').classList.toggle('d-none', !canManage); + $('counts').innerHTML = countCard('Models', d.counts.model) + countCard('Cars and rockets', d.counts.modular); + $('gallery').classList.toggle('d-none', view !== 'gallery'); + $('listCard').classList.toggle('d-none', view !== 'list'); + if (view === 'gallery') $('gallery').innerHTML = d.items.map(tile).join('') || '
Nothing here.
'; + else $('rows').innerHTML = d.items.map(row).join('') || 'Nothing here.'; + $('prevPage').disabled = page === 0; + $('nextPage').disabled = !d.more; }).catch(function () {}); } + function meter(label, value, limit, text) { + var share = limit > 0 ? Math.min(100, value / limit * 100) : 0; + var colour = share > 90 ? 'danger' : share > 70 ? 'warning' : 'success'; + return '
' + esc(label) + '' + text + '
' + + (limit > 0 ? '
' + + '
' : '') + '
'; + } + function loadStatus() { - var box = document.getElementById('serverStatus'); - fetch(serverUrl() + '/status').then(function (r) { return r.json(); }).then(function (s) { - var mb = function (b) { return (b / 1048576).toFixed(0) + ' MB'; }; - var last = s.recent && s.recent[0]; - box.innerHTML = fmt.badge('Online', 'success') + ' ' + esc(s.workers) + ' workers, ' + esc(s.active) + ' working, ' + esc(s.queued) + ' queued. ' + + var box = $('serverStatus'); + api.get('/api/ugc/server/status').then(function (d) { + if (!d.success) { + box.innerHTML = fmt.badge('Unreachable', 'warning') + ' The dashboard can\'t reach the UGC server at ' + esc(d.url || '') + (d.error ? ' (' + esc(d.error) + ')' : '') + + '. Is enable_ugc_server on, and ugc_internal_url (dashboard settings) right?'; + return; + } + var s = d.status, u = s.usage || {}, l = s.limits || {}, last = s.recent && s.recent[0]; + var cores = u.cores || 1, machinePercent = (u.cpuPercent || 0) / cores; + var badges = fmt.badge('Online', 'success') + (u.throttled ? ' ' + fmt.badge('Throttled', 'warning') : '') + (u.paused ? ' ' + fmt.badge('Paused (' + esc(l.pauseHours) + ')', 'info') : ''); + box.innerHTML = '
' + badges + ' ' + esc(s.workers) + ' workers, ' + esc(s.active) + ' working, ' + esc(s.queued) + ' queued. ' + 'Since it started: ' + esc(s.made) + ' made, ' + esc(s.failed) + ' failed attempts. Files: ' + esc(mb(s.storedBytes)) + - (s.maxStorageBytes ? ' of ' + esc(mb(s.maxStorageBytes)) : '') + (s.evicted ? ', ' + esc(s.evicted) + ' deleted to save space' : '') + '.' + + (s.maxStorageBytes ? ' of ' + esc(mb(s.maxStorageBytes)) : '') + (s.evicted ? ', ' + esc(s.evicted) + ' deleted to save space' : '') + '.
' + + '
' + + meter('CPU', machinePercent, l.maxCpus ? l.maxCpus / cores * 100 : 0, machinePercent.toFixed(0) + '% of ' + cores + ' cores' + + (l.maxCpus ? ' (limit ' + (l.maxCpus / cores * 100).toFixed(0) + '%)' : ', no limit') + (u.throttledMs ? ', paused ' + (u.throttledMs / 1000).toFixed(0) + ' s to stay under it' : '')) + + meter('Memory', u.residentBytes || 0, 0, esc(mb(u.residentBytes || 0)) + ' in use') + + meter('Jobs\' memory', s.jobMemoryBytes || 0, l.maxMemoryBytes || 0, esc(mb(s.jobMemoryBytes || 0)) + ' estimated' + + (l.maxMemoryBytes ? ' of ' + esc(mb(l.maxMemoryBytes)) : ', no limit') + (s.memoryWaits ? ', ' + esc(s.memoryWaits) + ' waits' : '')) + + '
' + (l.nice ? '
Workers run at priority ' + esc(l.nice) + '.
' : '') + (last ? '
Last: ' + esc(last.kind) + ' ' + esc(last.id) + (last.ok ? ' made in ' + esc(last.ms) + ' ms' : ' failed') + (last.message ? ' (' + esc(last.message) + ')' : '') + '
' : ''); - }).catch(function () { - box.innerHTML = fmt.badge('Unreachable', 'warning') + ' The UGC server doesn\'t answer at ' + esc(serverUrl()) + - '. Is enable_ugc_server on, and ugc_public_url right?'; - }); + }).catch(function () { box.innerHTML = fmt.badge('Unknown', 'secondary') + ' Couldn\'t ask the dashboard for the UGC server\'s status.'; }); } function remake(body, question) { @@ -75,23 +118,102 @@ }); } - function preview(id) { - document.getElementById('previewTitle').textContent = 'Model ' + id; - document.getElementById('previewIcon').src = fileUrl(id, 'icon.png'); - document.getElementById('previewLinks').innerHTML = 'Download the mesh (.nif)
' + - 'Download the LXFML'; - bootstrap.Modal.getOrCreateInstance(document.getElementById('previewModal')).show(); - var container = document.getElementById('previewViewer'); - container.textContent = 'Loading…'; - import('/js/lddviewer.js').then(function (module) { - if (viewer && viewer.dispose) viewer.dispose(); - container.textContent = ''; - viewer = module.createViewer(container, {}); - return viewer.load([{ id: id, name: 'Model ' + id, position: [0, 0, 0], rotation: [0, 0, 0, 1], url: '/api/ugc/' + id + '/lxfml' }], 1); - }).catch(function (e) { container.textContent = 'The 3D view could not load: ' + e.message; }); + // ---- the viewer ---- + + function fetchJson(url) { + return fetch(url, { credentials: 'same-origin' }).then(function (r) { return r.ok ? r.json() : null; }).catch(function () { return null; }); } - document.getElementById('kindButtons').addEventListener('click', function (e) { + function statsTable(now, before) { + if (!now) return '
No stats for this model (made before the UGC server wrote them).
'; + var lods = now.lods || [], old = before && before.lods || []; + var diff = function (a, b) { + if (b === undefined || b === null || a === b) return ''; + var d = a - b; + return ' (' + (d > 0 ? '+' : '') + d + ')'; + }; + var rows = lods.map(function (l, i) { + var o = old[i] || {}; + var removed = l.opaqueBefore ? (100 - l.opaqueAfter / l.opaqueBefore * 100).toFixed(0) + '%' : ''; + return 'LOD ' + esc(l.lod) + '' + esc(l.near) + ' - ' + esc(l.far) + '' + esc(l.opaqueBefore) + '' + esc(l.opaqueAfter) + diff(l.opaqueAfter, o.opaqueAfter) + + '' + esc(removed) + '' + esc(l.transparent) + '' + esc(l.vertices) + diff(l.vertices, o.vertices) + '' + esc(l.shapes) + ''; + }).join(''); + var ms = now.ms || {}, set = now.settings || {}; + return '
' + + '' + rows + '
DetailDrawn from - toOpaque trianglesAfter removing hidden facesRemovedTransparent trianglesVerticesShapes
' + + '
' + esc(now.bricks) + ' bricks' + (now.missingDesigns ? ', no geometry for ' + esc(now.missingDesigns.join(', ')) : '') + + '. Took ' + esc(ms.total) + ' ms (build ' + esc(ms.build) + ', hidden faces ' + esc(ms.hiddenSurfaces) + ', occlusion ' + esc(ms.ambientOcclusion) + ', icon ' + esc(ms.icon) + ')' + + (before && before.ms ? ', before ' + esc(before.ms.total) + ' ms' : '') + '. Colors: ' + esc(set.palette) + ', variation ' + esc(set.colorVariation) + '%' + + (set.ao ? ', occlusion ' + esc(set.aoSamples) + ' rays to ' + esc(set.aoDistance) : ', no occlusion') + '.
'; + } + + function loadMesh(reframe) { + if (!current || !nifViewer) return; + var lod = $('lodSelect').value || '0', version = $('versionSelect').value, ao = $('aoSwitch').checked ? '1' : '0'; + var stats = $('nifStats'); + stats.textContent = 'Loading…'; + nifViewer.load('/api/ugc/mesh/' + encodeURIComponent(current.id) + '?lod=' + lod + '&version=' + version + '&ao=' + ao, reframe).then(function (r) { + var lodStats = current.stats && current.stats.lods && current.stats.lods[+lod]; + stats.textContent = r.triangles.toLocaleString() + ' triangles, ' + r.vertices.toLocaleString() + ' vertices in ' + r.shapes + ' shape' + (r.shapes === 1 ? '' : 's') + + (lodStats && version === 'current' ? ' (' + lodStats.opaqueBefore.toLocaleString() + ' opaque triangles before hidden faces were removed)' : ''); + }).catch(function (e) { stats.textContent = 'Could not load the mesh: ' + e.message; }); + } + + function preview(id) { + var item = items.find(function (i) { return i.id === id; }) || { id: id, state: 'done' }; + current = { id: id, kind: kind, stats: null }; + $('previewTitle').textContent = (kind === 'model' ? 'Model ' : 'Car or rocket ') + id + (item.characterName ? ' by ' + item.characterName : ''); + $('previewIcon').src = fileUrl(kind, id, 'icon.png'); + $('previousIconBox').classList.add('d-none'); + var previous = $('previousIcon'); + previous.onload = function () { $('previousIconBox').classList.remove('d-none'); }; + previous.src = fileUrl(kind, id, 'previous.icon.png'); + var links = ''; + if (kind === 'model') { + links += 'Download the mesh (.nif)
' + + 'Download the LXFML
'; + } + if (publicUrl) links += 'Open on the UGC server'; + if (canManage) links += '
'; + if (item.error) links += '
' + esc(item.error) + '
'; + $('previewLinks').innerHTML = links; + var model = kind === 'model'; + ['nifColumn', 'lxfmlColumn', 'meshControls'].forEach(function (x) { $(x).classList.toggle('d-none', !model); }); + $('statsBox').innerHTML = model ? '' : '
Modules: ' + esc(item.modules || '') + '. The game client puts cars and rockets together itself; only the icon is made.
'; + bootstrap.Modal.getOrCreateInstance($('previewModal')).show(); + if (!model) return; + + Promise.all([fetchJson(fileUrl('model', id, 'stats.json')), fetchJson(fileUrl('model', id, 'previous.stats.json'))]).then(function (s) { + if (!current || current.id !== id) return; + current.stats = s[0]; + current.statsLoaded = true; + $('statsBox').innerHTML = statsTable(s[0], s[1]); + var lods = (s[0] && s[0].lods) || [{ lod: 0 }]; + $('lodSelect').innerHTML = lods.map(function (l, i) { return ''; }).join(''); + $('versionSelect').querySelector('[value=previous]').disabled = !s[1]; + $('versionSelect').value = 'current'; + loadMesh(true); + }); + + var nifBox = $('nifViewer'), lxfmlBox = $('lxfmlViewer'); + import('/js/ugc-viewer.js').then(function (module) { + if (!nifViewer) { + nifViewer = module.createNifViewer(nifBox); + nifViewer.setWireframe($('wireframeSwitch').checked); + nifViewer.setVertexColors($('colorsSwitch').checked); + } + if (current && current.id === id && current.statsLoaded) loadMesh(true); + }).catch(function (e) { nifBox.textContent = 'The 3D view could not load: ' + e.message; }); + lxfmlBox.textContent = 'Loading…'; + import('/js/lddviewer.js').then(function (module) { + if (lxfmlViewer && lxfmlViewer.dispose) lxfmlViewer.dispose(); + lxfmlBox.textContent = ''; + lxfmlViewer = module.createViewer(lxfmlBox, {}); + return lxfmlViewer.load([{ id: id, name: 'Model ' + id, position: [0, 0, 0], rotation: [0, 0, 0, 1], url: '/api/ugc/' + id + '/lxfml' }], 1); + }).catch(function (e) { lxfmlBox.textContent = 'The 3D view could not load: ' + e.message; }); + } + + $('kindButtons').addEventListener('click', function (e) { var button = e.target.closest('[data-kind]'); if (!button) return; kind = button.dataset.kind; @@ -99,20 +221,55 @@ this.querySelectorAll('[data-kind]').forEach(function (b) { b.classList.toggle('active', b === button); }); load(); }); - document.getElementById('stateFilter').addEventListener('change', function () { state = this.value; page = 0; load(); }); - document.getElementById('prevPage').addEventListener('click', function () { if (page > 0) { page--; load(); } }); - document.getElementById('nextPage').addEventListener('click', function () { page++; load(); }); - document.getElementById('retryFailed').addEventListener('click', function () { remake({ failedOnly: true }); }); - document.getElementById('remakeAll').addEventListener('click', function () { + $('viewButtons').addEventListener('click', function (e) { + var button = e.target.closest('[data-view]'); + if (!button) return; + view = button.dataset.view; + page = 0; + try { localStorage.setItem('ugcView', view); } catch (err) { /* storage blocked */ } + this.querySelectorAll('[data-view]').forEach(function (b) { b.classList.toggle('active', b === button); }); + load(); + }); + $('viewButtons').querySelectorAll('[data-view]').forEach(function (b) { b.classList.toggle('active', b.dataset.view === view); }); + $('stateFilter').addEventListener('change', function () { state = this.value; page = 0; load(); }); + var searchTimer = null; + $('search').addEventListener('input', function () { + var value = this.value.trim(); + clearTimeout(searchTimer); + searchTimer = setTimeout(function () { search = value; page = 0; load(); }, 300); + }); + $('prevPage').addEventListener('click', function () { if (page > 0) { page--; load(); } }); + $('nextPage').addEventListener('click', function () { page++; load(); }); + $('retryFailed').addEventListener('click', function () { remake({ failedOnly: true }); }); + $('remakeAll').addEventListener('click', function () { remake({}, 'Make every ' + (kind === 'model' ? 'model' : 'car and rocket') + ' again? This can take a long time.'); }); - document.getElementById('rows').addEventListener('click', function (e) { + function onItemClick(e) { var remakeButton = e.target.closest('[data-remake]'), previewButton = e.target.closest('[data-preview]'); if (remakeButton) remake({ id: remakeButton.dataset.remake }); else if (previewButton) preview(previewButton.dataset.preview); + } + $('rows').addEventListener('click', onItemClick); + $('gallery').addEventListener('click', onItemClick); + $('gallery').addEventListener('keydown', function (e) { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); onItemClick(e); } }); + $('previewLinks').addEventListener('click', function (e) { + var button = e.target.closest('[data-remake-open]'); + if (button) remake({ id: button.dataset.remakeOpen }); + }); + $('lodSelect').addEventListener('change', function () { loadMesh(false); }); + $('versionSelect').addEventListener('change', function () { loadMesh(false); }); + $('aoSwitch').addEventListener('change', function () { loadMesh(false); }); + $('wireframeSwitch').addEventListener('change', function () { if (nifViewer) nifViewer.setWireframe(this.checked); }); + $('colorsSwitch').addEventListener('change', function () { if (nifViewer) nifViewer.setVertexColors(this.checked); }); + $('reframeButton').addEventListener('click', function () { if (nifViewer) nifViewer.frame(); }); + $('previewModal').addEventListener('hidden.bs.modal', function () { + current = null; + if (lxfmlViewer && lxfmlViewer.dispose) lxfmlViewer.dispose(); + lxfmlViewer = null; }); if (window.Live) Live.on('ugc', load); load(); + loadStatus(); setInterval(loadStatus, 10000); })(); diff --git a/dDashboardServer/templates/ugc.jinja2 b/dDashboardServer/templates/ugc.jinja2 index 8a06d72ff..d5713f6d1 100644 --- a/dDashboardServer/templates/ugc.jinja2 +++ b/dDashboardServer/templates/ugc.jinja2 @@ -22,16 +22,22 @@ - + +
+ + +
-
+ +
@@ -43,19 +49,62 @@
IconIDOwnerStateLast attemptDetails