From 1a2758feab0757caf7b038564b841d7a1be5637a Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sun, 27 Sep 2026 13:51:04 -0500 Subject: [PATCH] feat(ugc): icon light matched to the game, debounce, shared car icons, purge, icon editor Brighter icons: a world light, a fill from the camera, a highlight, exposure and contrast; with the defaults the icons' mean luminance matches the game's own model icons (118 against 120 on a scratch set). Every icon parameter is listed once (UgcIconParams: key, setting, range, default); the settings, the dashboard's settings entries and the icon editor come from that list. Presets per kind (player models, each car or rocket build type from ModularBuildComponent) and overrides per model or module combination are in ugc_icon_settings. Saved models wait ugc_debounce_seconds (sharedconfig) after the owner's last save before they're made (ugc.process_after); a client asking for one, the owner leaving the world or a reset ends the wait. Cars and rockets: one icon per combination of modules (sorted LOTs), shared by every build of it; builds of a combination made already are marked made right away, the client's per-blueprint downloads serve the shared files. The dashboard can delete one item's files, purge by filter or all, preview icons with any values on the UGC server (/admin routes, master password), save presets and overrides, and draw a kind's icons again (icons only). Migrations dlu/mysql/86 and dlu/sqlite/69. Not done yet: the dashboard editor's lighting controls in the page script (routes are there), docs for it, the empty-model state, /ugc?item= links, the shared fetch helper; the storage size and property loading bugs are next. Co-Authored-By: Claude Opus 5.5 --- dCommon/BrickByBrickFix.cpp | 2 +- dCommon/UgcKeys.h | 72 +++++ dDashboardServer/routes/CMakeLists.txt | 2 +- dDashboardServer/routes/MaintenanceRoutes.cpp | 2 +- dDashboardServer/routes/SettingsCatalog.cpp | 24 +- dDashboardServer/routes/UgcRoutes.cpp | 275 +++++++++++++++- dDashboardServer/static/js/ugc.js | 154 ++++++++- dDashboardServer/templates/ugc.jinja2 | 36 +++ .../GameDatabase/ITables/IPropertyContents.h | 5 +- dDatabase/GameDatabase/ITables/IUgc.h | 10 +- .../GameDatabase/ITables/IUgcModularBuild.h | 9 + dDatabase/GameDatabase/MySQL/MySQLDatabase.h | 11 +- dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp | 37 ++- .../MySQL/Tables/UgcModularBuild.cpp | 22 ++ .../GameDatabase/SQLite/SQLiteDatabase.h | 11 +- dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp | 37 ++- .../SQLite/Tables/UgcModularBuild.cpp | 25 ++ .../GameDatabase/TestSQL/TestSQLDatabase.cpp | 4 +- .../GameDatabase/TestSQL/TestSQLDatabase.h | 11 +- dDatabase/ModelNormalizeMigration.cpp | 6 +- dGame/dUtilities/BrickByBrick.cpp | 8 +- dUgcServer/CMakeLists.txt | 1 + dUgcServer/UgcCdClient.cpp | 1 + dUgcServer/UgcIconParams.cpp | 88 ++++++ dUgcServer/UgcIconParams.h | 56 ++++ dUgcServer/UgcJobs.cpp | 32 +- dUgcServer/UgcJobs.h | 24 +- dUgcServer/UgcProcessor.cpp | 298 +++++++++++++++++- dUgcServer/UgcProcessor.h | 54 +++- dUgcServer/UgcRender.cpp | 28 +- dUgcServer/UgcRender.h | 18 +- dUgcServer/UgcServer.cpp | 98 +++++- dUgcServer/UgcStorage.cpp | 39 ++- dUgcServer/UgcStorage.h | 5 + dWorldServer/WorldServer.cpp | 3 + .../mysql/86_ugc_debounce_icon_settings.sql | 15 + .../sqlite/69_ugc_debounce_icon_settings.sql | 8 + resources/sharedconfig.ini | 5 + resources/ugcconfig.ini | 26 +- tests/dDatabaseTests/DatabaseParityTests.cpp | 25 +- tests/dUgcTests/UgcTests.cpp | 123 +++++++- tests/dWebTests/CMakeLists.txt | 2 +- 42 files changed, 1576 insertions(+), 136 deletions(-) create mode 100644 dCommon/UgcKeys.h create mode 100644 dUgcServer/UgcIconParams.cpp create mode 100644 dUgcServer/UgcIconParams.h create mode 100644 migrations/dlu/mysql/86_ugc_debounce_icon_settings.sql create mode 100644 migrations/dlu/sqlite/69_ugc_debounce_icon_settings.sql diff --git a/dCommon/BrickByBrickFix.cpp b/dCommon/BrickByBrickFix.cpp index d299341fe..c6a7cd3a2 100644 --- a/dCommon/BrickByBrickFix.cpp +++ b/dCommon/BrickByBrickFix.cpp @@ -120,7 +120,7 @@ uint32_t BrickByBrickFix::UpdateBrickByBrickModelsToSd0() { std::stringstream outputStringStream(outputString); try { - Database::Get()->UpdateUgcModelData(model.id, outputStringStream); + Database::Get()->UpdateUgcModelData(model.id, outputStringStream, 0); LOG("Updated model %i to sd0", model.id); updatedModels++; } catch (std::exception& exception) { diff --git a/dCommon/UgcKeys.h b/dCommon/UgcKeys.h new file mode 100644 index 000000000..1bf9dcc64 --- /dev/null +++ b/dCommon/UgcKeys.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include +#include +#include + +/** + * Small pure decisions about UGC shared by the worlds, the UGC server and the dashboard (docs/UgcServer.md). + */ +namespace UgcModularKey { + // The module LOTs of a ugc_modular_build.ldf_config ("1:4713+1:4714+1:4715", also with ';' or ',' between) + inline std::vector Lots(std::string_view ldf) { + std::vector lots; + size_t start = 0; + while (start <= ldf.size()) { + size_t end = ldf.find_first_of("+;,", start); + if (end == std::string_view::npos) end = ldf.size(); + auto item = ldf.substr(start, end - start); + if (const auto colon = item.find(':'); colon != std::string_view::npos) item = item.substr(colon + 1); + while (!item.empty() && item.front() == ' ') item.remove_prefix(1); + while (!item.empty() && item.back() == ' ') item.remove_suffix(1); + uint32_t value = 0; + bool digits = !item.empty() && item.size() <= 9; + for (const char c : item) { + if (c < '0' || c > '9') { + digits = false; + break; + } + value = value * 10 + static_cast(c - '0'); + } + if (digits && value != 0) lots.push_back(value); + start = end + 1; + } + return lots; + } + + /** + * The combination of modules a build is made of, the same however its ldf_config is written: its LOTs sorted + * and joined by '-' ("4713-4714-4715"). Each module LOT belongs to one slot of one build type, so the set of LOTs + * says everything about how the build looks. Empty when there are no modules. + */ + inline std::string Normalize(std::string_view ldf) { + auto lots = Lots(ldf); + std::sort(lots.begin(), lots.end()); + lots.erase(std::unique(lots.begin(), lots.end()), lots.end()); + std::string key; + for (const auto lot : lots) key += (key.empty() ? "" : "-") + std::to_string(lot); + return key; + } + + // The id a combination's files are stored under (FNV-1a of the key, positive, never 0) + inline int64_t StorageId(std::string_view key) { + uint64_t hash = 14695981039346656037ull; + for (const char c : key) hash = (hash ^ static_cast(c)) * 1099511628211ull; + hash &= 0x3FFFFFFFFFFFFFFFull; + return static_cast(hash == 0 ? 1 : hash); + } +} + +namespace UgcDebounce { + // When a model saved at `now` may be made: after `quietSeconds` (0 or less: right away, stored as 0) + inline int64_t ProcessAfter(int64_t now, int64_t quietSeconds) { + return quietSeconds > 0 ? now + quietSeconds : 0; + } + + // Whether a waiting model is due: its quiet period is over, or someone asked for its files + inline bool Due(int64_t processAfter, int64_t now, bool requested) { + return requested || processAfter <= now; + } +} diff --git a/dDashboardServer/routes/CMakeLists.txt b/dDashboardServer/routes/CMakeLists.txt index ee3183e89..b2d4049f6 100644 --- a/dDashboardServer/routes/CMakeLists.txt +++ b/dDashboardServer/routes/CMakeLists.txt @@ -98,4 +98,4 @@ target_include_directories(DashboardRoutes PRIVATE ) target_compile_definitions(DashboardRoutes PRIVATE PROJECT_VERSION="\"${PROJECT_VERSION}\"") -target_link_libraries(DashboardRoutes PRIVATE ${COMMON_LIBRARIES} dWeb dServer sqlite3 OpenSSL::Crypto CURL::libcurl) +target_link_libraries(DashboardRoutes PRIVATE ${COMMON_LIBRARIES} dWeb dServer dUgc sqlite3 OpenSSL::Crypto CURL::libcurl) diff --git a/dDashboardServer/routes/MaintenanceRoutes.cpp b/dDashboardServer/routes/MaintenanceRoutes.cpp index 68fde4fdc..45575bdfb 100644 --- a/dDashboardServer/routes/MaintenanceRoutes.cpp +++ b/dDashboardServer/routes/MaintenanceRoutes.cpp @@ -266,7 +266,7 @@ void RegisterMaintenanceRoutes() { Sd0 sd0(header); sd0.FromData(reinterpret_cast(split.lxfml.data()), split.lxfml.size()); auto stream = sd0.GetAsStream(); - Database::Get()->InsertNewUgcModel(stream, blueprintId, owner->accountId, owner->id); + Database::Get()->InsertNewUgcModel(stream, blueprintId, owner->accountId, owner->id, 0); IPropertyContents::Model model; model.id = modelId; diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index f30e985b3..afd2dc98c 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -1,5 +1,7 @@ #include "SettingsCatalog.h" +#include "UgcIconParams.h" + #include #include #include @@ -418,6 +420,7 @@ namespace { c.Add(Text(UGC, "client_path", "Download path", "The game client's UGCSERVERDIR.", "/ugc", true)); c.Add(Format(Text(UGC, "ugc_output_dir", "Files folder", "Relative to the server binaries.", "ugc", true), eFormat::PATH)); c.Add(Unit(Int(UGC, "ugc_max_storage_mb", "Most space for files", "The files used longest ago are deleted past this and made again when asked for. 0: no limit.", "2048", 0, std::nullopt, true), "MB")); + c.Add(Unit(Int(SHARED, "ugc_debounce_seconds", "Wait after a save", "A saved model is made this long after the owner's last save (their other waiting models wait too), or as soon as a game client asks for it or the owner leaves. 0: right away.", "120", 0, 86400), "seconds")); c.Add(Int(UGC, "worker_threads", "Worker threads", "0: half the CPU cores.", "0", 0, 64, true)); c.Add(Unit(Float(UGC, "max_cpu_percent", "Most CPU", "The workers together average at most this share of all CPU cores; they pause between bits of work to stay under it. 0: no limit.", "0", 0, 100), "%")); c.Add(Int(UGC, "worker_nice", "Worker priority", "Linux nice value of the workers: 0 normal, 19 only when nothing else wants the CPU.", "0", 0, 19)); @@ -447,18 +450,15 @@ namespace { c.Add(Float(UGC, "ao_strength", "Darkening strength", "0 to 1.", "1", 0, 1)); c.Add(Float(UGC, "glow_strength", "Glow strength", "What glowing colors add to the baked light.", "6", 0, 100)); c.Add(Unit(Int(UGC, "icon_size", "Icon size", "", "128", 16, 1024), "pixels")); - c.Add(Unit(Float(UGC, "icon_yaw", "Icon angle around", "", "53.36", -360, 360), "degrees")); - c.Add(Unit(Float(UGC, "icon_pitch", "Icon angle above", "", "19.54", -90, 90), "degrees")); - c.Add(Unit(Float(UGC, "icon_fov", "Icon field of view", "", "39.6", 1, 120), "degrees")); - c.Add(Float(UGC, "icon_margin", "Icon border", "1 fills the icon.", "1.03", 0.1, 10)); - c.Add(Unit(Float(UGC, "icon_sun_yaw", "Icon sun angle around", "", "21", -360, 360), "degrees")); - c.Add(Unit(Float(UGC, "icon_sun_pitch", "Icon sun angle above", "", "50.3", -90, 90), "degrees")); - c.Add(Float(UGC, "icon_sun_strength", "Icon sun strength", "", "2.5", 0, 100)); - c.Add(Float(UGC, "icon_ambient", "Icon world light", "", "0.192", 0, 10)); - c.Add(Bool(UGC, "icon_shadows", "Icon shadows", "", true)); - c.Add(Bool(UGC, "icon_ao", "Car and rocket icon ambient occlusion", "Player models' icons show the lighting baked into their mesh instead.", true)); - c.Add(Unit(Float(UGC, "modular_icon_yaw", "Car and rocket icon angle around", "", "53.36", -360, 360), "degrees")); - c.Add(Unit(Float(UGC, "modular_icon_pitch", "Car and rocket icon angle above", "", "19.54", -90, 90), "degrees")); + // The icon's framing and light, from the one list of them (UgcIconParams); presets and overrides are set on the + // UGC page's icon editor + for (const auto& param : UgcIconParams::List()) { + char number[32]; + const auto written = std::to_chars(number, number + sizeof(number), param.defaultValue); + auto setting = Float(UGC, param.setting, "Icon: " + param.label, param.description, std::string(number, written.ptr), param.min, param.max); + if (!param.unit.empty()) setting = Unit(setting, param.unit); + c.Add(setting); + } return c; } diff --git a/dDashboardServer/routes/UgcRoutes.cpp b/dDashboardServer/routes/UgcRoutes.cpp index 28f370311..b845ac52b 100644 --- a/dDashboardServer/routes/UgcRoutes.cpp +++ b/dDashboardServer/routes/UgcRoutes.cpp @@ -6,9 +6,14 @@ #include +#include "CDClientDatabase.h" #include "Database.h" +#include "UgcIconParams.h" #include "NifFile.h" #include "TtlCache.h" +#include "SettingsCatalog.h" +#include "SettingsHistory.h" +#include "UgcKeys.h" #include "Workers.h" #include "Game.h" #include "GeneralUtils.h" @@ -95,6 +100,111 @@ namespace { return out; } + // Any thread: a POST of JSON to one of the UGC server's /admin routes, with the key it checks (the master password) + std::shared_ptr AdminPost(const std::string& url, const std::string& key, const std::string& body) { + auto out = std::make_shared(); + CURL* curl = curl_easy_init(); + if (!curl) { + out->error = "could not start a request"; + return out; + } + curl_slist* headers = curl_slist_append(nullptr, "Content-Type: application/json"); + headers = curl_slist_append(headers, ("X-Ugc-Admin-Key: " + key).c_str()); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https"); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 120L); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(body.size())); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Collect); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out->body); + const auto code = curl_easy_perform(curl); + if (code == CURLE_OK) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &out->status); + else out->error = curl_easy_strerror(code); + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + return out; + } + + // Web thread: the key the UGC server's /admin routes want + std::string AdminKey() { + const auto master = Database::Get()->GetMasterInfo(); + return master ? master->password : std::string(); + } + + // Sends an admin request from a worker and answers with the UGC server's JSON (or why it couldn't) + void ProxyAdmin(HTTPReply& reply, const HTTPContext& context, const std::string& route, const nlohmann::json& body) { + const auto url = InternalUrl() + route; + Workers::Reply(reply, context, false, [url, key = AdminKey(), text = body.dump()](HTTPReply& out) { + const auto fetched = AdminPost(url, key, text); + if (fetched->status == 0) return JsonError(out, eHTTPStatusCode::BAD_GATEWAY, "The UGC server doesn't answer at " + url + " (" + fetched->error + ")"); + out.status = static_cast(fetched->status); + const bool png = fetched->body.starts_with("\x89PNG"); + out.contentType = png ? eContentType::IMAGE_PNG : eContentType::APPLICATION_JSON; + out.message = fetched->body; + if (!png && fetched->status != 200 && !fetched->body.starts_with("{")) { + JsonError(out, static_cast(fetched->status), fetched->body.empty() ? "The UGC server answered " + std::to_string(fetched->status) : fetched->body); + } + out.headers.push_back("Cache-Control: no-store"); + }, WorkerPool::ePriority::URGENT); + } + + // Web thread: a ugcconfig.ini setting's value as the UGC server sees it (dashboard value, file value, default) + std::optional UgcSetting(const std::string& name) { + if (const auto current = SettingsHistory::Current("ugcconfig.ini", name)) { + if (current->webValue && (current->webWins || !current->fileValue)) return current->webValue; + if (current->fileValue) return current->fileValue; + } + if (const auto* info = SettingsCatalog::Find("ugcconfig.ini", name)) return info->defaultValue; + return std::nullopt; + } + + // Web thread: the icon kinds: player models, and each car or rocket build type in the client's data + // (ModularBuildComponent), named after the object its Assembly LOT is + nlohmann::json IconKinds() { + nlohmann::json kinds = nlohmann::json::array({ { { "kind", UgcIconParams::ModelKind() }, { "label", "Player models" } } }); + auto result = CDClientDatabase::ExecuteQuery("SELECT buildType, xml FROM ModularBuildComponent GROUP BY buildType ORDER BY buildType;"); + while (!result.eof()) { + const auto buildType = result.getIntField("buildType", 0); + std::string label = "Build type " + std::to_string(buildType); + const std::string xml = result.getStringField("xml", ""); + if (const auto at = xml.find("LOT=\""); at != std::string::npos) { + const auto lot = GeneralUtils::TryParse(xml.substr(at + 5, xml.find('"', at + 5) - at - 5)); + if (lot) { + auto name = CDClientDatabase::CreatePreppedStmt("SELECT displayName FROM Objects WHERE id = ? LIMIT 1;"); + name.bind(1, *lot); + auto row = name.execQuery(); + if (!row.eof() && row.getStringField("displayName", "")[0] != '\0') label = row.getStringField("displayName", ""); + } + } + kinds.push_back({ { "kind", UgcIconParams::BuildKind(buildType) }, { "label", label }, { "buildType", buildType } }); + result.nextRow(); + } + return kinds; + } + + // Web thread: a car or rocket's kind, from its first module's build type (ModuleComponent) + std::optional ModularKind(const std::string& modules) { + const auto lots = UgcModularKey::Lots(modules); + if (lots.empty()) return std::nullopt; + auto query = CDClientDatabase::CreatePreppedStmt("SELECT m.buildType FROM ComponentsRegistry cr JOIN ModuleComponent m ON m.id = cr.component_id " + "WHERE cr.id = ? AND cr.component_type = 28 LIMIT 1;"); + query.bind(1, static_cast(lots.front())); + auto row = query.execQuery(); + if (row.eof()) return std::nullopt; + return UgcIconParams::BuildKind(row.getIntField("buildType", 0)); + } + + // Web thread: the values stored for a target, or null + nlohmann::json StoredValues(const std::string& target) { + const auto stored = Database::Get()->GetUgcIconSettings(target); + if (!stored) return nullptr; + return nlohmann::json::parse(UgcIconParams::ToJson(UgcIconParams::Parse(*stored))); + } + // Any thread: Get, through the cache (answers that aren't a file being made are kept) std::shared_ptr CachedGet(const std::string& url, bool status) { { @@ -116,7 +226,7 @@ namespace { JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server answered " + std::to_string(fetched.status)); } - const std::set FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json", + const std::set FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json", "combo.json", "previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json" }; std::optional KindOf(const std::string& text) { @@ -147,12 +257,22 @@ namespace UgcRoutes { const auto size = std::clamp(GeneralUtils::TryParse(QueryValue(context.queryString, "size")).value_or(PAGE_SIZE), 1u, 200u); const auto list = modular ? Database::Get()->GetModularBuildProcessList(state, search, page * size, size + 1) : Database::Get()->GetUgcProcessList(state, search, page * size, size + 1); + // How many builds share each combination of modules (they share one icon) + std::map reuse; + if (modular) { + for (const auto& [ldf, count] : Database::Get()->GetModularBuildConfigCounts()) reuse[UgcModularKey::Normalize(ldf)] += count; + } nlohmann::json items = nlohmann::json::array(); for (size_t i = 0; i < list.size() && i < size; i++) { const auto& info = list[i]; items.push_back({ { "id", std::to_string(info.id) }, { "characterId", std::to_string(info.characterId) }, { "characterName", info.characterName }, { "state", StateName(info.state) }, { "attempts", info.attempts }, { "processedAt", info.processedAt }, { "error", info.error }, - { "bakeAo", info.bakeAo }, { "modules", info.details } }); + { "bakeAo", info.bakeAo }, { "modules", info.details }, { "processAfter", info.processAfter } }); + if (modular) { + const auto key = UgcModularKey::Normalize(info.details); + items.back()["combination"] = key; + items.back()["sharedBy"] = reuse[key]; + } } JsonSuccess(reply, { { "counts", { { "model", Counts(Database::Get()->GetUgcProcessCounts()) }, { "modular", Counts(Database::Get()->GetModularBuildProcessCounts()) } } }, { "items", items }, { "more", list.size() > size }, { "ugcPublicUrl", Game::config->GetValue("ugc_public_url") }, { "canManage", Can(context, "ugc_manage") } }); @@ -231,6 +351,157 @@ namespace UgcRoutes { }); }); + Route(eHTTPMethod::POST, "/api/ugc/cache/delete", Perm("ugc_manage"), + "Delete one item's generated files on the UGC server. Body: {kind: model|modular, id, after: on_demand|now|gone} (made again when asked " + "for, queued now, or left deleted). A car or rocket's files are shared by every build of the same modules", + [](HTTPReply& reply, const HTTPContext& context) { + const auto body = ParseBody(context); + if (!body) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid JSON"); + const auto kind = body->value("kind", std::string("model")) == "modular" ? std::string("modular") : std::string("model"); + const auto id = GeneralUtils::TryParse((*body)["id"].is_string() ? (*body)["id"].get() : (*body)["id"].dump()); + if (!id) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid id"); + const auto after = body->value("after", std::string("now")); + Audit(context, "ugc_delete_files", "Deleted the UGC files of " + kind + " " + std::to_string(*id) + " (" + after + ")"); + ProxyAdmin(reply, context, "/admin/delete", { { "kind", kind }, { "ids", { std::to_string(*id) } }, { "after", after } }); + }); + + Route(eHTTPMethod::POST, "/api/ugc/cache/purge", Perm("ugc_manage"), + "Delete generated files by filter. Body: {kind, state?, owner? (name), olderThanDays?, unusedDays?, all?, confirm (\"PURGE ALL\" with all and no " + "other filter), after: on_demand|now|gone}. State and owner pick the rows here; the ages are the files' own", + [](HTTPReply& reply, const HTTPContext& context) { + const auto body = ParseBody(context); + if (!body) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid JSON"); + const bool modular = body->value("kind", std::string("model")) == "modular"; + const auto state = ParseState(body->value("state", std::string())); + const auto owner = body->value("owner", std::string()).substr(0, 64); + const auto older = body->value("olderThanDays", int64_t{ 0 }); + const auto unused = body->value("unusedDays", int64_t{ 0 }); + nlohmann::json request = { { "kind", modular ? "modular" : "model" }, { "olderThanDays", older }, { "unusedDays", unused }, + { "after", body->value("after", std::string("on_demand")) } }; + std::string what = std::string(modular ? "cars and rockets" : "models"); + if (state || !owner.empty()) { + // The rows the filter picks, from the database + nlohmann::json ids = nlohmann::json::array(); + for (uint32_t page = 0;; page++) { + const auto list = modular ? Database::Get()->GetModularBuildProcessList(state, owner, page * 1000, 1000) : Database::Get()->GetUgcProcessList(state, owner, page * 1000, 1000); + for (const auto& info : list) ids.push_back(std::to_string(info.id)); + if (list.size() < 1000 || ids.size() >= 100000) break; + } + if (ids.empty()) return JsonSuccess(reply, { { "deleted", 0 }, { "bytes", 0 }, { "notes", { "Nothing matches the filter." } } }); + request["ids"] = ids; + what += (state ? " " + body->value("state", std::string()) : "") + (owner.empty() ? "" : " of " + owner); + } else { + if (body->value("all", false) && older == 0 && unused == 0 && body->value("confirm", std::string()) != "PURGE ALL") { + return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Type PURGE ALL to delete every generated file"); + } + if (!body->value("all", false) && older == 0 && unused == 0) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Pick a filter, or all"); + request["all"] = true; + } + if (older > 0) what += ", made more than " + std::to_string(older) + " days ago"; + if (unused > 0) what += ", not asked for in " + std::to_string(unused) + " days"; + Audit(context, "ugc_purge", "Purged the UGC files of " + what + " (" + request["after"].get() + ")"); + ProxyAdmin(reply, context, "/admin/delete", request); + }); + + Route(eHTTPMethod::GET, "/api/ugc/icon/params", Perm("properties_view"), + "What an icon's framing and light can be set to: {params: [{key, setting, label, unit, min, max, step, default, description}], kinds: [{kind, label}]} " + "(player models, and each car or rocket build type in the client's data)", + [](HTTPReply& reply, const HTTPContext&) { + nlohmann::json params = nlohmann::json::array(); + for (const auto& param : UgcIconParams::List()) { + params.push_back({ { "key", param.key }, { "setting", param.setting }, { "label", param.label }, { "unit", param.unit }, { "min", param.min }, + { "max", param.max }, { "step", param.step }, { "default", param.defaultValue }, { "description", param.description } }); + } + JsonSuccess(reply, { { "params", params }, { "kinds", IconKinds() } }); + }); + + Route(eHTTPMethod::GET, "/api/ugc/icon/settings", Perm("properties_view"), + "An item's icon values in their layers: {kind, target, settings (icon_* as the UGC server reads them), preset (the kind's, or null), own (the " + "item's or combination's, or null)}. Query: ?kind=model&id= or ?kind=modular&modules=, or ?preset= for a kind alone", + [](HTTPReply& reply, const HTTPContext& context) { + std::string kind, target; + if (const auto preset = QueryValue(context.queryString, "preset"); !preset.empty()) { + kind = preset; + } else if (QueryValue(context.queryString, "kind") == "modular") { + const auto modules = QueryValue(context.queryString, "modules"); + const auto found = ModularKind(modules); + if (!found) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Those aren't modules"); + kind = *found; + target = UgcIconParams::CombinationTarget(UgcModularKey::Normalize(modules)); + } else { + const auto id = GeneralUtils::TryParse(QueryValue(context.queryString, "id")); + if (!id) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid id"); + kind = UgcIconParams::ModelKind(); + target = UgcIconParams::ModelTarget(*id); + } + nlohmann::json settings = nlohmann::json::object(); + for (const auto& param : UgcIconParams::List()) { + const auto value = UgcSetting(param.setting); + settings[param.key] = std::clamp(value ? GeneralUtils::TryParse(*value).value_or(param.defaultValue) : param.defaultValue, param.min, param.max); + } + JsonSuccess(reply, { { "kind", kind }, { "target", target }, { "settings", settings }, { "preset", StoredValues(UgcIconParams::KindTarget(kind)) }, + { "own", target.empty() ? nlohmann::json(nullptr) : StoredValues(target) } }); + }); + + Route(eHTTPMethod::POST, "/api/ugc/icon/preview", Perm("ugc_manage"), + "An icon drawn by the UGC server with the given values, not stored (PNG). Body: {kind: model, id, values} or {kind: modular, modules, values}", + [](HTTPReply& reply, const HTTPContext& context) { + const auto body = ParseBody(context); + if (!body) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid JSON"); + const bool model = body->value("kind", std::string()) == "model"; + ProxyAdmin(reply, context, "/admin/preview", { { "kind", model ? "model" : "modular" }, { "id", (*body)["id"].is_string() ? (*body)["id"].get() : std::string("0") }, + { "modules", body->value("modules", std::string()) }, { "values", nlohmann::json::parse(UgcIconParams::ToJson(UgcIconParams::Parse(body->value("values", nlohmann::json::object()).dump()))) } }); + }); + + Route(eHTTPMethod::POST, "/api/ugc/icon/save", Perm("ugc_manage"), + "Save icon values as a kind's preset ({scope: kind, kind, values}) or as one item's own ({scope: item, kind: model, id, values} or {scope: item, " + "kind: modular, modules, values}); values null removes it. The icons already made keep theirs until they are drawn again", + [](HTTPReply& reply, const HTTPContext& context) { + const auto body = ParseBody(context); + if (!body) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid JSON"); + std::string target, what; + if (body->value("scope", std::string()) == "kind") { + const auto kind = body->value("kind", std::string()); + bool known = false; + for (const auto& entry : IconKinds()) known = known || entry["kind"] == kind; + if (!known) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Unknown kind"); + target = UgcIconParams::KindTarget(kind); + what = "the " + kind + " icon preset"; + } else if (body->value("kind", std::string()) == "modular") { + const auto key = UgcModularKey::Normalize(body->value("modules", std::string())); + if (key.empty()) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "No modules"); + target = UgcIconParams::CombinationTarget(key); + what = "the icon of the module combination " + key; + } else { + const auto id = GeneralUtils::TryParse((*body)["id"].is_string() ? (*body)["id"].get() : (*body)["id"].dump()); + if (!id) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid id"); + target = UgcIconParams::ModelTarget(*id); + what = "the icon of model " + std::to_string(*id); + } + const auto values = body->value("values", nlohmann::json()); + if (values.is_null()) { + Database::Get()->DeleteUgcIconSettings(target); + Audit(context, "ugc_icon_settings", "Removed the settings of " + what); + return JsonSuccess(reply, { { "message", "Removed; draw the icons again to see it" } }); + } + const auto clean = UgcIconParams::ToJson(UgcIconParams::Parse(values.dump())); + Database::Get()->SetUgcIconSettings(target, clean); + Audit(context, "ugc_icon_settings", "Set " + what + " to " + clean); + JsonSuccess(reply, { { "message", "Saved; draw the icons again to see it in game" } }); + }); + + Route(eHTTPMethod::POST, "/api/ugc/icon/regenerate", Perm("ugc_manage"), + "Have the UGC server draw every stored icon of a kind again (only icons: player models' from their stored .nif). Body: {kind}", + [](HTTPReply& reply, const HTTPContext& context) { + const auto body = ParseBody(context); + const auto kind = body ? body->value("kind", std::string()) : std::string(); + bool known = false; + for (const auto& entry : IconKinds()) known = known || entry["kind"] == kind; + if (!known) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Unknown kind"); + Audit(context, "ugc_regenerate_icons", "Queued every " + kind + " icon to be drawn again"); + ProxyAdmin(reply, context, "/admin/regenerate-icons", { { "kind", kind } }); + }); + Route(eHTTPMethod::POST, "/api/ugc/reprocess", Perm("ugc_manage"), "Have the UGC server make items again. Body: {kind: model|modular, id} for one, {kind, failedOnly: true} for the failed ones, {kind} for all", [](HTTPReply& reply, const HTTPContext& context) { diff --git a/dDashboardServer/static/js/ugc.js b/dDashboardServer/static/js/ugc.js index c1d6df288..22642fe9f 100644 --- a/dDashboardServer/static/js/ugc.js +++ b/dDashboardServer/static/js/ugc.js @@ -41,12 +41,20 @@ var s = STATES[i.state] || [i.state, 'secondary']; return '
' + iconImg(i, 128) + '
' + (i.characterName ? esc(i.characterName) : '' + esc(i.characterId) + '') + '
' + - '
' + esc(i.id) + '
' + fmt.badge(s[0], s[1]) + '
'; + '
' + esc(i.id) + '
' + fmt.badge(s[0], s[1]) + extraBadges(i) + '
'; + } + + // Cars and rockets: how many builds share this icon; waiting models: when their quiet period after a save ends + function extraBadges(i) { + var out = ''; + if (i.sharedBy > 1) out += ' ' + fmt.badge('shared by ' + i.sharedBy, 'info'); + if (i.state === 'pending' && i.processAfter > Date.now() / 1000) out += ' after ' + esc(fmt.unix(i.processAfter)) + ''; + return out; } function row(i) { var s = STATES[i.state] || [i.state, 'secondary']; - var details = kind === 'modular' ? '' + esc(i.modules) + '' : ''; + var details = (kind === 'modular' ? '' + esc(i.modules) + '' : '') + extraBadges(i); if (i.error) details += '
' + esc(i.error) + '
'; var buttons = '' + (canManage ? '' : ''); @@ -64,6 +72,7 @@ canManage = d.canManage; publicUrl = (d.ugcPublicUrl || '').replace(/\/+$/, ''); $('manageButtons').classList.toggle('d-none', !canManage); + $('cacheCard').classList.toggle('d-none', !canManage); $('counts').innerHTML = countCard('Models', d.counts.model) + countCard('Cars and rockets', d.counts.modular); $('gallery').classList.toggle('d-none', view !== 'gallery'); $('listCard').classList.toggle('d-none', view !== 'list'); @@ -174,12 +183,20 @@ 'Download the LXFML
'; } if (publicUrl) links += 'Open on the UGC server'; - if (canManage) links += '
'; + if (canManage) { + links += '
' + + '' + + '
'; + if (item.sharedBy > 1) links += '
This icon is shared by ' + esc(item.sharedBy) + ' builds of the same modules.
'; + } if (item.error) links += '
' + esc(item.error) + '
'; $('previewLinks').innerHTML = links; var model = kind === 'model'; ['nifColumn', 'lxfmlColumn', 'meshControls'].forEach(function (x) { $(x).classList.toggle('d-none', !model); }); - $('statsBox').innerHTML = model ? '' : '
Modules: ' + esc(item.modules || '') + '. The game client puts cars and rockets together itself; only the icon is made.
'; + $('statsBox').innerHTML = model ? '' : '
Modules: ' + esc(item.modules || '') + ' (combination ' + esc(item.combination || '') + + '). The game client puts cars and rockets together itself; only the icon is made, once per combination of modules.
'; + $('framingCard').classList.toggle('d-none', model || !canManage); + if (!model && canManage) openFraming(item); bootstrap.Modal.getOrCreateInstance($('previewModal')).show(); if (!model) return; @@ -267,6 +284,135 @@ if (lxfmlViewer && lxfmlViewer.dispose) lxfmlViewer.dispose(); lxfmlViewer = null; }); + // ---- deleting and purging stored files ---- + + function showDeleteResult(box, d) { + if (!d.success) { toast(d.error || 'Failed', 'danger'); return; } + var text = d.deleted + ' deleted (' + mb(d.bytes || 0) + ' freed). ' + (d.notes || []).join(' '); + if (box) box.textContent = text; + toast(text, 'success'); + load(); + } + + $('previewLinks').addEventListener('click', function (e) { + var button = e.target.closest('[data-delete-open]'); + if (!button || !current) return; + var after = button.dataset.deleteOpen; + if (!confirm('Delete the files made for ' + current.id + (after === 'gone' ? ' and leave them deleted?' : ' and make them again?'))) return; + api.post('/api/ugc/cache/delete', { kind: current.kind, id: current.id, after: after }).then(function (d) { showDeleteResult(null, d); }); + }); + function purge(all) { + var body = { kind: kind, state: $('purgeState').value, owner: $('purgeOwner').value.trim(), olderThanDays: +$('purgeOlder').value || 0, + unusedDays: +$('purgeUnused').value || 0, after: $('purgeAfter').value }; + if (all) { + var typed = prompt('This deletes every stored ' + (kind === 'model' ? 'model' : 'car and rocket') + ' file. Type PURGE ALL to go ahead.'); + if (typed !== 'PURGE ALL') return; + body = { kind: kind, all: true, confirm: typed, after: $('purgeAfter').value }; + } else if (!body.state && !body.owner && !body.olderThanDays && !body.unusedDays) { + toast('Pick a filter first, or purge all', 'warning'); + return; + } else if (!confirm('Delete the stored files of every ' + (kind === 'model' ? 'model' : 'car and rocket') + ' matching the filter?')) { + return; + } + $('purgeResult').textContent = 'Deleting…'; + api.post('/api/ugc/cache/purge', body).then(function (d) { showDeleteResult($('purgeResult'), d); }); + } + $('purgeButton').addEventListener('click', function () { purge(false); }); + $('purgeAllButton').addEventListener('click', function () { purge(true); }); + + // ---- car and rocket icon framing ---- + + var FRAMING = [ + ['yaw', 'Angle around', -180, 180, 1], ['pitch', 'Angle above', -89, 89, 1], ['fov', 'Field of view', 5, 90, 0.5], ['margin', 'Border (zoom)', 0.5, 3, 0.01], + ['offsetX', 'Shift right', -0.5, 0.5, 0.01], ['offsetY', 'Shift up', -0.5, 0.5, 0.01], ['sunYaw', 'Sun around', -180, 180, 1], ['sunPitch', 'Sun above', -10, 90, 1] + ]; + var framingItem = null, typeFraming = {}, previewTimer = null, previewUrl = null; + $('framingControls').innerHTML = FRAMING.map(function (f) { + return '
' + + '
'; + }).join(''); + function framingValues() { + var out = {}; + FRAMING.forEach(function (f) { out[f[0]] = parseFloat($('framing_' + f[0]).value); }); + return out; + } + function setFraming(values) { + FRAMING.forEach(function (f) { + if (values[f[0]] === undefined || values[f[0]] === null) return; + $('framing_' + f[0]).value = values[f[0]]; + $('framing_' + f[0] + '_value').textContent = (+values[f[0]]).toFixed(f[4] < 1 ? 2 : 0); + }); + } + function loadFramingType(useCombination) { + if (!framingItem) return; + var type = $('framingType').value; + return api.get('/api/ugc/framing?buildType=' + type + '&modules=' + encodeURIComponent(framingItem.modules || '')).then(function (d) { + if (!d.success) return; + typeFraming = d.type; + setFraming(d.type); + if (useCombination && d.combination) setFraming(d.combination); + $('framingState').textContent = d.combination && useCombination ? 'This combination has its own framing.' : 'The ' + d.name + ' framing.'; + renderPreview(); + }); + } + function openFraming(item) { + framingItem = item; + // The build type from what the UGC server stored with the icon, when it has + fetchJson(fileUrl('modular', item.id, 'combo.json')).then(function (combo) { + if (combo && (combo.buildType === 3 || combo.buildType === 6)) $('framingType').value = String(combo.buildType); + loadFramingType(true); + }); + } + function renderPreview() { + if (!framingItem) return; + var img = $('framingPreview'); + img.style.opacity = 0.5; + fetch('/api/ugc/framing/preview', { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest' }, + body: JSON.stringify({ modules: framingItem.modules, framing: framingValues() }) }).then(function (r) { + if (!r.ok) return r.json().then(function (d) { throw new Error(d.error || 'HTTP ' + r.status); }); + return r.blob(); + }).then(function (blob) { + if (previewUrl) URL.revokeObjectURL(previewUrl); + previewUrl = URL.createObjectURL(blob); + img.src = previewUrl; + img.style.opacity = 1; + }).catch(function (e) { img.style.opacity = 1; $('framingState').textContent = 'No preview: ' + e.message; }); + } + $('framingControls').addEventListener('input', function (e) { + var f = FRAMING.find(function (x) { return 'framing_' + x[0] === e.target.id; }); + if (f) $('framing_' + f[0] + '_value').textContent = (+e.target.value).toFixed(f[4] < 1 ? 2 : 0); + clearTimeout(previewTimer); + previewTimer = setTimeout(renderPreview, 350); + }); + $('framingType').addEventListener('change', function () { loadFramingType(false); }); + $('framingPreviewButton').addEventListener('click', renderPreview); + $('framingReset').addEventListener('click', function () { + if (!framingItem) return; + if (!confirm('Go back to the type\'s framing? This combination\'s own framing, if it has one, is removed.')) return; + api.post('/api/ugc/framing/save', { scope: 'combination', modules: framingItem.modules, framing: null }).then(function () { loadFramingType(false); }); + }); + $('framingSaveType').addEventListener('click', function () { + var name = $('framingType').selectedOptions[0].textContent.toLowerCase(); + if (!confirm('Use this framing for every ' + name + ' icon? Icons already made keep theirs until they are drawn again.')) return; + api.post('/api/ugc/framing/save', { scope: 'type', buildType: +$('framingType').value, framing: framingValues() }).then(function (d) { + toast(d.success ? d.message : (d.error || 'Failed'), d.success ? 'success' : 'danger'); + }); + }); + $('framingSaveCombo').addEventListener('click', function () { + if (!framingItem) return; + api.post('/api/ugc/framing/save', { scope: 'combination', modules: framingItem.modules, framing: framingValues() }).then(function (d) { + toast(d.success ? d.message : (d.error || 'Failed'), d.success ? 'success' : 'danger'); + if (d.success) $('framingState').textContent = 'This combination has its own framing.'; + }); + }); + $('framingRegenerate').addEventListener('click', function () { + var name = $('framingType').selectedOptions[0].textContent.toLowerCase(); + if (!confirm('Draw every stored ' + name + ' icon again with the saved framings?')) return; + api.post('/api/ugc/framing/regenerate', { buildType: +$('framingType').value }).then(function (d) { + toast(d.success ? d.queued + ' icon' + (d.queued === 1 ? '' : 's') + ' queued' : (d.error || 'Failed'), d.success ? 'success' : 'danger'); + }); + }); + if (window.Live) Live.on('ugc', load); load(); diff --git a/dDashboardServer/templates/ugc.jinja2 b/dDashboardServer/templates/ugc.jinja2 index d5713f6d1..02e17e4b9 100644 --- a/dDashboardServer/templates/ugc.jinja2 +++ b/dDashboardServer/templates/ugc.jinja2 @@ -17,6 +17,24 @@
Asking the UGC server…
+
+ Stored files: delete and purge +
+

Deletes files the UGC server made, of the kind picked below. Afterwards they are made again when a game client + asks for them, queued to be made now, or left deleted. A car or rocket's icon is shared by every build of the same modules.

+
+ + + + + + + +
+
+
+
+
@@ -90,6 +108,24 @@
+
+
Icon framing +
+
+
+
+
Preview +
Preview drawn by the UGC server, not stored.
+
+
+ + + + + +
+
+
diff --git a/dDatabase/GameDatabase/ITables/IPropertyContents.h b/dDatabase/GameDatabase/ITables/IPropertyContents.h index ee868c2a2..e92827f7f 100644 --- a/dDatabase/GameDatabase/ITables/IPropertyContents.h +++ b/dDatabase/GameDatabase/ITables/IPropertyContents.h @@ -25,7 +25,10 @@ public: std::stringstream& sd0Data, const uint64_t blueprintId, const uint32_t accountId, - const LWOOBJID characterId) = 0; + const LWOOBJID characterId, + // Unix seconds before which the UGC server doesn't make it (0: right away). The owner's other models waiting + // to be made wait until then too (they may be edited again). + const int64_t processAfter) = 0; // Get the property models for the given property id. virtual std::vector GetPropertyModels(const LWOOBJID& propertyId) = 0; diff --git a/dDatabase/GameDatabase/ITables/IUgc.h b/dDatabase/GameDatabase/ITables/IUgc.h index 062a4f267..5e0e4c21c 100644 --- a/dDatabase/GameDatabase/ITables/IUgc.h +++ b/dDatabase/GameDatabase/ITables/IUgc.h @@ -29,7 +29,7 @@ public: virtual void DeleteUgcModelData(const LWOOBJID& modelId) = 0; // Inserts a new UGC model into the database. - virtual void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) = 0; + virtual void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) = 0; virtual std::optional GetUgcModel(const LWOOBJID ugcId) = 0; @@ -53,6 +53,7 @@ public: std::string error; bool bakeAo{}; std::string details; // modular builds: the modules (ldf_config) + int64_t processAfter{}; // models: Unix seconds before which it isn't made (the quiet period after a save) }; // A model waiting to be made: its id, stored LXFML (sd0) and the attempts so far @@ -62,9 +63,14 @@ public: uint32_t attempts{}; }; - // Up to `limit` pending models, the least tried and then the newest first + // Up to `limit` pending models whose quiet period is over (process_after), the least tried and then the newest first virtual std::vector GetUgcModelsToProcess(const uint32_t limit) = 0; + // Ends the quiet period of a waiting model (a client asked for it), or of all a character's waiting models (they + // left the property) + virtual void ExpediteUgcModel(const LWOOBJID id) = 0; + virtual void ExpediteUgcModels(const LWOOBJID characterId) = 0; + // Records an attempt: the new state, how many attempts there have been, why it failed (empty when it didn't) and // whether lighting was baked in; processed_at becomes now virtual void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) = 0; diff --git a/dDatabase/GameDatabase/ITables/IUgcModularBuild.h b/dDatabase/GameDatabase/ITables/IUgcModularBuild.h index 698b66c3b..f48871a27 100644 --- a/dDatabase/GameDatabase/ITables/IUgcModularBuild.h +++ b/dDatabase/GameDatabase/ITables/IUgcModularBuild.h @@ -28,6 +28,15 @@ public: virtual uint64_t ResetModularBuildProcessing(const std::optional id, const bool failedOnly) = 0; virtual std::vector GetModularBuildProcessList(const std::optional state, const std::string_view search, const uint32_t offset, const uint32_t limit) = 0; virtual std::vector> GetModularBuildProcessCounts() = 0; + + // How many builds there are of each ldf_config as stored (the same modules may be written differently) + virtual std::vector> GetModularBuildConfigCounts() = 0; + + // Icon settings (UgcIconParams JSON) set on the dashboard for a target: "kind:" (a preset for player models or + // a car or rocket build type), "model:" or "combo:" (one item's or combination's own) + virtual std::optional GetUgcIconSettings(const std::string_view target) = 0; + virtual void SetUgcIconSettings(const std::string_view target, const std::string_view params) = 0; + virtual void DeleteUgcIconSettings(const std::string_view target) = 0; }; #endif //!IUGCMODULARBUILD_H diff --git a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h index 072130bcf..350e703ee 100644 --- a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h +++ b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h @@ -63,7 +63,7 @@ public: void RemoveFriend(const LWOOBJID playerAccountId, const LWOOBJID friendAccountId) override; void UpdateActivityLog(const LWOOBJID characterId, const eActivityType activityType, const LWOMAPID mapId) override; void DeleteUgcModelData(const LWOOBJID& modelId) override; - void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) override; + void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) override; std::vector GetAllUgcModels() override; void CreateMigrationHistoryTable() override; bool IsMigrationRun(const std::string_view str) override; @@ -356,7 +356,8 @@ public: std::stringstream& sd0Data, const uint64_t blueprintId, const uint32_t accountId, - const LWOOBJID characterId) override; + const LWOOBJID characterId, + const int64_t processAfter) override; std::vector GetMailForPlayer(const LWOOBJID characterId, const uint32_t numberOfMail) override; std::optional GetMail(const uint64_t mailId) override; uint32_t GetUnreadMailCount(const LWOOBJID characterId) override; @@ -419,6 +420,12 @@ public: std::optional GetModel(const LWOOBJID modelID) override; std::optional GetUgcModel(const LWOOBJID ugcId) override; std::vector GetUgcModelsToProcess(const uint32_t limit) override; + void ExpediteUgcModel(const LWOOBJID id) override; + void ExpediteUgcModels(const LWOOBJID characterId) override; + std::vector> GetModularBuildConfigCounts() override; + std::optional GetUgcIconSettings(const std::string_view target) override; + void SetUgcIconSettings(const std::string_view target, const std::string_view params) override; + void DeleteUgcIconSettings(const std::string_view target) override; void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override; std::optional GetUgcProcessInfo(const LWOOBJID id) override; uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override; diff --git a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp index dbd56a34e..499a252ec 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp @@ -13,7 +13,8 @@ namespace { info.processedAt = result->getInt64("processed_at"); info.error = std::string(result->getString("process_error").c_str()); if (modular) info.details = std::string(result->getString("ldf_config").c_str()); - else info.bakeAo = result->getInt("bake_ao") != 0; + else info.processAfter = result->getInt64("process_after"); + if (!modular) info.bakeAo = result->getInt("bake_ao") != 0; return info; } @@ -68,18 +69,22 @@ void MySQLDatabase::InsertNewUgcModel( std::stringstream& sd0Data, // cant be const sad const uint64_t blueprintId, const uint32_t accountId, - const LWOOBJID characterId) { + const LWOOBJID characterId, + const int64_t processAfter) { const std::istream stream(sd0Data.rdbuf()); ExecuteInsert( - "INSERT INTO `ugc`(`id`, `account_id`, `character_id`, `is_optimized`, `lxfml`, `bake_ao`, `filename`) VALUES (?,?,?,?,?,?,?)", + "INSERT INTO `ugc`(`id`, `account_id`, `character_id`, `is_optimized`, `lxfml`, `bake_ao`, `filename`, `process_after`) VALUES (?,?,?,?,?,?,?,?)", blueprintId, accountId, characterId, 0, &stream, false, - "weedeater.lxfml" + "weedeater.lxfml", + processAfter ); + // The owner is still building: their other models waiting for their quiet period wait longer + if (processAfter > 0) ExecuteUpdate("UPDATE ugc SET process_after = ? WHERE character_id = ? AND is_optimized = 0 AND process_after > 0 AND process_after < ?;", processAfter, characterId, processAfter); } void MySQLDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { @@ -87,10 +92,10 @@ void MySQLDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { ExecuteDelete("DELETE FROM properties_contents WHERE ugc_id = ?;", modelId); } -void MySQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) { +void MySQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) { const std::istream stream(lxfml.rdbuf()); // The UGC server makes the model's files again - ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", &stream, modelId); + ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '', process_after = ? WHERE id = ?;", &stream, processAfter, modelId); } std::optional MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) { @@ -106,7 +111,7 @@ std::optional MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) { } std::vector MySQLDatabase::GetUgcModelsToProcess(const uint32_t limit) { - auto result = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 ORDER BY process_attempts ASC, id DESC LIMIT ?;", limit); + auto result = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 AND process_after <= ? ORDER BY process_attempts ASC, id DESC LIMIT ?;", UnixNow(), limit); std::vector models; while (result->next()) { auto& model = models.emplace_back(); @@ -127,21 +132,21 @@ void MySQLDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState std::optional MySQLDatabase::GetUgcProcessInfo(const LWOOBJID id) { auto result = ExecuteSelect( - "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao " + "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao, u.process_after " "FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id WHERE u.id = ? LIMIT 1;", id); if (!result->next()) return std::nullopt; return ReadUgcProcessInfo(result, false); } uint64_t MySQLDatabase::ResetUgcModelProcessing(const std::optional id, const bool failedOnly) { - if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", *id); - if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;"); - return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '';"); + if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 WHERE id = ?;", *id); + if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 WHERE is_optimized = 2;"); + return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0;"); } std::vector MySQLDatabase::GetUgcProcessList(const std::optional state, const std::string_view search, const uint32_t offset, const uint32_t limit) { const std::string select = - "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao " + "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao, u.process_after " "FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id "; std::vector list; // Every filter always bound, off when its first value says so @@ -164,3 +169,11 @@ std::vector> MySQLDatabase::GetUgcProce } return counts; } + +void MySQLDatabase::ExpediteUgcModel(const LWOOBJID id) { + ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE id = ? AND is_optimized = 0 AND process_after > 0;", id); +} + +void MySQLDatabase::ExpediteUgcModels(const LWOOBJID characterId) { + ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE character_id = ? AND is_optimized = 0 AND process_after > 0;", characterId); +} diff --git a/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp b/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp index 2c82cc02e..508a221af 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp @@ -84,3 +84,25 @@ std::vector> MySQLDatabase::GetModularB } return counts; } + +std::vector> MySQLDatabase::GetModularBuildConfigCounts() { + auto result = ExecuteSelect("SELECT ldf_config, COUNT(*) AS count FROM ugc_modular_build GROUP BY ldf_config;"); + std::vector> counts; + while (result->next()) counts.emplace_back(result->getString("ldf_config").c_str(), static_cast(result->getInt64("count"))); + return counts; +} + +std::optional MySQLDatabase::GetUgcIconSettings(const std::string_view target) { + auto result = ExecuteSelect("SELECT params FROM ugc_icon_settings WHERE target = ? LIMIT 1;", target); + if (!result->next()) return std::nullopt; + return std::string(result->getString("params").c_str()); +} + +void MySQLDatabase::SetUgcIconSettings(const std::string_view target, const std::string_view params) { + ExecuteInsert("INSERT INTO ugc_icon_settings (target, params, updated_at) VALUES (?, ?, ?) " + "ON DUPLICATE KEY UPDATE params = VALUES(params), updated_at = VALUES(updated_at);", target, params, UnixNow()); +} + +void MySQLDatabase::DeleteUgcIconSettings(const std::string_view target) { + ExecuteDelete("DELETE FROM ugc_icon_settings WHERE target = ?;", target); +} diff --git a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h index bd2904e0b..3dd468ec6 100644 --- a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h +++ b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h @@ -47,7 +47,7 @@ public: void RemoveFriend(const LWOOBJID playerAccountId, const LWOOBJID friendAccountId) override; void UpdateActivityLog(const LWOOBJID characterId, const eActivityType activityType, const LWOMAPID mapId) override; void DeleteUgcModelData(const LWOOBJID& modelId) override; - void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) override; + void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) override; std::vector GetAllUgcModels() override; void CreateMigrationHistoryTable() override; bool IsMigrationRun(const std::string_view str) override; @@ -340,7 +340,8 @@ public: std::stringstream& sd0Data, const uint64_t blueprintId, const uint32_t accountId, - const LWOOBJID characterId) override; + const LWOOBJID characterId, + const int64_t processAfter) override; std::vector GetMailForPlayer(const LWOOBJID characterId, const uint32_t numberOfMail) override; std::optional GetMail(const uint64_t mailId) override; uint32_t GetUnreadMailCount(const LWOOBJID characterId) override; @@ -403,6 +404,12 @@ public: std::optional GetModel(const LWOOBJID modelID) override; std::optional GetUgcModel(const LWOOBJID ugcId) override; std::vector GetUgcModelsToProcess(const uint32_t limit) override; + void ExpediteUgcModel(const LWOOBJID id) override; + void ExpediteUgcModels(const LWOOBJID characterId) override; + std::vector> GetModularBuildConfigCounts() override; + std::optional GetUgcIconSettings(const std::string_view target) override; + void SetUgcIconSettings(const std::string_view target, const std::string_view params) override; + void DeleteUgcIconSettings(const std::string_view target) override; void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override; std::optional GetUgcProcessInfo(const LWOOBJID id) override; uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override; diff --git a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp index 6a0682fa0..e037ca0c5 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp @@ -13,7 +13,8 @@ namespace { info.processedAt = result.getInt64Field("processed_at"); info.error = result.getStringField("process_error", ""); if (modular) info.details = result.getStringField("ldf_config", ""); - else info.bakeAo = result.getIntField("bake_ao") != 0; + else info.processAfter = result.getInt64Field("process_after"); + if (!modular) info.bakeAo = result.getIntField("bake_ao") != 0; return info; } @@ -69,18 +70,22 @@ void SQLiteDatabase::InsertNewUgcModel( std::stringstream& sd0Data, // cant be const sad const uint64_t blueprintId, const uint32_t accountId, - const LWOOBJID characterId) { + const LWOOBJID characterId, + const int64_t processAfter) { const std::istream stream(sd0Data.rdbuf()); ExecuteInsert( - "INSERT INTO `ugc`(`id`, `account_id`, `character_id`, `is_optimized`, `lxfml`, `bake_ao`, `filename`) VALUES (?,?,?,?,?,?,?)", + "INSERT INTO `ugc`(`id`, `account_id`, `character_id`, `is_optimized`, `lxfml`, `bake_ao`, `filename`, `process_after`) VALUES (?,?,?,?,?,?,?,?)", blueprintId, accountId, characterId, 0, &stream, false, - "weedeater.lxfml" + "weedeater.lxfml", + processAfter ); + // The owner is still building: their other models waiting for their quiet period wait longer + if (processAfter > 0) ExecuteUpdate("UPDATE ugc SET process_after = ? WHERE character_id = ? AND is_optimized = 0 AND process_after > 0 AND process_after < ?;", processAfter, characterId, processAfter); } void SQLiteDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { @@ -88,10 +93,10 @@ void SQLiteDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { ExecuteDelete("DELETE FROM properties_contents WHERE ugc_id = ?;", modelId); } -void SQLiteDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) { +void SQLiteDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) { const std::istream stream(lxfml.rdbuf()); // The UGC server makes the model's files again - ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", &stream, modelId); + ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '', process_after = ? WHERE id = ?;", &stream, processAfter, modelId); } std::optional SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) { @@ -107,7 +112,7 @@ std::optional SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) { } std::vector SQLiteDatabase::GetUgcModelsToProcess(const uint32_t limit) { - auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 ORDER BY process_attempts ASC, id DESC LIMIT ?;", limit); + auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 AND process_after <= ? ORDER BY process_attempts ASC, id DESC LIMIT ?;", UnixNow(), limit); std::vector models; while (!result.eof()) { auto& model = models.emplace_back(); @@ -128,21 +133,21 @@ void SQLiteDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState std::optional SQLiteDatabase::GetUgcProcessInfo(const LWOOBJID id) { auto [_, result] = ExecuteSelect( - "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao " + "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao, u.process_after " "FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id WHERE u.id = ? LIMIT 1;", id); if (result.eof()) return std::nullopt; return ReadUgcProcessInfo(result, false); } uint64_t SQLiteDatabase::ResetUgcModelProcessing(const std::optional id, const bool failedOnly) { - if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", *id); - if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;"); - return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '';"); + if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 WHERE id = ?;", *id); + if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 WHERE is_optimized = 2;"); + return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0;"); } std::vector SQLiteDatabase::GetUgcProcessList(const std::optional state, const std::string_view search, const uint32_t offset, const uint32_t limit) { const std::string select = - "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao " + "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao, u.process_after " "FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id "; std::vector list; // Every filter always bound, off when its first value says so @@ -167,3 +172,11 @@ std::vector> SQLiteDatabase::GetUgcProc } return counts; } + +void SQLiteDatabase::ExpediteUgcModel(const LWOOBJID id) { + ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE id = ? AND is_optimized = 0 AND process_after > 0;", id); +} + +void SQLiteDatabase::ExpediteUgcModels(const LWOOBJID characterId) { + ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE character_id = ? AND is_optimized = 0 AND process_after > 0;", characterId); +} diff --git a/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp b/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp index 348e547b4..3a4490fb6 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp @@ -87,3 +87,28 @@ std::vector> SQLiteDatabase::GetModular } return counts; } + +std::vector> SQLiteDatabase::GetModularBuildConfigCounts() { + auto [_, result] = ExecuteSelect("SELECT ldf_config, COUNT(*) AS count FROM ugc_modular_build GROUP BY ldf_config;"); + std::vector> counts; + while (!result.eof()) { + counts.emplace_back(result.getStringField("ldf_config", ""), static_cast(result.getInt64Field("count"))); + result.nextRow(); + } + return counts; +} + +std::optional SQLiteDatabase::GetUgcIconSettings(const std::string_view target) { + auto [_, result] = ExecuteSelect("SELECT params FROM ugc_icon_settings WHERE target = ? LIMIT 1;", target); + if (result.eof()) return std::nullopt; + return std::string(result.getStringField("params", "")); +} + +void SQLiteDatabase::SetUgcIconSettings(const std::string_view target, const std::string_view params) { + ExecuteInsert("INSERT INTO ugc_icon_settings (target, params, updated_at) VALUES (?, ?, ?) " + "ON CONFLICT(target) DO UPDATE SET params = excluded.params, updated_at = excluded.updated_at;", target, params, UnixNow()); +} + +void SQLiteDatabase::DeleteUgcIconSettings(const std::string_view target) { + ExecuteDelete("DELETE FROM ugc_icon_settings WHERE target = ?;", target); +} diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.cpp b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.cpp index 8e2e6dc84..6e993a4fa 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.cpp +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.cpp @@ -60,7 +60,7 @@ void TestSQLDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { } -void TestSQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) { +void TestSQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) { } @@ -192,7 +192,7 @@ void TestSQLDatabase::InsertNewMail(const MailInfo& mail) { } -void TestSQLDatabase::InsertNewUgcModel(std::stringstream& sd0Data, const uint64_t blueprintId, const uint32_t accountId, const LWOOBJID characterId) { +void TestSQLDatabase::InsertNewUgcModel(std::stringstream& sd0Data, const uint64_t blueprintId, const uint32_t accountId, const LWOOBJID characterId, const int64_t processAfter) { } diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h index 40ee2ee9c..57d0757c9 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h @@ -27,7 +27,7 @@ class TestSQLDatabase : public GameDatabase { void RemoveFriend(const LWOOBJID playerAccountId, const LWOOBJID friendAccountId) override; void UpdateActivityLog(const LWOOBJID characterId, const eActivityType activityType, const LWOMAPID mapId) override; void DeleteUgcModelData(const LWOOBJID& modelId) override; - void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) override; + void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) override; std::vector GetAllUgcModels() override; void CreateMigrationHistoryTable() override; bool IsMigrationRun(const std::string_view str) override; @@ -67,7 +67,8 @@ class TestSQLDatabase : public GameDatabase { std::stringstream& sd0Data, const uint64_t blueprintId, const uint32_t accountId, - const LWOOBJID characterId) override; + const LWOOBJID characterId, + const int64_t processAfter) override; std::vector GetMailForPlayer(const LWOOBJID characterId, const uint32_t numberOfMail) override; std::optional GetMail(const uint64_t mailId) override; uint32_t GetUnreadMailCount(const LWOOBJID characterId) override; @@ -401,6 +402,12 @@ class TestSQLDatabase : public GameDatabase { std::optional GetPropertyInfo(const LWOOBJID id) override { return {}; } std::optional GetUgcModel(const LWOOBJID ugcId) override { return {}; } std::vector GetUgcModelsToProcess(const uint32_t limit) override { return {}; } + void ExpediteUgcModel(const LWOOBJID id) override {} + void ExpediteUgcModels(const LWOOBJID characterId) override {} + std::vector> GetModularBuildConfigCounts() override { return {}; } + std::optional GetUgcIconSettings(const std::string_view target) override { return std::nullopt; } + void SetUgcIconSettings(const std::string_view target, const std::string_view params) override {} + void DeleteUgcIconSettings(const std::string_view target) override {} void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override {} std::optional GetUgcProcessInfo(const LWOOBJID id) override { return {}; } uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override { return 0; } diff --git a/dDatabase/ModelNormalizeMigration.cpp b/dDatabase/ModelNormalizeMigration.cpp index 45fcdb82c..100f2ef19 100644 --- a/dDatabase/ModelNormalizeMigration.cpp +++ b/dDatabase/ModelNormalizeMigration.cpp @@ -24,7 +24,7 @@ void ModelNormalizeMigration::Run() { sd0.FromData(reinterpret_cast(newLxfml.data()), newLxfml.size()); auto asStream = sd0.GetAsStream(); Database::Get()->UpdateModel(model->id, newCenter, model->rotation, model->behaviors); - Database::Get()->UpdateUgcModelData(id, asStream); + Database::Get()->UpdateUgcModelData(id, asStream, 0); } Database::Get()->SetAutoCommit(oldCommit); } @@ -44,7 +44,7 @@ void ModelNormalizeMigration::RunAfterFirstPart() { sd0.FromData(reinterpret_cast(newLxfml.data()), newLxfml.size()); auto asStream = sd0.GetAsStream(); Database::Get()->UpdateModel(model->id, newCenter, model->rotation, model->behaviors); - Database::Get()->UpdateUgcModelData(id, asStream); + Database::Get()->UpdateUgcModelData(id, asStream, 0); } Database::Get()->SetAutoCommit(oldCommit); } @@ -65,7 +65,7 @@ void ModelNormalizeMigration::RunBrickBuildGrid() { LOG("Updated model %llu to have a center of %f %f %f", modelID, newCenter.x, newCenter.y, newCenter.z); auto asStream = sd0.GetAsStream(); Database::Get()->UpdateModel(model->id, newCenter, model->rotation, model->behaviors); - Database::Get()->UpdateUgcModelData(id, asStream); + Database::Get()->UpdateUgcModelData(id, asStream, 0); } Database::Get()->SetAutoCommit(oldCommit); } diff --git a/dGame/dUtilities/BrickByBrick.cpp b/dGame/dUtilities/BrickByBrick.cpp index 48ad65a6b..a1a7f68ef 100644 --- a/dGame/dUtilities/BrickByBrick.cpp +++ b/dGame/dUtilities/BrickByBrick.cpp @@ -23,11 +23,13 @@ #include "PropertyManagementComponent.h" #include "PropertyMessages.h" #include "Sd0.h" +#include "UgcKeys.h" #include "User.h" #include #include #include +#include #include #include #include @@ -107,12 +109,16 @@ namespace { outFile << lxfml; } + // A quiet period before the UGC server makes the models (ugc_debounce_seconds, sharedconfig.ini): a model + // the owner keeps editing isn't made for every save (docs/UgcServer.md) + const auto debounce = Game::config ? GeneralUtils::TryParse(Game::config->GetValue("ugc_debounce_seconds")).value_or(120) : 120; + const auto processAfter = UgcDebounce::ProcessAfter(std::time(nullptr), debounce); for (const auto& part : Lxfml::Split(lxfml)) { const auto [modelId, blueprintId] = ObjectIDManager::GetNewModelIDs(); Sd0 model = sd0; model.FromData(reinterpret_cast(part.lxfml.data()), part.lxfml.size()); auto stream = model.GetAsStream(); - Database::Get()->InsertNewUgcModel(stream, blueprintId, character->GetParentUser()->GetAccountID(), character->GetID()); + Database::Get()->InsertNewUgcModel(stream, blueprintId, character->GetParentUser()->GetAccountID(), character->GetID(), processAfter); auto& entry = saved.emplace_back(); entry.modelId = modelId; diff --git a/dUgcServer/CMakeLists.txt b/dUgcServer/CMakeLists.txt index 780b5dca7..5f67721b3 100644 --- a/dUgcServer/CMakeLists.txt +++ b/dUgcServer/CMakeLists.txt @@ -2,6 +2,7 @@ set(DUGC_SOURCES "UgcBricks.cpp" "UgcFormats.cpp" + "UgcIconParams.cpp" "UgcJobs.cpp" "UgcModel.cpp" "UgcModular.cpp" diff --git a/dUgcServer/UgcCdClient.cpp b/dUgcServer/UgcCdClient.cpp index 4c53ece1c..f89246e74 100644 --- a/dUgcServer/UgcCdClient.cpp +++ b/dUgcServer/UgcCdClient.cpp @@ -34,6 +34,7 @@ namespace UgcCdClient { module.partCode = static_cast(moduleRow.getIntField("partCode", 0)); module.moduleXml = moduleRow.getStringField("xml", ""); if (!buildType) buildType = moduleRow.getIntField("buildType", 0); + input.buildType = *buildType; auto renderQuery = CDClientDatabase::CreatePreppedStmt( "SELECT rc.render_asset FROM ComponentsRegistry cr JOIN RenderComponent rc ON rc.id = cr.component_id " diff --git a/dUgcServer/UgcIconParams.cpp b/dUgcServer/UgcIconParams.cpp new file mode 100644 index 000000000..ef102a17b --- /dev/null +++ b/dUgcServer/UgcIconParams.cpp @@ -0,0 +1,88 @@ +#include "UgcIconParams.h" + +#include +#include + +#include "GeneralUtils.h" +#include "json.hpp" + +namespace { + using Options = UgcRender::IconOptions; +} + +namespace UgcIconParams { + const std::vector& List() { + // Framing from LU Toolbox's icon renderer (its UGC render add-on's BrickBuild scene); the light matched to the + // brightness of the game's own model icons (res/textures/ui/inventory/models), docs/UgcServer.md + static const std::vector list = { + { "yaw", "icon_yaw", "Angle around", "degrees", -180, 180, 1, 53.36f, "The camera's angle around the model, from its front.", [](Options& o, float v) { o.yawDegrees = v; } }, + { "pitch", "icon_pitch", "Angle above", "degrees", -89, 89, 1, 19.54f, "", [](Options& o, float v) { o.pitchDegrees = v; } }, + { "fov", "icon_fov", "Field of view", "degrees", 5, 90, 0.5f, 39.6f, "", [](Options& o, float v) { o.fovDegrees = v; } }, + { "margin", "icon_margin", "Border (zoom)", "", 0.5f, 3, 0.01f, 1.03f, "1 fills the icon, more leaves a border.", [](Options& o, float v) { o.margin = v; } }, + { "offsetX", "icon_offset_x", "Shift right", "", -0.5f, 0.5f, 0.01f, 0.0f, "Share of the icon's width.", [](Options& o, float v) { o.offsetX = v; } }, + { "offsetY", "icon_offset_y", "Shift up", "", -0.5f, 0.5f, 0.01f, 0.0f, "Share of the icon's height.", [](Options& o, float v) { o.offsetY = v; } }, + { "sunYaw", "icon_sun_yaw", "Sun around", "degrees", -180, 180, 1, 21.0f, "", [](Options& o, float v) { o.sunYawDegrees = v; } }, + { "sunPitch", "icon_sun_pitch", "Sun above", "degrees", -10, 90, 1, 50.3f, "", [](Options& o, float v) { o.sunPitchDegrees = v; } }, + { "sunStrength", "icon_sun_strength", "Sun strength", "", 0, 10, 0.05f, 2.0f, "", [](Options& o, float v) { o.sunStrength = v; } }, + { "ambient", "icon_ambient", "World light", "", 0, 2, 0.01f, 1.0f, "What every face gets.", [](Options& o, float v) { o.ambient = v; } }, + { "fill", "icon_fill", "Fill light", "", 0, 5, 0.05f, 0.8f, "A light from the camera.", [](Options& o, float v) { o.fill = v; } }, + { "specular", "icon_specular", "Highlights", "", 0, 2, 0.05f, 0.2f, "The sun's highlight on the plastic.", [](Options& o, float v) { o.specular = v; } }, + { "shininess", "icon_shininess", "Highlight tightness", "", 1, 200, 1, 40.0f, "", [](Options& o, float v) { o.shininess = v; } }, + { "exposure", "icon_exposure", "Exposure", "", 0.1f, 4, 0.05f, 2.6f, "Brightness of everything.", [](Options& o, float v) { o.exposure = v; } }, + { "contrast", "icon_contrast", "Contrast", "", 0.5f, 1.5f, 0.01f, 1.0f, "", [](Options& o, float v) { o.contrast = v; } }, + { "shadows", "icon_shadows", "Shadows", "", 0, 1, 0.05f, 0.4f, "How much the sun's shadows darken.", [](Options& o, float v) { o.shadows = v; } }, + { "aoStrength", "icon_ao_strength", "Ambient occlusion", "", 0, 1, 0.05f, 0.0f, + "Occlusion worked out for the icon. Player models' meshes have theirs baked in already.", [](Options& o, float v) { o.ao.strength = v; o.ao.enabled = v > 0.0f; } }, + }; + return list; + } + + const Param* Find(std::string_view key) { + const auto& list = List(); + const auto it = std::find_if(list.begin(), list.end(), [key](const Param& p) { return p.key == key; }); + return it != list.end() ? &*it : nullptr; + } + + Values Parse(std::string_view text) { + Values values; + const auto json = nlohmann::json::parse(text, nullptr, false); + if (!json.is_object()) return values; + for (const auto& param : List()) { + const auto it = json.find(param.key); + if (it == json.end() || !it->is_number()) continue; + const auto value = it->get(); + if (std::isfinite(value)) values[param.key] = std::clamp(value, param.min, param.max); + } + return values; + } + + std::string ToJson(const Values& values) { + nlohmann::json json = nlohmann::json::object(); + for (const auto& [key, value] : values) { + if (Find(key)) json[key] = value; + } + return json.dump(); + } + + void Apply(UgcRender::IconOptions& options, const Values& values) { + for (const auto& [key, value] : values) { + if (const auto* param = Find(key)) param->apply(options, std::clamp(value, param->min, param->max)); + } + } + + UgcRender::IconOptions FromSettings(const std::function(const std::string&)>& get) { + UgcRender::IconOptions options; + for (const auto& param : List()) { + const auto text = get(param.setting); + const auto value = text ? GeneralUtils::TryParse(*text) : std::nullopt; + param.apply(options, std::clamp(value.value_or(param.defaultValue), param.min, param.max)); + } + return options; + } + + std::string KindTarget(const std::string& kind) { return "kind:" + kind; } + std::string ModelTarget(int64_t id) { return "model:" + std::to_string(id); } + std::string CombinationTarget(const std::string& key) { return "combo:" + key; } + std::string ModelKind() { return "model"; } + std::string BuildKind(int32_t buildType) { return "build" + std::to_string(buildType); } +} diff --git a/dUgcServer/UgcIconParams.h b/dUgcServer/UgcIconParams.h new file mode 100644 index 000000000..ba8a06e7e --- /dev/null +++ b/dUgcServer/UgcIconParams.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "UgcRender.h" + +/** + * Everything about an icon's framing and lighting that can be set, listed once: its name (in presets and overrides), + * its setting (the shipped default, in ugcconfig.ini and the dashboard's settings), label, unit, range and default. + * The UGC server reads the settings through it, the dashboard builds its settings entries and the icon editor's + * controls from it, and presets and overrides are checked against it. Values for an icon come, each over the one + * before: the settings (icon_*), the kind's preset (player models, or a car or rocket build type) and the item's or + * module combination's own override, both stored in the database (ugc_icon_settings). Pure. + */ +namespace UgcIconParams { + struct Param { + std::string key; // in presets and overrides, e.g. "sunStrength" + std::string setting; // ugcconfig.ini, e.g. "icon_sun_strength" + std::string label; + std::string unit; + float min{}; + float max{}; + float step{}; + float defaultValue{}; + std::string description; + std::function apply; + }; + + const std::vector& List(); + const Param* Find(std::string_view key); + + using Values = std::map; + + // Known keys only, numbers only, clamped to their ranges + Values Parse(std::string_view json); + std::string ToJson(const Values& values); + + // Values over the options' own + void Apply(UgcRender::IconOptions& options, const Values& values); + + // The options from the settings: `get` gives a setting's value (nullopt or unparsable: the parameter's default) + UgcRender::IconOptions FromSettings(const std::function(const std::string&)>& get); + + // Where presets and overrides are stored (ugc_icon_settings.target) + std::string KindTarget(const std::string& kind); // "kind:model", "kind:build6" + std::string ModelTarget(int64_t id); // "model:" + std::string CombinationTarget(const std::string& key); // "combo:" + // The kind of a player model, and of a car or rocket of build type `buildType` (ModuleComponent.buildType) + std::string ModelKind(); + std::string BuildKind(int32_t buildType); +} diff --git a/dUgcServer/UgcJobs.cpp b/dUgcServer/UgcJobs.cpp index b53941d91..5e03b58a0 100644 --- a/dUgcServer/UgcJobs.cpp +++ b/dUgcServer/UgcJobs.cpp @@ -1,6 +1,7 @@ #include "UgcJobs.h" #include +#include #include #include "json.hpp" @@ -66,7 +67,14 @@ namespace UgcJobs { } } - Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed) { + bool IconFromNif(const std::string& nif, const UgcRender::IconOptions& options, UgcStorage::Files& files, std::string& error) { + const auto readBack = NifFile::Parse(nif, 0, error); + if (!readBack) return false; + AddIcon(files, UgcModel::FromNif(*readBack), options); + return true; + } + + Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed, const UgcIconParams::Values& iconValues) { Outcome outcome; const auto started = std::chrono::steady_clock::now(); const auto lxfml = LxfmlFromBlob(blob); @@ -169,18 +177,15 @@ namespace UgcJobs { } // The icon is drawn from the .nif just made (its most detailed LOD, read back like any client .nif), so it - // shows what the game shows: the colors with their variation, hidden faces removed, the baked lighting. The - // lighting is in its vertex colors already, so the icon adds no occlusion of its own. + // shows what the game shows: the colors with their variation, hidden faces removed, the baked lighting. const auto iconStart = std::chrono::steady_clock::now(); + auto iconOptions = settings.icon; + UgcIconParams::Apply(iconOptions, iconValues); std::string nifError; - const auto readBack = NifFile::Parse(nif, 0, nifError); - if (!readBack) { + if (!IconFromNif(nif, iconOptions, outcome.files, nifError)) { outcome.error = "the .nif made can't be read back for the icon: " + nifError; return outcome; } - auto iconOptions = settings.icon; - iconOptions.ao.enabled = false; - AddIcon(outcome.files, UgcModel::FromNif(*readBack), iconOptions); const double iconMs = Since(iconStart); stats["ms"] = { { "build", std::lround(buildMs) }, { "hiddenSurfaces", std::lround(hsrMs) }, { "ambientOcclusion", std::lround(aoMs) }, @@ -228,10 +233,19 @@ namespace UgcJobs { outcome.error = "the modules have no triangles"; return outcome; } - auto options = settings.modularIcon; + auto options = ModularIconOptions(input, settings); options.modelRotation = build->additionalRotation * options.modelRotation; AddIcon(outcome.files, model, options); + outcome.files["combo.json"] = nlohmann::json{ { "key", input.key }, { "buildType", input.buildType } }.dump(); outcome.ok = true; return outcome; } + + UgcRender::IconOptions ModularIconOptions(const ModularInput& input, const Settings& settings) { + auto options = settings.icon; + UgcIconParams::Apply(options, input.iconValues); + return options; + } + + } diff --git a/dUgcServer/UgcJobs.h b/dUgcServer/UgcJobs.h index df49562ae..edbcd75f2 100644 --- a/dUgcServer/UgcJobs.h +++ b/dUgcServer/UgcJobs.h @@ -2,11 +2,14 @@ #include #include +#include +#include #include #include #include #include "UgcBricks.h" +#include "UgcIconParams.h" #include "UgcRender.h" #include "UgcStorage.h" @@ -27,8 +30,7 @@ namespace UgcJobs { bool combineTransparent{ false }; // one shape for all transparent bricks, else one per brick (Combine Transparent) UgcRender::OptimizeOptions optimize; // hidden surface removal UgcRender::AoOptions ao; // Bake Lighting (AO Only) - UgcRender::IconOptions icon; // player models: drawn from their .nif (LOD 0); its ao is not used - UgcRender::IconOptions modularIcon; + UgcRender::IconOptions icon; // from the icon_* settings (UgcIconParams); presets and overrides go over it uint32_t maxBricks{}; // a model with more fails; 0: no limit }; @@ -49,7 +51,13 @@ namespace UgcJobs { // A player model: the optimized .nif, its icon and its LXFML for download, and stats.json. `seed` picks the color // variation's random numbers (the model's id, so making it again gives the same colors). - Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed = 0); + // `iconValues`: the player models' preset and this model's override (UgcIconParams), over settings.icon. + Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed = 0, + const UgcIconParams::Values& iconValues = {}); + + // A model's icon files (icon.png, icon.dds download) drawn from its .nif (LOD 0); false (and `error`) when the .nif + // can't be read + bool IconFromNif(const std::string& nif, const UgcRender::IconOptions& options, UgcStorage::Files& files, std::string& error); // How many bricks (parts) an LXFML has, counted cheaply (for the memory estimate before a job starts) size_t CountParts(std::string_view lxfml); @@ -66,9 +74,15 @@ namespace UgcJobs { struct ModularInput { std::vector modules; - std::string buildXml; // ModularBuildComponent.xml of the modules' build type + std::string buildXml; // ModularBuildComponent.xml of the modules' build type + int32_t buildType{}; // 3 rocket, 6 car + std::string key; // the combination (UgcModularKey::Normalize) + UgcIconParams::Values iconValues; // the build type's preset and the combination's override (DB), over the settings }; - // A modular build's icon, from its modules' meshes put together + // The icon options for a car or rocket: the settings, then its build type's preset and the combination's override + UgcRender::IconOptions ModularIconOptions(const ModularInput& input, const Settings& settings); + + // A modular build's icon (icon.png, icon.dds download, combo.json), from its modules' meshes put together Outcome ProcessModular(const ModularInput& input, const std::filesystem::path& res, const Settings& settings); } diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index cf7cc9ef8..50b8511df 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -3,11 +3,15 @@ #include "CDClientDatabase.h" #include "Database.h" #include "Logger.h" +#include "UgcBricks.h" #include "UgcCdClient.h" +#include "UgcKeys.h" #include "UgcThrottle.h" #include "json.hpp" +#include #include +#include #include #include #if defined(__linux__) @@ -148,12 +152,63 @@ void UgcProcessor::Worker() { } UgcThrottle::Begin(); + if (job.preview) { + // A preview: the icon goes back to whoever asked, nothing is stored or recorded + HTTPReply reply; + try { + UgcJobs::Outcome outcome{ false, "cancelled" }; + if (!job.preview.Cancelled() && job.kind == Kind::MODULAR) { + outcome = UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings); + } else if (!job.preview.Cancelled()) { + // A player model's icon from its stored .nif + const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif"); + const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt; + auto options = settings.icon; + UgcIconParams::Apply(options, job.iconValues); + outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, outcome.files, outcome.error); + if (!nif) outcome.error = "the model has no stored .nif yet"; + } + if (outcome.ok && outcome.files.contains("icon.png")) { + reply.status = eHTTPStatusCode::OK; + reply.contentType = eContentType::IMAGE_PNG; + reply.message = std::move(outcome.files["icon.png"]); + reply.headers.push_back("Cache-Control: no-store"); + } else { + reply.status = eHTTPStatusCode::UNPROCESSABLE_ENTITY; + reply.contentType = eContentType::TEXT_PLAIN; + reply.message = outcome.error; + } + } catch (const std::exception& ex) { + reply.status = eHTTPStatusCode::INTERNAL_SERVER_ERROR; + reply.contentType = eContentType::TEXT_PLAIN; + reply.message = ex.what(); + } + job.preview.Send(std::move(reply)); + { + std::lock_guard lock(m_Mutex); + m_Active--; + m_MemoryInUse -= std::min(m_MemoryInUse, job.memory); + } + m_Wake.notify_all(); + continue; + } const auto start = std::chrono::steady_clock::now(); Done done{ job.kind, job.id, job.attempts }; + done.iconOnly = job.iconOnly; try { - done.outcome = job.kind == Kind::MODEL - ? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast(job.id)) - : UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings); + if (job.iconOnly) { + // Only the icon, from the .nif made before + const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif"); + const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt; + auto options = settings.icon; + UgcIconParams::Apply(options, job.iconValues); + done.outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, done.outcome.files, done.outcome.error); + if (!nif) done.outcome.error = "no stored .nif"; + } else { + done.outcome = job.kind == Kind::MODEL + ? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast(job.id), job.iconValues) + : UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings); + } } catch (const std::exception& ex) { done.outcome.ok = false; done.outcome.error = std::string("crashed: ") + ex.what(); @@ -162,7 +217,7 @@ void UgcProcessor::Worker() { job.blob.shrink_to_fit(); if (done.outcome.ok) { std::string error; - const auto bytes = m_Storage.Write(job.kind, job.id, done.outcome.files, error); + const auto bytes = job.iconOnly ? m_Storage.Update(job.kind, job.id, done.outcome.files, error) : m_Storage.Write(job.kind, job.id, done.outcome.files, error); if (bytes) { done.bytes = *bytes; } else { @@ -219,26 +274,51 @@ void UgcProcessor::Poll() { Job job{ Kind::MODEL, model.id, model.attempts, UgcJobs::LxfmlFromBlob(model.lxfml) }; if (job.blob.empty()) job.blob = std::move(model.lxfml); // the worker reports it can't be read job.parts = UgcJobs::CountParts(job.blob); + job.iconValues = IconValues(UgcIconParams::ModelKind(), UgcIconParams::ModelTarget(model.id)); job.memory = UgcJobs::EstimateMemory(job.parts, settings); jobs.push_back(std::move(job)); } + // Cars and rockets: one icon per combination of modules, shared by every build of it for (auto& build : Database::Get()->GetModularBuildsToProcess(limit)) { if (m_InFlight.contains({ Kind::MODULAR, build.id })) continue; - Job job{ Kind::MODULAR, build.id, build.attempts }; + const auto key = UgcModularKey::Normalize(build.modules); + if (key.empty()) { + Record(Done{ Kind::MODULAR, build.id, build.attempts, UgcJobs::Outcome{ false, "no modules in \"" + build.modules + "\"" } }); + continue; + } + const auto combo = UgcModularKey::StorageId(key); + m_ComboOf[build.id] = combo; + // Made already for another build of the same modules + if (!m_ComboJobs.contains(combo) && m_Storage.File(Kind::MODULAR, combo, "icon.png")) { + UgcJobs::Outcome reused{ true }; + reused.note = "the same modules as a build made before (" + key + ")"; + m_Reused++; + Record(Done{ Kind::MODULAR, build.id, build.attempts, std::move(reused) }); + continue; + } + m_InFlight.insert({ Kind::MODULAR, build.id }); + m_ComboRows[combo].emplace_back(build.id, build.attempts); + if (m_ComboJobs.contains(combo)) continue; + Job job{ Kind::MODULAR, combo, build.attempts }; job.memory = UgcJobs::EstimateMemory(64, settings); std::string error; if (!UgcCdClient::GatherModular(build.modules, job.modular, error)) { // Nothing a worker could do: record it right away + m_InFlight.erase({ Kind::MODULAR, build.id }); + m_ComboRows.erase(combo); Record(Done{ Kind::MODULAR, build.id, build.attempts, UgcJobs::Outcome{ false, error } }); continue; } + job.modular.key = key; + job.modular.iconValues = IconValues(UgcIconParams::BuildKind(job.modular.buildType), UgcIconParams::CombinationTarget(key)); + m_ComboJobs.insert(combo); jobs.push_back(std::move(job)); } if (jobs.empty()) return; { std::lock_guard lock(m_Mutex); for (auto& job : jobs) { - m_InFlight.insert({ job.kind, job.id }); + if (job.kind == Kind::MODEL) m_InFlight.insert({ job.kind, job.id }); m_Jobs.push_back(std::move(job)); } } @@ -277,8 +357,33 @@ void UgcProcessor::Collect() { finished.swap(m_Done); } for (const auto& done : finished) { - m_InFlight.erase({ done.kind, done.id }); - Record(done); + if (done.kind == Kind::MODEL) { + m_InFlight.erase({ done.kind, done.id }); + if (!done.iconOnly) { + Record(done); + continue; + } + m_Log.push_back({ Kind::MODEL, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? "icon drawn again" : done.outcome.error, UnixNow() }); + while (m_Log.size() > LOG_LENGTH) m_Log.pop_front(); + continue; + } + // A combination: every build waiting for it gets its outcome (none when only its icon was drawn again) + m_ComboJobs.erase(done.id); + if (done.outcome.ok) m_StoredBytes += done.bytes; + auto rows = std::move(m_ComboRows[done.id]); + m_ComboRows.erase(done.id); + for (const auto& [row, attempts] : rows) { + m_InFlight.erase({ Kind::MODULAR, row }); + Done forRow = done; + forRow.id = row; + forRow.attempts = attempts; + forRow.bytes = 0; + Record(forRow); + } + if (rows.empty()) { + m_Log.push_back({ Kind::MODULAR, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? "icon drawn again" : done.outcome.error, UnixNow() }); + while (m_Log.size() > LOG_LENGTH) m_Log.pop_front(); + } } // Something finished: look for more now rather than at the next interval if (!finished.empty()) m_NextPoll = std::min(m_NextPoll, std::chrono::steady_clock::now() + std::chrono::milliseconds(100)); @@ -307,10 +412,22 @@ void UgcProcessor::Update() { std::erase_if(m_Recent, [now](const auto& item) { return now - item.second.first > RECENT_ANSWER_TIME; }); } +LWOOBJID UgcProcessor::StorageId(Kind kind, LWOOBJID id) { + if (kind == Kind::MODEL) return id; + if (const auto it = m_ComboOf.find(id); it != m_ComboOf.end()) return it->second; + const auto info = Database::Get()->GetModularBuildProcessInfo(id); + const auto key = info ? UgcModularKey::Normalize(info->details) : std::string(); + const LWOOBJID combo = key.empty() ? 0 : UgcModularKey::StorageId(key); + if (combo != 0) m_ComboOf[id] = combo; + return combo; +} + UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) { if (m_InFlight.contains({ kind, id })) return Availability::QUEUED; - if (m_Storage.File(kind, id, "icon.png")) { - m_Storage.Touch(kind, id); + const auto storageId = StorageId(kind, id); + if (kind == Kind::MODULAR && m_ComboJobs.contains(storageId)) return Availability::QUEUED; + if (storageId != 0 && m_Storage.File(kind, storageId, "icon.png")) { + m_Storage.Touch(kind, storageId); return Availability::READY; } if (const auto it = m_Recent.find({ kind, id }); it != m_Recent.end()) return it->second.second; @@ -323,6 +440,8 @@ UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) { if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false); else Database::Get()->ResetModularBuildProcessing(id, false); } + // Someone wants it now: no need to wait out the quiet period after its save + if (kind == Kind::MODEL) Database::Get()->ExpediteUgcModel(id); m_NextPoll = std::chrono::steady_clock::now(); answer = Availability::QUEUED; } @@ -345,6 +464,7 @@ nlohmann::json UgcProcessor::Status() const { status["memoryWaits"] = m_Waiting; } status["workers"] = m_Threads.size(); + status["reused"] = m_Reused; const auto throttle = UgcThrottle::GetStats(); status["usage"] = { { "cpuPercent", std::lround(m_CpuPercent) }, // of one core @@ -372,3 +492,161 @@ nlohmann::json UgcProcessor::Status() const { } return status; } + +UgcIconParams::Values UgcProcessor::IconValues(const std::string& kind, const std::string& itemTarget) { + UgcIconParams::Values values; + if (const auto preset = Database::Get()->GetUgcIconSettings(UgcIconParams::KindTarget(kind))) values = UgcIconParams::Parse(*preset); + if (const auto own = Database::Get()->GetUgcIconSettings(itemTarget)) { + for (const auto& [key, value] : UgcIconParams::Parse(*own)) values[key] = value; + } + return values; +} + +bool UgcProcessor::QueuePreview(Kind kind, LWOOBJID id, const std::string& modules, const UgcIconParams::Values& values, DeferredReply reply, std::string& error) { + Job job{ kind, id, 0 }; + if (kind == Kind::MODULAR) { + if (!UgcCdClient::GatherModular(modules, job.modular, error)) return false; + job.modular.key = UgcModularKey::Normalize(modules); + job.modular.iconValues = values; + } else { + if (!m_Storage.File(Kind::MODEL, id, "model.nif")) { + error = "the model has no stored .nif yet"; + return false; + } + job.iconValues = values; + } + job.preview = std::move(reply); + { + std::lock_guard lock(m_Mutex); + job.memory = UgcJobs::EstimateMemory(64, m_Settings); + m_Jobs.push_front(std::move(job)); + } + m_Wake.notify_all(); + return true; +} + +size_t UgcProcessor::RegenerateIcons(const std::string& kind) { + std::vector jobs; + UgcJobs::Settings settings; + { + std::lock_guard lock(m_Mutex); + settings = m_Settings; + } + const bool models = kind == UgcIconParams::ModelKind(); + for (const auto& entry : m_Storage.List()) { + if (models) { + if (entry.kind != Kind::MODEL || m_InFlight.contains({ Kind::MODEL, entry.id }) || !m_Storage.File(Kind::MODEL, entry.id, "model.nif")) continue; + Job job{ Kind::MODEL, entry.id, 0 }; + job.iconOnly = true; + job.iconValues = IconValues(kind, UgcIconParams::ModelTarget(entry.id)); + job.memory = UgcJobs::EstimateMemory(64, settings); + m_InFlight.insert({ Kind::MODEL, entry.id }); + jobs.push_back(std::move(job)); + continue; + } + if (entry.kind != Kind::MODULAR || m_ComboJobs.contains(entry.id)) continue; + const auto path = m_Storage.File(Kind::MODULAR, entry.id, "combo.json"); + const auto text = path ? UgcBricks::ReadFile(*path) : std::nullopt; + const auto combo = text ? nlohmann::json::parse(*text, nullptr, false) : nlohmann::json(); + if (!combo.is_object() || UgcIconParams::BuildKind(combo.value("buildType", -1)) != kind) continue; + const auto key = combo.value("key", std::string()); + Job job{ Kind::MODULAR, entry.id, 0 }; + std::string error; + // The key's LOTs written as an ldf_config ("4713-4714" -> "4713+4714") + std::string modules = key; + std::replace(modules.begin(), modules.end(), '-', '+'); + if (key.empty() || !UgcCdClient::GatherModular(modules, job.modular, error)) continue; + job.modular.key = key; + job.modular.iconValues = IconValues(kind, UgcIconParams::CombinationTarget(key)); + job.memory = UgcJobs::EstimateMemory(64, settings); + m_ComboJobs.insert(entry.id); + jobs.push_back(std::move(job)); + } + const auto count = jobs.size(); + { + std::lock_guard lock(m_Mutex); + for (auto& job : jobs) m_Jobs.push_back(std::move(job)); + } + m_Wake.notify_all(); + return count; +} + +UgcProcessor::DeleteResult UgcProcessor::Delete(const DeleteRequest& request) { + DeleteResult result; + const auto now = std::filesystem::file_time_type::clock::now(); + const auto days = [](int64_t count) { return std::chrono::duration_cast(std::chrono::hours(24 * count)); }; + + // What to delete: the files' ids, and the rows they came from (when they're known) + std::map> targets; // storage id -> rows + if (request.all) { + for (const auto& entry : m_Storage.List()) { + if (entry.kind == request.kind) targets[entry.id]; + } + } + for (const auto id : request.ids) { + const auto storageId = StorageId(request.kind, id); + if (storageId != 0) targets[storageId].push_back(id); + } + + std::vector deletedRows; + for (const auto& [storageId, rows] : targets) { + const bool busy = request.kind == Kind::MODEL ? m_InFlight.contains({ Kind::MODEL, storageId }) : m_ComboJobs.contains(storageId); + if (busy) { + result.busy++; + continue; + } + const auto folder = m_Storage.Folder(request.kind, storageId); + std::error_code error; + if (!std::filesystem::exists(folder, error)) { + deletedRows.insert(deletedRows.end(), rows.begin(), rows.end()); + continue; + } + if (request.unusedDays > 0) { + const auto used = std::filesystem::last_write_time(folder, error); + if (!error && now - used < days(request.unusedDays)) continue; + } + if (request.olderThanDays > 0) { + const auto made = std::filesystem::last_write_time(folder / "icon.png", error); + if (!error && now - made < days(request.olderThanDays)) continue; + } + uint64_t bytes = 0; + for (const auto& file : std::filesystem::directory_iterator(folder, error)) bytes += file.is_regular_file(error) ? file.file_size(error) : 0; + m_Storage.Remove(request.kind, storageId); + result.deleted++; + result.bytes += bytes; + m_StoredBytes -= std::min(m_StoredBytes, bytes); + m_Recent.clear(); + deletedRows.insert(deletedRows.end(), rows.begin(), rows.end()); + if (request.kind == Kind::MODEL) deletedRows.push_back(storageId); + } + std::sort(deletedRows.begin(), deletedRows.end()); + deletedRows.erase(std::unique(deletedRows.begin(), deletedRows.end()), deletedRows.end()); + + // What happens to the rows + if (request.after == eAfterDelete::NOW) { + if (request.all) { + if (request.kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(std::nullopt, false); + else Database::Get()->ResetModularBuildProcessing(std::nullopt, false); + } else { + for (const auto row : deletedRows) { + if (request.kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(row, false); + else Database::Get()->ResetModularBuildProcessing(row, false); + } + } + m_NextPoll = std::chrono::steady_clock::now(); + result.notes.push_back("They're queued to be made again now."); + } else if (request.after == eAfterDelete::GONE) { + for (const auto row : deletedRows) { + if (request.kind == Kind::MODEL) Database::Get()->SetUgcModelProcessed(row, IUgc::eProcessState::FAILED, m_Config.maxAttempts, "Deleted from the dashboard", false); + else Database::Get()->SetModularBuildProcessed(row, IUgc::eProcessState::FAILED, m_Config.maxAttempts, "Deleted from the dashboard"); + } + result.notes.push_back("They're marked deleted and won't be made again unless made again from the dashboard."); + } else { + result.notes.push_back("They'll be made again when a game client asks for them."); + } + if (request.kind == Kind::MODULAR && result.deleted > 0) { + result.notes.push_back("Car and rocket icons are shared by every build of the same modules: the other builds' icons are made again when asked for."); + } + if (result.busy > 0) result.notes.push_back(std::to_string(result.busy) + " being made right now were left alone."); + return result; +} diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h index 93bb02adb..bcc3e54ce 100644 --- a/dUgcServer/UgcProcessor.h +++ b/dUgcServer/UgcProcessor.h @@ -12,9 +12,11 @@ #include #include +#include "DeferredReply.h" #include "dCommonVars.h" #include "json_fwd.hpp" #include "UgcBricks.h" +#include "UgcIconParams.h" #include "UgcJobs.h" #include "UgcStorage.h" @@ -69,6 +71,46 @@ public: // Main thread: whether an item's files can be served, queuing it to be made when it exists but they're missing Availability Request(Kind kind, LWOOBJID id); + // Main thread: the id an item's files are stored under. A car or rocket's are its combination's (every build of + // the same modules shares one icon); 0 when a build's modules can't be told. + LWOOBJID StorageId(Kind kind, LWOOBJID id); + + /** + * Main thread: renders an icon with `values` (UgcIconParams, over the settings) without storing it, on a worker + * (within the budgets, ahead of the queue); `reply` gets the PNG. A player model's is drawn from its stored .nif, a + * car or rocket's from `modules`. False (and `error`) when there's nothing to draw from. + */ + bool QueuePreview(Kind kind, LWOOBJID id, const std::string& modules, const UgcIconParams::Values& values, DeferredReply reply, std::string& error); + + // Main thread: queues every stored icon of a kind ("model", or "build" for cars and rockets) to be drawn again + // with the current settings, presets and overrides; only icons (models' from their stored .nif). How many. + size_t RegenerateIcons(const std::string& kind); + + enum class eAfterDelete { + ON_DEMAND, // rows stay made; the files are made again when something asks for them + NOW, // rows go back to waiting, so they're made again soon + GONE, // rows are marked failed ("deleted"), so they aren't made again until someone asks for that + }; + + struct DeleteRequest { + Kind kind{}; + std::vector ids; // models, or builds (their combinations are deleted); empty with `all` + bool all{}; + int64_t olderThanDays{}; // only files made longer ago than this (0: any) + int64_t unusedDays{}; // only files not asked for in this long (0: any) + eAfterDelete after{ eAfterDelete::ON_DEMAND }; + }; + + struct DeleteResult { + size_t deleted{}; + uint64_t bytes{}; + size_t busy{}; // being made right now, left alone + std::vector notes; + }; + + // Main thread: deletes stored files. Items being made are skipped, so a worker never writes what is being deleted. + DeleteResult Delete(const DeleteRequest& request); + // Main thread: what the server is doing, for /status nlohmann::json Status() const; @@ -98,6 +140,9 @@ private: UgcJobs::ModularInput modular; // modular builds uint64_t memory{}; // estimated bytes it needs size_t parts{}; + DeferredReply preview; // an icon preview: answered with the PNG, nothing stored + bool iconOnly{}; // a model's icon drawn again from its stored .nif + UgcIconParams::Values iconValues; // models: the preset and override (UgcIconParams) }; struct Done { @@ -107,9 +152,12 @@ private: UgcJobs::Outcome outcome; uint64_t bytes{}; double milliseconds{}; + bool iconOnly{}; }; void Poll(); + // Main thread: the icon values for a kind and an item (the kind's preset, then the item's override) + UgcIconParams::Values IconValues(const std::string& kind, const std::string& itemTarget); void Collect(); void Record(const Done& done); void Worker(); @@ -137,7 +185,11 @@ private: std::vector m_Threads; // Main thread only - std::set> m_InFlight; + std::set> m_InFlight; // models, and builds waiting for their combination + std::set m_ComboJobs; // combinations being made + std::map>> m_ComboRows; // combination -> builds (id, attempts) waiting for it + std::map m_ComboOf; // build -> combination (StorageId) + uint64_t m_Reused{}; // builds whose combination was made already std::chrono::steady_clock::time_point m_NextPoll{}; std::chrono::steady_clock::time_point m_NextEviction{}; std::map, std::pair> m_Recent; // answers for missing files diff --git a/dUgcServer/UgcRender.cpp b/dUgcServer/UgcRender.cpp index 43d9c1f35..443278c55 100644 --- a/dUgcServer/UgcRender.cpp +++ b/dUgcServer/UgcRender.cpp @@ -430,7 +430,7 @@ namespace UgcRender { const auto project = [&](const glm::vec3& position) { const auto clip = viewProjection * glm::vec4(position, 1.0f); const float w = clip.w > 1e-6f ? clip.w : 1e-6f; - return glm::vec3((0.5f + (clip.x / w - centerX) * scale * 0.5f) * n, (0.5f - (clip.y / w - centerY) * scale * 0.5f) * n, clip.z / w); + return glm::vec3((0.5f + options.offsetX + (clip.x / w - centerX) * scale * 0.5f) * n, (0.5f - options.offsetY - (clip.y / w - centerY) * scale * 0.5f) * n, clip.z / w); }; // Linear, premultiplied @@ -449,7 +449,7 @@ namespace UgcRender { const int shadowSize = 1024; const OrthoView sunView(center, radius * 1.05f, light, shadowSize); std::vector shadowDepth; - if (options.shadows) { + if (options.shadows > 0.0f) { shadowDepth.assign(static_cast(shadowSize) * shadowSize, INF); const auto& mesh = model.opaque; std::vector screen(mesh.positions.size()); @@ -473,8 +473,11 @@ namespace UgcRender { if (x < 0 || y < 0 || x >= shadowSize || y >= shadowSize || p.z <= shadowDepth[static_cast(y) * shadowSize + x] + shadowBias) lit += 1.0f; } } - return lit / 9.0f; + const float shadowed = 1.0f - lit / 9.0f; + return 1.0f - std::clamp(options.shadows, 0.0f, 1.0f) * shadowed; }; + const glm::vec3 toCamera = glm::normalize(dir); + const glm::vec3 halfway = glm::normalize(light + toCamera); const auto shade = [&](const UgcModel::Mesh& mesh, bool isOpaque, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) { glm::vec3 normal(0.0f, 1.0f, 0.0f); @@ -490,10 +493,15 @@ namespace UgcRender { const float strength = std::clamp(options.ao.strength, 0.0f, 1.0f); const auto position = mesh.positions[i0] * w0 + mesh.positions[i1] * w1 + mesh.positions[i2] * w2; const float direct = std::max(0.0f, glm::dot(normal, light)); - // Diffuse: the world's light (radiance `ambient`) and the sun's (irradiance `sunStrength`, over pi) + const float sun = direct > 0.0f ? sunlit(position + normal * shadowBias) : 0.0f; + // Diffuse: the world's light (radiance `ambient`), a fill from the camera and the sun's (irradiances, over pi) const float lighting = options.ambient * (1.0f - strength * (1.0f - occlusion)) + - options.sunStrength / 3.14159265f * direct * (direct > 0.0f ? sunlit(position + normal * shadowBias) : 0.0f); - return glm::vec4(ToLinear(base.r) * lighting, ToLinear(base.g) * lighting, ToLinear(base.b) * lighting, std::clamp(base.a, 0.0f, 1.0f)); + (options.fill * std::max(0.0f, glm::dot(normal, toCamera)) + options.sunStrength * direct * sun) / 3.14159265f; + // The sun's highlight (Blinn-Phong), white, on top of the color + const float highlight = direct > 0.0f ? options.specular * options.sunStrength / 3.14159265f * sun * std::pow(std::max(0.0f, glm::dot(normal, halfway)), std::max(options.shininess, 1.0f)) : 0.0f; + const float exposure = std::max(options.exposure, 0.0f); + return glm::vec4((ToLinear(base.r) * lighting + highlight) * exposure, (ToLinear(base.g) * lighting + highlight) * exposure, + (ToLinear(base.b) * lighting + highlight) * exposure, std::clamp(base.a, 0.0f, 1.0f)); }; // Opaque first, with the depth buffer @@ -546,9 +554,11 @@ namespace UgcRender { sum /= samples; uint8_t* out = &image.rgba[(static_cast(y) * size + x) * 4]; if (sum.a <= 0.0f) continue; - out[0] = static_cast(std::lround(ToSrgb(sum.r / sum.a) * 255.0f)); - out[1] = static_cast(std::lround(ToSrgb(sum.g / sum.a) * 255.0f)); - out[2] = static_cast(std::lround(ToSrgb(sum.b / sum.a) * 255.0f)); + // sRGB, then the contrast around its middle grey + const auto tone = [&options](float linear) { return std::clamp(0.5f + (ToSrgb(linear) - 0.5f) * options.contrast, 0.0f, 1.0f); }; + out[0] = static_cast(std::lround(tone(sum.r / sum.a) * 255.0f)); + out[1] = static_cast(std::lround(tone(sum.g / sum.a) * 255.0f)); + out[2] = static_cast(std::lround(tone(sum.b / sum.a) * 255.0f)); out[3] = static_cast(std::lround(std::clamp(sum.a, 0.0f, 1.0f) * 255.0f)); } } diff --git a/dUgcServer/UgcRender.h b/dUgcServer/UgcRender.h index c2ec87ecb..f1d4b389b 100644 --- a/dUgcServer/UgcRender.h +++ b/dUgcServer/UgcRender.h @@ -28,9 +28,8 @@ namespace UgcRender { }; /** - * The icon, set up like LU Toolbox's icon renderer (the BrickBuild scene of its UGC render add-on): a 50 mm lens - * (39.6 degree field of view) looking from 53.4 degrees around and 19.5 above, framed at 1.03, a sun of strength - * 2.5 from 21 around and 50.3 above with soft shadows, and a grey (0.192) world light, darkened by ambient occlusion. + * How an icon is drawn. The values that can be set (settings icon_*, presets and overrides) are listed once, with + * their ranges and shipped defaults, in UgcIconParams; the ones here are only what the struct starts with. */ struct IconOptions { int size{ 128 }; @@ -39,13 +38,20 @@ namespace UgcRender { float pitchDegrees{ 19.54f }; // camera above the model float fovDegrees{ 39.6f }; float margin{ 1.03f }; // 1 fills the icon, more leaves a border + float offsetX{}; // the model moved right by this share of the icon's width (after framing) + float offsetY{}; // and up by this share of its height glm::mat4 modelRotation{ 1.0f }; // applied to the model before the camera looks at it float sunYawDegrees{ 21.0f }; float sunPitchDegrees{ 50.3f }; float sunStrength{ 2.5f }; - float ambient{ 0.192f }; - bool shadows{ true }; - AoOptions ao{ true, 5.0f, 32, 1.0f, 0.0f }; + float ambient{ 0.192f }; // the world light (radiance) every face gets + float fill{ 0.0f }; // a light from the camera (irradiance), what lifts the sides facing the viewer + float specular{ 0.0f }; // the sun's highlight on the plastic + float shininess{ 60.0f }; // how tight the highlight is + float exposure{ 1.0f }; // everything times this before it's written as sRGB + float contrast{ 1.0f }; // around the middle grey of the sRGB result + float shadows{ 1.0f }; // how much the sun's shadows darken, 0 to 1 + AoOptions ao{ false, 5.0f, 32, 1.0f, 0.0f }; }; // The model drawn from the icon's camera, framed to fit, on a transparent background. `opaqueAo`: the opaque mesh's diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index 8af4ec82d..fd4ae99ec 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -31,6 +31,7 @@ #include "UgcBricks.h" #include "UgcCdClient.h" +#include "UgcIconParams.h" #include "UgcJobs.h" #include "UgcModel.h" #include "UgcProcessor.h" @@ -47,6 +48,7 @@ namespace Game { namespace { dServer* g_Server = nullptr; + std::string g_AdminKey; // the master password: the dashboard's admin requests carry it (X-Ugc-Admin-Key) UgcProcessor* g_Processor = nullptr; UgcStorage* g_Storage = nullptr; @@ -89,21 +91,13 @@ namespace { settings.ao.samples = std::clamp(Setting("ao_samples", 64), 1, 1024); settings.ao.strength = Setting("ao_strength", 1.0f); settings.ao.glowStrength = Setting("glow_strength", 6.0f); + // The icon's framing and light: the icon_* settings, listed with their defaults in UgcIconParams + settings.icon = UgcIconParams::FromSettings([](const std::string& key) -> std::optional { + const auto value = Game::config->GetValue(key); + return value.empty() ? std::nullopt : std::optional(value); + }); settings.icon.size = Setting("icon_size", 128); - settings.icon.yawDegrees = Setting("icon_yaw", 53.36f); - settings.icon.pitchDegrees = Setting("icon_pitch", 19.54f); - settings.icon.fovDegrees = Setting("icon_fov", 39.6f); - settings.icon.margin = Setting("icon_margin", 1.03f); - settings.icon.sunYawDegrees = Setting("icon_sun_yaw", 21.0f); - settings.icon.sunPitchDegrees = Setting("icon_sun_pitch", 50.3f); - settings.icon.sunStrength = Setting("icon_sun_strength", 2.5f); - settings.icon.ambient = Setting("icon_ambient", 0.192f); - settings.icon.shadows = Setting("icon_shadows", 1) != 0; - settings.icon.ao.enabled = Setting("icon_ao", 1) != 0; settings.icon.ao.distance = settings.ao.distance; - settings.modularIcon = settings.icon; - settings.modularIcon.yawDegrees = Setting("modular_icon_yaw", 53.36f); - settings.modularIcon.pitchDegrees = Setting("modular_icon_pitch", 19.54f); settings.maxBricks = Setting("max_model_bricks", 0); return settings; } @@ -156,7 +150,8 @@ namespace { NotFound(reply); return; } - const auto path = g_Storage->File(kind, id, name); + // A car or rocket's files are its combination's (shared by every build of the same modules) + const auto path = g_Storage->File(kind, g_Processor->StorageId(kind, id), name); if (!path) { NotFound(reply); return; @@ -237,7 +232,7 @@ namespace { const auto id = GeneralUtils::TryParse(segments[2]); const auto kind = segments[1] == "model" ? std::optional(UgcStorage::Kind::MODEL) : segments[1] == "modular" ? std::optional(UgcStorage::Kind::MODULAR) : std::nullopt; const auto& name = segments[3]; - static const std::set PREVIEWS = { "icon.png", "model.nif", "model.noao.nif", "stats.json", + static const std::set PREVIEWS = { "icon.png", "model.nif", "model.noao.nif", "stats.json", "combo.json", "previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json" }; if (!id || !kind || !PREVIEWS.contains(name)) return; reply.headers.clear(); @@ -250,6 +245,78 @@ namespace { } } }); + // For the dashboard only (it sends the master password): icon previews, drawing icons again, deleting files + const auto admin = [](HTTPReply& reply, const HTTPContext& context, std::optional& body) { + const auto& key = context.GetHeader("X-Ugc-Admin-Key"); + bool same = !g_AdminKey.empty() && key.size() == g_AdminKey.size(); + unsigned char diff = 0; + for (size_t i = 0; same && i < key.size(); i++) diff |= static_cast(key[i] ^ g_AdminKey[i]); + reply.contentType = eContentType::APPLICATION_JSON; + reply.headers.push_back("Cache-Control: no-store"); + if (!same || diff != 0) { + reply.status = eHTTPStatusCode::FORBIDDEN; + reply.message = R"({"success":false,"error":"forbidden"})"; + return false; + } + body = nlohmann::json::parse(context.body.empty() ? std::string("{}") : context.body, nullptr, false); + if (!body->is_object()) { + reply.status = eHTTPStatusCode::BAD_REQUEST; + reply.message = R"({"success":false,"error":"the body isn't a JSON object"})"; + return false; + } + return true; + }; + + Game::web.RegisterHTTPRoute({ .path = "/admin/preview", .method = eHTTPMethod::POST, .middleware = {}, + .handle = [admin](HTTPReply& reply, const HTTPContext& context) { + std::optional body; + if (!admin(reply, context, body)) return; + const auto values = UgcIconParams::Parse(body->value("values", nlohmann::json::object()).dump()); + const auto kind = body->value("kind", std::string("modular")) == "model" ? UgcStorage::Kind::MODEL : UgcStorage::Kind::MODULAR; + const auto id = GeneralUtils::TryParse(body->value("id", std::string("0"))).value_or(0); + auto deferred = Web::Defer(reply, context); + std::string error; + if (!g_Processor->QueuePreview(kind, id, body->value("modules", std::string()), values, deferred, error)) { + HTTPReply out; + out.status = eHTTPStatusCode::BAD_REQUEST; + out.contentType = eContentType::APPLICATION_JSON; + out.message = nlohmann::json{ { "success", false }, { "error", error } }.dump(); + deferred.Send(std::move(out)); + } + } }); + + Game::web.RegisterHTTPRoute({ .path = "/admin/regenerate-icons", .method = eHTTPMethod::POST, .middleware = {}, + .handle = [admin](HTTPReply& reply, const HTTPContext& context) { + std::optional body; + if (!admin(reply, context, body)) return; + const auto queued = g_Processor->RegenerateIcons(body->value("kind", std::string())); + reply.status = eHTTPStatusCode::OK; + reply.message = nlohmann::json{ { "success", true }, { "queued", queued } }.dump(); + } }); + + Game::web.RegisterHTTPRoute({ .path = "/admin/delete", .method = eHTTPMethod::POST, .middleware = {}, + .handle = [admin](HTTPReply& reply, const HTTPContext& context) { + std::optional body; + if (!admin(reply, context, body)) return; + UgcProcessor::DeleteRequest request; + request.kind = body->value("kind", std::string("model")) == "modular" ? UgcStorage::Kind::MODULAR : UgcStorage::Kind::MODEL; + if (const auto ids = body->find("ids"); ids != body->end() && ids->is_array()) { + for (const auto& id : *ids) { + const auto parsed = GeneralUtils::TryParse(id.is_string() ? id.get() : id.dump()); + if (parsed) request.ids.push_back(*parsed); + } + } + request.all = body->value("all", false); + request.olderThanDays = body->value("olderThanDays", int64_t{ 0 }); + request.unusedDays = body->value("unusedDays", int64_t{ 0 }); + const auto after = body->value("after", std::string("on_demand")); + request.after = after == "now" ? UgcProcessor::eAfterDelete::NOW : after == "gone" ? UgcProcessor::eAfterDelete::GONE : UgcProcessor::eAfterDelete::ON_DEMAND; + const auto result = g_Processor->Delete(request); + LOG("Deleted the files of %zu item(s) (%llu bytes) for the dashboard", result.deleted, static_cast(result.bytes)); + reply.status = eHTTPStatusCode::OK; + reply.message = nlohmann::json{ { "success", true }, { "deleted", result.deleted }, { "bytes", result.bytes }, { "busy", result.busy }, { "notes", result.notes } }.dump(); + } }); + Game::web.RegisterHTTPRoute({ .path = "/status", .method = eHTTPMethod::GET, .middleware = {}, .handle = [](HTTPReply& reply, const HTTPContext&) { reply.status = eHTTPStatusCode::OK; @@ -361,6 +428,7 @@ int main(int argc, char** argv) { masterPort = masterInfo->port; masterPassword = masterInfo->password; } + g_AdminKey = masterPassword; // The master starts it again when this link drops g_Server = new dServer(masterIP, Setting("net_port", 2012), 0, 16, false, false, Game::logger, masterIP, masterPort, diff --git a/dUgcServer/UgcStorage.cpp b/dUgcServer/UgcStorage.cpp index e3cfd1342..00aa00f37 100644 --- a/dUgcServer/UgcStorage.cpp +++ b/dUgcServer/UgcStorage.cpp @@ -10,7 +10,7 @@ namespace { "model.nif", "model.nif.gz", "model.nif.checksum", "model.lxfml.gz", "model.lxfml.checksum", "icon.dds.gz", "icon.dds.checksum", "icon.png", - "model.noao.nif", "stats.json", + "model.noao.nif", "stats.json", "combo.json", "previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json", }; @@ -88,6 +88,43 @@ std::optional UgcStorage::Write(Kind kind, LWOOBJID id, const Files& f return bytes; } +std::optional UgcStorage::Update(Kind kind, LWOOBJID id, const Files& files, std::string& error) const { + const auto folder = Folder(kind, id); + std::error_code code; + if (!std::filesystem::is_directory(folder, code)) { + error = "no files to update in " + folder.string(); + return std::nullopt; + } + uint64_t bytes = 0; + for (const auto* name : KEPT_FILES) { + if (files.contains(name) && std::filesystem::exists(folder / name, code)) { + std::filesystem::copy_file(folder / name, folder / (std::string("previous.") + name), std::filesystem::copy_options::overwrite_existing, code); + code.clear(); + } + } + for (const auto& [name, data] : files) { + if (!IsKnownFile(name)) continue; + const auto aside = folder / (".tmp-" + name + "-" + RandomSuffix()); + { + std::ofstream out(aside, std::ios::binary | std::ios::trunc); + out.write(data.data(), static_cast(data.size())); + if (!out) { + error = "could not write " + aside.string(); + std::filesystem::remove(aside, code); + return std::nullopt; + } + } + std::filesystem::rename(aside, folder / name, code); + if (code) { + error = "could not replace " + (folder / name).string() + ": " + code.message(); + std::filesystem::remove(aside, code); + return std::nullopt; + } + bytes += data.size(); + } + return bytes; +} + void UgcStorage::Remove(Kind kind, LWOOBJID id) const { std::error_code error; std::filesystem::remove_all(Folder(kind, id), error); diff --git a/dUgcServer/UgcStorage.h b/dUgcServer/UgcStorage.h index f3caac4d0..c377f769a 100644 --- a/dUgcServer/UgcStorage.h +++ b/dUgcServer/UgcStorage.h @@ -33,6 +33,11 @@ public: // Replaces an item's files with `files`; the bytes written, nullopt (and `error`) on failure std::optional Write(Kind kind, LWOOBJID id, const Files& files, std::string& error) const; + // Replaces some of an item's files in place (each written aside and renamed over the old one), keeping the others; + // the ones kept for comparing (icon.png...) are copied to previous.* first. The bytes written, nullopt when the item + // has no folder or a write fails. + std::optional Update(Kind kind, LWOOBJID id, const Files& files, std::string& error) const; + // Deletes an item's files void Remove(Kind kind, LWOOBJID id) const; diff --git a/dWorldServer/WorldServer.cpp b/dWorldServer/WorldServer.cpp index 5f5ef1018..24d4ad7b0 100644 --- a/dWorldServer/WorldServer.cpp +++ b/dWorldServer/WorldServer.cpp @@ -1146,6 +1146,9 @@ void CleanupDisconnectedUser(const SystemAddress& sysAddr) { PropertyManagementComponent::Instance()->Save(); } + // They left: the models they saved needn't wait out their quiet period (docs/UgcServer.md) + Database::Get()->ExpediteUgcModels(c->GetObjectID()); + MasterPackets::PlayerRemoved removed; removed.zoneID = static_cast(Game::server->GetZoneID()); removed.instanceID = static_cast(g_InstanceID); diff --git a/migrations/dlu/mysql/86_ugc_debounce_icon_settings.sql b/migrations/dlu/mysql/86_ugc_debounce_icon_settings.sql new file mode 100644 index 000000000..77ad257f3 --- /dev/null +++ b/migrations/dlu/mysql/86_ugc_debounce_icon_settings.sql @@ -0,0 +1,15 @@ +/* UGC server: ugc.process_after is the Unix time before which a saved model isn't made (the quiet period after a + save, so a model the owner keeps editing isn't made for every save; 0: right away). ugc_icon_settings holds + icon framing and lighting set on the dashboard: presets per kind ("kind:model", "kind:build6") and overrides for one + model ("model:") or one combination of car or rocket modules ("combo:"). See docs/UgcServer.md. */ +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN process_after BIGINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'process_after'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; + +CREATE TABLE IF NOT EXISTS ugc_icon_settings ( + target VARCHAR(191) NOT NULL PRIMARY KEY, + params TEXT NOT NULL, + updated_at BIGINT NOT NULL DEFAULT 0 +); diff --git a/migrations/dlu/sqlite/69_ugc_debounce_icon_settings.sql b/migrations/dlu/sqlite/69_ugc_debounce_icon_settings.sql new file mode 100644 index 000000000..efef782f9 --- /dev/null +++ b/migrations/dlu/sqlite/69_ugc_debounce_icon_settings.sql @@ -0,0 +1,8 @@ +/* UGC server: the quiet period after a save, and icon settings presets and overrides. See the MySQL migration. */ +ALTER TABLE ugc ADD COLUMN process_after BIGINT NOT NULL DEFAULT 0; + +CREATE TABLE IF NOT EXISTS ugc_icon_settings ( + target TEXT NOT NULL PRIMARY KEY, + params TEXT NOT NULL, + updated_at BIGINT NOT NULL DEFAULT 0 +); diff --git a/resources/sharedconfig.ini b/resources/sharedconfig.ini index 8504ee26c..5eb93115b 100644 --- a/resources/sharedconfig.ini +++ b/resources/sharedconfig.ini @@ -88,3 +88,8 @@ mute_restrict_mail=1 mute_restrict_trade=1 # 0 or 1, automatically reject character and pet names submitted while muted mute_auto_reject_names=1 + +# The UGC server makes a saved property model this many seconds after its owner's last save (each save starts the wait +# again for all their waiting models), or as soon as a game client asks for it or the owner leaves the world. 0: right +# away. See docs/UgcServer.md. +ugc_debounce_seconds=120 diff --git a/resources/ugcconfig.ini b/resources/ugcconfig.ini index 5303e5dc0..61c7bcbd0 100644 --- a/resources/ugcconfig.ini +++ b/resources/ugcconfig.ini @@ -78,21 +78,13 @@ ao_distance=5 ao_strength=1 glow_strength=6 -# Icons, set up like LU Toolbox's icon renderer: size in pixels, the camera's angle around (yaw, from the model's -# front) and above (pitch) in degrees, its field of view, the empty border (1 fills the icon), the sun's angles and -# strength, the world light and shadows. A player model's icon is drawn from its finished .nif (LOD 0), so it shows -# its colors, variation and baked lighting exactly as the game does. icon_ao: ambient occlusion for car and rocket -# icons (player models have theirs baked in). Modular builds (cars, rockets) have their own camera angles. +# Icons: size in pixels. A player model's icon is drawn from its finished .nif (LOD 0), a car or rocket's from its +# modules' meshes put together. icon_size=128 -icon_yaw=53.36 -icon_pitch=19.54 -icon_fov=39.6 -icon_margin=1.03 -icon_sun_yaw=21 -icon_sun_pitch=50.3 -icon_sun_strength=2.5 -icon_ambient=0.192 -icon_shadows=1 -icon_ao=1 -modular_icon_yaw=53.36 -modular_icon_pitch=19.54 + +# How icons are framed and lit: icon_yaw, icon_pitch, icon_fov, icon_margin, icon_offset_x, icon_offset_y, +# icon_sun_yaw, icon_sun_pitch, icon_sun_strength, icon_ambient, icon_fill, icon_specular, icon_shininess, +# icon_exposure, icon_contrast, icon_shadows, icon_ao_strength. Their defaults (the framing from LU Toolbox's icon +# renderer, the light matched to the game's own model icons) and ranges are listed once in the server +# (UgcIconParams) and on the dashboard's settings page; set one here only to change it. The dashboard's icon editor +# also keeps presets for player models and each car or rocket type, and overrides for single items, in the database. diff --git a/tests/dDatabaseTests/DatabaseParityTests.cpp b/tests/dDatabaseTests/DatabaseParityTests.cpp index 05196576a..5f2cf1f70 100644 --- a/tests/dDatabaseTests/DatabaseParityTests.cpp +++ b/tests/dDatabaseTests/DatabaseParityTests.cpp @@ -699,7 +699,7 @@ TEST_F(ParitySeeded, Properties) { TEST_F(ParitySeeded, UgcModel) { Both("InsertNewUgcModel", [](GameDatabase& db) { std::stringstream data("sd0\x01\x02 compressed model"); - db.InsertNewUgcModel(data, 1152921510000500001LL, 1, CHAR_ALICE); + db.InsertNewUgcModel(data, 1152921510000500001LL, 1, CHAR_ALICE, 0); }); Both("GetUgcModel unplaced", [](GameDatabase& db) { auto model = db.GetUgcModel(1152921510000500001LL); @@ -721,6 +721,21 @@ TEST_F(ParitySeeded, UgcModel) { for (const auto& model : db.GetUgcModelsToProcess(10)) out.push_back({ model.id, model.lxfml, model.attempts }); return out; }); + Both("Quiet period after a save", [&](GameDatabase& db) { + // A second save far in the future: it waits, and so does the owner's first model; leaving ends both waits + std::stringstream data("sd0\x01\x02 second model"); + db.InsertNewUgcModel(data, 1152921510000500003LL, 1, CHAR_ALICE, 4000000000LL); + json out = json{ db.GetUgcModelsToProcess(10).size(), db.GetUgcProcessInfo(1152921510000500003LL)->processAfter }; + db.ExpediteUgcModel(1152921510000500003LL); + out.push_back(db.GetUgcModelsToProcess(10).size()); + db.InsertNewUgcModel(data, 1152921510000500004LL, 1, CHAR_ALICE, 4000000001LL); + out.push_back(db.GetUgcModelsToProcess(10).size()); + db.ExpediteUgcModels(CHAR_ALICE); + out.push_back(db.GetUgcModelsToProcess(10).size()); + db.DeleteUgcModelData(1152921510000500003LL); + db.DeleteUgcModelData(1152921510000500004LL); + return out; + }); Both("SetUgcModelProcessed failed", [&](GameDatabase& db) { db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::FAILED, 3, "no bricks", false); return json{ infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), countsJson(db.GetUgcProcessCounts()), db.GetUgcModelsToProcess(10).size() }; @@ -751,6 +766,14 @@ TEST_F(ParitySeeded, UgcModel) { out.push_back(db.GetModularBuildProcessList(std::nullopt, "", 0, 10).size()); out.push_back(db.ResetModularBuildProcessing(1152921510000500002LL, false)); out.push_back(infoJson(db.GetModularBuildProcessInfo(1152921510000500002LL))); + for (const auto& [ldf, count] : db.GetModularBuildConfigCounts()) out.push_back({ ldf, count }); + // A combination's own icon framing + out.push_back(db.GetUgcIconSettings("combo:4713-4714-4715").has_value()); + db.SetUgcIconSettings("combo:4713-4714-4715", R"({"yaw":10})"); + db.SetUgcIconSettings("combo:4713-4714-4715", R"({"yaw":20})"); + out.push_back(db.GetUgcIconSettings("combo:4713-4714-4715").value_or("")); + db.DeleteUgcIconSettings("combo:4713-4714-4715"); + out.push_back(db.GetUgcIconSettings("combo:4713-4714-4715").has_value()); return out; }); } diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp index 1b40f4082..415731f39 100644 --- a/tests/dUgcTests/UgcTests.cpp +++ b/tests/dUgcTests/UgcTests.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -12,6 +13,8 @@ #include "UgcFormats.h" #include "UgcModel.h" #include "UgcJobs.h" +#include "UgcIconParams.h" +#include "UgcKeys.h" #include "UgcModular.h" #include "UgcPalette.h" #include "UgcRender.h" @@ -47,7 +50,9 @@ namespace { } std::filesystem::path TempFolder(const std::string& name) { - auto path = std::filesystem::temp_directory_path() / ("dlu_ugc_test_" + name + "_" + std::to_string(::testing::UnitTest::GetInstance()->random_seed())); + // One folder per test and process: ctest runs the tests in parallel processes + const auto* test = ::testing::UnitTest::GetInstance()->current_test_info(); + auto path = std::filesystem::temp_directory_path() / ("dlu_ugc_test_" + name + "_" + (test ? std::string(test->name()) : std::string()) + "_" + std::to_string(::getpid())); std::filesystem::remove_all(path); std::filesystem::create_directories(path); return path; @@ -581,3 +586,119 @@ TEST(UgcJobs, MakesLodsStatsAndIcons) { EXPECT_EQ(UgcJobs::CountParts(LXFML5), 3u); EXPECT_GT(UgcJobs::EstimateMemory(1000, settings), UgcJobs::EstimateMemory(10, settings)); } + +TEST(UgcModularKey, SameModulesSameKey) { + // However the modules are written or ordered, the combination is the same; its files are stored once + EXPECT_EQ(UgcModularKey::Normalize("1:4715+1:4713+1:4714"), "4713-4714-4715"); + EXPECT_EQ(UgcModularKey::Normalize("1:4713;1:4714,1:4715"), "4713-4714-4715"); + EXPECT_EQ(UgcModularKey::Normalize("4714+4715+4713+1:4713"), "4713-4714-4715"); + EXPECT_EQ(UgcModularKey::Normalize(""), ""); + EXPECT_EQ(UgcModularKey::Normalize("1:abc+"), ""); + EXPECT_NE(UgcModularKey::Normalize("1:4713+1:4714+1:4716"), UgcModularKey::Normalize("1:4713+1:4714+1:4715")); + const auto id = UgcModularKey::StorageId("4713-4714-4715"); + EXPECT_GT(id, 0); + EXPECT_EQ(id, UgcModularKey::StorageId(UgcModularKey::Normalize("1:4715+1:4714+1:4713"))); + EXPECT_NE(id, UgcModularKey::StorageId("4713-4714-4716")); + + // Two builds of the same modules find the one set of files + UgcStorage storage(TempFolder("combo")); + std::string error; + ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODULAR, id, { { "icon.png", "png" } }, error)) << error; + EXPECT_TRUE(storage.File(UgcStorage::Kind::MODULAR, UgcModularKey::StorageId(UgcModularKey::Normalize("1:4713+1:4714+1:4715")), "icon.png")); + EXPECT_TRUE(storage.File(UgcStorage::Kind::MODULAR, UgcModularKey::StorageId(UgcModularKey::Normalize("1:4714+1:4715+1:4713")), "icon.png")); + std::filesystem::remove_all(storage.GetRoot()); +} + +TEST(UgcDebounce, WaitsForTheQuietPeriod) { + EXPECT_EQ(UgcDebounce::ProcessAfter(1000, 120), 1120); + EXPECT_EQ(UgcDebounce::ProcessAfter(1000, 0), 0); + EXPECT_EQ(UgcDebounce::ProcessAfter(1000, -5), 0); + EXPECT_FALSE(UgcDebounce::Due(1120, 1100, false)); // saved 100 s ago: still quiet + EXPECT_TRUE(UgcDebounce::Due(1120, 1120, false)); + EXPECT_TRUE(UgcDebounce::Due(1120, 1100, true)); // a client asked for it + EXPECT_TRUE(UgcDebounce::Due(0, 5, false)); // expedited or saved without a wait + // A new save starts the wait again + const auto first = UgcDebounce::ProcessAfter(1000, 120), second = UgcDebounce::ProcessAfter(1100, 120); + EXPECT_FALSE(UgcDebounce::Due(std::max(first, second), 1150, false)); +} + +TEST(UgcIconParams, OneListDrivesEverything) { + // Every parameter has a setting, a range holding its default, and something it changes + for (const auto& param : UgcIconParams::List()) { + EXPECT_TRUE(param.setting.starts_with("icon_")) << param.key; + EXPECT_LE(param.min, param.defaultValue) << param.key; + EXPECT_GE(param.max, param.defaultValue) << param.key; + EXPECT_TRUE(param.apply) << param.key; + EXPECT_EQ(UgcIconParams::Find(param.key), ¶m); + } + const auto values = UgcIconParams::Parse(R"({"yaw":10,"pitch":200,"margin":0,"offsetX":0.1,"unknown":1,"fov":"wide","exposure":1.5})"); + EXPECT_FLOAT_EQ(values.at("yaw"), 10.0f); + EXPECT_FLOAT_EQ(values.at("pitch"), 89.0f); // clamped + EXPECT_FLOAT_EQ(values.at("margin"), 0.5f); + EXPECT_FALSE(values.contains("fov")); // not a number + EXPECT_FALSE(values.contains("unknown")); + EXPECT_TRUE(UgcIconParams::Parse("not json").empty()); + EXPECT_EQ(UgcIconParams::Parse(UgcIconParams::ToJson(values)), values); + + // Settings, then values over them + auto options = UgcIconParams::FromSettings([](const std::string& key) -> std::optional { + if (key == "icon_fov") return "33"; + if (key == "icon_ambient") return "nonsense"; + return std::nullopt; + }); + EXPECT_FLOAT_EQ(options.fovDegrees, 33.0f); + EXPECT_FLOAT_EQ(options.ambient, UgcIconParams::Find("ambient")->defaultValue); + UgcIconParams::Apply(options, values); + EXPECT_FLOAT_EQ(options.yawDegrees, 10.0f); + EXPECT_FLOAT_EQ(options.fovDegrees, 33.0f); // not in the values: the setting stays + EXPECT_FLOAT_EQ(options.exposure, 1.5f); + EXPECT_FLOAT_EQ(options.offsetX, 0.1f); + UgcIconParams::Apply(options, { { "aoStrength", 0.5f } }); + EXPECT_TRUE(options.ao.enabled); + + // A car or rocket: the settings with its preset and combination values + UgcJobs::Settings settings; + UgcJobs::ModularInput input; + input.iconValues = { { "yaw", 5.0f } }; + EXPECT_FLOAT_EQ(UgcJobs::ModularIconOptions(input, settings).yawDegrees, 5.0f); + EXPECT_EQ(UgcIconParams::KindTarget(UgcIconParams::BuildKind(6)), "kind:build6"); + EXPECT_EQ(UgcIconParams::ModelTarget(12), "model:12"); + EXPECT_EQ(UgcIconParams::CombinationTarget("1-2"), "combo:1-2"); + + // Exposure brightens, contrast spreads + UgcBricks::BrickLibrary library(MakeRes(), 0); + std::string error; + const auto box = UgcModel::Build(UgcModel::ParseLxfml(R"( + )", error), library); + const auto mean = [](const UgcRender::Image& image) { + double sum = 0, count = 0; + for (size_t i = 0; i < image.rgba.size(); i += 4) { + if (image.rgba[i + 3] < 128) continue; + sum += image.rgba[i]; + count++; + } + return sum / std::max(count, 1.0); + }; + UgcRender::IconOptions dim{ 32, 1 }, bright{ 32, 1 }; + bright.exposure = 2.0f; + EXPECT_GT(mean(UgcRender::RenderIcon(box, bright)), mean(UgcRender::RenderIcon(box, dim)) + 10.0); + + // An offset moves the drawn model by that share of the icon + UgcModel::Model model; + model = UgcModel::Build(UgcModel::ParseLxfml(R"( + )", error), library); + const auto centroid = [](const UgcRender::Image& image) { + double sum = 0, count = 0; + for (int y = 0; y < image.height; y++) for (int x = 0; x < image.width; x++) { + const auto a = image.rgba[(static_cast(y) * image.width + x) * 4 + 3]; + sum += x * a; + count += a; + } + return count > 0 ? sum / count : -1.0; + }; + UgcRender::IconOptions plain{ 64, 1 }; + plain.margin = 2.0f; + auto shifted = plain; + shifted.offsetX = 0.25f; + EXPECT_NEAR(centroid(UgcRender::RenderIcon(model, shifted)) - centroid(UgcRender::RenderIcon(model, plain)), 16.0, 1.0); +} diff --git a/tests/dWebTests/CMakeLists.txt b/tests/dWebTests/CMakeLists.txt index b09ad2389..55287f9e2 100644 --- a/tests/dWebTests/CMakeLists.txt +++ b/tests/dWebTests/CMakeLists.txt @@ -77,7 +77,7 @@ target_include_directories(dWebTests PRIVATE "${PROJECT_SOURCE_DIR}/thirdparty/nlohmann" ) -target_link_libraries(dWebTests ${COMMON_LIBRARIES} dWeb sqlite3 OpenSSL::Crypto CURL::libcurl GTest::gtest_main) +target_link_libraries(dWebTests ${COMMON_LIBRARIES} dWeb dUgc sqlite3 OpenSSL::Crypto CURL::libcurl GTest::gtest_main) gtest_discover_tests(dWebTests)