diff --git a/dCommon/BrickByBrickFix.cpp b/dCommon/BrickByBrickFix.cpp index d299341fe..c6a7cd3a2 100644 --- a/dCommon/BrickByBrickFix.cpp +++ b/dCommon/BrickByBrickFix.cpp @@ -120,7 +120,7 @@ uint32_t BrickByBrickFix::UpdateBrickByBrickModelsToSd0() { std::stringstream outputStringStream(outputString); try { - Database::Get()->UpdateUgcModelData(model.id, outputStringStream); + Database::Get()->UpdateUgcModelData(model.id, outputStringStream, 0); LOG("Updated model %i to sd0", model.id); updatedModels++; } catch (std::exception& exception) { diff --git a/dCommon/UgcKeys.h b/dCommon/UgcKeys.h new file mode 100644 index 000000000..1bf9dcc64 --- /dev/null +++ b/dCommon/UgcKeys.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include +#include +#include + +/** + * Small pure decisions about UGC shared by the worlds, the UGC server and the dashboard (docs/UgcServer.md). + */ +namespace UgcModularKey { + // The module LOTs of a ugc_modular_build.ldf_config ("1:4713+1:4714+1:4715", also with ';' or ',' between) + inline std::vector Lots(std::string_view ldf) { + std::vector lots; + size_t start = 0; + while (start <= ldf.size()) { + size_t end = ldf.find_first_of("+;,", start); + if (end == std::string_view::npos) end = ldf.size(); + auto item = ldf.substr(start, end - start); + if (const auto colon = item.find(':'); colon != std::string_view::npos) item = item.substr(colon + 1); + while (!item.empty() && item.front() == ' ') item.remove_prefix(1); + while (!item.empty() && item.back() == ' ') item.remove_suffix(1); + uint32_t value = 0; + bool digits = !item.empty() && item.size() <= 9; + for (const char c : item) { + if (c < '0' || c > '9') { + digits = false; + break; + } + value = value * 10 + static_cast(c - '0'); + } + if (digits && value != 0) lots.push_back(value); + start = end + 1; + } + return lots; + } + + /** + * The combination of modules a build is made of, the same however its ldf_config is written: its LOTs sorted + * and joined by '-' ("4713-4714-4715"). Each module LOT belongs to one slot of one build type, so the set of LOTs + * says everything about how the build looks. Empty when there are no modules. + */ + inline std::string Normalize(std::string_view ldf) { + auto lots = Lots(ldf); + std::sort(lots.begin(), lots.end()); + lots.erase(std::unique(lots.begin(), lots.end()), lots.end()); + std::string key; + for (const auto lot : lots) key += (key.empty() ? "" : "-") + std::to_string(lot); + return key; + } + + // The id a combination's files are stored under (FNV-1a of the key, positive, never 0) + inline int64_t StorageId(std::string_view key) { + uint64_t hash = 14695981039346656037ull; + for (const char c : key) hash = (hash ^ static_cast(c)) * 1099511628211ull; + hash &= 0x3FFFFFFFFFFFFFFFull; + return static_cast(hash == 0 ? 1 : hash); + } +} + +namespace UgcDebounce { + // When a model saved at `now` may be made: after `quietSeconds` (0 or less: right away, stored as 0) + inline int64_t ProcessAfter(int64_t now, int64_t quietSeconds) { + return quietSeconds > 0 ? now + quietSeconds : 0; + } + + // Whether a waiting model is due: its quiet period is over, or someone asked for its files + inline bool Due(int64_t processAfter, int64_t now, bool requested) { + return requested || processAfter <= now; + } +} diff --git a/dDashboardServer/routes/CMakeLists.txt b/dDashboardServer/routes/CMakeLists.txt index ee3183e89..b2d4049f6 100644 --- a/dDashboardServer/routes/CMakeLists.txt +++ b/dDashboardServer/routes/CMakeLists.txt @@ -98,4 +98,4 @@ target_include_directories(DashboardRoutes PRIVATE ) target_compile_definitions(DashboardRoutes PRIVATE PROJECT_VERSION="\"${PROJECT_VERSION}\"") -target_link_libraries(DashboardRoutes PRIVATE ${COMMON_LIBRARIES} dWeb dServer sqlite3 OpenSSL::Crypto CURL::libcurl) +target_link_libraries(DashboardRoutes PRIVATE ${COMMON_LIBRARIES} dWeb dServer dUgc sqlite3 OpenSSL::Crypto CURL::libcurl) diff --git a/dDashboardServer/routes/MaintenanceRoutes.cpp b/dDashboardServer/routes/MaintenanceRoutes.cpp index 68fde4fdc..45575bdfb 100644 --- a/dDashboardServer/routes/MaintenanceRoutes.cpp +++ b/dDashboardServer/routes/MaintenanceRoutes.cpp @@ -266,7 +266,7 @@ void RegisterMaintenanceRoutes() { Sd0 sd0(header); sd0.FromData(reinterpret_cast(split.lxfml.data()), split.lxfml.size()); auto stream = sd0.GetAsStream(); - Database::Get()->InsertNewUgcModel(stream, blueprintId, owner->accountId, owner->id); + Database::Get()->InsertNewUgcModel(stream, blueprintId, owner->accountId, owner->id, 0); IPropertyContents::Model model; model.id = modelId; diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index f30e985b3..afd2dc98c 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -1,5 +1,7 @@ #include "SettingsCatalog.h" +#include "UgcIconParams.h" + #include #include #include @@ -418,6 +420,7 @@ namespace { c.Add(Text(UGC, "client_path", "Download path", "The game client's UGCSERVERDIR.", "/ugc", true)); c.Add(Format(Text(UGC, "ugc_output_dir", "Files folder", "Relative to the server binaries.", "ugc", true), eFormat::PATH)); c.Add(Unit(Int(UGC, "ugc_max_storage_mb", "Most space for files", "The files used longest ago are deleted past this and made again when asked for. 0: no limit.", "2048", 0, std::nullopt, true), "MB")); + c.Add(Unit(Int(SHARED, "ugc_debounce_seconds", "Wait after a save", "A saved model is made this long after the owner's last save (their other waiting models wait too), or as soon as a game client asks for it or the owner leaves. 0: right away.", "120", 0, 86400), "seconds")); c.Add(Int(UGC, "worker_threads", "Worker threads", "0: half the CPU cores.", "0", 0, 64, true)); c.Add(Unit(Float(UGC, "max_cpu_percent", "Most CPU", "The workers together average at most this share of all CPU cores; they pause between bits of work to stay under it. 0: no limit.", "0", 0, 100), "%")); c.Add(Int(UGC, "worker_nice", "Worker priority", "Linux nice value of the workers: 0 normal, 19 only when nothing else wants the CPU.", "0", 0, 19)); @@ -447,18 +450,15 @@ namespace { c.Add(Float(UGC, "ao_strength", "Darkening strength", "0 to 1.", "1", 0, 1)); c.Add(Float(UGC, "glow_strength", "Glow strength", "What glowing colors add to the baked light.", "6", 0, 100)); c.Add(Unit(Int(UGC, "icon_size", "Icon size", "", "128", 16, 1024), "pixels")); - c.Add(Unit(Float(UGC, "icon_yaw", "Icon angle around", "", "53.36", -360, 360), "degrees")); - c.Add(Unit(Float(UGC, "icon_pitch", "Icon angle above", "", "19.54", -90, 90), "degrees")); - c.Add(Unit(Float(UGC, "icon_fov", "Icon field of view", "", "39.6", 1, 120), "degrees")); - c.Add(Float(UGC, "icon_margin", "Icon border", "1 fills the icon.", "1.03", 0.1, 10)); - c.Add(Unit(Float(UGC, "icon_sun_yaw", "Icon sun angle around", "", "21", -360, 360), "degrees")); - c.Add(Unit(Float(UGC, "icon_sun_pitch", "Icon sun angle above", "", "50.3", -90, 90), "degrees")); - c.Add(Float(UGC, "icon_sun_strength", "Icon sun strength", "", "2.5", 0, 100)); - c.Add(Float(UGC, "icon_ambient", "Icon world light", "", "0.192", 0, 10)); - c.Add(Bool(UGC, "icon_shadows", "Icon shadows", "", true)); - c.Add(Bool(UGC, "icon_ao", "Car and rocket icon ambient occlusion", "Player models' icons show the lighting baked into their mesh instead.", true)); - c.Add(Unit(Float(UGC, "modular_icon_yaw", "Car and rocket icon angle around", "", "53.36", -360, 360), "degrees")); - c.Add(Unit(Float(UGC, "modular_icon_pitch", "Car and rocket icon angle above", "", "19.54", -90, 90), "degrees")); + // The icon's framing and light, from the one list of them (UgcIconParams); presets and overrides are set on the + // UGC page's icon editor + for (const auto& param : UgcIconParams::List()) { + char number[32]; + const auto written = std::to_chars(number, number + sizeof(number), param.defaultValue); + auto setting = Float(UGC, param.setting, "Icon: " + param.label, param.description, std::string(number, written.ptr), param.min, param.max); + if (!param.unit.empty()) setting = Unit(setting, param.unit); + c.Add(setting); + } return c; } diff --git a/dDashboardServer/routes/UgcRoutes.cpp b/dDashboardServer/routes/UgcRoutes.cpp index 28f370311..b845ac52b 100644 --- a/dDashboardServer/routes/UgcRoutes.cpp +++ b/dDashboardServer/routes/UgcRoutes.cpp @@ -6,9 +6,14 @@ #include +#include "CDClientDatabase.h" #include "Database.h" +#include "UgcIconParams.h" #include "NifFile.h" #include "TtlCache.h" +#include "SettingsCatalog.h" +#include "SettingsHistory.h" +#include "UgcKeys.h" #include "Workers.h" #include "Game.h" #include "GeneralUtils.h" @@ -95,6 +100,111 @@ namespace { return out; } + // Any thread: a POST of JSON to one of the UGC server's /admin routes, with the key it checks (the master password) + std::shared_ptr AdminPost(const std::string& url, const std::string& key, const std::string& body) { + auto out = std::make_shared(); + CURL* curl = curl_easy_init(); + if (!curl) { + out->error = "could not start a request"; + return out; + } + curl_slist* headers = curl_slist_append(nullptr, "Content-Type: application/json"); + headers = curl_slist_append(headers, ("X-Ugc-Admin-Key: " + key).c_str()); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https"); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 120L); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(body.size())); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Collect); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out->body); + const auto code = curl_easy_perform(curl); + if (code == CURLE_OK) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &out->status); + else out->error = curl_easy_strerror(code); + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + return out; + } + + // Web thread: the key the UGC server's /admin routes want + std::string AdminKey() { + const auto master = Database::Get()->GetMasterInfo(); + return master ? master->password : std::string(); + } + + // Sends an admin request from a worker and answers with the UGC server's JSON (or why it couldn't) + void ProxyAdmin(HTTPReply& reply, const HTTPContext& context, const std::string& route, const nlohmann::json& body) { + const auto url = InternalUrl() + route; + Workers::Reply(reply, context, false, [url, key = AdminKey(), text = body.dump()](HTTPReply& out) { + const auto fetched = AdminPost(url, key, text); + if (fetched->status == 0) return JsonError(out, eHTTPStatusCode::BAD_GATEWAY, "The UGC server doesn't answer at " + url + " (" + fetched->error + ")"); + out.status = static_cast(fetched->status); + const bool png = fetched->body.starts_with("\x89PNG"); + out.contentType = png ? eContentType::IMAGE_PNG : eContentType::APPLICATION_JSON; + out.message = fetched->body; + if (!png && fetched->status != 200 && !fetched->body.starts_with("{")) { + JsonError(out, static_cast(fetched->status), fetched->body.empty() ? "The UGC server answered " + std::to_string(fetched->status) : fetched->body); + } + out.headers.push_back("Cache-Control: no-store"); + }, WorkerPool::ePriority::URGENT); + } + + // Web thread: a ugcconfig.ini setting's value as the UGC server sees it (dashboard value, file value, default) + std::optional UgcSetting(const std::string& name) { + if (const auto current = SettingsHistory::Current("ugcconfig.ini", name)) { + if (current->webValue && (current->webWins || !current->fileValue)) return current->webValue; + if (current->fileValue) return current->fileValue; + } + if (const auto* info = SettingsCatalog::Find("ugcconfig.ini", name)) return info->defaultValue; + return std::nullopt; + } + + // Web thread: the icon kinds: player models, and each car or rocket build type in the client's data + // (ModularBuildComponent), named after the object its Assembly LOT is + nlohmann::json IconKinds() { + nlohmann::json kinds = nlohmann::json::array({ { { "kind", UgcIconParams::ModelKind() }, { "label", "Player models" } } }); + auto result = CDClientDatabase::ExecuteQuery("SELECT buildType, xml FROM ModularBuildComponent GROUP BY buildType ORDER BY buildType;"); + while (!result.eof()) { + const auto buildType = result.getIntField("buildType", 0); + std::string label = "Build type " + std::to_string(buildType); + const std::string xml = result.getStringField("xml", ""); + if (const auto at = xml.find("LOT=\""); at != std::string::npos) { + const auto lot = GeneralUtils::TryParse(xml.substr(at + 5, xml.find('"', at + 5) - at - 5)); + if (lot) { + auto name = CDClientDatabase::CreatePreppedStmt("SELECT displayName FROM Objects WHERE id = ? LIMIT 1;"); + name.bind(1, *lot); + auto row = name.execQuery(); + if (!row.eof() && row.getStringField("displayName", "")[0] != '\0') label = row.getStringField("displayName", ""); + } + } + kinds.push_back({ { "kind", UgcIconParams::BuildKind(buildType) }, { "label", label }, { "buildType", buildType } }); + result.nextRow(); + } + return kinds; + } + + // Web thread: a car or rocket's kind, from its first module's build type (ModuleComponent) + std::optional ModularKind(const std::string& modules) { + const auto lots = UgcModularKey::Lots(modules); + if (lots.empty()) return std::nullopt; + auto query = CDClientDatabase::CreatePreppedStmt("SELECT m.buildType FROM ComponentsRegistry cr JOIN ModuleComponent m ON m.id = cr.component_id " + "WHERE cr.id = ? AND cr.component_type = 28 LIMIT 1;"); + query.bind(1, static_cast(lots.front())); + auto row = query.execQuery(); + if (row.eof()) return std::nullopt; + return UgcIconParams::BuildKind(row.getIntField("buildType", 0)); + } + + // Web thread: the values stored for a target, or null + nlohmann::json StoredValues(const std::string& target) { + const auto stored = Database::Get()->GetUgcIconSettings(target); + if (!stored) return nullptr; + return nlohmann::json::parse(UgcIconParams::ToJson(UgcIconParams::Parse(*stored))); + } + // Any thread: Get, through the cache (answers that aren't a file being made are kept) std::shared_ptr CachedGet(const std::string& url, bool status) { { @@ -116,7 +226,7 @@ namespace { JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server answered " + std::to_string(fetched.status)); } - const std::set FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json", + const std::set FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json", "combo.json", "previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json" }; std::optional KindOf(const std::string& text) { @@ -147,12 +257,22 @@ namespace UgcRoutes { const auto size = std::clamp(GeneralUtils::TryParse(QueryValue(context.queryString, "size")).value_or(PAGE_SIZE), 1u, 200u); const auto list = modular ? Database::Get()->GetModularBuildProcessList(state, search, page * size, size + 1) : Database::Get()->GetUgcProcessList(state, search, page * size, size + 1); + // How many builds share each combination of modules (they share one icon) + std::map reuse; + if (modular) { + for (const auto& [ldf, count] : Database::Get()->GetModularBuildConfigCounts()) reuse[UgcModularKey::Normalize(ldf)] += count; + } nlohmann::json items = nlohmann::json::array(); for (size_t i = 0; i < list.size() && i < size; i++) { const auto& info = list[i]; items.push_back({ { "id", std::to_string(info.id) }, { "characterId", std::to_string(info.characterId) }, { "characterName", info.characterName }, { "state", StateName(info.state) }, { "attempts", info.attempts }, { "processedAt", info.processedAt }, { "error", info.error }, - { "bakeAo", info.bakeAo }, { "modules", info.details } }); + { "bakeAo", info.bakeAo }, { "modules", info.details }, { "processAfter", info.processAfter } }); + if (modular) { + const auto key = UgcModularKey::Normalize(info.details); + items.back()["combination"] = key; + items.back()["sharedBy"] = reuse[key]; + } } JsonSuccess(reply, { { "counts", { { "model", Counts(Database::Get()->GetUgcProcessCounts()) }, { "modular", Counts(Database::Get()->GetModularBuildProcessCounts()) } } }, { "items", items }, { "more", list.size() > size }, { "ugcPublicUrl", Game::config->GetValue("ugc_public_url") }, { "canManage", Can(context, "ugc_manage") } }); @@ -231,6 +351,157 @@ namespace UgcRoutes { }); }); + Route(eHTTPMethod::POST, "/api/ugc/cache/delete", Perm("ugc_manage"), + "Delete one item's generated files on the UGC server. Body: {kind: model|modular, id, after: on_demand|now|gone} (made again when asked " + "for, queued now, or left deleted). A car or rocket's files are shared by every build of the same modules", + [](HTTPReply& reply, const HTTPContext& context) { + const auto body = ParseBody(context); + if (!body) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid JSON"); + const auto kind = body->value("kind", std::string("model")) == "modular" ? std::string("modular") : std::string("model"); + const auto id = GeneralUtils::TryParse((*body)["id"].is_string() ? (*body)["id"].get() : (*body)["id"].dump()); + if (!id) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid id"); + const auto after = body->value("after", std::string("now")); + Audit(context, "ugc_delete_files", "Deleted the UGC files of " + kind + " " + std::to_string(*id) + " (" + after + ")"); + ProxyAdmin(reply, context, "/admin/delete", { { "kind", kind }, { "ids", { std::to_string(*id) } }, { "after", after } }); + }); + + Route(eHTTPMethod::POST, "/api/ugc/cache/purge", Perm("ugc_manage"), + "Delete generated files by filter. Body: {kind, state?, owner? (name), olderThanDays?, unusedDays?, all?, confirm (\"PURGE ALL\" with all and no " + "other filter), after: on_demand|now|gone}. State and owner pick the rows here; the ages are the files' own", + [](HTTPReply& reply, const HTTPContext& context) { + const auto body = ParseBody(context); + if (!body) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid JSON"); + const bool modular = body->value("kind", std::string("model")) == "modular"; + const auto state = ParseState(body->value("state", std::string())); + const auto owner = body->value("owner", std::string()).substr(0, 64); + const auto older = body->value("olderThanDays", int64_t{ 0 }); + const auto unused = body->value("unusedDays", int64_t{ 0 }); + nlohmann::json request = { { "kind", modular ? "modular" : "model" }, { "olderThanDays", older }, { "unusedDays", unused }, + { "after", body->value("after", std::string("on_demand")) } }; + std::string what = std::string(modular ? "cars and rockets" : "models"); + if (state || !owner.empty()) { + // The rows the filter picks, from the database + nlohmann::json ids = nlohmann::json::array(); + for (uint32_t page = 0;; page++) { + const auto list = modular ? Database::Get()->GetModularBuildProcessList(state, owner, page * 1000, 1000) : Database::Get()->GetUgcProcessList(state, owner, page * 1000, 1000); + for (const auto& info : list) ids.push_back(std::to_string(info.id)); + if (list.size() < 1000 || ids.size() >= 100000) break; + } + if (ids.empty()) return JsonSuccess(reply, { { "deleted", 0 }, { "bytes", 0 }, { "notes", { "Nothing matches the filter." } } }); + request["ids"] = ids; + what += (state ? " " + body->value("state", std::string()) : "") + (owner.empty() ? "" : " of " + owner); + } else { + if (body->value("all", false) && older == 0 && unused == 0 && body->value("confirm", std::string()) != "PURGE ALL") { + return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Type PURGE ALL to delete every generated file"); + } + if (!body->value("all", false) && older == 0 && unused == 0) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Pick a filter, or all"); + request["all"] = true; + } + if (older > 0) what += ", made more than " + std::to_string(older) + " days ago"; + if (unused > 0) what += ", not asked for in " + std::to_string(unused) + " days"; + Audit(context, "ugc_purge", "Purged the UGC files of " + what + " (" + request["after"].get() + ")"); + ProxyAdmin(reply, context, "/admin/delete", request); + }); + + Route(eHTTPMethod::GET, "/api/ugc/icon/params", Perm("properties_view"), + "What an icon's framing and light can be set to: {params: [{key, setting, label, unit, min, max, step, default, description}], kinds: [{kind, label}]} " + "(player models, and each car or rocket build type in the client's data)", + [](HTTPReply& reply, const HTTPContext&) { + nlohmann::json params = nlohmann::json::array(); + for (const auto& param : UgcIconParams::List()) { + params.push_back({ { "key", param.key }, { "setting", param.setting }, { "label", param.label }, { "unit", param.unit }, { "min", param.min }, + { "max", param.max }, { "step", param.step }, { "default", param.defaultValue }, { "description", param.description } }); + } + JsonSuccess(reply, { { "params", params }, { "kinds", IconKinds() } }); + }); + + Route(eHTTPMethod::GET, "/api/ugc/icon/settings", Perm("properties_view"), + "An item's icon values in their layers: {kind, target, settings (icon_* as the UGC server reads them), preset (the kind's, or null), own (the " + "item's or combination's, or null)}. Query: ?kind=model&id= or ?kind=modular&modules=, or ?preset= for a kind alone", + [](HTTPReply& reply, const HTTPContext& context) { + std::string kind, target; + if (const auto preset = QueryValue(context.queryString, "preset"); !preset.empty()) { + kind = preset; + } else if (QueryValue(context.queryString, "kind") == "modular") { + const auto modules = QueryValue(context.queryString, "modules"); + const auto found = ModularKind(modules); + if (!found) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Those aren't modules"); + kind = *found; + target = UgcIconParams::CombinationTarget(UgcModularKey::Normalize(modules)); + } else { + const auto id = GeneralUtils::TryParse(QueryValue(context.queryString, "id")); + if (!id) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid id"); + kind = UgcIconParams::ModelKind(); + target = UgcIconParams::ModelTarget(*id); + } + nlohmann::json settings = nlohmann::json::object(); + for (const auto& param : UgcIconParams::List()) { + const auto value = UgcSetting(param.setting); + settings[param.key] = std::clamp(value ? GeneralUtils::TryParse(*value).value_or(param.defaultValue) : param.defaultValue, param.min, param.max); + } + JsonSuccess(reply, { { "kind", kind }, { "target", target }, { "settings", settings }, { "preset", StoredValues(UgcIconParams::KindTarget(kind)) }, + { "own", target.empty() ? nlohmann::json(nullptr) : StoredValues(target) } }); + }); + + Route(eHTTPMethod::POST, "/api/ugc/icon/preview", Perm("ugc_manage"), + "An icon drawn by the UGC server with the given values, not stored (PNG). Body: {kind: model, id, values} or {kind: modular, modules, values}", + [](HTTPReply& reply, const HTTPContext& context) { + const auto body = ParseBody(context); + if (!body) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid JSON"); + const bool model = body->value("kind", std::string()) == "model"; + ProxyAdmin(reply, context, "/admin/preview", { { "kind", model ? "model" : "modular" }, { "id", (*body)["id"].is_string() ? (*body)["id"].get() : std::string("0") }, + { "modules", body->value("modules", std::string()) }, { "values", nlohmann::json::parse(UgcIconParams::ToJson(UgcIconParams::Parse(body->value("values", nlohmann::json::object()).dump()))) } }); + }); + + Route(eHTTPMethod::POST, "/api/ugc/icon/save", Perm("ugc_manage"), + "Save icon values as a kind's preset ({scope: kind, kind, values}) or as one item's own ({scope: item, kind: model, id, values} or {scope: item, " + "kind: modular, modules, values}); values null removes it. The icons already made keep theirs until they are drawn again", + [](HTTPReply& reply, const HTTPContext& context) { + const auto body = ParseBody(context); + if (!body) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid JSON"); + std::string target, what; + if (body->value("scope", std::string()) == "kind") { + const auto kind = body->value("kind", std::string()); + bool known = false; + for (const auto& entry : IconKinds()) known = known || entry["kind"] == kind; + if (!known) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Unknown kind"); + target = UgcIconParams::KindTarget(kind); + what = "the " + kind + " icon preset"; + } else if (body->value("kind", std::string()) == "modular") { + const auto key = UgcModularKey::Normalize(body->value("modules", std::string())); + if (key.empty()) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "No modules"); + target = UgcIconParams::CombinationTarget(key); + what = "the icon of the module combination " + key; + } else { + const auto id = GeneralUtils::TryParse((*body)["id"].is_string() ? (*body)["id"].get() : (*body)["id"].dump()); + if (!id) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid id"); + target = UgcIconParams::ModelTarget(*id); + what = "the icon of model " + std::to_string(*id); + } + const auto values = body->value("values", nlohmann::json()); + if (values.is_null()) { + Database::Get()->DeleteUgcIconSettings(target); + Audit(context, "ugc_icon_settings", "Removed the settings of " + what); + return JsonSuccess(reply, { { "message", "Removed; draw the icons again to see it" } }); + } + const auto clean = UgcIconParams::ToJson(UgcIconParams::Parse(values.dump())); + Database::Get()->SetUgcIconSettings(target, clean); + Audit(context, "ugc_icon_settings", "Set " + what + " to " + clean); + JsonSuccess(reply, { { "message", "Saved; draw the icons again to see it in game" } }); + }); + + Route(eHTTPMethod::POST, "/api/ugc/icon/regenerate", Perm("ugc_manage"), + "Have the UGC server draw every stored icon of a kind again (only icons: player models' from their stored .nif). Body: {kind}", + [](HTTPReply& reply, const HTTPContext& context) { + const auto body = ParseBody(context); + const auto kind = body ? body->value("kind", std::string()) : std::string(); + bool known = false; + for (const auto& entry : IconKinds()) known = known || entry["kind"] == kind; + if (!known) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Unknown kind"); + Audit(context, "ugc_regenerate_icons", "Queued every " + kind + " icon to be drawn again"); + ProxyAdmin(reply, context, "/admin/regenerate-icons", { { "kind", kind } }); + }); + Route(eHTTPMethod::POST, "/api/ugc/reprocess", Perm("ugc_manage"), "Have the UGC server make items again. Body: {kind: model|modular, id} for one, {kind, failedOnly: true} for the failed ones, {kind} for all", [](HTTPReply& reply, const HTTPContext& context) { diff --git a/dDashboardServer/static/js/ugc.js b/dDashboardServer/static/js/ugc.js index c1d6df288..22642fe9f 100644 --- a/dDashboardServer/static/js/ugc.js +++ b/dDashboardServer/static/js/ugc.js @@ -41,12 +41,20 @@ var s = STATES[i.state] || [i.state, 'secondary']; return '
' + iconImg(i, 128) + '
' + (i.characterName ? esc(i.characterName) : '' + esc(i.characterId) + '') + '
' + - '
' + esc(i.id) + '
' + fmt.badge(s[0], s[1]) + '
'; + '
' + esc(i.id) + '
' + fmt.badge(s[0], s[1]) + extraBadges(i) + '
'; + } + + // Cars and rockets: how many builds share this icon; waiting models: when their quiet period after a save ends + function extraBadges(i) { + var out = ''; + if (i.sharedBy > 1) out += ' ' + fmt.badge('shared by ' + i.sharedBy, 'info'); + if (i.state === 'pending' && i.processAfter > Date.now() / 1000) out += ' after ' + esc(fmt.unix(i.processAfter)) + ''; + return out; } function row(i) { var s = STATES[i.state] || [i.state, 'secondary']; - var details = kind === 'modular' ? '' + esc(i.modules) + '' : ''; + var details = (kind === 'modular' ? '' + esc(i.modules) + '' : '') + extraBadges(i); if (i.error) details += '
' + esc(i.error) + '
'; var buttons = '' + (canManage ? '' : ''); @@ -64,6 +72,7 @@ canManage = d.canManage; publicUrl = (d.ugcPublicUrl || '').replace(/\/+$/, ''); $('manageButtons').classList.toggle('d-none', !canManage); + $('cacheCard').classList.toggle('d-none', !canManage); $('counts').innerHTML = countCard('Models', d.counts.model) + countCard('Cars and rockets', d.counts.modular); $('gallery').classList.toggle('d-none', view !== 'gallery'); $('listCard').classList.toggle('d-none', view !== 'list'); @@ -174,12 +183,20 @@ 'Download the LXFML
'; } if (publicUrl) links += 'Open on the UGC server'; - if (canManage) links += '
'; + if (canManage) { + links += '
' + + '' + + '
'; + if (item.sharedBy > 1) links += '
This icon is shared by ' + esc(item.sharedBy) + ' builds of the same modules.
'; + } if (item.error) links += '
' + esc(item.error) + '
'; $('previewLinks').innerHTML = links; var model = kind === 'model'; ['nifColumn', 'lxfmlColumn', 'meshControls'].forEach(function (x) { $(x).classList.toggle('d-none', !model); }); - $('statsBox').innerHTML = model ? '' : '
Modules: ' + esc(item.modules || '') + '. The game client puts cars and rockets together itself; only the icon is made.
'; + $('statsBox').innerHTML = model ? '' : '
Modules: ' + esc(item.modules || '') + ' (combination ' + esc(item.combination || '') + + '). The game client puts cars and rockets together itself; only the icon is made, once per combination of modules.
'; + $('framingCard').classList.toggle('d-none', model || !canManage); + if (!model && canManage) openFraming(item); bootstrap.Modal.getOrCreateInstance($('previewModal')).show(); if (!model) return; @@ -267,6 +284,135 @@ if (lxfmlViewer && lxfmlViewer.dispose) lxfmlViewer.dispose(); lxfmlViewer = null; }); + // ---- deleting and purging stored files ---- + + function showDeleteResult(box, d) { + if (!d.success) { toast(d.error || 'Failed', 'danger'); return; } + var text = d.deleted + ' deleted (' + mb(d.bytes || 0) + ' freed). ' + (d.notes || []).join(' '); + if (box) box.textContent = text; + toast(text, 'success'); + load(); + } + + $('previewLinks').addEventListener('click', function (e) { + var button = e.target.closest('[data-delete-open]'); + if (!button || !current) return; + var after = button.dataset.deleteOpen; + if (!confirm('Delete the files made for ' + current.id + (after === 'gone' ? ' and leave them deleted?' : ' and make them again?'))) return; + api.post('/api/ugc/cache/delete', { kind: current.kind, id: current.id, after: after }).then(function (d) { showDeleteResult(null, d); }); + }); + function purge(all) { + var body = { kind: kind, state: $('purgeState').value, owner: $('purgeOwner').value.trim(), olderThanDays: +$('purgeOlder').value || 0, + unusedDays: +$('purgeUnused').value || 0, after: $('purgeAfter').value }; + if (all) { + var typed = prompt('This deletes every stored ' + (kind === 'model' ? 'model' : 'car and rocket') + ' file. Type PURGE ALL to go ahead.'); + if (typed !== 'PURGE ALL') return; + body = { kind: kind, all: true, confirm: typed, after: $('purgeAfter').value }; + } else if (!body.state && !body.owner && !body.olderThanDays && !body.unusedDays) { + toast('Pick a filter first, or purge all', 'warning'); + return; + } else if (!confirm('Delete the stored files of every ' + (kind === 'model' ? 'model' : 'car and rocket') + ' matching the filter?')) { + return; + } + $('purgeResult').textContent = 'Deleting…'; + api.post('/api/ugc/cache/purge', body).then(function (d) { showDeleteResult($('purgeResult'), d); }); + } + $('purgeButton').addEventListener('click', function () { purge(false); }); + $('purgeAllButton').addEventListener('click', function () { purge(true); }); + + // ---- car and rocket icon framing ---- + + var FRAMING = [ + ['yaw', 'Angle around', -180, 180, 1], ['pitch', 'Angle above', -89, 89, 1], ['fov', 'Field of view', 5, 90, 0.5], ['margin', 'Border (zoom)', 0.5, 3, 0.01], + ['offsetX', 'Shift right', -0.5, 0.5, 0.01], ['offsetY', 'Shift up', -0.5, 0.5, 0.01], ['sunYaw', 'Sun around', -180, 180, 1], ['sunPitch', 'Sun above', -10, 90, 1] + ]; + var framingItem = null, typeFraming = {}, previewTimer = null, previewUrl = null; + $('framingControls').innerHTML = FRAMING.map(function (f) { + return '
' + + '
'; + }).join(''); + function framingValues() { + var out = {}; + FRAMING.forEach(function (f) { out[f[0]] = parseFloat($('framing_' + f[0]).value); }); + return out; + } + function setFraming(values) { + FRAMING.forEach(function (f) { + if (values[f[0]] === undefined || values[f[0]] === null) return; + $('framing_' + f[0]).value = values[f[0]]; + $('framing_' + f[0] + '_value').textContent = (+values[f[0]]).toFixed(f[4] < 1 ? 2 : 0); + }); + } + function loadFramingType(useCombination) { + if (!framingItem) return; + var type = $('framingType').value; + return api.get('/api/ugc/framing?buildType=' + type + '&modules=' + encodeURIComponent(framingItem.modules || '')).then(function (d) { + if (!d.success) return; + typeFraming = d.type; + setFraming(d.type); + if (useCombination && d.combination) setFraming(d.combination); + $('framingState').textContent = d.combination && useCombination ? 'This combination has its own framing.' : 'The ' + d.name + ' framing.'; + renderPreview(); + }); + } + function openFraming(item) { + framingItem = item; + // The build type from what the UGC server stored with the icon, when it has + fetchJson(fileUrl('modular', item.id, 'combo.json')).then(function (combo) { + if (combo && (combo.buildType === 3 || combo.buildType === 6)) $('framingType').value = String(combo.buildType); + loadFramingType(true); + }); + } + function renderPreview() { + if (!framingItem) return; + var img = $('framingPreview'); + img.style.opacity = 0.5; + fetch('/api/ugc/framing/preview', { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest' }, + body: JSON.stringify({ modules: framingItem.modules, framing: framingValues() }) }).then(function (r) { + if (!r.ok) return r.json().then(function (d) { throw new Error(d.error || 'HTTP ' + r.status); }); + return r.blob(); + }).then(function (blob) { + if (previewUrl) URL.revokeObjectURL(previewUrl); + previewUrl = URL.createObjectURL(blob); + img.src = previewUrl; + img.style.opacity = 1; + }).catch(function (e) { img.style.opacity = 1; $('framingState').textContent = 'No preview: ' + e.message; }); + } + $('framingControls').addEventListener('input', function (e) { + var f = FRAMING.find(function (x) { return 'framing_' + x[0] === e.target.id; }); + if (f) $('framing_' + f[0] + '_value').textContent = (+e.target.value).toFixed(f[4] < 1 ? 2 : 0); + clearTimeout(previewTimer); + previewTimer = setTimeout(renderPreview, 350); + }); + $('framingType').addEventListener('change', function () { loadFramingType(false); }); + $('framingPreviewButton').addEventListener('click', renderPreview); + $('framingReset').addEventListener('click', function () { + if (!framingItem) return; + if (!confirm('Go back to the type\'s framing? This combination\'s own framing, if it has one, is removed.')) return; + api.post('/api/ugc/framing/save', { scope: 'combination', modules: framingItem.modules, framing: null }).then(function () { loadFramingType(false); }); + }); + $('framingSaveType').addEventListener('click', function () { + var name = $('framingType').selectedOptions[0].textContent.toLowerCase(); + if (!confirm('Use this framing for every ' + name + ' icon? Icons already made keep theirs until they are drawn again.')) return; + api.post('/api/ugc/framing/save', { scope: 'type', buildType: +$('framingType').value, framing: framingValues() }).then(function (d) { + toast(d.success ? d.message : (d.error || 'Failed'), d.success ? 'success' : 'danger'); + }); + }); + $('framingSaveCombo').addEventListener('click', function () { + if (!framingItem) return; + api.post('/api/ugc/framing/save', { scope: 'combination', modules: framingItem.modules, framing: framingValues() }).then(function (d) { + toast(d.success ? d.message : (d.error || 'Failed'), d.success ? 'success' : 'danger'); + if (d.success) $('framingState').textContent = 'This combination has its own framing.'; + }); + }); + $('framingRegenerate').addEventListener('click', function () { + var name = $('framingType').selectedOptions[0].textContent.toLowerCase(); + if (!confirm('Draw every stored ' + name + ' icon again with the saved framings?')) return; + api.post('/api/ugc/framing/regenerate', { buildType: +$('framingType').value }).then(function (d) { + toast(d.success ? d.queued + ' icon' + (d.queued === 1 ? '' : 's') + ' queued' : (d.error || 'Failed'), d.success ? 'success' : 'danger'); + }); + }); + if (window.Live) Live.on('ugc', load); load(); diff --git a/dDashboardServer/templates/ugc.jinja2 b/dDashboardServer/templates/ugc.jinja2 index d5713f6d1..02e17e4b9 100644 --- a/dDashboardServer/templates/ugc.jinja2 +++ b/dDashboardServer/templates/ugc.jinja2 @@ -17,6 +17,24 @@
Asking the UGC server…
+
+ Stored files: delete and purge +
+

Deletes files the UGC server made, of the kind picked below. Afterwards they are made again when a game client + asks for them, queued to be made now, or left deleted. A car or rocket's icon is shared by every build of the same modules.

+
+ + + + + + + +
+
+
+
+
@@ -90,6 +108,24 @@
+
+
Icon framing +
+
+
+
+
Preview +
Preview drawn by the UGC server, not stored.
+
+
+ + + + + +
+
+
diff --git a/dDatabase/GameDatabase/ITables/IPropertyContents.h b/dDatabase/GameDatabase/ITables/IPropertyContents.h index ee868c2a2..e92827f7f 100644 --- a/dDatabase/GameDatabase/ITables/IPropertyContents.h +++ b/dDatabase/GameDatabase/ITables/IPropertyContents.h @@ -25,7 +25,10 @@ public: std::stringstream& sd0Data, const uint64_t blueprintId, const uint32_t accountId, - const LWOOBJID characterId) = 0; + const LWOOBJID characterId, + // Unix seconds before which the UGC server doesn't make it (0: right away). The owner's other models waiting + // to be made wait until then too (they may be edited again). + const int64_t processAfter) = 0; // Get the property models for the given property id. virtual std::vector GetPropertyModels(const LWOOBJID& propertyId) = 0; diff --git a/dDatabase/GameDatabase/ITables/IUgc.h b/dDatabase/GameDatabase/ITables/IUgc.h index 062a4f267..5e0e4c21c 100644 --- a/dDatabase/GameDatabase/ITables/IUgc.h +++ b/dDatabase/GameDatabase/ITables/IUgc.h @@ -29,7 +29,7 @@ public: virtual void DeleteUgcModelData(const LWOOBJID& modelId) = 0; // Inserts a new UGC model into the database. - virtual void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) = 0; + virtual void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) = 0; virtual std::optional GetUgcModel(const LWOOBJID ugcId) = 0; @@ -53,6 +53,7 @@ public: std::string error; bool bakeAo{}; std::string details; // modular builds: the modules (ldf_config) + int64_t processAfter{}; // models: Unix seconds before which it isn't made (the quiet period after a save) }; // A model waiting to be made: its id, stored LXFML (sd0) and the attempts so far @@ -62,9 +63,14 @@ public: uint32_t attempts{}; }; - // Up to `limit` pending models, the least tried and then the newest first + // Up to `limit` pending models whose quiet period is over (process_after), the least tried and then the newest first virtual std::vector GetUgcModelsToProcess(const uint32_t limit) = 0; + // Ends the quiet period of a waiting model (a client asked for it), or of all a character's waiting models (they + // left the property) + virtual void ExpediteUgcModel(const LWOOBJID id) = 0; + virtual void ExpediteUgcModels(const LWOOBJID characterId) = 0; + // Records an attempt: the new state, how many attempts there have been, why it failed (empty when it didn't) and // whether lighting was baked in; processed_at becomes now virtual void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) = 0; diff --git a/dDatabase/GameDatabase/ITables/IUgcModularBuild.h b/dDatabase/GameDatabase/ITables/IUgcModularBuild.h index 698b66c3b..f48871a27 100644 --- a/dDatabase/GameDatabase/ITables/IUgcModularBuild.h +++ b/dDatabase/GameDatabase/ITables/IUgcModularBuild.h @@ -28,6 +28,15 @@ public: virtual uint64_t ResetModularBuildProcessing(const std::optional id, const bool failedOnly) = 0; virtual std::vector GetModularBuildProcessList(const std::optional state, const std::string_view search, const uint32_t offset, const uint32_t limit) = 0; virtual std::vector> GetModularBuildProcessCounts() = 0; + + // How many builds there are of each ldf_config as stored (the same modules may be written differently) + virtual std::vector> GetModularBuildConfigCounts() = 0; + + // Icon settings (UgcIconParams JSON) set on the dashboard for a target: "kind:" (a preset for player models or + // a car or rocket build type), "model:" or "combo:" (one item's or combination's own) + virtual std::optional GetUgcIconSettings(const std::string_view target) = 0; + virtual void SetUgcIconSettings(const std::string_view target, const std::string_view params) = 0; + virtual void DeleteUgcIconSettings(const std::string_view target) = 0; }; #endif //!IUGCMODULARBUILD_H diff --git a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h index 072130bcf..350e703ee 100644 --- a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h +++ b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h @@ -63,7 +63,7 @@ public: void RemoveFriend(const LWOOBJID playerAccountId, const LWOOBJID friendAccountId) override; void UpdateActivityLog(const LWOOBJID characterId, const eActivityType activityType, const LWOMAPID mapId) override; void DeleteUgcModelData(const LWOOBJID& modelId) override; - void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) override; + void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) override; std::vector GetAllUgcModels() override; void CreateMigrationHistoryTable() override; bool IsMigrationRun(const std::string_view str) override; @@ -356,7 +356,8 @@ public: std::stringstream& sd0Data, const uint64_t blueprintId, const uint32_t accountId, - const LWOOBJID characterId) override; + const LWOOBJID characterId, + const int64_t processAfter) override; std::vector GetMailForPlayer(const LWOOBJID characterId, const uint32_t numberOfMail) override; std::optional GetMail(const uint64_t mailId) override; uint32_t GetUnreadMailCount(const LWOOBJID characterId) override; @@ -419,6 +420,12 @@ public: std::optional GetModel(const LWOOBJID modelID) override; std::optional GetUgcModel(const LWOOBJID ugcId) override; std::vector GetUgcModelsToProcess(const uint32_t limit) override; + void ExpediteUgcModel(const LWOOBJID id) override; + void ExpediteUgcModels(const LWOOBJID characterId) override; + std::vector> GetModularBuildConfigCounts() override; + std::optional GetUgcIconSettings(const std::string_view target) override; + void SetUgcIconSettings(const std::string_view target, const std::string_view params) override; + void DeleteUgcIconSettings(const std::string_view target) override; void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override; std::optional GetUgcProcessInfo(const LWOOBJID id) override; uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override; diff --git a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp index dbd56a34e..499a252ec 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp @@ -13,7 +13,8 @@ namespace { info.processedAt = result->getInt64("processed_at"); info.error = std::string(result->getString("process_error").c_str()); if (modular) info.details = std::string(result->getString("ldf_config").c_str()); - else info.bakeAo = result->getInt("bake_ao") != 0; + else info.processAfter = result->getInt64("process_after"); + if (!modular) info.bakeAo = result->getInt("bake_ao") != 0; return info; } @@ -68,18 +69,22 @@ void MySQLDatabase::InsertNewUgcModel( std::stringstream& sd0Data, // cant be const sad const uint64_t blueprintId, const uint32_t accountId, - const LWOOBJID characterId) { + const LWOOBJID characterId, + const int64_t processAfter) { const std::istream stream(sd0Data.rdbuf()); ExecuteInsert( - "INSERT INTO `ugc`(`id`, `account_id`, `character_id`, `is_optimized`, `lxfml`, `bake_ao`, `filename`) VALUES (?,?,?,?,?,?,?)", + "INSERT INTO `ugc`(`id`, `account_id`, `character_id`, `is_optimized`, `lxfml`, `bake_ao`, `filename`, `process_after`) VALUES (?,?,?,?,?,?,?,?)", blueprintId, accountId, characterId, 0, &stream, false, - "weedeater.lxfml" + "weedeater.lxfml", + processAfter ); + // The owner is still building: their other models waiting for their quiet period wait longer + if (processAfter > 0) ExecuteUpdate("UPDATE ugc SET process_after = ? WHERE character_id = ? AND is_optimized = 0 AND process_after > 0 AND process_after < ?;", processAfter, characterId, processAfter); } void MySQLDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { @@ -87,10 +92,10 @@ void MySQLDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { ExecuteDelete("DELETE FROM properties_contents WHERE ugc_id = ?;", modelId); } -void MySQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) { +void MySQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) { const std::istream stream(lxfml.rdbuf()); // The UGC server makes the model's files again - ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", &stream, modelId); + ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '', process_after = ? WHERE id = ?;", &stream, processAfter, modelId); } std::optional MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) { @@ -106,7 +111,7 @@ std::optional MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) { } std::vector MySQLDatabase::GetUgcModelsToProcess(const uint32_t limit) { - auto result = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 ORDER BY process_attempts ASC, id DESC LIMIT ?;", limit); + auto result = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 AND process_after <= ? ORDER BY process_attempts ASC, id DESC LIMIT ?;", UnixNow(), limit); std::vector models; while (result->next()) { auto& model = models.emplace_back(); @@ -127,21 +132,21 @@ void MySQLDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState std::optional MySQLDatabase::GetUgcProcessInfo(const LWOOBJID id) { auto result = ExecuteSelect( - "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao " + "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao, u.process_after " "FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id WHERE u.id = ? LIMIT 1;", id); if (!result->next()) return std::nullopt; return ReadUgcProcessInfo(result, false); } uint64_t MySQLDatabase::ResetUgcModelProcessing(const std::optional id, const bool failedOnly) { - if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", *id); - if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;"); - return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '';"); + if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 WHERE id = ?;", *id); + if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 WHERE is_optimized = 2;"); + return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0;"); } std::vector MySQLDatabase::GetUgcProcessList(const std::optional state, const std::string_view search, const uint32_t offset, const uint32_t limit) { const std::string select = - "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao " + "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao, u.process_after " "FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id "; std::vector list; // Every filter always bound, off when its first value says so @@ -164,3 +169,11 @@ std::vector> MySQLDatabase::GetUgcProce } return counts; } + +void MySQLDatabase::ExpediteUgcModel(const LWOOBJID id) { + ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE id = ? AND is_optimized = 0 AND process_after > 0;", id); +} + +void MySQLDatabase::ExpediteUgcModels(const LWOOBJID characterId) { + ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE character_id = ? AND is_optimized = 0 AND process_after > 0;", characterId); +} diff --git a/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp b/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp index 2c82cc02e..508a221af 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp @@ -84,3 +84,25 @@ std::vector> MySQLDatabase::GetModularB } return counts; } + +std::vector> MySQLDatabase::GetModularBuildConfigCounts() { + auto result = ExecuteSelect("SELECT ldf_config, COUNT(*) AS count FROM ugc_modular_build GROUP BY ldf_config;"); + std::vector> counts; + while (result->next()) counts.emplace_back(result->getString("ldf_config").c_str(), static_cast(result->getInt64("count"))); + return counts; +} + +std::optional MySQLDatabase::GetUgcIconSettings(const std::string_view target) { + auto result = ExecuteSelect("SELECT params FROM ugc_icon_settings WHERE target = ? LIMIT 1;", target); + if (!result->next()) return std::nullopt; + return std::string(result->getString("params").c_str()); +} + +void MySQLDatabase::SetUgcIconSettings(const std::string_view target, const std::string_view params) { + ExecuteInsert("INSERT INTO ugc_icon_settings (target, params, updated_at) VALUES (?, ?, ?) " + "ON DUPLICATE KEY UPDATE params = VALUES(params), updated_at = VALUES(updated_at);", target, params, UnixNow()); +} + +void MySQLDatabase::DeleteUgcIconSettings(const std::string_view target) { + ExecuteDelete("DELETE FROM ugc_icon_settings WHERE target = ?;", target); +} diff --git a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h index bd2904e0b..3dd468ec6 100644 --- a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h +++ b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h @@ -47,7 +47,7 @@ public: void RemoveFriend(const LWOOBJID playerAccountId, const LWOOBJID friendAccountId) override; void UpdateActivityLog(const LWOOBJID characterId, const eActivityType activityType, const LWOMAPID mapId) override; void DeleteUgcModelData(const LWOOBJID& modelId) override; - void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) override; + void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) override; std::vector GetAllUgcModels() override; void CreateMigrationHistoryTable() override; bool IsMigrationRun(const std::string_view str) override; @@ -340,7 +340,8 @@ public: std::stringstream& sd0Data, const uint64_t blueprintId, const uint32_t accountId, - const LWOOBJID characterId) override; + const LWOOBJID characterId, + const int64_t processAfter) override; std::vector GetMailForPlayer(const LWOOBJID characterId, const uint32_t numberOfMail) override; std::optional GetMail(const uint64_t mailId) override; uint32_t GetUnreadMailCount(const LWOOBJID characterId) override; @@ -403,6 +404,12 @@ public: std::optional GetModel(const LWOOBJID modelID) override; std::optional GetUgcModel(const LWOOBJID ugcId) override; std::vector GetUgcModelsToProcess(const uint32_t limit) override; + void ExpediteUgcModel(const LWOOBJID id) override; + void ExpediteUgcModels(const LWOOBJID characterId) override; + std::vector> GetModularBuildConfigCounts() override; + std::optional GetUgcIconSettings(const std::string_view target) override; + void SetUgcIconSettings(const std::string_view target, const std::string_view params) override; + void DeleteUgcIconSettings(const std::string_view target) override; void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override; std::optional GetUgcProcessInfo(const LWOOBJID id) override; uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override; diff --git a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp index 6a0682fa0..e037ca0c5 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp @@ -13,7 +13,8 @@ namespace { info.processedAt = result.getInt64Field("processed_at"); info.error = result.getStringField("process_error", ""); if (modular) info.details = result.getStringField("ldf_config", ""); - else info.bakeAo = result.getIntField("bake_ao") != 0; + else info.processAfter = result.getInt64Field("process_after"); + if (!modular) info.bakeAo = result.getIntField("bake_ao") != 0; return info; } @@ -69,18 +70,22 @@ void SQLiteDatabase::InsertNewUgcModel( std::stringstream& sd0Data, // cant be const sad const uint64_t blueprintId, const uint32_t accountId, - const LWOOBJID characterId) { + const LWOOBJID characterId, + const int64_t processAfter) { const std::istream stream(sd0Data.rdbuf()); ExecuteInsert( - "INSERT INTO `ugc`(`id`, `account_id`, `character_id`, `is_optimized`, `lxfml`, `bake_ao`, `filename`) VALUES (?,?,?,?,?,?,?)", + "INSERT INTO `ugc`(`id`, `account_id`, `character_id`, `is_optimized`, `lxfml`, `bake_ao`, `filename`, `process_after`) VALUES (?,?,?,?,?,?,?,?)", blueprintId, accountId, characterId, 0, &stream, false, - "weedeater.lxfml" + "weedeater.lxfml", + processAfter ); + // The owner is still building: their other models waiting for their quiet period wait longer + if (processAfter > 0) ExecuteUpdate("UPDATE ugc SET process_after = ? WHERE character_id = ? AND is_optimized = 0 AND process_after > 0 AND process_after < ?;", processAfter, characterId, processAfter); } void SQLiteDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { @@ -88,10 +93,10 @@ void SQLiteDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { ExecuteDelete("DELETE FROM properties_contents WHERE ugc_id = ?;", modelId); } -void SQLiteDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) { +void SQLiteDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) { const std::istream stream(lxfml.rdbuf()); // The UGC server makes the model's files again - ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", &stream, modelId); + ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '', process_after = ? WHERE id = ?;", &stream, processAfter, modelId); } std::optional SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) { @@ -107,7 +112,7 @@ std::optional SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) { } std::vector SQLiteDatabase::GetUgcModelsToProcess(const uint32_t limit) { - auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 ORDER BY process_attempts ASC, id DESC LIMIT ?;", limit); + auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 AND process_after <= ? ORDER BY process_attempts ASC, id DESC LIMIT ?;", UnixNow(), limit); std::vector models; while (!result.eof()) { auto& model = models.emplace_back(); @@ -128,21 +133,21 @@ void SQLiteDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState std::optional SQLiteDatabase::GetUgcProcessInfo(const LWOOBJID id) { auto [_, result] = ExecuteSelect( - "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao " + "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao, u.process_after " "FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id WHERE u.id = ? LIMIT 1;", id); if (result.eof()) return std::nullopt; return ReadUgcProcessInfo(result, false); } uint64_t SQLiteDatabase::ResetUgcModelProcessing(const std::optional id, const bool failedOnly) { - if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", *id); - if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;"); - return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '';"); + if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 WHERE id = ?;", *id); + if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0 WHERE is_optimized = 2;"); + return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '', process_after = 0;"); } std::vector SQLiteDatabase::GetUgcProcessList(const std::optional state, const std::string_view search, const uint32_t offset, const uint32_t limit) { const std::string select = - "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao " + "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao, u.process_after " "FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id "; std::vector list; // Every filter always bound, off when its first value says so @@ -167,3 +172,11 @@ std::vector> SQLiteDatabase::GetUgcProc } return counts; } + +void SQLiteDatabase::ExpediteUgcModel(const LWOOBJID id) { + ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE id = ? AND is_optimized = 0 AND process_after > 0;", id); +} + +void SQLiteDatabase::ExpediteUgcModels(const LWOOBJID characterId) { + ExecuteUpdate("UPDATE ugc SET process_after = 0 WHERE character_id = ? AND is_optimized = 0 AND process_after > 0;", characterId); +} diff --git a/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp b/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp index 348e547b4..3a4490fb6 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp @@ -87,3 +87,28 @@ std::vector> SQLiteDatabase::GetModular } return counts; } + +std::vector> SQLiteDatabase::GetModularBuildConfigCounts() { + auto [_, result] = ExecuteSelect("SELECT ldf_config, COUNT(*) AS count FROM ugc_modular_build GROUP BY ldf_config;"); + std::vector> counts; + while (!result.eof()) { + counts.emplace_back(result.getStringField("ldf_config", ""), static_cast(result.getInt64Field("count"))); + result.nextRow(); + } + return counts; +} + +std::optional SQLiteDatabase::GetUgcIconSettings(const std::string_view target) { + auto [_, result] = ExecuteSelect("SELECT params FROM ugc_icon_settings WHERE target = ? LIMIT 1;", target); + if (result.eof()) return std::nullopt; + return std::string(result.getStringField("params", "")); +} + +void SQLiteDatabase::SetUgcIconSettings(const std::string_view target, const std::string_view params) { + ExecuteInsert("INSERT INTO ugc_icon_settings (target, params, updated_at) VALUES (?, ?, ?) " + "ON CONFLICT(target) DO UPDATE SET params = excluded.params, updated_at = excluded.updated_at;", target, params, UnixNow()); +} + +void SQLiteDatabase::DeleteUgcIconSettings(const std::string_view target) { + ExecuteDelete("DELETE FROM ugc_icon_settings WHERE target = ?;", target); +} diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.cpp b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.cpp index 8e2e6dc84..6e993a4fa 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.cpp +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.cpp @@ -60,7 +60,7 @@ void TestSQLDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { } -void TestSQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) { +void TestSQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) { } @@ -192,7 +192,7 @@ void TestSQLDatabase::InsertNewMail(const MailInfo& mail) { } -void TestSQLDatabase::InsertNewUgcModel(std::stringstream& sd0Data, const uint64_t blueprintId, const uint32_t accountId, const LWOOBJID characterId) { +void TestSQLDatabase::InsertNewUgcModel(std::stringstream& sd0Data, const uint64_t blueprintId, const uint32_t accountId, const LWOOBJID characterId, const int64_t processAfter) { } diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h index 40ee2ee9c..57d0757c9 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h @@ -27,7 +27,7 @@ class TestSQLDatabase : public GameDatabase { void RemoveFriend(const LWOOBJID playerAccountId, const LWOOBJID friendAccountId) override; void UpdateActivityLog(const LWOOBJID characterId, const eActivityType activityType, const LWOMAPID mapId) override; void DeleteUgcModelData(const LWOOBJID& modelId) override; - void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) override; + void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml, const int64_t processAfter) override; std::vector GetAllUgcModels() override; void CreateMigrationHistoryTable() override; bool IsMigrationRun(const std::string_view str) override; @@ -67,7 +67,8 @@ class TestSQLDatabase : public GameDatabase { std::stringstream& sd0Data, const uint64_t blueprintId, const uint32_t accountId, - const LWOOBJID characterId) override; + const LWOOBJID characterId, + const int64_t processAfter) override; std::vector GetMailForPlayer(const LWOOBJID characterId, const uint32_t numberOfMail) override; std::optional GetMail(const uint64_t mailId) override; uint32_t GetUnreadMailCount(const LWOOBJID characterId) override; @@ -401,6 +402,12 @@ class TestSQLDatabase : public GameDatabase { std::optional GetPropertyInfo(const LWOOBJID id) override { return {}; } std::optional GetUgcModel(const LWOOBJID ugcId) override { return {}; } std::vector GetUgcModelsToProcess(const uint32_t limit) override { return {}; } + void ExpediteUgcModel(const LWOOBJID id) override {} + void ExpediteUgcModels(const LWOOBJID characterId) override {} + std::vector> GetModularBuildConfigCounts() override { return {}; } + std::optional GetUgcIconSettings(const std::string_view target) override { return std::nullopt; } + void SetUgcIconSettings(const std::string_view target, const std::string_view params) override {} + void DeleteUgcIconSettings(const std::string_view target) override {} void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override {} std::optional GetUgcProcessInfo(const LWOOBJID id) override { return {}; } uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override { return 0; } diff --git a/dDatabase/ModelNormalizeMigration.cpp b/dDatabase/ModelNormalizeMigration.cpp index 45fcdb82c..100f2ef19 100644 --- a/dDatabase/ModelNormalizeMigration.cpp +++ b/dDatabase/ModelNormalizeMigration.cpp @@ -24,7 +24,7 @@ void ModelNormalizeMigration::Run() { sd0.FromData(reinterpret_cast(newLxfml.data()), newLxfml.size()); auto asStream = sd0.GetAsStream(); Database::Get()->UpdateModel(model->id, newCenter, model->rotation, model->behaviors); - Database::Get()->UpdateUgcModelData(id, asStream); + Database::Get()->UpdateUgcModelData(id, asStream, 0); } Database::Get()->SetAutoCommit(oldCommit); } @@ -44,7 +44,7 @@ void ModelNormalizeMigration::RunAfterFirstPart() { sd0.FromData(reinterpret_cast(newLxfml.data()), newLxfml.size()); auto asStream = sd0.GetAsStream(); Database::Get()->UpdateModel(model->id, newCenter, model->rotation, model->behaviors); - Database::Get()->UpdateUgcModelData(id, asStream); + Database::Get()->UpdateUgcModelData(id, asStream, 0); } Database::Get()->SetAutoCommit(oldCommit); } @@ -65,7 +65,7 @@ void ModelNormalizeMigration::RunBrickBuildGrid() { LOG("Updated model %llu to have a center of %f %f %f", modelID, newCenter.x, newCenter.y, newCenter.z); auto asStream = sd0.GetAsStream(); Database::Get()->UpdateModel(model->id, newCenter, model->rotation, model->behaviors); - Database::Get()->UpdateUgcModelData(id, asStream); + Database::Get()->UpdateUgcModelData(id, asStream, 0); } Database::Get()->SetAutoCommit(oldCommit); } diff --git a/dGame/dUtilities/BrickByBrick.cpp b/dGame/dUtilities/BrickByBrick.cpp index 48ad65a6b..a1a7f68ef 100644 --- a/dGame/dUtilities/BrickByBrick.cpp +++ b/dGame/dUtilities/BrickByBrick.cpp @@ -23,11 +23,13 @@ #include "PropertyManagementComponent.h" #include "PropertyMessages.h" #include "Sd0.h" +#include "UgcKeys.h" #include "User.h" #include #include #include +#include #include #include #include @@ -107,12 +109,16 @@ namespace { outFile << lxfml; } + // A quiet period before the UGC server makes the models (ugc_debounce_seconds, sharedconfig.ini): a model + // the owner keeps editing isn't made for every save (docs/UgcServer.md) + const auto debounce = Game::config ? GeneralUtils::TryParse(Game::config->GetValue("ugc_debounce_seconds")).value_or(120) : 120; + const auto processAfter = UgcDebounce::ProcessAfter(std::time(nullptr), debounce); for (const auto& part : Lxfml::Split(lxfml)) { const auto [modelId, blueprintId] = ObjectIDManager::GetNewModelIDs(); Sd0 model = sd0; model.FromData(reinterpret_cast(part.lxfml.data()), part.lxfml.size()); auto stream = model.GetAsStream(); - Database::Get()->InsertNewUgcModel(stream, blueprintId, character->GetParentUser()->GetAccountID(), character->GetID()); + Database::Get()->InsertNewUgcModel(stream, blueprintId, character->GetParentUser()->GetAccountID(), character->GetID(), processAfter); auto& entry = saved.emplace_back(); entry.modelId = modelId; diff --git a/dUgcServer/CMakeLists.txt b/dUgcServer/CMakeLists.txt index 780b5dca7..5f67721b3 100644 --- a/dUgcServer/CMakeLists.txt +++ b/dUgcServer/CMakeLists.txt @@ -2,6 +2,7 @@ set(DUGC_SOURCES "UgcBricks.cpp" "UgcFormats.cpp" + "UgcIconParams.cpp" "UgcJobs.cpp" "UgcModel.cpp" "UgcModular.cpp" diff --git a/dUgcServer/UgcCdClient.cpp b/dUgcServer/UgcCdClient.cpp index 4c53ece1c..f89246e74 100644 --- a/dUgcServer/UgcCdClient.cpp +++ b/dUgcServer/UgcCdClient.cpp @@ -34,6 +34,7 @@ namespace UgcCdClient { module.partCode = static_cast(moduleRow.getIntField("partCode", 0)); module.moduleXml = moduleRow.getStringField("xml", ""); if (!buildType) buildType = moduleRow.getIntField("buildType", 0); + input.buildType = *buildType; auto renderQuery = CDClientDatabase::CreatePreppedStmt( "SELECT rc.render_asset FROM ComponentsRegistry cr JOIN RenderComponent rc ON rc.id = cr.component_id " diff --git a/dUgcServer/UgcIconParams.cpp b/dUgcServer/UgcIconParams.cpp new file mode 100644 index 000000000..ef102a17b --- /dev/null +++ b/dUgcServer/UgcIconParams.cpp @@ -0,0 +1,88 @@ +#include "UgcIconParams.h" + +#include +#include + +#include "GeneralUtils.h" +#include "json.hpp" + +namespace { + using Options = UgcRender::IconOptions; +} + +namespace UgcIconParams { + const std::vector& List() { + // Framing from LU Toolbox's icon renderer (its UGC render add-on's BrickBuild scene); the light matched to the + // brightness of the game's own model icons (res/textures/ui/inventory/models), docs/UgcServer.md + static const std::vector list = { + { "yaw", "icon_yaw", "Angle around", "degrees", -180, 180, 1, 53.36f, "The camera's angle around the model, from its front.", [](Options& o, float v) { o.yawDegrees = v; } }, + { "pitch", "icon_pitch", "Angle above", "degrees", -89, 89, 1, 19.54f, "", [](Options& o, float v) { o.pitchDegrees = v; } }, + { "fov", "icon_fov", "Field of view", "degrees", 5, 90, 0.5f, 39.6f, "", [](Options& o, float v) { o.fovDegrees = v; } }, + { "margin", "icon_margin", "Border (zoom)", "", 0.5f, 3, 0.01f, 1.03f, "1 fills the icon, more leaves a border.", [](Options& o, float v) { o.margin = v; } }, + { "offsetX", "icon_offset_x", "Shift right", "", -0.5f, 0.5f, 0.01f, 0.0f, "Share of the icon's width.", [](Options& o, float v) { o.offsetX = v; } }, + { "offsetY", "icon_offset_y", "Shift up", "", -0.5f, 0.5f, 0.01f, 0.0f, "Share of the icon's height.", [](Options& o, float v) { o.offsetY = v; } }, + { "sunYaw", "icon_sun_yaw", "Sun around", "degrees", -180, 180, 1, 21.0f, "", [](Options& o, float v) { o.sunYawDegrees = v; } }, + { "sunPitch", "icon_sun_pitch", "Sun above", "degrees", -10, 90, 1, 50.3f, "", [](Options& o, float v) { o.sunPitchDegrees = v; } }, + { "sunStrength", "icon_sun_strength", "Sun strength", "", 0, 10, 0.05f, 2.0f, "", [](Options& o, float v) { o.sunStrength = v; } }, + { "ambient", "icon_ambient", "World light", "", 0, 2, 0.01f, 1.0f, "What every face gets.", [](Options& o, float v) { o.ambient = v; } }, + { "fill", "icon_fill", "Fill light", "", 0, 5, 0.05f, 0.8f, "A light from the camera.", [](Options& o, float v) { o.fill = v; } }, + { "specular", "icon_specular", "Highlights", "", 0, 2, 0.05f, 0.2f, "The sun's highlight on the plastic.", [](Options& o, float v) { o.specular = v; } }, + { "shininess", "icon_shininess", "Highlight tightness", "", 1, 200, 1, 40.0f, "", [](Options& o, float v) { o.shininess = v; } }, + { "exposure", "icon_exposure", "Exposure", "", 0.1f, 4, 0.05f, 2.6f, "Brightness of everything.", [](Options& o, float v) { o.exposure = v; } }, + { "contrast", "icon_contrast", "Contrast", "", 0.5f, 1.5f, 0.01f, 1.0f, "", [](Options& o, float v) { o.contrast = v; } }, + { "shadows", "icon_shadows", "Shadows", "", 0, 1, 0.05f, 0.4f, "How much the sun's shadows darken.", [](Options& o, float v) { o.shadows = v; } }, + { "aoStrength", "icon_ao_strength", "Ambient occlusion", "", 0, 1, 0.05f, 0.0f, + "Occlusion worked out for the icon. Player models' meshes have theirs baked in already.", [](Options& o, float v) { o.ao.strength = v; o.ao.enabled = v > 0.0f; } }, + }; + return list; + } + + const Param* Find(std::string_view key) { + const auto& list = List(); + const auto it = std::find_if(list.begin(), list.end(), [key](const Param& p) { return p.key == key; }); + return it != list.end() ? &*it : nullptr; + } + + Values Parse(std::string_view text) { + Values values; + const auto json = nlohmann::json::parse(text, nullptr, false); + if (!json.is_object()) return values; + for (const auto& param : List()) { + const auto it = json.find(param.key); + if (it == json.end() || !it->is_number()) continue; + const auto value = it->get(); + if (std::isfinite(value)) values[param.key] = std::clamp(value, param.min, param.max); + } + return values; + } + + std::string ToJson(const Values& values) { + nlohmann::json json = nlohmann::json::object(); + for (const auto& [key, value] : values) { + if (Find(key)) json[key] = value; + } + return json.dump(); + } + + void Apply(UgcRender::IconOptions& options, const Values& values) { + for (const auto& [key, value] : values) { + if (const auto* param = Find(key)) param->apply(options, std::clamp(value, param->min, param->max)); + } + } + + UgcRender::IconOptions FromSettings(const std::function(const std::string&)>& get) { + UgcRender::IconOptions options; + for (const auto& param : List()) { + const auto text = get(param.setting); + const auto value = text ? GeneralUtils::TryParse(*text) : std::nullopt; + param.apply(options, std::clamp(value.value_or(param.defaultValue), param.min, param.max)); + } + return options; + } + + std::string KindTarget(const std::string& kind) { return "kind:" + kind; } + std::string ModelTarget(int64_t id) { return "model:" + std::to_string(id); } + std::string CombinationTarget(const std::string& key) { return "combo:" + key; } + std::string ModelKind() { return "model"; } + std::string BuildKind(int32_t buildType) { return "build" + std::to_string(buildType); } +} diff --git a/dUgcServer/UgcIconParams.h b/dUgcServer/UgcIconParams.h new file mode 100644 index 000000000..ba8a06e7e --- /dev/null +++ b/dUgcServer/UgcIconParams.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "UgcRender.h" + +/** + * Everything about an icon's framing and lighting that can be set, listed once: its name (in presets and overrides), + * its setting (the shipped default, in ugcconfig.ini and the dashboard's settings), label, unit, range and default. + * The UGC server reads the settings through it, the dashboard builds its settings entries and the icon editor's + * controls from it, and presets and overrides are checked against it. Values for an icon come, each over the one + * before: the settings (icon_*), the kind's preset (player models, or a car or rocket build type) and the item's or + * module combination's own override, both stored in the database (ugc_icon_settings). Pure. + */ +namespace UgcIconParams { + struct Param { + std::string key; // in presets and overrides, e.g. "sunStrength" + std::string setting; // ugcconfig.ini, e.g. "icon_sun_strength" + std::string label; + std::string unit; + float min{}; + float max{}; + float step{}; + float defaultValue{}; + std::string description; + std::function apply; + }; + + const std::vector& List(); + const Param* Find(std::string_view key); + + using Values = std::map; + + // Known keys only, numbers only, clamped to their ranges + Values Parse(std::string_view json); + std::string ToJson(const Values& values); + + // Values over the options' own + void Apply(UgcRender::IconOptions& options, const Values& values); + + // The options from the settings: `get` gives a setting's value (nullopt or unparsable: the parameter's default) + UgcRender::IconOptions FromSettings(const std::function(const std::string&)>& get); + + // Where presets and overrides are stored (ugc_icon_settings.target) + std::string KindTarget(const std::string& kind); // "kind:model", "kind:build6" + std::string ModelTarget(int64_t id); // "model:" + std::string CombinationTarget(const std::string& key); // "combo:" + // The kind of a player model, and of a car or rocket of build type `buildType` (ModuleComponent.buildType) + std::string ModelKind(); + std::string BuildKind(int32_t buildType); +} diff --git a/dUgcServer/UgcJobs.cpp b/dUgcServer/UgcJobs.cpp index b53941d91..5e03b58a0 100644 --- a/dUgcServer/UgcJobs.cpp +++ b/dUgcServer/UgcJobs.cpp @@ -1,6 +1,7 @@ #include "UgcJobs.h" #include +#include #include #include "json.hpp" @@ -66,7 +67,14 @@ namespace UgcJobs { } } - Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed) { + bool IconFromNif(const std::string& nif, const UgcRender::IconOptions& options, UgcStorage::Files& files, std::string& error) { + const auto readBack = NifFile::Parse(nif, 0, error); + if (!readBack) return false; + AddIcon(files, UgcModel::FromNif(*readBack), options); + return true; + } + + Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed, const UgcIconParams::Values& iconValues) { Outcome outcome; const auto started = std::chrono::steady_clock::now(); const auto lxfml = LxfmlFromBlob(blob); @@ -169,18 +177,15 @@ namespace UgcJobs { } // The icon is drawn from the .nif just made (its most detailed LOD, read back like any client .nif), so it - // shows what the game shows: the colors with their variation, hidden faces removed, the baked lighting. The - // lighting is in its vertex colors already, so the icon adds no occlusion of its own. + // shows what the game shows: the colors with their variation, hidden faces removed, the baked lighting. const auto iconStart = std::chrono::steady_clock::now(); + auto iconOptions = settings.icon; + UgcIconParams::Apply(iconOptions, iconValues); std::string nifError; - const auto readBack = NifFile::Parse(nif, 0, nifError); - if (!readBack) { + if (!IconFromNif(nif, iconOptions, outcome.files, nifError)) { outcome.error = "the .nif made can't be read back for the icon: " + nifError; return outcome; } - auto iconOptions = settings.icon; - iconOptions.ao.enabled = false; - AddIcon(outcome.files, UgcModel::FromNif(*readBack), iconOptions); const double iconMs = Since(iconStart); stats["ms"] = { { "build", std::lround(buildMs) }, { "hiddenSurfaces", std::lround(hsrMs) }, { "ambientOcclusion", std::lround(aoMs) }, @@ -228,10 +233,19 @@ namespace UgcJobs { outcome.error = "the modules have no triangles"; return outcome; } - auto options = settings.modularIcon; + auto options = ModularIconOptions(input, settings); options.modelRotation = build->additionalRotation * options.modelRotation; AddIcon(outcome.files, model, options); + outcome.files["combo.json"] = nlohmann::json{ { "key", input.key }, { "buildType", input.buildType } }.dump(); outcome.ok = true; return outcome; } + + UgcRender::IconOptions ModularIconOptions(const ModularInput& input, const Settings& settings) { + auto options = settings.icon; + UgcIconParams::Apply(options, input.iconValues); + return options; + } + + } diff --git a/dUgcServer/UgcJobs.h b/dUgcServer/UgcJobs.h index df49562ae..edbcd75f2 100644 --- a/dUgcServer/UgcJobs.h +++ b/dUgcServer/UgcJobs.h @@ -2,11 +2,14 @@ #include #include +#include +#include #include #include #include #include "UgcBricks.h" +#include "UgcIconParams.h" #include "UgcRender.h" #include "UgcStorage.h" @@ -27,8 +30,7 @@ namespace UgcJobs { bool combineTransparent{ false }; // one shape for all transparent bricks, else one per brick (Combine Transparent) UgcRender::OptimizeOptions optimize; // hidden surface removal UgcRender::AoOptions ao; // Bake Lighting (AO Only) - UgcRender::IconOptions icon; // player models: drawn from their .nif (LOD 0); its ao is not used - UgcRender::IconOptions modularIcon; + UgcRender::IconOptions icon; // from the icon_* settings (UgcIconParams); presets and overrides go over it uint32_t maxBricks{}; // a model with more fails; 0: no limit }; @@ -49,7 +51,13 @@ namespace UgcJobs { // A player model: the optimized .nif, its icon and its LXFML for download, and stats.json. `seed` picks the color // variation's random numbers (the model's id, so making it again gives the same colors). - Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed = 0); + // `iconValues`: the player models' preset and this model's override (UgcIconParams), over settings.icon. + Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed = 0, + const UgcIconParams::Values& iconValues = {}); + + // A model's icon files (icon.png, icon.dds download) drawn from its .nif (LOD 0); false (and `error`) when the .nif + // can't be read + bool IconFromNif(const std::string& nif, const UgcRender::IconOptions& options, UgcStorage::Files& files, std::string& error); // How many bricks (parts) an LXFML has, counted cheaply (for the memory estimate before a job starts) size_t CountParts(std::string_view lxfml); @@ -66,9 +74,15 @@ namespace UgcJobs { struct ModularInput { std::vector modules; - std::string buildXml; // ModularBuildComponent.xml of the modules' build type + std::string buildXml; // ModularBuildComponent.xml of the modules' build type + int32_t buildType{}; // 3 rocket, 6 car + std::string key; // the combination (UgcModularKey::Normalize) + UgcIconParams::Values iconValues; // the build type's preset and the combination's override (DB), over the settings }; - // A modular build's icon, from its modules' meshes put together + // The icon options for a car or rocket: the settings, then its build type's preset and the combination's override + UgcRender::IconOptions ModularIconOptions(const ModularInput& input, const Settings& settings); + + // A modular build's icon (icon.png, icon.dds download, combo.json), from its modules' meshes put together Outcome ProcessModular(const ModularInput& input, const std::filesystem::path& res, const Settings& settings); } diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index cf7cc9ef8..50b8511df 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -3,11 +3,15 @@ #include "CDClientDatabase.h" #include "Database.h" #include "Logger.h" +#include "UgcBricks.h" #include "UgcCdClient.h" +#include "UgcKeys.h" #include "UgcThrottle.h" #include "json.hpp" +#include #include +#include #include #include #if defined(__linux__) @@ -148,12 +152,63 @@ void UgcProcessor::Worker() { } UgcThrottle::Begin(); + if (job.preview) { + // A preview: the icon goes back to whoever asked, nothing is stored or recorded + HTTPReply reply; + try { + UgcJobs::Outcome outcome{ false, "cancelled" }; + if (!job.preview.Cancelled() && job.kind == Kind::MODULAR) { + outcome = UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings); + } else if (!job.preview.Cancelled()) { + // A player model's icon from its stored .nif + const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif"); + const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt; + auto options = settings.icon; + UgcIconParams::Apply(options, job.iconValues); + outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, outcome.files, outcome.error); + if (!nif) outcome.error = "the model has no stored .nif yet"; + } + if (outcome.ok && outcome.files.contains("icon.png")) { + reply.status = eHTTPStatusCode::OK; + reply.contentType = eContentType::IMAGE_PNG; + reply.message = std::move(outcome.files["icon.png"]); + reply.headers.push_back("Cache-Control: no-store"); + } else { + reply.status = eHTTPStatusCode::UNPROCESSABLE_ENTITY; + reply.contentType = eContentType::TEXT_PLAIN; + reply.message = outcome.error; + } + } catch (const std::exception& ex) { + reply.status = eHTTPStatusCode::INTERNAL_SERVER_ERROR; + reply.contentType = eContentType::TEXT_PLAIN; + reply.message = ex.what(); + } + job.preview.Send(std::move(reply)); + { + std::lock_guard lock(m_Mutex); + m_Active--; + m_MemoryInUse -= std::min(m_MemoryInUse, job.memory); + } + m_Wake.notify_all(); + continue; + } const auto start = std::chrono::steady_clock::now(); Done done{ job.kind, job.id, job.attempts }; + done.iconOnly = job.iconOnly; try { - done.outcome = job.kind == Kind::MODEL - ? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast(job.id)) - : UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings); + if (job.iconOnly) { + // Only the icon, from the .nif made before + const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif"); + const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt; + auto options = settings.icon; + UgcIconParams::Apply(options, job.iconValues); + done.outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, done.outcome.files, done.outcome.error); + if (!nif) done.outcome.error = "no stored .nif"; + } else { + done.outcome = job.kind == Kind::MODEL + ? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast(job.id), job.iconValues) + : UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings); + } } catch (const std::exception& ex) { done.outcome.ok = false; done.outcome.error = std::string("crashed: ") + ex.what(); @@ -162,7 +217,7 @@ void UgcProcessor::Worker() { job.blob.shrink_to_fit(); if (done.outcome.ok) { std::string error; - const auto bytes = m_Storage.Write(job.kind, job.id, done.outcome.files, error); + const auto bytes = job.iconOnly ? m_Storage.Update(job.kind, job.id, done.outcome.files, error) : m_Storage.Write(job.kind, job.id, done.outcome.files, error); if (bytes) { done.bytes = *bytes; } else { @@ -219,26 +274,51 @@ void UgcProcessor::Poll() { Job job{ Kind::MODEL, model.id, model.attempts, UgcJobs::LxfmlFromBlob(model.lxfml) }; if (job.blob.empty()) job.blob = std::move(model.lxfml); // the worker reports it can't be read job.parts = UgcJobs::CountParts(job.blob); + job.iconValues = IconValues(UgcIconParams::ModelKind(), UgcIconParams::ModelTarget(model.id)); job.memory = UgcJobs::EstimateMemory(job.parts, settings); jobs.push_back(std::move(job)); } + // Cars and rockets: one icon per combination of modules, shared by every build of it for (auto& build : Database::Get()->GetModularBuildsToProcess(limit)) { if (m_InFlight.contains({ Kind::MODULAR, build.id })) continue; - Job job{ Kind::MODULAR, build.id, build.attempts }; + const auto key = UgcModularKey::Normalize(build.modules); + if (key.empty()) { + Record(Done{ Kind::MODULAR, build.id, build.attempts, UgcJobs::Outcome{ false, "no modules in \"" + build.modules + "\"" } }); + continue; + } + const auto combo = UgcModularKey::StorageId(key); + m_ComboOf[build.id] = combo; + // Made already for another build of the same modules + if (!m_ComboJobs.contains(combo) && m_Storage.File(Kind::MODULAR, combo, "icon.png")) { + UgcJobs::Outcome reused{ true }; + reused.note = "the same modules as a build made before (" + key + ")"; + m_Reused++; + Record(Done{ Kind::MODULAR, build.id, build.attempts, std::move(reused) }); + continue; + } + m_InFlight.insert({ Kind::MODULAR, build.id }); + m_ComboRows[combo].emplace_back(build.id, build.attempts); + if (m_ComboJobs.contains(combo)) continue; + Job job{ Kind::MODULAR, combo, build.attempts }; job.memory = UgcJobs::EstimateMemory(64, settings); std::string error; if (!UgcCdClient::GatherModular(build.modules, job.modular, error)) { // Nothing a worker could do: record it right away + m_InFlight.erase({ Kind::MODULAR, build.id }); + m_ComboRows.erase(combo); Record(Done{ Kind::MODULAR, build.id, build.attempts, UgcJobs::Outcome{ false, error } }); continue; } + job.modular.key = key; + job.modular.iconValues = IconValues(UgcIconParams::BuildKind(job.modular.buildType), UgcIconParams::CombinationTarget(key)); + m_ComboJobs.insert(combo); jobs.push_back(std::move(job)); } if (jobs.empty()) return; { std::lock_guard lock(m_Mutex); for (auto& job : jobs) { - m_InFlight.insert({ job.kind, job.id }); + if (job.kind == Kind::MODEL) m_InFlight.insert({ job.kind, job.id }); m_Jobs.push_back(std::move(job)); } } @@ -277,8 +357,33 @@ void UgcProcessor::Collect() { finished.swap(m_Done); } for (const auto& done : finished) { - m_InFlight.erase({ done.kind, done.id }); - Record(done); + if (done.kind == Kind::MODEL) { + m_InFlight.erase({ done.kind, done.id }); + if (!done.iconOnly) { + Record(done); + continue; + } + m_Log.push_back({ Kind::MODEL, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? "icon drawn again" : done.outcome.error, UnixNow() }); + while (m_Log.size() > LOG_LENGTH) m_Log.pop_front(); + continue; + } + // A combination: every build waiting for it gets its outcome (none when only its icon was drawn again) + m_ComboJobs.erase(done.id); + if (done.outcome.ok) m_StoredBytes += done.bytes; + auto rows = std::move(m_ComboRows[done.id]); + m_ComboRows.erase(done.id); + for (const auto& [row, attempts] : rows) { + m_InFlight.erase({ Kind::MODULAR, row }); + Done forRow = done; + forRow.id = row; + forRow.attempts = attempts; + forRow.bytes = 0; + Record(forRow); + } + if (rows.empty()) { + m_Log.push_back({ Kind::MODULAR, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? "icon drawn again" : done.outcome.error, UnixNow() }); + while (m_Log.size() > LOG_LENGTH) m_Log.pop_front(); + } } // Something finished: look for more now rather than at the next interval if (!finished.empty()) m_NextPoll = std::min(m_NextPoll, std::chrono::steady_clock::now() + std::chrono::milliseconds(100)); @@ -307,10 +412,22 @@ void UgcProcessor::Update() { std::erase_if(m_Recent, [now](const auto& item) { return now - item.second.first > RECENT_ANSWER_TIME; }); } +LWOOBJID UgcProcessor::StorageId(Kind kind, LWOOBJID id) { + if (kind == Kind::MODEL) return id; + if (const auto it = m_ComboOf.find(id); it != m_ComboOf.end()) return it->second; + const auto info = Database::Get()->GetModularBuildProcessInfo(id); + const auto key = info ? UgcModularKey::Normalize(info->details) : std::string(); + const LWOOBJID combo = key.empty() ? 0 : UgcModularKey::StorageId(key); + if (combo != 0) m_ComboOf[id] = combo; + return combo; +} + UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) { if (m_InFlight.contains({ kind, id })) return Availability::QUEUED; - if (m_Storage.File(kind, id, "icon.png")) { - m_Storage.Touch(kind, id); + const auto storageId = StorageId(kind, id); + if (kind == Kind::MODULAR && m_ComboJobs.contains(storageId)) return Availability::QUEUED; + if (storageId != 0 && m_Storage.File(kind, storageId, "icon.png")) { + m_Storage.Touch(kind, storageId); return Availability::READY; } if (const auto it = m_Recent.find({ kind, id }); it != m_Recent.end()) return it->second.second; @@ -323,6 +440,8 @@ UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) { if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false); else Database::Get()->ResetModularBuildProcessing(id, false); } + // Someone wants it now: no need to wait out the quiet period after its save + if (kind == Kind::MODEL) Database::Get()->ExpediteUgcModel(id); m_NextPoll = std::chrono::steady_clock::now(); answer = Availability::QUEUED; } @@ -345,6 +464,7 @@ nlohmann::json UgcProcessor::Status() const { status["memoryWaits"] = m_Waiting; } status["workers"] = m_Threads.size(); + status["reused"] = m_Reused; const auto throttle = UgcThrottle::GetStats(); status["usage"] = { { "cpuPercent", std::lround(m_CpuPercent) }, // of one core @@ -372,3 +492,161 @@ nlohmann::json UgcProcessor::Status() const { } return status; } + +UgcIconParams::Values UgcProcessor::IconValues(const std::string& kind, const std::string& itemTarget) { + UgcIconParams::Values values; + if (const auto preset = Database::Get()->GetUgcIconSettings(UgcIconParams::KindTarget(kind))) values = UgcIconParams::Parse(*preset); + if (const auto own = Database::Get()->GetUgcIconSettings(itemTarget)) { + for (const auto& [key, value] : UgcIconParams::Parse(*own)) values[key] = value; + } + return values; +} + +bool UgcProcessor::QueuePreview(Kind kind, LWOOBJID id, const std::string& modules, const UgcIconParams::Values& values, DeferredReply reply, std::string& error) { + Job job{ kind, id, 0 }; + if (kind == Kind::MODULAR) { + if (!UgcCdClient::GatherModular(modules, job.modular, error)) return false; + job.modular.key = UgcModularKey::Normalize(modules); + job.modular.iconValues = values; + } else { + if (!m_Storage.File(Kind::MODEL, id, "model.nif")) { + error = "the model has no stored .nif yet"; + return false; + } + job.iconValues = values; + } + job.preview = std::move(reply); + { + std::lock_guard lock(m_Mutex); + job.memory = UgcJobs::EstimateMemory(64, m_Settings); + m_Jobs.push_front(std::move(job)); + } + m_Wake.notify_all(); + return true; +} + +size_t UgcProcessor::RegenerateIcons(const std::string& kind) { + std::vector jobs; + UgcJobs::Settings settings; + { + std::lock_guard lock(m_Mutex); + settings = m_Settings; + } + const bool models = kind == UgcIconParams::ModelKind(); + for (const auto& entry : m_Storage.List()) { + if (models) { + if (entry.kind != Kind::MODEL || m_InFlight.contains({ Kind::MODEL, entry.id }) || !m_Storage.File(Kind::MODEL, entry.id, "model.nif")) continue; + Job job{ Kind::MODEL, entry.id, 0 }; + job.iconOnly = true; + job.iconValues = IconValues(kind, UgcIconParams::ModelTarget(entry.id)); + job.memory = UgcJobs::EstimateMemory(64, settings); + m_InFlight.insert({ Kind::MODEL, entry.id }); + jobs.push_back(std::move(job)); + continue; + } + if (entry.kind != Kind::MODULAR || m_ComboJobs.contains(entry.id)) continue; + const auto path = m_Storage.File(Kind::MODULAR, entry.id, "combo.json"); + const auto text = path ? UgcBricks::ReadFile(*path) : std::nullopt; + const auto combo = text ? nlohmann::json::parse(*text, nullptr, false) : nlohmann::json(); + if (!combo.is_object() || UgcIconParams::BuildKind(combo.value("buildType", -1)) != kind) continue; + const auto key = combo.value("key", std::string()); + Job job{ Kind::MODULAR, entry.id, 0 }; + std::string error; + // The key's LOTs written as an ldf_config ("4713-4714" -> "4713+4714") + std::string modules = key; + std::replace(modules.begin(), modules.end(), '-', '+'); + if (key.empty() || !UgcCdClient::GatherModular(modules, job.modular, error)) continue; + job.modular.key = key; + job.modular.iconValues = IconValues(kind, UgcIconParams::CombinationTarget(key)); + job.memory = UgcJobs::EstimateMemory(64, settings); + m_ComboJobs.insert(entry.id); + jobs.push_back(std::move(job)); + } + const auto count = jobs.size(); + { + std::lock_guard lock(m_Mutex); + for (auto& job : jobs) m_Jobs.push_back(std::move(job)); + } + m_Wake.notify_all(); + return count; +} + +UgcProcessor::DeleteResult UgcProcessor::Delete(const DeleteRequest& request) { + DeleteResult result; + const auto now = std::filesystem::file_time_type::clock::now(); + const auto days = [](int64_t count) { return std::chrono::duration_cast(std::chrono::hours(24 * count)); }; + + // What to delete: the files' ids, and the rows they came from (when they're known) + std::map> targets; // storage id -> rows + if (request.all) { + for (const auto& entry : m_Storage.List()) { + if (entry.kind == request.kind) targets[entry.id]; + } + } + for (const auto id : request.ids) { + const auto storageId = StorageId(request.kind, id); + if (storageId != 0) targets[storageId].push_back(id); + } + + std::vector deletedRows; + for (const auto& [storageId, rows] : targets) { + const bool busy = request.kind == Kind::MODEL ? m_InFlight.contains({ Kind::MODEL, storageId }) : m_ComboJobs.contains(storageId); + if (busy) { + result.busy++; + continue; + } + const auto folder = m_Storage.Folder(request.kind, storageId); + std::error_code error; + if (!std::filesystem::exists(folder, error)) { + deletedRows.insert(deletedRows.end(), rows.begin(), rows.end()); + continue; + } + if (request.unusedDays > 0) { + const auto used = std::filesystem::last_write_time(folder, error); + if (!error && now - used < days(request.unusedDays)) continue; + } + if (request.olderThanDays > 0) { + const auto made = std::filesystem::last_write_time(folder / "icon.png", error); + if (!error && now - made < days(request.olderThanDays)) continue; + } + uint64_t bytes = 0; + for (const auto& file : std::filesystem::directory_iterator(folder, error)) bytes += file.is_regular_file(error) ? file.file_size(error) : 0; + m_Storage.Remove(request.kind, storageId); + result.deleted++; + result.bytes += bytes; + m_StoredBytes -= std::min(m_StoredBytes, bytes); + m_Recent.clear(); + deletedRows.insert(deletedRows.end(), rows.begin(), rows.end()); + if (request.kind == Kind::MODEL) deletedRows.push_back(storageId); + } + std::sort(deletedRows.begin(), deletedRows.end()); + deletedRows.erase(std::unique(deletedRows.begin(), deletedRows.end()), deletedRows.end()); + + // What happens to the rows + if (request.after == eAfterDelete::NOW) { + if (request.all) { + if (request.kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(std::nullopt, false); + else Database::Get()->ResetModularBuildProcessing(std::nullopt, false); + } else { + for (const auto row : deletedRows) { + if (request.kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(row, false); + else Database::Get()->ResetModularBuildProcessing(row, false); + } + } + m_NextPoll = std::chrono::steady_clock::now(); + result.notes.push_back("They're queued to be made again now."); + } else if (request.after == eAfterDelete::GONE) { + for (const auto row : deletedRows) { + if (request.kind == Kind::MODEL) Database::Get()->SetUgcModelProcessed(row, IUgc::eProcessState::FAILED, m_Config.maxAttempts, "Deleted from the dashboard", false); + else Database::Get()->SetModularBuildProcessed(row, IUgc::eProcessState::FAILED, m_Config.maxAttempts, "Deleted from the dashboard"); + } + result.notes.push_back("They're marked deleted and won't be made again unless made again from the dashboard."); + } else { + result.notes.push_back("They'll be made again when a game client asks for them."); + } + if (request.kind == Kind::MODULAR && result.deleted > 0) { + result.notes.push_back("Car and rocket icons are shared by every build of the same modules: the other builds' icons are made again when asked for."); + } + if (result.busy > 0) result.notes.push_back(std::to_string(result.busy) + " being made right now were left alone."); + return result; +} diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h index 93bb02adb..bcc3e54ce 100644 --- a/dUgcServer/UgcProcessor.h +++ b/dUgcServer/UgcProcessor.h @@ -12,9 +12,11 @@ #include #include +#include "DeferredReply.h" #include "dCommonVars.h" #include "json_fwd.hpp" #include "UgcBricks.h" +#include "UgcIconParams.h" #include "UgcJobs.h" #include "UgcStorage.h" @@ -69,6 +71,46 @@ public: // Main thread: whether an item's files can be served, queuing it to be made when it exists but they're missing Availability Request(Kind kind, LWOOBJID id); + // Main thread: the id an item's files are stored under. A car or rocket's are its combination's (every build of + // the same modules shares one icon); 0 when a build's modules can't be told. + LWOOBJID StorageId(Kind kind, LWOOBJID id); + + /** + * Main thread: renders an icon with `values` (UgcIconParams, over the settings) without storing it, on a worker + * (within the budgets, ahead of the queue); `reply` gets the PNG. A player model's is drawn from its stored .nif, a + * car or rocket's from `modules`. False (and `error`) when there's nothing to draw from. + */ + bool QueuePreview(Kind kind, LWOOBJID id, const std::string& modules, const UgcIconParams::Values& values, DeferredReply reply, std::string& error); + + // Main thread: queues every stored icon of a kind ("model", or "build" for cars and rockets) to be drawn again + // with the current settings, presets and overrides; only icons (models' from their stored .nif). How many. + size_t RegenerateIcons(const std::string& kind); + + enum class eAfterDelete { + ON_DEMAND, // rows stay made; the files are made again when something asks for them + NOW, // rows go back to waiting, so they're made again soon + GONE, // rows are marked failed ("deleted"), so they aren't made again until someone asks for that + }; + + struct DeleteRequest { + Kind kind{}; + std::vector ids; // models, or builds (their combinations are deleted); empty with `all` + bool all{}; + int64_t olderThanDays{}; // only files made longer ago than this (0: any) + int64_t unusedDays{}; // only files not asked for in this long (0: any) + eAfterDelete after{ eAfterDelete::ON_DEMAND }; + }; + + struct DeleteResult { + size_t deleted{}; + uint64_t bytes{}; + size_t busy{}; // being made right now, left alone + std::vector notes; + }; + + // Main thread: deletes stored files. Items being made are skipped, so a worker never writes what is being deleted. + DeleteResult Delete(const DeleteRequest& request); + // Main thread: what the server is doing, for /status nlohmann::json Status() const; @@ -98,6 +140,9 @@ private: UgcJobs::ModularInput modular; // modular builds uint64_t memory{}; // estimated bytes it needs size_t parts{}; + DeferredReply preview; // an icon preview: answered with the PNG, nothing stored + bool iconOnly{}; // a model's icon drawn again from its stored .nif + UgcIconParams::Values iconValues; // models: the preset and override (UgcIconParams) }; struct Done { @@ -107,9 +152,12 @@ private: UgcJobs::Outcome outcome; uint64_t bytes{}; double milliseconds{}; + bool iconOnly{}; }; void Poll(); + // Main thread: the icon values for a kind and an item (the kind's preset, then the item's override) + UgcIconParams::Values IconValues(const std::string& kind, const std::string& itemTarget); void Collect(); void Record(const Done& done); void Worker(); @@ -137,7 +185,11 @@ private: std::vector m_Threads; // Main thread only - std::set> m_InFlight; + std::set> m_InFlight; // models, and builds waiting for their combination + std::set m_ComboJobs; // combinations being made + std::map>> m_ComboRows; // combination -> builds (id, attempts) waiting for it + std::map m_ComboOf; // build -> combination (StorageId) + uint64_t m_Reused{}; // builds whose combination was made already std::chrono::steady_clock::time_point m_NextPoll{}; std::chrono::steady_clock::time_point m_NextEviction{}; std::map, std::pair> m_Recent; // answers for missing files diff --git a/dUgcServer/UgcRender.cpp b/dUgcServer/UgcRender.cpp index 43d9c1f35..443278c55 100644 --- a/dUgcServer/UgcRender.cpp +++ b/dUgcServer/UgcRender.cpp @@ -430,7 +430,7 @@ namespace UgcRender { const auto project = [&](const glm::vec3& position) { const auto clip = viewProjection * glm::vec4(position, 1.0f); const float w = clip.w > 1e-6f ? clip.w : 1e-6f; - return glm::vec3((0.5f + (clip.x / w - centerX) * scale * 0.5f) * n, (0.5f - (clip.y / w - centerY) * scale * 0.5f) * n, clip.z / w); + return glm::vec3((0.5f + options.offsetX + (clip.x / w - centerX) * scale * 0.5f) * n, (0.5f - options.offsetY - (clip.y / w - centerY) * scale * 0.5f) * n, clip.z / w); }; // Linear, premultiplied @@ -449,7 +449,7 @@ namespace UgcRender { const int shadowSize = 1024; const OrthoView sunView(center, radius * 1.05f, light, shadowSize); std::vector shadowDepth; - if (options.shadows) { + if (options.shadows > 0.0f) { shadowDepth.assign(static_cast(shadowSize) * shadowSize, INF); const auto& mesh = model.opaque; std::vector screen(mesh.positions.size()); @@ -473,8 +473,11 @@ namespace UgcRender { if (x < 0 || y < 0 || x >= shadowSize || y >= shadowSize || p.z <= shadowDepth[static_cast(y) * shadowSize + x] + shadowBias) lit += 1.0f; } } - return lit / 9.0f; + const float shadowed = 1.0f - lit / 9.0f; + return 1.0f - std::clamp(options.shadows, 0.0f, 1.0f) * shadowed; }; + const glm::vec3 toCamera = glm::normalize(dir); + const glm::vec3 halfway = glm::normalize(light + toCamera); const auto shade = [&](const UgcModel::Mesh& mesh, bool isOpaque, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) { glm::vec3 normal(0.0f, 1.0f, 0.0f); @@ -490,10 +493,15 @@ namespace UgcRender { const float strength = std::clamp(options.ao.strength, 0.0f, 1.0f); const auto position = mesh.positions[i0] * w0 + mesh.positions[i1] * w1 + mesh.positions[i2] * w2; const float direct = std::max(0.0f, glm::dot(normal, light)); - // Diffuse: the world's light (radiance `ambient`) and the sun's (irradiance `sunStrength`, over pi) + const float sun = direct > 0.0f ? sunlit(position + normal * shadowBias) : 0.0f; + // Diffuse: the world's light (radiance `ambient`), a fill from the camera and the sun's (irradiances, over pi) const float lighting = options.ambient * (1.0f - strength * (1.0f - occlusion)) + - options.sunStrength / 3.14159265f * direct * (direct > 0.0f ? sunlit(position + normal * shadowBias) : 0.0f); - return glm::vec4(ToLinear(base.r) * lighting, ToLinear(base.g) * lighting, ToLinear(base.b) * lighting, std::clamp(base.a, 0.0f, 1.0f)); + (options.fill * std::max(0.0f, glm::dot(normal, toCamera)) + options.sunStrength * direct * sun) / 3.14159265f; + // The sun's highlight (Blinn-Phong), white, on top of the color + const float highlight = direct > 0.0f ? options.specular * options.sunStrength / 3.14159265f * sun * std::pow(std::max(0.0f, glm::dot(normal, halfway)), std::max(options.shininess, 1.0f)) : 0.0f; + const float exposure = std::max(options.exposure, 0.0f); + return glm::vec4((ToLinear(base.r) * lighting + highlight) * exposure, (ToLinear(base.g) * lighting + highlight) * exposure, + (ToLinear(base.b) * lighting + highlight) * exposure, std::clamp(base.a, 0.0f, 1.0f)); }; // Opaque first, with the depth buffer @@ -546,9 +554,11 @@ namespace UgcRender { sum /= samples; uint8_t* out = &image.rgba[(static_cast(y) * size + x) * 4]; if (sum.a <= 0.0f) continue; - out[0] = static_cast(std::lround(ToSrgb(sum.r / sum.a) * 255.0f)); - out[1] = static_cast(std::lround(ToSrgb(sum.g / sum.a) * 255.0f)); - out[2] = static_cast(std::lround(ToSrgb(sum.b / sum.a) * 255.0f)); + // sRGB, then the contrast around its middle grey + const auto tone = [&options](float linear) { return std::clamp(0.5f + (ToSrgb(linear) - 0.5f) * options.contrast, 0.0f, 1.0f); }; + out[0] = static_cast(std::lround(tone(sum.r / sum.a) * 255.0f)); + out[1] = static_cast(std::lround(tone(sum.g / sum.a) * 255.0f)); + out[2] = static_cast(std::lround(tone(sum.b / sum.a) * 255.0f)); out[3] = static_cast(std::lround(std::clamp(sum.a, 0.0f, 1.0f) * 255.0f)); } } diff --git a/dUgcServer/UgcRender.h b/dUgcServer/UgcRender.h index c2ec87ecb..f1d4b389b 100644 --- a/dUgcServer/UgcRender.h +++ b/dUgcServer/UgcRender.h @@ -28,9 +28,8 @@ namespace UgcRender { }; /** - * The icon, set up like LU Toolbox's icon renderer (the BrickBuild scene of its UGC render add-on): a 50 mm lens - * (39.6 degree field of view) looking from 53.4 degrees around and 19.5 above, framed at 1.03, a sun of strength - * 2.5 from 21 around and 50.3 above with soft shadows, and a grey (0.192) world light, darkened by ambient occlusion. + * How an icon is drawn. The values that can be set (settings icon_*, presets and overrides) are listed once, with + * their ranges and shipped defaults, in UgcIconParams; the ones here are only what the struct starts with. */ struct IconOptions { int size{ 128 }; @@ -39,13 +38,20 @@ namespace UgcRender { float pitchDegrees{ 19.54f }; // camera above the model float fovDegrees{ 39.6f }; float margin{ 1.03f }; // 1 fills the icon, more leaves a border + float offsetX{}; // the model moved right by this share of the icon's width (after framing) + float offsetY{}; // and up by this share of its height glm::mat4 modelRotation{ 1.0f }; // applied to the model before the camera looks at it float sunYawDegrees{ 21.0f }; float sunPitchDegrees{ 50.3f }; float sunStrength{ 2.5f }; - float ambient{ 0.192f }; - bool shadows{ true }; - AoOptions ao{ true, 5.0f, 32, 1.0f, 0.0f }; + float ambient{ 0.192f }; // the world light (radiance) every face gets + float fill{ 0.0f }; // a light from the camera (irradiance), what lifts the sides facing the viewer + float specular{ 0.0f }; // the sun's highlight on the plastic + float shininess{ 60.0f }; // how tight the highlight is + float exposure{ 1.0f }; // everything times this before it's written as sRGB + float contrast{ 1.0f }; // around the middle grey of the sRGB result + float shadows{ 1.0f }; // how much the sun's shadows darken, 0 to 1 + AoOptions ao{ false, 5.0f, 32, 1.0f, 0.0f }; }; // The model drawn from the icon's camera, framed to fit, on a transparent background. `opaqueAo`: the opaque mesh's diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index 8af4ec82d..fd4ae99ec 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -31,6 +31,7 @@ #include "UgcBricks.h" #include "UgcCdClient.h" +#include "UgcIconParams.h" #include "UgcJobs.h" #include "UgcModel.h" #include "UgcProcessor.h" @@ -47,6 +48,7 @@ namespace Game { namespace { dServer* g_Server = nullptr; + std::string g_AdminKey; // the master password: the dashboard's admin requests carry it (X-Ugc-Admin-Key) UgcProcessor* g_Processor = nullptr; UgcStorage* g_Storage = nullptr; @@ -89,21 +91,13 @@ namespace { settings.ao.samples = std::clamp(Setting("ao_samples", 64), 1, 1024); settings.ao.strength = Setting("ao_strength", 1.0f); settings.ao.glowStrength = Setting("glow_strength", 6.0f); + // The icon's framing and light: the icon_* settings, listed with their defaults in UgcIconParams + settings.icon = UgcIconParams::FromSettings([](const std::string& key) -> std::optional { + const auto value = Game::config->GetValue(key); + return value.empty() ? std::nullopt : std::optional(value); + }); settings.icon.size = Setting("icon_size", 128); - settings.icon.yawDegrees = Setting("icon_yaw", 53.36f); - settings.icon.pitchDegrees = Setting("icon_pitch", 19.54f); - settings.icon.fovDegrees = Setting("icon_fov", 39.6f); - settings.icon.margin = Setting("icon_margin", 1.03f); - settings.icon.sunYawDegrees = Setting("icon_sun_yaw", 21.0f); - settings.icon.sunPitchDegrees = Setting("icon_sun_pitch", 50.3f); - settings.icon.sunStrength = Setting("icon_sun_strength", 2.5f); - settings.icon.ambient = Setting("icon_ambient", 0.192f); - settings.icon.shadows = Setting("icon_shadows", 1) != 0; - settings.icon.ao.enabled = Setting("icon_ao", 1) != 0; settings.icon.ao.distance = settings.ao.distance; - settings.modularIcon = settings.icon; - settings.modularIcon.yawDegrees = Setting("modular_icon_yaw", 53.36f); - settings.modularIcon.pitchDegrees = Setting("modular_icon_pitch", 19.54f); settings.maxBricks = Setting("max_model_bricks", 0); return settings; } @@ -156,7 +150,8 @@ namespace { NotFound(reply); return; } - const auto path = g_Storage->File(kind, id, name); + // A car or rocket's files are its combination's (shared by every build of the same modules) + const auto path = g_Storage->File(kind, g_Processor->StorageId(kind, id), name); if (!path) { NotFound(reply); return; @@ -237,7 +232,7 @@ namespace { const auto id = GeneralUtils::TryParse(segments[2]); const auto kind = segments[1] == "model" ? std::optional(UgcStorage::Kind::MODEL) : segments[1] == "modular" ? std::optional(UgcStorage::Kind::MODULAR) : std::nullopt; const auto& name = segments[3]; - static const std::set PREVIEWS = { "icon.png", "model.nif", "model.noao.nif", "stats.json", + static const std::set PREVIEWS = { "icon.png", "model.nif", "model.noao.nif", "stats.json", "combo.json", "previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json" }; if (!id || !kind || !PREVIEWS.contains(name)) return; reply.headers.clear(); @@ -250,6 +245,78 @@ namespace { } } }); + // For the dashboard only (it sends the master password): icon previews, drawing icons again, deleting files + const auto admin = [](HTTPReply& reply, const HTTPContext& context, std::optional& body) { + const auto& key = context.GetHeader("X-Ugc-Admin-Key"); + bool same = !g_AdminKey.empty() && key.size() == g_AdminKey.size(); + unsigned char diff = 0; + for (size_t i = 0; same && i < key.size(); i++) diff |= static_cast(key[i] ^ g_AdminKey[i]); + reply.contentType = eContentType::APPLICATION_JSON; + reply.headers.push_back("Cache-Control: no-store"); + if (!same || diff != 0) { + reply.status = eHTTPStatusCode::FORBIDDEN; + reply.message = R"({"success":false,"error":"forbidden"})"; + return false; + } + body = nlohmann::json::parse(context.body.empty() ? std::string("{}") : context.body, nullptr, false); + if (!body->is_object()) { + reply.status = eHTTPStatusCode::BAD_REQUEST; + reply.message = R"({"success":false,"error":"the body isn't a JSON object"})"; + return false; + } + return true; + }; + + Game::web.RegisterHTTPRoute({ .path = "/admin/preview", .method = eHTTPMethod::POST, .middleware = {}, + .handle = [admin](HTTPReply& reply, const HTTPContext& context) { + std::optional body; + if (!admin(reply, context, body)) return; + const auto values = UgcIconParams::Parse(body->value("values", nlohmann::json::object()).dump()); + const auto kind = body->value("kind", std::string("modular")) == "model" ? UgcStorage::Kind::MODEL : UgcStorage::Kind::MODULAR; + const auto id = GeneralUtils::TryParse(body->value("id", std::string("0"))).value_or(0); + auto deferred = Web::Defer(reply, context); + std::string error; + if (!g_Processor->QueuePreview(kind, id, body->value("modules", std::string()), values, deferred, error)) { + HTTPReply out; + out.status = eHTTPStatusCode::BAD_REQUEST; + out.contentType = eContentType::APPLICATION_JSON; + out.message = nlohmann::json{ { "success", false }, { "error", error } }.dump(); + deferred.Send(std::move(out)); + } + } }); + + Game::web.RegisterHTTPRoute({ .path = "/admin/regenerate-icons", .method = eHTTPMethod::POST, .middleware = {}, + .handle = [admin](HTTPReply& reply, const HTTPContext& context) { + std::optional body; + if (!admin(reply, context, body)) return; + const auto queued = g_Processor->RegenerateIcons(body->value("kind", std::string())); + reply.status = eHTTPStatusCode::OK; + reply.message = nlohmann::json{ { "success", true }, { "queued", queued } }.dump(); + } }); + + Game::web.RegisterHTTPRoute({ .path = "/admin/delete", .method = eHTTPMethod::POST, .middleware = {}, + .handle = [admin](HTTPReply& reply, const HTTPContext& context) { + std::optional body; + if (!admin(reply, context, body)) return; + UgcProcessor::DeleteRequest request; + request.kind = body->value("kind", std::string("model")) == "modular" ? UgcStorage::Kind::MODULAR : UgcStorage::Kind::MODEL; + if (const auto ids = body->find("ids"); ids != body->end() && ids->is_array()) { + for (const auto& id : *ids) { + const auto parsed = GeneralUtils::TryParse(id.is_string() ? id.get() : id.dump()); + if (parsed) request.ids.push_back(*parsed); + } + } + request.all = body->value("all", false); + request.olderThanDays = body->value("olderThanDays", int64_t{ 0 }); + request.unusedDays = body->value("unusedDays", int64_t{ 0 }); + const auto after = body->value("after", std::string("on_demand")); + request.after = after == "now" ? UgcProcessor::eAfterDelete::NOW : after == "gone" ? UgcProcessor::eAfterDelete::GONE : UgcProcessor::eAfterDelete::ON_DEMAND; + const auto result = g_Processor->Delete(request); + LOG("Deleted the files of %zu item(s) (%llu bytes) for the dashboard", result.deleted, static_cast(result.bytes)); + reply.status = eHTTPStatusCode::OK; + reply.message = nlohmann::json{ { "success", true }, { "deleted", result.deleted }, { "bytes", result.bytes }, { "busy", result.busy }, { "notes", result.notes } }.dump(); + } }); + Game::web.RegisterHTTPRoute({ .path = "/status", .method = eHTTPMethod::GET, .middleware = {}, .handle = [](HTTPReply& reply, const HTTPContext&) { reply.status = eHTTPStatusCode::OK; @@ -361,6 +428,7 @@ int main(int argc, char** argv) { masterPort = masterInfo->port; masterPassword = masterInfo->password; } + g_AdminKey = masterPassword; // The master starts it again when this link drops g_Server = new dServer(masterIP, Setting("net_port", 2012), 0, 16, false, false, Game::logger, masterIP, masterPort, diff --git a/dUgcServer/UgcStorage.cpp b/dUgcServer/UgcStorage.cpp index e3cfd1342..00aa00f37 100644 --- a/dUgcServer/UgcStorage.cpp +++ b/dUgcServer/UgcStorage.cpp @@ -10,7 +10,7 @@ namespace { "model.nif", "model.nif.gz", "model.nif.checksum", "model.lxfml.gz", "model.lxfml.checksum", "icon.dds.gz", "icon.dds.checksum", "icon.png", - "model.noao.nif", "stats.json", + "model.noao.nif", "stats.json", "combo.json", "previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json", }; @@ -88,6 +88,43 @@ std::optional UgcStorage::Write(Kind kind, LWOOBJID id, const Files& f return bytes; } +std::optional UgcStorage::Update(Kind kind, LWOOBJID id, const Files& files, std::string& error) const { + const auto folder = Folder(kind, id); + std::error_code code; + if (!std::filesystem::is_directory(folder, code)) { + error = "no files to update in " + folder.string(); + return std::nullopt; + } + uint64_t bytes = 0; + for (const auto* name : KEPT_FILES) { + if (files.contains(name) && std::filesystem::exists(folder / name, code)) { + std::filesystem::copy_file(folder / name, folder / (std::string("previous.") + name), std::filesystem::copy_options::overwrite_existing, code); + code.clear(); + } + } + for (const auto& [name, data] : files) { + if (!IsKnownFile(name)) continue; + const auto aside = folder / (".tmp-" + name + "-" + RandomSuffix()); + { + std::ofstream out(aside, std::ios::binary | std::ios::trunc); + out.write(data.data(), static_cast(data.size())); + if (!out) { + error = "could not write " + aside.string(); + std::filesystem::remove(aside, code); + return std::nullopt; + } + } + std::filesystem::rename(aside, folder / name, code); + if (code) { + error = "could not replace " + (folder / name).string() + ": " + code.message(); + std::filesystem::remove(aside, code); + return std::nullopt; + } + bytes += data.size(); + } + return bytes; +} + void UgcStorage::Remove(Kind kind, LWOOBJID id) const { std::error_code error; std::filesystem::remove_all(Folder(kind, id), error); diff --git a/dUgcServer/UgcStorage.h b/dUgcServer/UgcStorage.h index f3caac4d0..c377f769a 100644 --- a/dUgcServer/UgcStorage.h +++ b/dUgcServer/UgcStorage.h @@ -33,6 +33,11 @@ public: // Replaces an item's files with `files`; the bytes written, nullopt (and `error`) on failure std::optional Write(Kind kind, LWOOBJID id, const Files& files, std::string& error) const; + // Replaces some of an item's files in place (each written aside and renamed over the old one), keeping the others; + // the ones kept for comparing (icon.png...) are copied to previous.* first. The bytes written, nullopt when the item + // has no folder or a write fails. + std::optional Update(Kind kind, LWOOBJID id, const Files& files, std::string& error) const; + // Deletes an item's files void Remove(Kind kind, LWOOBJID id) const; diff --git a/dWorldServer/WorldServer.cpp b/dWorldServer/WorldServer.cpp index 5f5ef1018..24d4ad7b0 100644 --- a/dWorldServer/WorldServer.cpp +++ b/dWorldServer/WorldServer.cpp @@ -1146,6 +1146,9 @@ void CleanupDisconnectedUser(const SystemAddress& sysAddr) { PropertyManagementComponent::Instance()->Save(); } + // They left: the models they saved needn't wait out their quiet period (docs/UgcServer.md) + Database::Get()->ExpediteUgcModels(c->GetObjectID()); + MasterPackets::PlayerRemoved removed; removed.zoneID = static_cast(Game::server->GetZoneID()); removed.instanceID = static_cast(g_InstanceID); diff --git a/migrations/dlu/mysql/86_ugc_debounce_icon_settings.sql b/migrations/dlu/mysql/86_ugc_debounce_icon_settings.sql new file mode 100644 index 000000000..77ad257f3 --- /dev/null +++ b/migrations/dlu/mysql/86_ugc_debounce_icon_settings.sql @@ -0,0 +1,15 @@ +/* UGC server: ugc.process_after is the Unix time before which a saved model isn't made (the quiet period after a + save, so a model the owner keeps editing isn't made for every save; 0: right away). ugc_icon_settings holds + icon framing and lighting set on the dashboard: presets per kind ("kind:model", "kind:build6") and overrides for one + model ("model:") or one combination of car or rocket modules ("combo:"). See docs/UgcServer.md. */ +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN process_after BIGINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'process_after'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; + +CREATE TABLE IF NOT EXISTS ugc_icon_settings ( + target VARCHAR(191) NOT NULL PRIMARY KEY, + params TEXT NOT NULL, + updated_at BIGINT NOT NULL DEFAULT 0 +); diff --git a/migrations/dlu/sqlite/69_ugc_debounce_icon_settings.sql b/migrations/dlu/sqlite/69_ugc_debounce_icon_settings.sql new file mode 100644 index 000000000..efef782f9 --- /dev/null +++ b/migrations/dlu/sqlite/69_ugc_debounce_icon_settings.sql @@ -0,0 +1,8 @@ +/* UGC server: the quiet period after a save, and icon settings presets and overrides. See the MySQL migration. */ +ALTER TABLE ugc ADD COLUMN process_after BIGINT NOT NULL DEFAULT 0; + +CREATE TABLE IF NOT EXISTS ugc_icon_settings ( + target TEXT NOT NULL PRIMARY KEY, + params TEXT NOT NULL, + updated_at BIGINT NOT NULL DEFAULT 0 +); diff --git a/resources/sharedconfig.ini b/resources/sharedconfig.ini index 8504ee26c..5eb93115b 100644 --- a/resources/sharedconfig.ini +++ b/resources/sharedconfig.ini @@ -88,3 +88,8 @@ mute_restrict_mail=1 mute_restrict_trade=1 # 0 or 1, automatically reject character and pet names submitted while muted mute_auto_reject_names=1 + +# The UGC server makes a saved property model this many seconds after its owner's last save (each save starts the wait +# again for all their waiting models), or as soon as a game client asks for it or the owner leaves the world. 0: right +# away. See docs/UgcServer.md. +ugc_debounce_seconds=120 diff --git a/resources/ugcconfig.ini b/resources/ugcconfig.ini index 5303e5dc0..61c7bcbd0 100644 --- a/resources/ugcconfig.ini +++ b/resources/ugcconfig.ini @@ -78,21 +78,13 @@ ao_distance=5 ao_strength=1 glow_strength=6 -# Icons, set up like LU Toolbox's icon renderer: size in pixels, the camera's angle around (yaw, from the model's -# front) and above (pitch) in degrees, its field of view, the empty border (1 fills the icon), the sun's angles and -# strength, the world light and shadows. A player model's icon is drawn from its finished .nif (LOD 0), so it shows -# its colors, variation and baked lighting exactly as the game does. icon_ao: ambient occlusion for car and rocket -# icons (player models have theirs baked in). Modular builds (cars, rockets) have their own camera angles. +# Icons: size in pixels. A player model's icon is drawn from its finished .nif (LOD 0), a car or rocket's from its +# modules' meshes put together. icon_size=128 -icon_yaw=53.36 -icon_pitch=19.54 -icon_fov=39.6 -icon_margin=1.03 -icon_sun_yaw=21 -icon_sun_pitch=50.3 -icon_sun_strength=2.5 -icon_ambient=0.192 -icon_shadows=1 -icon_ao=1 -modular_icon_yaw=53.36 -modular_icon_pitch=19.54 + +# How icons are framed and lit: icon_yaw, icon_pitch, icon_fov, icon_margin, icon_offset_x, icon_offset_y, +# icon_sun_yaw, icon_sun_pitch, icon_sun_strength, icon_ambient, icon_fill, icon_specular, icon_shininess, +# icon_exposure, icon_contrast, icon_shadows, icon_ao_strength. Their defaults (the framing from LU Toolbox's icon +# renderer, the light matched to the game's own model icons) and ranges are listed once in the server +# (UgcIconParams) and on the dashboard's settings page; set one here only to change it. The dashboard's icon editor +# also keeps presets for player models and each car or rocket type, and overrides for single items, in the database. diff --git a/tests/dDatabaseTests/DatabaseParityTests.cpp b/tests/dDatabaseTests/DatabaseParityTests.cpp index 05196576a..5f2cf1f70 100644 --- a/tests/dDatabaseTests/DatabaseParityTests.cpp +++ b/tests/dDatabaseTests/DatabaseParityTests.cpp @@ -699,7 +699,7 @@ TEST_F(ParitySeeded, Properties) { TEST_F(ParitySeeded, UgcModel) { Both("InsertNewUgcModel", [](GameDatabase& db) { std::stringstream data("sd0\x01\x02 compressed model"); - db.InsertNewUgcModel(data, 1152921510000500001LL, 1, CHAR_ALICE); + db.InsertNewUgcModel(data, 1152921510000500001LL, 1, CHAR_ALICE, 0); }); Both("GetUgcModel unplaced", [](GameDatabase& db) { auto model = db.GetUgcModel(1152921510000500001LL); @@ -721,6 +721,21 @@ TEST_F(ParitySeeded, UgcModel) { for (const auto& model : db.GetUgcModelsToProcess(10)) out.push_back({ model.id, model.lxfml, model.attempts }); return out; }); + Both("Quiet period after a save", [&](GameDatabase& db) { + // A second save far in the future: it waits, and so does the owner's first model; leaving ends both waits + std::stringstream data("sd0\x01\x02 second model"); + db.InsertNewUgcModel(data, 1152921510000500003LL, 1, CHAR_ALICE, 4000000000LL); + json out = json{ db.GetUgcModelsToProcess(10).size(), db.GetUgcProcessInfo(1152921510000500003LL)->processAfter }; + db.ExpediteUgcModel(1152921510000500003LL); + out.push_back(db.GetUgcModelsToProcess(10).size()); + db.InsertNewUgcModel(data, 1152921510000500004LL, 1, CHAR_ALICE, 4000000001LL); + out.push_back(db.GetUgcModelsToProcess(10).size()); + db.ExpediteUgcModels(CHAR_ALICE); + out.push_back(db.GetUgcModelsToProcess(10).size()); + db.DeleteUgcModelData(1152921510000500003LL); + db.DeleteUgcModelData(1152921510000500004LL); + return out; + }); Both("SetUgcModelProcessed failed", [&](GameDatabase& db) { db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::FAILED, 3, "no bricks", false); return json{ infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), countsJson(db.GetUgcProcessCounts()), db.GetUgcModelsToProcess(10).size() }; @@ -751,6 +766,14 @@ TEST_F(ParitySeeded, UgcModel) { out.push_back(db.GetModularBuildProcessList(std::nullopt, "", 0, 10).size()); out.push_back(db.ResetModularBuildProcessing(1152921510000500002LL, false)); out.push_back(infoJson(db.GetModularBuildProcessInfo(1152921510000500002LL))); + for (const auto& [ldf, count] : db.GetModularBuildConfigCounts()) out.push_back({ ldf, count }); + // A combination's own icon framing + out.push_back(db.GetUgcIconSettings("combo:4713-4714-4715").has_value()); + db.SetUgcIconSettings("combo:4713-4714-4715", R"({"yaw":10})"); + db.SetUgcIconSettings("combo:4713-4714-4715", R"({"yaw":20})"); + out.push_back(db.GetUgcIconSettings("combo:4713-4714-4715").value_or("")); + db.DeleteUgcIconSettings("combo:4713-4714-4715"); + out.push_back(db.GetUgcIconSettings("combo:4713-4714-4715").has_value()); return out; }); } diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp index 1b40f4082..415731f39 100644 --- a/tests/dUgcTests/UgcTests.cpp +++ b/tests/dUgcTests/UgcTests.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -12,6 +13,8 @@ #include "UgcFormats.h" #include "UgcModel.h" #include "UgcJobs.h" +#include "UgcIconParams.h" +#include "UgcKeys.h" #include "UgcModular.h" #include "UgcPalette.h" #include "UgcRender.h" @@ -47,7 +50,9 @@ namespace { } std::filesystem::path TempFolder(const std::string& name) { - auto path = std::filesystem::temp_directory_path() / ("dlu_ugc_test_" + name + "_" + std::to_string(::testing::UnitTest::GetInstance()->random_seed())); + // One folder per test and process: ctest runs the tests in parallel processes + const auto* test = ::testing::UnitTest::GetInstance()->current_test_info(); + auto path = std::filesystem::temp_directory_path() / ("dlu_ugc_test_" + name + "_" + (test ? std::string(test->name()) : std::string()) + "_" + std::to_string(::getpid())); std::filesystem::remove_all(path); std::filesystem::create_directories(path); return path; @@ -581,3 +586,119 @@ TEST(UgcJobs, MakesLodsStatsAndIcons) { EXPECT_EQ(UgcJobs::CountParts(LXFML5), 3u); EXPECT_GT(UgcJobs::EstimateMemory(1000, settings), UgcJobs::EstimateMemory(10, settings)); } + +TEST(UgcModularKey, SameModulesSameKey) { + // However the modules are written or ordered, the combination is the same; its files are stored once + EXPECT_EQ(UgcModularKey::Normalize("1:4715+1:4713+1:4714"), "4713-4714-4715"); + EXPECT_EQ(UgcModularKey::Normalize("1:4713;1:4714,1:4715"), "4713-4714-4715"); + EXPECT_EQ(UgcModularKey::Normalize("4714+4715+4713+1:4713"), "4713-4714-4715"); + EXPECT_EQ(UgcModularKey::Normalize(""), ""); + EXPECT_EQ(UgcModularKey::Normalize("1:abc+"), ""); + EXPECT_NE(UgcModularKey::Normalize("1:4713+1:4714+1:4716"), UgcModularKey::Normalize("1:4713+1:4714+1:4715")); + const auto id = UgcModularKey::StorageId("4713-4714-4715"); + EXPECT_GT(id, 0); + EXPECT_EQ(id, UgcModularKey::StorageId(UgcModularKey::Normalize("1:4715+1:4714+1:4713"))); + EXPECT_NE(id, UgcModularKey::StorageId("4713-4714-4716")); + + // Two builds of the same modules find the one set of files + UgcStorage storage(TempFolder("combo")); + std::string error; + ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODULAR, id, { { "icon.png", "png" } }, error)) << error; + EXPECT_TRUE(storage.File(UgcStorage::Kind::MODULAR, UgcModularKey::StorageId(UgcModularKey::Normalize("1:4713+1:4714+1:4715")), "icon.png")); + EXPECT_TRUE(storage.File(UgcStorage::Kind::MODULAR, UgcModularKey::StorageId(UgcModularKey::Normalize("1:4714+1:4715+1:4713")), "icon.png")); + std::filesystem::remove_all(storage.GetRoot()); +} + +TEST(UgcDebounce, WaitsForTheQuietPeriod) { + EXPECT_EQ(UgcDebounce::ProcessAfter(1000, 120), 1120); + EXPECT_EQ(UgcDebounce::ProcessAfter(1000, 0), 0); + EXPECT_EQ(UgcDebounce::ProcessAfter(1000, -5), 0); + EXPECT_FALSE(UgcDebounce::Due(1120, 1100, false)); // saved 100 s ago: still quiet + EXPECT_TRUE(UgcDebounce::Due(1120, 1120, false)); + EXPECT_TRUE(UgcDebounce::Due(1120, 1100, true)); // a client asked for it + EXPECT_TRUE(UgcDebounce::Due(0, 5, false)); // expedited or saved without a wait + // A new save starts the wait again + const auto first = UgcDebounce::ProcessAfter(1000, 120), second = UgcDebounce::ProcessAfter(1100, 120); + EXPECT_FALSE(UgcDebounce::Due(std::max(first, second), 1150, false)); +} + +TEST(UgcIconParams, OneListDrivesEverything) { + // Every parameter has a setting, a range holding its default, and something it changes + for (const auto& param : UgcIconParams::List()) { + EXPECT_TRUE(param.setting.starts_with("icon_")) << param.key; + EXPECT_LE(param.min, param.defaultValue) << param.key; + EXPECT_GE(param.max, param.defaultValue) << param.key; + EXPECT_TRUE(param.apply) << param.key; + EXPECT_EQ(UgcIconParams::Find(param.key), ¶m); + } + const auto values = UgcIconParams::Parse(R"({"yaw":10,"pitch":200,"margin":0,"offsetX":0.1,"unknown":1,"fov":"wide","exposure":1.5})"); + EXPECT_FLOAT_EQ(values.at("yaw"), 10.0f); + EXPECT_FLOAT_EQ(values.at("pitch"), 89.0f); // clamped + EXPECT_FLOAT_EQ(values.at("margin"), 0.5f); + EXPECT_FALSE(values.contains("fov")); // not a number + EXPECT_FALSE(values.contains("unknown")); + EXPECT_TRUE(UgcIconParams::Parse("not json").empty()); + EXPECT_EQ(UgcIconParams::Parse(UgcIconParams::ToJson(values)), values); + + // Settings, then values over them + auto options = UgcIconParams::FromSettings([](const std::string& key) -> std::optional { + if (key == "icon_fov") return "33"; + if (key == "icon_ambient") return "nonsense"; + return std::nullopt; + }); + EXPECT_FLOAT_EQ(options.fovDegrees, 33.0f); + EXPECT_FLOAT_EQ(options.ambient, UgcIconParams::Find("ambient")->defaultValue); + UgcIconParams::Apply(options, values); + EXPECT_FLOAT_EQ(options.yawDegrees, 10.0f); + EXPECT_FLOAT_EQ(options.fovDegrees, 33.0f); // not in the values: the setting stays + EXPECT_FLOAT_EQ(options.exposure, 1.5f); + EXPECT_FLOAT_EQ(options.offsetX, 0.1f); + UgcIconParams::Apply(options, { { "aoStrength", 0.5f } }); + EXPECT_TRUE(options.ao.enabled); + + // A car or rocket: the settings with its preset and combination values + UgcJobs::Settings settings; + UgcJobs::ModularInput input; + input.iconValues = { { "yaw", 5.0f } }; + EXPECT_FLOAT_EQ(UgcJobs::ModularIconOptions(input, settings).yawDegrees, 5.0f); + EXPECT_EQ(UgcIconParams::KindTarget(UgcIconParams::BuildKind(6)), "kind:build6"); + EXPECT_EQ(UgcIconParams::ModelTarget(12), "model:12"); + EXPECT_EQ(UgcIconParams::CombinationTarget("1-2"), "combo:1-2"); + + // Exposure brightens, contrast spreads + UgcBricks::BrickLibrary library(MakeRes(), 0); + std::string error; + const auto box = UgcModel::Build(UgcModel::ParseLxfml(R"( + )", error), library); + const auto mean = [](const UgcRender::Image& image) { + double sum = 0, count = 0; + for (size_t i = 0; i < image.rgba.size(); i += 4) { + if (image.rgba[i + 3] < 128) continue; + sum += image.rgba[i]; + count++; + } + return sum / std::max(count, 1.0); + }; + UgcRender::IconOptions dim{ 32, 1 }, bright{ 32, 1 }; + bright.exposure = 2.0f; + EXPECT_GT(mean(UgcRender::RenderIcon(box, bright)), mean(UgcRender::RenderIcon(box, dim)) + 10.0); + + // An offset moves the drawn model by that share of the icon + UgcModel::Model model; + model = UgcModel::Build(UgcModel::ParseLxfml(R"( + )", error), library); + const auto centroid = [](const UgcRender::Image& image) { + double sum = 0, count = 0; + for (int y = 0; y < image.height; y++) for (int x = 0; x < image.width; x++) { + const auto a = image.rgba[(static_cast(y) * image.width + x) * 4 + 3]; + sum += x * a; + count += a; + } + return count > 0 ? sum / count : -1.0; + }; + UgcRender::IconOptions plain{ 64, 1 }; + plain.margin = 2.0f; + auto shifted = plain; + shifted.offsetX = 0.25f; + EXPECT_NEAR(centroid(UgcRender::RenderIcon(model, shifted)) - centroid(UgcRender::RenderIcon(model, plain)), 16.0, 1.0); +} diff --git a/tests/dWebTests/CMakeLists.txt b/tests/dWebTests/CMakeLists.txt index b09ad2389..55287f9e2 100644 --- a/tests/dWebTests/CMakeLists.txt +++ b/tests/dWebTests/CMakeLists.txt @@ -77,7 +77,7 @@ target_include_directories(dWebTests PRIVATE "${PROJECT_SOURCE_DIR}/thirdparty/nlohmann" ) -target_link_libraries(dWebTests ${COMMON_LIBRARIES} dWeb sqlite3 OpenSSL::Crypto CURL::libcurl GTest::gtest_main) +target_link_libraries(dWebTests ${COMMON_LIBRARIES} dWeb dUgc sqlite3 OpenSSL::Crypto CURL::libcurl GTest::gtest_main) gtest_discover_tests(dWebTests)