diff --git a/CMakeLists.txt b/CMakeLists.txt index 43a316e76..d526cb686 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -133,7 +133,7 @@ endif() message(STATUS "Variable: DLU_CONFIG_DIR = ${DLU_CONFIG_DIR}") # Copy resource files on first build -set(RESOURCE_FILES "sharedconfig.ini" "authconfig.ini" "chatconfig.ini" "worldconfig.ini" "masterconfig.ini" "dashboardconfig.ini" "blocklist.dcf") +set(RESOURCE_FILES "sharedconfig.ini" "authconfig.ini" "chatconfig.ini" "worldconfig.ini" "masterconfig.ini" "dashboardconfig.ini" "ugcconfig.ini" "blocklist.dcf") message(STATUS "Checking resource file integrity") include(Utils) @@ -330,6 +330,7 @@ add_subdirectory(dWorldServer) add_subdirectory(dAuthServer) add_subdirectory(dChatServer) add_subdirectory(dDashboardServer) +add_subdirectory(dUgcServer) add_subdirectory(dMasterServer) # Add MasterServer last so it can rely on the other binaries target_precompile_headers( diff --git a/dCommon/CMakeLists.txt b/dCommon/CMakeLists.txt index 92151ee65..3fb9d4d32 100644 --- a/dCommon/CMakeLists.txt +++ b/dCommon/CMakeLists.txt @@ -31,6 +31,7 @@ set(DCOMMON_SOURCES "Raw.cpp" "Process.cpp" "Permissions.cpp" + "NifFile.cpp" ) # Workaround for compiler bug where the optimized code could result in a memcpy of 0 bytes, even though that isnt possible. diff --git a/dDashboardServer/routes/NifFile.cpp b/dCommon/NifFile.cpp similarity index 98% rename from dDashboardServer/routes/NifFile.cpp rename to dCommon/NifFile.cpp index 41608a974..c737b0dd2 100644 --- a/dDashboardServer/routes/NifFile.cpp +++ b/dCommon/NifFile.cpp @@ -383,9 +383,17 @@ namespace { const auto children = reader.Array(childCount); const auto effectCount = reader.U32(); reader.Skip(static_cast(effectCount) * 4); - if (!reader.Ok() || (av.flags & APP_CULLED)) return; + if (!reader.Ok()) return; const auto world = parent.Then(av.transform); + // Recorded even when hidden: attach points often are + if (!av.name.empty() && !m_Model.nodes.contains(av.name)) { + NifFile::NodeTransform node; + for (int i = 0; i < 9; i++) node.rotation[i] = world.r[i] * world.s; + node.translation = world.t; + m_Model.nodes.emplace(av.name, node); + } + if (av.flags & APP_CULLED) return; properties = Inherit(properties, av.properties); if (const auto tag = NifFile::ShaderTag(av.name); tag >= 0) properties.shaderTag = tag; diff --git a/dDashboardServer/routes/NifFile.h b/dCommon/NifFile.h similarity index 93% rename from dDashboardServer/routes/NifFile.h rename to dCommon/NifFile.h index b84d48c2e..582f00099 100644 --- a/dDashboardServer/routes/NifFile.h +++ b/dCommon/NifFile.h @@ -72,9 +72,16 @@ namespace NifFile { // What a texture's alpha does under a shader (mapShaders.gameValue); -1 is fixed function (opacity) eTextureAlpha TextureAlphaFor(int32_t shader); + // Where a node is in the model's space: p' = rotation * p + translation (row-major, scale folded in) + struct NodeTransform { + std::array rotation{ 1, 0, 0, 0, 1, 0, 0, 0, 1 }; + std::array translation{}; + }; + struct Model { uint32_t version{}; std::vector meshes; + std::map nodes; // named nodes that are drawn (the first of each name), e.g. attach points std::array min{}; std::array max{}; std::map skipped; // block types in the file that aren't drawn, with how many diff --git a/dCommon/ZCompression.cpp b/dCommon/ZCompression.cpp index 11ec4e833..344ac80a2 100644 --- a/dCommon/ZCompression.cpp +++ b/dCommon/ZCompression.cpp @@ -46,5 +46,55 @@ namespace ZCompression { inflateEnd(&zInfo); // zlib function return(nRet); } -} + namespace { + // Inflate everything in `data` with zlib's `windowBits` (31: gzip, -15: raw); nullopt on any error + std::optional Inflate(std::string_view data, int windowBits, size_t sizeHint) { + z_stream stream{}; + if (inflateInit2(&stream, windowBits) != Z_OK) return std::nullopt; + std::string out; + out.reserve(sizeHint); + stream.next_in = reinterpret_cast(const_cast(data.data())); + stream.avail_in = static_cast(data.size()); + char buffer[65536]; + int result = Z_OK; + while (result == Z_OK) { + stream.next_out = reinterpret_cast(buffer); + stream.avail_out = sizeof(buffer); + result = inflate(&stream, Z_NO_FLUSH); + if (result != Z_OK && result != Z_STREAM_END) break; + out.append(buffer, sizeof(buffer) - stream.avail_out); + if (result == Z_OK && stream.avail_in == 0 && stream.avail_out != 0) break; // truncated + } + inflateEnd(&stream); + if (result != Z_STREAM_END) return std::nullopt; + return out; + } + } + + std::string Gzip(std::string_view data) { + z_stream stream{}; + // 31: a gzip header and trailer instead of zlib's + if (deflateInit2(&stream, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY) != Z_OK) return {}; + std::string out; + out.resize(deflateBound(&stream, static_cast(data.size())) + 32); + stream.next_in = reinterpret_cast(const_cast(data.data())); + stream.avail_in = static_cast(data.size()); + stream.next_out = reinterpret_cast(out.data()); + stream.avail_out = static_cast(out.size()); + const auto result = deflate(&stream, Z_FINISH); + out.resize(stream.total_out); + deflateEnd(&stream); + return result == Z_STREAM_END ? out : std::string{}; + } + + std::optional Gunzip(std::string_view data) { + return Inflate(data, 31, data.size() * 4); + } + + std::optional InflateRaw(std::string_view data, size_t size) { + auto out = Inflate(data, -15, size); + if (!out || out->size() != size) return std::nullopt; + return out; + } +} diff --git a/dCommon/ZCompression.h b/dCommon/ZCompression.h index d58a8778f..57d399571 100644 --- a/dCommon/ZCompression.h +++ b/dCommon/ZCompression.h @@ -1,6 +1,9 @@ #pragma once #include +#include +#include +#include namespace ZCompression { uint32_t GetMaxCompressedLength(uint32_t nLenSrc); @@ -8,5 +11,14 @@ namespace ZCompression { int32_t Compress(const uint8_t* abSrc, uint32_t nLenSrc, uint8_t* abDst, uint32_t nLenDst); int32_t Decompress(const uint8_t* abSrc, uint32_t nLenSrc, uint8_t* abDst, uint32_t nLenDst, int32_t& nErr); + + // `data` as a gzip file (RFC 1952), what HTTP clients and the game client's UGC downloads inflate + std::string Gzip(std::string_view data); + + // A gzip file's contents; nullopt when it isn't one or is damaged + std::optional Gunzip(std::string_view data); + + // Raw deflate data (no zlib or gzip header, as in zip files) that inflates to `size` bytes; nullopt when it doesn't + std::optional InflateRaw(std::string_view data, size_t size); } diff --git a/dCommon/dEnums/ServiceType.h b/dCommon/dEnums/ServiceType.h index 2fdee209b..830bbb801 100644 --- a/dCommon/dEnums/ServiceType.h +++ b/dCommon/dEnums/ServiceType.h @@ -9,7 +9,8 @@ enum class ServiceType : uint16_t { WORLD, CLIENT, MASTER, - UNKNOWN + UNKNOWN, + UGC // the UGC server (dUgcServer): appended, the values above are on the wire }; #endif //!__SERVICETYPE__H__ diff --git a/dDashboardServer/routes/CMakeLists.txt b/dDashboardServer/routes/CMakeLists.txt index 884cb9ccd..a99d16a6e 100644 --- a/dDashboardServer/routes/CMakeLists.txt +++ b/dDashboardServer/routes/CMakeLists.txt @@ -40,7 +40,6 @@ set(DASHBOARDROUTES_SOURCES "ChallengeRoutes.cpp" "InstanceLoad.cpp" "WorldView.cpp" - "NifFile.cpp" "Scenery.cpp" "WorkerPool.cpp" "Workers.cpp" diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index 813fea104..42107dcc8 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -16,6 +16,7 @@ namespace { const std::string CHAT = "chatconfig.ini"; const std::string WORLD = "worldconfig.ini"; const std::string DASHBOARD = "dashboardconfig.ini"; + const std::string UGC = "ugcconfig.ini"; constexpr double PORT_MIN = 1; constexpr double PORT_MAX = 65535; @@ -127,6 +128,7 @@ namespace { c.AddSection("Startup"); c.Add(Bool(MASTER, "prestart_servers", "Start auth, chat and char servers", "Master starts the other servers itself.", true, true)); c.Add(Bool(MASTER, "enable_dashboard", "Start the web dashboard", "", false, true)); + c.Add(Bool(MASTER, "enable_ugc_server", "Start the UGC server", "Makes and serves the meshes and icons of what players build (docs/UgcServer.md).", false, true)); c.Add(Bool(SHARED, "skip_account_creation", "Skip the first-account prompt", "For non-interactive setups: master doesn't ask for an account when there are none.", false, true)); c.AddSection("Game client"); @@ -400,6 +402,30 @@ namespace { c.AddSection("Character history"); c.Add(Days("snapshot_days", "Snapshots", "Older snapshots are deleted, but each character keeps its newest few (below).", "90", 1)); c.Add(Unit(Int(DASHBOARD, "snapshot_keep", "Always keep per character", "", "10", 1, 1000), "snapshots")); + c.AddSection("UGC server", "Makes the meshes and icons of what players build and serves them to the game client. See docs/UgcServer.md.", + eLayout::ROWS, Condition{ MASTER, "enable_ugc_server", { "1" } }); + c.Add(Port(UGC, "port", "Download port", "The game client's UGCSERVERPORT.", "2008")); + c.Add(Format(Text(UGC, "listen_ip", "Listen address", "The game client has to reach it.", "0.0.0.0", true), eFormat::HOST)); + c.Add(Port(UGC, "net_port", "Master connection port", "UDP; the next port is used too.", "2012")); + c.Add(Text(UGC, "client_path", "Download path", "The game client's UGCSERVERDIR.", "/ugc", true)); + c.Add(Format(Text(UGC, "ugc_output_dir", "Files folder", "Relative to the server binaries.", "ugc", true), eFormat::PATH)); + c.Add(Unit(Int(UGC, "ugc_max_storage_mb", "Most space for files", "The files used longest ago are deleted past this and made again when asked for. 0: no limit.", "2048", 0, std::nullopt, true), "MB")); + c.Add(Int(UGC, "worker_threads", "Worker threads", "0: half the CPU cores.", "0", 0, 64, true)); + c.Add(Unit(Int(UGC, "poll_interval_ms", "Look for new models every", "", "2000", 100, 600000, true), "ms")); + c.Add(Int(UGC, "poll_batch", "Models taken at once", "", "32", 1, 1000, true)); + c.Add(Int(UGC, "max_attempts", "Attempts before giving up", "", "3", 1, 100, true)); + c.Add(Int(UGC, "brick_lod", "Brick detail", "0 is the most detailed, 2 the least.", "0", 0, 2, true)); + c.Add(Bool(UGC, "remove_hidden_faces", "Remove faces nobody can see", "", true, true)); + c.Add(Bool(UGC, "bake_ao", "Darken hidden corners", "Ambient occlusion baked into the vertex colors.", true, true)); + c.Add(Float(UGC, "ao_strength", "Darkening strength", "0 to 1.", "0.6", 0, 1)); + c.Add(Unit(Int(UGC, "optimize_resolution", "Detail of the visibility renders", "", "1024", 64, 4096, true), "pixels")); + c.Add(Unit(Int(UGC, "icon_size", "Icon size", "", "128", 16, 1024, true), "pixels")); + c.Add(Unit(Float(UGC, "icon_yaw", "Icon angle around", "", "35", -360, 360), "degrees")); + c.Add(Unit(Float(UGC, "icon_pitch", "Icon angle above", "", "25", -90, 90), "degrees")); + c.Add(Float(UGC, "icon_margin", "Icon border", "1 fills the icon.", "1.03", 0.1, 10)); + c.Add(Unit(Float(UGC, "modular_icon_yaw", "Car and rocket icon angle around", "", "35", -360, 360), "degrees")); + c.Add(Unit(Float(UGC, "modular_icon_pitch", "Car and rocket icon angle above", "", "20", -90, 90), "degrees")); + return c; } @@ -456,7 +482,7 @@ namespace SettingsCatalog { const std::vector>& Files() { static const std::vector> files{ - { SHARED, "Shared" }, { WORLD, "World" }, { AUTH, "Auth" }, { CHAT, "Chat" }, { MASTER, "Master" }, { DASHBOARD, "Dashboard" } + { SHARED, "Shared" }, { WORLD, "World" }, { AUTH, "Auth" }, { CHAT, "Chat" }, { MASTER, "Master" }, { DASHBOARD, "Dashboard" }, { UGC, "UGC" } }; return files; } diff --git a/dDatabase/GameDatabase/ITables/IUgc.h b/dDatabase/GameDatabase/ITables/IUgc.h index 9f6d2ef17..37b71273f 100644 --- a/dDatabase/GameDatabase/ITables/IUgc.h +++ b/dDatabase/GameDatabase/ITables/IUgc.h @@ -6,6 +6,7 @@ #include #include #include +#include class IUgc { public: @@ -31,5 +32,53 @@ public: virtual void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) = 0; virtual std::optional GetUgcModel(const LWOOBJID ugcId) = 0; + + // ---- Processing by the UGC server (is_optimized, processed_at, process_attempts, process_error) ---- + + // is_optimized's values + enum class eProcessState : int32_t { + PENDING = 0, // not made yet (or to be made again) + DONE = 1, // the UGC server's files are made + FAILED = 2, // gave up after the allowed attempts + }; + + // A row's processing state, for the UGC server and the dashboard + struct ProcessInfo { + LWOOBJID id{}; + LWOOBJID characterId{}; + std::string characterName; // empty when the character is gone + eProcessState state{}; + uint32_t attempts{}; + int64_t processedAt{}; // Unix seconds of the last attempt, 0 for none + std::string error; + bool bakeAo{}; + std::string details; // modular builds: the modules (ldf_config) + }; + + // A model waiting to be made: its id, stored LXFML (sd0) and the attempts so far + struct PendingModel { + LWOOBJID id{}; + std::string lxfml; + uint32_t attempts{}; + }; + + // Up to `limit` pending models, the least tried and then the newest first + virtual std::vector GetUgcModelsToProcess(const uint32_t limit) = 0; + + // Records an attempt: the new state, how many attempts there have been, why it failed (empty when it didn't) and + // whether lighting was baked in; processed_at becomes now + virtual void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) = 0; + + virtual std::optional GetUgcProcessInfo(const LWOOBJID id) = 0; + + // Sets models back to pending with no attempts: one (`id`), or all of them (`id` nullopt; only the failed ones + // with `failedOnly`). Returns how many rows changed. + virtual uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) = 0; + + // A page of models (all, or those in `state`), the newest first + virtual std::vector GetUgcProcessList(const std::optional state, const uint32_t offset, const uint32_t limit) = 0; + + // How many models are in each state + virtual std::vector> GetUgcProcessCounts() = 0; }; #endif //!__IUGC__H__ diff --git a/dDatabase/GameDatabase/ITables/IUgcModularBuild.h b/dDatabase/GameDatabase/ITables/IUgcModularBuild.h index 7ac69fe3d..9d738bda4 100644 --- a/dDatabase/GameDatabase/ITables/IUgcModularBuild.h +++ b/dDatabase/GameDatabase/ITables/IUgcModularBuild.h @@ -4,11 +4,30 @@ #include #include #include +#include + +#include "IUgc.h" class IUgcModularBuild { public: virtual void InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional characterId) = 0; virtual void DeleteUgcBuild(const LWOOBJID bigId) = 0; + + // ---- Icons made by the UGC server; the same columns and meanings as IUgc's processing ---- + + // A modular build waiting for its icon: its id, modules (ldf_config) and the attempts so far + struct PendingBuild { + LWOOBJID id{}; + std::string modules; + uint32_t attempts{}; + }; + + virtual std::vector GetModularBuildsToProcess(const uint32_t limit) = 0; + virtual void SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) = 0; + virtual std::optional GetModularBuildProcessInfo(const LWOOBJID id) = 0; + virtual uint64_t ResetModularBuildProcessing(const std::optional id, const bool failedOnly) = 0; + virtual std::vector GetModularBuildProcessList(const std::optional state, const uint32_t offset, const uint32_t limit) = 0; + virtual std::vector> GetModularBuildProcessCounts() = 0; }; #endif //!IUGCMODULARBUILD_H diff --git a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h index 73f96e6f2..721ae54e7 100644 --- a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h +++ b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h @@ -393,6 +393,18 @@ public: nlohmann::json GetCharacterById(const LWOOBJID charId) override; std::optional GetModel(const LWOOBJID modelID) override; std::optional GetUgcModel(const LWOOBJID ugcId) override; + std::vector GetUgcModelsToProcess(const uint32_t limit) override; + void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override; + std::optional GetUgcProcessInfo(const LWOOBJID id) override; + uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override; + std::vector GetUgcProcessList(const std::optional state, const uint32_t offset, const uint32_t limit) override; + std::vector> GetUgcProcessCounts() override; + std::vector GetModularBuildsToProcess(const uint32_t limit) override; + void SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) override; + std::optional GetModularBuildProcessInfo(const LWOOBJID id) override; + uint64_t ResetModularBuildProcessing(const std::optional id, const bool failedOnly) override; + std::vector GetModularBuildProcessList(const std::optional state, const uint32_t offset, const uint32_t limit) override; + std::vector> GetModularBuildProcessCounts() override; std::optional GetPropertyInfo(const LWOOBJID id) override; std::string GetPropertiesTable(uint32_t start, uint32_t length, const std::string_view search = "", uint32_t orderColumn = 0, bool orderAsc = true, bool pendingOnly = false) override; uint32_t GetPropertyCount() override; diff --git a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp index 981077ccc..02bb27d5b 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp @@ -1,5 +1,27 @@ #include "MySQLDatabase.h" +#include + +namespace { + IUgc::ProcessInfo ReadUgcProcessInfo(PreparedStmtResultSet& result, bool modular) { + IUgc::ProcessInfo info; + info.id = result->getInt64("id"); + info.characterId = result->getInt64("character_id"); + info.characterName = std::string(result->getString("character_name").c_str()); + info.state = static_cast(result->getInt("is_optimized")); + info.attempts = static_cast(result->getInt("process_attempts")); + info.processedAt = result->getInt64("processed_at"); + info.error = std::string(result->getString("process_error").c_str()); + if (modular) info.details = std::string(result->getString("ldf_config").c_str()); + else info.bakeAo = result->getInt("bake_ao") != 0; + return info; + } + + int64_t UnixNow() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + } +} + IUgc::Model ReadModel(PreparedStmtResultSet& result) { IUgc::Model model; @@ -67,7 +89,8 @@ void MySQLDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { void MySQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) { const std::istream stream(lxfml.rdbuf()); - ExecuteUpdate("UPDATE ugc SET lxfml = ? WHERE id = ?;", &stream, modelId); + // The UGC server makes the model's files again + ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", &stream, modelId); } std::optional MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) { @@ -81,3 +104,66 @@ std::optional MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) { return toReturn; } + +std::vector MySQLDatabase::GetUgcModelsToProcess(const uint32_t limit) { + auto result = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 ORDER BY process_attempts ASC, id DESC LIMIT ?;", limit); + std::vector models; + while (result->next()) { + auto& model = models.emplace_back(); + model.id = result->getInt64("id"); + std::unique_ptr blob(result->getBlob("lxfml")); + std::stringstream contents; + contents << blob->rdbuf(); + model.lxfml = contents.str(); + model.attempts = static_cast(result->getInt("process_attempts")); + } + return models; +} + +void MySQLDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) { + ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ? WHERE id = ?;", + static_cast(state), attempts, error, bakeAo, UnixNow(), id); +} + +std::optional MySQLDatabase::GetUgcProcessInfo(const LWOOBJID id) { + auto result = ExecuteSelect( + "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao " + "FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id WHERE u.id = ? LIMIT 1;", id); + if (!result->next()) return std::nullopt; + return ReadUgcProcessInfo(result, false); +} + +uint64_t MySQLDatabase::ResetUgcModelProcessing(const std::optional id, const bool failedOnly) { + if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", *id); + if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;"); + return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '';"); +} + +std::vector MySQLDatabase::GetUgcProcessList(const std::optional state, const uint32_t offset, const uint32_t limit) { + const std::string select = + "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao " + "FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id "; + std::vector list; + auto read = [&list](PreparedStmtResultSet& result) { + while (result->next()) { + list.push_back(ReadUgcProcessInfo(result, false)); + } + }; + if (state) { + auto result = ExecuteSelect(select + "WHERE u.is_optimized = ? ORDER BY u.id DESC LIMIT ? OFFSET ?;", static_cast(*state), limit, offset); + read(result); + } else { + auto result = ExecuteSelect(select + "ORDER BY u.id DESC LIMIT ? OFFSET ?;", limit, offset); + read(result); + } + return list; +} + +std::vector> MySQLDatabase::GetUgcProcessCounts() { + auto result = ExecuteSelect("SELECT is_optimized, COUNT(*) AS count FROM ugc GROUP BY is_optimized;"); + std::vector> counts; + while (result->next()) { + counts.emplace_back(static_cast(result->getInt("is_optimized")), static_cast(result->getInt64("count"))); + } + return counts; +} diff --git a/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp b/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp index 6336a32f8..6d6096af8 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp @@ -1,5 +1,27 @@ #include "MySQLDatabase.h" +#include + +namespace { + IUgc::ProcessInfo ReadModularProcessInfo(PreparedStmtResultSet& result, bool modular) { + IUgc::ProcessInfo info; + info.id = result->getInt64("id"); + info.characterId = result->getInt64("character_id"); + info.characterName = std::string(result->getString("character_name").c_str()); + info.state = static_cast(result->getInt("is_optimized")); + info.attempts = static_cast(result->getInt("process_attempts")); + info.processedAt = result->getInt64("processed_at"); + info.error = std::string(result->getString("process_error").c_str()); + if (modular) info.details = std::string(result->getString("ldf_config").c_str()); + else info.bakeAo = result->getInt("bake_ao") != 0; + return info; + } + + int64_t UnixNow() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + } +} + void MySQLDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional characterId) { ExecuteInsert("INSERT INTO ugc_modular_build (ugc_id, ldf_config, character_id) VALUES (?,?,?)", bigId, modules, characterId); } @@ -7,3 +29,61 @@ void MySQLDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID bi void MySQLDatabase::DeleteUgcBuild(const LWOOBJID bigId) { ExecuteDelete("DELETE FROM ugc_modular_build WHERE ugc_id = ?;", bigId); } + +std::vector MySQLDatabase::GetModularBuildsToProcess(const uint32_t limit) { + auto result = ExecuteSelect("SELECT ugc_id, ldf_config, process_attempts FROM ugc_modular_build WHERE is_optimized = 0 ORDER BY process_attempts ASC, ugc_id DESC LIMIT ?;", limit); + std::vector builds; + while (result->next()) { + builds.push_back({ result->getInt64("ugc_id"), std::string(result->getString("ldf_config").c_str()), static_cast(result->getInt("process_attempts")) }); + } + return builds; +} + +void MySQLDatabase::SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) { + ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = ?, process_attempts = ?, process_error = ?, processed_at = ? WHERE ugc_id = ?;", + static_cast(state), attempts, error, UnixNow(), id); +} + +namespace { + const std::string MODULAR_SELECT = + "SELECT b.ugc_id AS id, b.character_id, c.name AS character_name, b.is_optimized, b.process_attempts, b.processed_at, b.process_error, b.ldf_config " + "FROM ugc_modular_build AS b LEFT JOIN charinfo AS c ON c.id = b.character_id "; +} + +std::optional MySQLDatabase::GetModularBuildProcessInfo(const LWOOBJID id) { + auto result = ExecuteSelect(MODULAR_SELECT + "WHERE b.ugc_id = ? LIMIT 1;", id); + if (!result->next()) return std::nullopt; + return ReadModularProcessInfo(result, true); +} + +uint64_t MySQLDatabase::ResetModularBuildProcessing(const std::optional id, const bool failedOnly) { + if (id) return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE ugc_id = ?;", *id); + if (failedOnly) return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;"); + return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '';"); +} + +std::vector MySQLDatabase::GetModularBuildProcessList(const std::optional state, const uint32_t offset, const uint32_t limit) { + std::vector list; + auto read = [&list](PreparedStmtResultSet& result) { + while (result->next()) { + list.push_back(ReadModularProcessInfo(result, true)); + } + }; + if (state) { + auto result = ExecuteSelect(MODULAR_SELECT + "WHERE b.is_optimized = ? ORDER BY b.ugc_id DESC LIMIT ? OFFSET ?;", static_cast(*state), limit, offset); + read(result); + } else { + auto result = ExecuteSelect(MODULAR_SELECT + "ORDER BY b.ugc_id DESC LIMIT ? OFFSET ?;", limit, offset); + read(result); + } + return list; +} + +std::vector> MySQLDatabase::GetModularBuildProcessCounts() { + auto result = ExecuteSelect("SELECT is_optimized, COUNT(*) AS count FROM ugc_modular_build GROUP BY is_optimized;"); + std::vector> counts; + while (result->next()) { + counts.emplace_back(static_cast(result->getInt("is_optimized")), static_cast(result->getInt64("count"))); + } + return counts; +} diff --git a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h index b7ba05302..58d90a7d8 100644 --- a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h +++ b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h @@ -377,6 +377,18 @@ public: nlohmann::json GetCharacterById(const LWOOBJID charId) override; std::optional GetModel(const LWOOBJID modelID) override; std::optional GetUgcModel(const LWOOBJID ugcId) override; + std::vector GetUgcModelsToProcess(const uint32_t limit) override; + void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override; + std::optional GetUgcProcessInfo(const LWOOBJID id) override; + uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override; + std::vector GetUgcProcessList(const std::optional state, const uint32_t offset, const uint32_t limit) override; + std::vector> GetUgcProcessCounts() override; + std::vector GetModularBuildsToProcess(const uint32_t limit) override; + void SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) override; + std::optional GetModularBuildProcessInfo(const LWOOBJID id) override; + uint64_t ResetModularBuildProcessing(const std::optional id, const bool failedOnly) override; + std::vector GetModularBuildProcessList(const std::optional state, const uint32_t offset, const uint32_t limit) override; + std::vector> GetModularBuildProcessCounts() override; std::optional GetPropertyInfo(const LWOOBJID id) override; std::string GetPropertiesTable(uint32_t start, uint32_t length, const std::string_view search = "", uint32_t orderColumn = 0, bool orderAsc = true, bool pendingOnly = false) override; uint32_t GetPropertyCount() override; diff --git a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp index f0c3b8378..eb2135023 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp @@ -1,5 +1,27 @@ #include "SQLiteDatabase.h" +#include + +namespace { + IUgc::ProcessInfo ReadUgcProcessInfo(CppSQLite3Query& result, bool modular) { + IUgc::ProcessInfo info; + info.id = result.getInt64Field("id"); + info.characterId = result.getInt64Field("character_id"); + info.characterName = result.getStringField("character_name", ""); + info.state = static_cast(result.getIntField("is_optimized")); + info.attempts = static_cast(result.getIntField("process_attempts")); + info.processedAt = result.getInt64Field("processed_at"); + info.error = result.getStringField("process_error", ""); + if (modular) info.details = result.getStringField("ldf_config", ""); + else info.bakeAo = result.getIntField("bake_ao") != 0; + return info; + } + + int64_t UnixNow() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + } +} + IUgc::Model ReadModel(CppSQLite3Query& result) { IUgc::Model model; @@ -68,7 +90,8 @@ void SQLiteDatabase::DeleteUgcModelData(const LWOOBJID& modelId) { void SQLiteDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) { const std::istream stream(lxfml.rdbuf()); - ExecuteUpdate("UPDATE ugc SET lxfml = ? WHERE id = ?;", &stream, modelId); + // The UGC server makes the model's files again + ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", &stream, modelId); } std::optional SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) { @@ -82,3 +105,68 @@ std::optional SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) { return toReturn; } + +std::vector SQLiteDatabase::GetUgcModelsToProcess(const uint32_t limit) { + auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 ORDER BY process_attempts ASC, id DESC LIMIT ?;", limit); + std::vector models; + while (!result.eof()) { + auto& model = models.emplace_back(); + model.id = result.getInt64Field("id"); + int blobSize{}; + const auto* blob = result.getBlobField("lxfml", blobSize); + model.lxfml.assign(reinterpret_cast(blob), blobSize); + model.attempts = static_cast(result.getIntField("process_attempts")); + result.nextRow(); + } + return models; +} + +void SQLiteDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) { + ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ? WHERE id = ?;", + static_cast(state), attempts, error, bakeAo, UnixNow(), id); +} + +std::optional SQLiteDatabase::GetUgcProcessInfo(const LWOOBJID id) { + auto [_, result] = ExecuteSelect( + "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao " + "FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id WHERE u.id = ? LIMIT 1;", id); + if (result.eof()) return std::nullopt; + return ReadUgcProcessInfo(result, false); +} + +uint64_t SQLiteDatabase::ResetUgcModelProcessing(const std::optional id, const bool failedOnly) { + if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", *id); + if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;"); + return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '';"); +} + +std::vector SQLiteDatabase::GetUgcProcessList(const std::optional state, const uint32_t offset, const uint32_t limit) { + const std::string select = + "SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao " + "FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id "; + std::vector list; + auto read = [&list](CppSQLite3Query& result) { + while (!result.eof()) { + list.push_back(ReadUgcProcessInfo(result, false)); + result.nextRow(); + } + }; + if (state) { + auto [_, result] = ExecuteSelect(select + "WHERE u.is_optimized = ? ORDER BY u.id DESC LIMIT ? OFFSET ?;", static_cast(*state), limit, offset); + read(result); + } else { + auto [_, result] = ExecuteSelect(select + "ORDER BY u.id DESC LIMIT ? OFFSET ?;", limit, offset); + read(result); + } + return list; +} + +std::vector> SQLiteDatabase::GetUgcProcessCounts() { + auto [_, result] = ExecuteSelect("SELECT is_optimized, COUNT(*) AS count FROM ugc GROUP BY is_optimized;"); + std::vector> counts; + while (!result.eof()) { + counts.emplace_back(static_cast(result.getIntField("is_optimized")), static_cast(result.getInt64Field("count"))); + result.nextRow(); + } + return counts; +} diff --git a/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp b/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp index f501d0e0f..3f81353d0 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp @@ -1,5 +1,27 @@ #include "SQLiteDatabase.h" +#include + +namespace { + IUgc::ProcessInfo ReadModularProcessInfo(CppSQLite3Query& result, bool modular) { + IUgc::ProcessInfo info; + info.id = result.getInt64Field("id"); + info.characterId = result.getInt64Field("character_id"); + info.characterName = result.getStringField("character_name", ""); + info.state = static_cast(result.getIntField("is_optimized")); + info.attempts = static_cast(result.getIntField("process_attempts")); + info.processedAt = result.getInt64Field("processed_at"); + info.error = result.getStringField("process_error", ""); + if (modular) info.details = result.getStringField("ldf_config", ""); + else info.bakeAo = result.getIntField("bake_ao") != 0; + return info; + } + + int64_t UnixNow() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + } +} + void SQLiteDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional characterId) { ExecuteInsert("INSERT INTO ugc_modular_build (ugc_id, ldf_config, character_id) VALUES (?,?,?)", bigId, modules, characterId); } @@ -7,3 +29,64 @@ void SQLiteDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID b void SQLiteDatabase::DeleteUgcBuild(const LWOOBJID bigId) { ExecuteDelete("DELETE FROM ugc_modular_build WHERE ugc_id = ?;", bigId); } + +std::vector SQLiteDatabase::GetModularBuildsToProcess(const uint32_t limit) { + auto [_, result] = ExecuteSelect("SELECT ugc_id, ldf_config, process_attempts FROM ugc_modular_build WHERE is_optimized = 0 ORDER BY process_attempts ASC, ugc_id DESC LIMIT ?;", limit); + std::vector builds; + while (!result.eof()) { + builds.push_back({ result.getInt64Field("ugc_id"), result.getStringField("ldf_config", ""), static_cast(result.getIntField("process_attempts")) }); + result.nextRow(); + } + return builds; +} + +void SQLiteDatabase::SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) { + ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = ?, process_attempts = ?, process_error = ?, processed_at = ? WHERE ugc_id = ?;", + static_cast(state), attempts, error, UnixNow(), id); +} + +namespace { + const std::string MODULAR_SELECT = + "SELECT b.ugc_id AS id, b.character_id, c.name AS character_name, b.is_optimized, b.process_attempts, b.processed_at, b.process_error, b.ldf_config " + "FROM ugc_modular_build AS b LEFT JOIN charinfo AS c ON c.id = b.character_id "; +} + +std::optional SQLiteDatabase::GetModularBuildProcessInfo(const LWOOBJID id) { + auto [_, result] = ExecuteSelect(MODULAR_SELECT + "WHERE b.ugc_id = ? LIMIT 1;", id); + if (result.eof()) return std::nullopt; + return ReadModularProcessInfo(result, true); +} + +uint64_t SQLiteDatabase::ResetModularBuildProcessing(const std::optional id, const bool failedOnly) { + if (id) return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE ugc_id = ?;", *id); + if (failedOnly) return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;"); + return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '';"); +} + +std::vector SQLiteDatabase::GetModularBuildProcessList(const std::optional state, const uint32_t offset, const uint32_t limit) { + std::vector list; + auto read = [&list](CppSQLite3Query& result) { + while (!result.eof()) { + list.push_back(ReadModularProcessInfo(result, true)); + result.nextRow(); + } + }; + if (state) { + auto [_, result] = ExecuteSelect(MODULAR_SELECT + "WHERE b.is_optimized = ? ORDER BY b.ugc_id DESC LIMIT ? OFFSET ?;", static_cast(*state), limit, offset); + read(result); + } else { + auto [_, result] = ExecuteSelect(MODULAR_SELECT + "ORDER BY b.ugc_id DESC LIMIT ? OFFSET ?;", limit, offset); + read(result); + } + return list; +} + +std::vector> SQLiteDatabase::GetModularBuildProcessCounts() { + auto [_, result] = ExecuteSelect("SELECT is_optimized, COUNT(*) AS count FROM ugc_modular_build GROUP BY is_optimized;"); + std::vector> counts; + while (!result.eof()) { + counts.emplace_back(static_cast(result.getIntField("is_optimized")), static_cast(result.getInt64Field("count"))); + result.nextRow(); + } + return counts; +} diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h index 0e3f39eed..804cc4d60 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h @@ -372,6 +372,18 @@ class TestSQLDatabase : public GameDatabase { std::optional GetModel(const LWOOBJID modelID) override { return {}; } std::optional GetPropertyInfo(const LWOOBJID id) override { return {}; } std::optional GetUgcModel(const LWOOBJID ugcId) override { return {}; } + std::vector GetUgcModelsToProcess(const uint32_t limit) override { return {}; } + void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override {} + std::optional GetUgcProcessInfo(const LWOOBJID id) override { return {}; } + uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override { return 0; } + std::vector GetUgcProcessList(const std::optional state, const uint32_t offset, const uint32_t limit) override { return {}; } + std::vector> GetUgcProcessCounts() override { return {}; } + std::vector GetModularBuildsToProcess(const uint32_t limit) override { return {}; } + void SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) override {} + std::optional GetModularBuildProcessInfo(const LWOOBJID id) override { return {}; } + uint64_t ResetModularBuildProcessing(const std::optional id, const bool failedOnly) override { return 0; } + std::vector GetModularBuildProcessList(const std::optional state, const uint32_t offset, const uint32_t limit) override { return {}; } + std::vector> GetModularBuildProcessCounts() override { return {}; } void DeleteAccount(const uint32_t accountId) override {}; void InsertAuditLog(uint32_t accountId, const std::string_view accountName, const std::string_view action, const std::string_view description, uint32_t targetAccountId, LWOOBJID targetCharacterId) override {}; std::string GetAuditLogTable(uint32_t start, uint32_t length, const std::string_view search = "", uint32_t orderColumn = 0, bool orderAsc = true) override { return "{}"; }; diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index 837fdabe4..a3e99a13c 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -82,6 +82,7 @@ std::map activeSessions; SystemAddress authServerMasterPeerSysAddr; SystemAddress chatServerMasterPeerSysAddr; SystemAddress dashboardServerMasterPeerSysAddr; +SystemAddress ugcServerMasterPeerSysAddr; namespace { // Dashboard player actions waiting for world servers to answer @@ -444,6 +445,11 @@ int main(int argc, char** argv) { StartDashboardServer(); } + // The UGC server makes and serves player models' meshes and icons (docs/UgcServer.md) + if (Game::config->GetValue("enable_ugc_server") == "1") { + StartUgcServer(); + } + auto t = std::chrono::high_resolution_clock::now(); Packet* packet = nullptr; constexpr uint32_t logFlushTime = 15 * masterFramerate; @@ -624,6 +630,9 @@ namespace { case ServiceType::DASHBOARD: dashboardServerMasterPeerSysAddr = sysAddr; break; + case ServiceType::UGC: + ugcServerMasterPeerSysAddr = sysAddr; + break; default: break; } @@ -1025,6 +1034,19 @@ void HandlePacket(Packet* packet) { dashboardServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS; StartDashboardServer(); } + + if (packet->systemAddress == ugcServerMasterPeerSysAddr) { + ugcServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS; + + if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) { + MasterPackets::ServerInfo offline; + offline.serverType = ServiceType::UGC; + offline.ip = LUString("offline"); + MasterPackets::SendTo(dashboardServerMasterPeerSysAddr, offline); + } + + StartUgcServer(); + } } if (packet->length < 4) return; diff --git a/dMasterServer/Start.cpp b/dMasterServer/Start.cpp index 01cb9f16e..b9655bbff 100644 --- a/dMasterServer/Start.cpp +++ b/dMasterServer/Start.cpp @@ -143,6 +143,42 @@ uint32_t StartDashboardServer() { return web_pid; } +uint32_t StartUgcServer() { + if (Game::ShouldShutdown()) { + LOG("Currently shutting down. UgcServer will not be restarted."); + return 0; + } + auto ugc_path = BinaryPathFinder::GetBinaryDir() / "UgcServer"; +#ifdef _WIN32 + ugc_path.replace_extension(".exe"); + auto ugc_startup = startup; + auto ugc_info = PROCESS_INFORMATION{}; + if (!CreateProcessW(ugc_path.wstring().data(), ugc_path.wstring().data(), + nullptr, nullptr, false, 0, nullptr, nullptr, + &ugc_startup, &ugc_info)) + { + LOG("Failed to launch UgcServer"); + return 0; + } + + // get pid and close unused handles + auto ugc_pid = ugc_info.dwProcessId; + CloseHandle(ugc_info.hProcess); + CloseHandle(ugc_info.hThread); +#else // *nix systems + const auto ugc_pid = fork(); + if (ugc_pid < 0) { + LOG("Failed to launch UgcServer"); + return 0; + } else if (ugc_pid == 0) { + // We are the child process + execl(ugc_path.string().c_str(), ugc_path.string().c_str(), nullptr); + } +#endif + LOG("UgcServer PID is %d", ugc_pid); + return ugc_pid; +} + uint32_t StartWorldServer(LWOMAPID mapID, uint16_t port, LWOINSTANCEID lastInstanceID, int maxPlayers, LWOCLONEID cloneID) { auto world_path = BinaryPathFinder::GetBinaryDir() / "WorldServer"; #ifdef _WIN32 diff --git a/dMasterServer/Start.h b/dMasterServer/Start.h index e8dd5b4dd..08c02096c 100644 --- a/dMasterServer/Start.h +++ b/dMasterServer/Start.h @@ -4,4 +4,5 @@ uint32_t StartAuthServer(); uint32_t StartChatServer(); uint32_t StartDashboardServer(); +uint32_t StartUgcServer(); uint32_t StartWorldServer(LWOMAPID mapID, uint16_t port, LWOINSTANCEID lastInstanceID, int maxPlayers, LWOCLONEID cloneID); diff --git a/dUgcServer/CMakeLists.txt b/dUgcServer/CMakeLists.txt new file mode 100644 index 000000000..43b9fdb97 --- /dev/null +++ b/dUgcServer/CMakeLists.txt @@ -0,0 +1,40 @@ +# The pure part of the UGC server (brick data, meshes, rendering, file formats, storage), shared with the tests +set(DUGC_SOURCES + "UgcBricks.cpp" + "UgcFormats.cpp" + "UgcJobs.cpp" + "UgcModel.cpp" + "UgcModular.cpp" + "UgcRender.cpp" + "UgcStorage.cpp" +) + +add_library(dUgc STATIC ${DUGC_SOURCES}) +target_include_directories(dUgc PUBLIC "." "${PROJECT_SOURCE_DIR}/thirdparty/MD5") +target_link_libraries(dUgc PUBLIC dCommon glm::glm tinyxml2 MD5) + +add_executable(UgcServer "UgcServer.cpp" "UgcProcessor.cpp" "UgcCdClient.cpp") + +target_include_directories(UgcServer PRIVATE + "${PROJECT_SOURCE_DIR}/dCommon" + "${PROJECT_SOURCE_DIR}/dCommon/dClient" + "${PROJECT_SOURCE_DIR}/dCommon/dEnums" + "${PROJECT_SOURCE_DIR}/dDatabase" + "${PROJECT_SOURCE_DIR}/dDatabase/CDClientDatabase" + "${PROJECT_SOURCE_DIR}/dDatabase/CDClientDatabase/CDClientTables" + "${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase" + "${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase/ITables" + "${PROJECT_SOURCE_DIR}/dNet" + "${PROJECT_SOURCE_DIR}/dWeb" + "${PROJECT_SOURCE_DIR}/dServer" + "${PROJECT_SOURCE_DIR}/thirdparty" + "${PROJECT_SOURCE_DIR}/thirdparty/nlohmann" +) + +target_compile_definitions(UgcServer PRIVATE PROJECT_VERSION="\"${PROJECT_VERSION}\"") +target_link_libraries(UgcServer ${COMMON_LIBRARIES} dWeb dServer dUgc) + +# Rendering is far too slow unoptimized (minutes a model instead of seconds), so it is optimized in every build type +if(NOT MSVC) + set_source_files_properties("UgcRender.cpp" "UgcModel.cpp" PROPERTIES COMPILE_OPTIONS "-O2") +endif() diff --git a/dUgcServer/UgcBricks.cpp b/dUgcServer/UgcBricks.cpp new file mode 100644 index 000000000..745c0214b --- /dev/null +++ b/dUgcServer/UgcBricks.cpp @@ -0,0 +1,208 @@ +#include "UgcBricks.h" + +#include +#include +#include +#include +#include + +#include "tinyxml2.h" +#include "ZCompression.h" + +namespace { + constexpr int32_t GEOMETRY_MAGIC = 0x42473031; // "10GB" + constexpr uint32_t MAX_GEOMETRY_PARTS = 64; + + template + bool ReadAt(std::string_view data, size_t offset, T& value) { + if (offset > data.size() || sizeof(T) > data.size() - offset) return false; + std::memcpy(&value, data.data() + offset, sizeof(T)); + return true; + } + + std::string Lower(std::string_view text) { + std::string out(text); + std::transform(out.begin(), out.end(), out.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + return out; + } +} + +namespace UgcBricks { + std::optional ParseGeometry(std::string_view data) { + int32_t magic{}, vertexCount{}, indexCount{}, options{}; + if (!ReadAt(data, 0, magic) || magic != GEOMETRY_MAGIC || !ReadAt(data, 4, vertexCount) || !ReadAt(data, 8, indexCount) || + !ReadAt(data, 12, options) || vertexCount < 0 || indexCount < 0 || indexCount % 3 != 0) { + return std::nullopt; + } + const auto vertices = static_cast(vertexCount); + const auto indices = static_cast(indexCount); + size_t offset = 16; + const bool hasUvs = (options & 3) == 3; + const size_t needed = offset + vertices * 24 + (hasUvs ? vertices * 8 : 0) + indices * 4; + if (needed > data.size()) return std::nullopt; + + Geometry geometry; + geometry.positions.resize(vertices * 3); + std::memcpy(geometry.positions.data(), data.data() + offset, vertices * 12); + offset += vertices * 12; + geometry.normals.resize(vertices * 3); + std::memcpy(geometry.normals.data(), data.data() + offset, vertices * 12); + offset += vertices * 12; + if (hasUvs) offset += vertices * 8; + geometry.indices.resize(indices); + std::memcpy(geometry.indices.data(), data.data() + offset, indices * 4); + for (const auto index : geometry.indices) { + if (index >= vertices) return std::nullopt; + } + return geometry; + } + + std::map ParseMaterials(std::string_view xml) { + std::map materials; + tinyxml2::XMLDocument doc; + if (doc.Parse(xml.data(), xml.size()) != tinyxml2::XML_SUCCESS) return materials; + const auto* root = doc.FirstChildElement("Materials"); + if (!root) return materials; + for (const auto* element = root->FirstChildElement("Material"); element; element = element->NextSiblingElement("Material")) { + const auto id = element->UnsignedAttribute("MatID", 0); + if (id == 0) continue; + const auto channel = [element](const char* name, uint32_t fallback) { + return static_cast(std::min(element->UnsignedAttribute(name, fallback), 255)); + }; + materials[id] = Material{ channel("Red", 160), channel("Green", 160), channel("Blue", 160), channel("Alpha", 255) }; + } + return materials; + } + + std::optional ReadZipEntry(std::string_view zip, std::string_view name) { + // The end of central directory record is in the last 64 KiB + 22 bytes + constexpr uint32_t END_SIGNATURE = 0x06054b50; + constexpr uint32_t CENTRAL_SIGNATURE = 0x02014b50; + constexpr uint32_t LOCAL_SIGNATURE = 0x04034b50; + if (zip.size() < 22) return std::nullopt; + size_t end = std::string_view::npos; + const size_t lowest = zip.size() > 22 + 65535 ? zip.size() - 22 - 65535 : 0; + for (size_t i = zip.size() - 22 + 1; i-- > lowest;) { + uint32_t signature{}; + if (ReadAt(zip, i, signature) && signature == END_SIGNATURE) { + end = i; + break; + } + } + if (end == std::string_view::npos) return std::nullopt; + uint16_t entries{}; + uint32_t directoryOffset{}; + if (!ReadAt(zip, end + 10, entries) || !ReadAt(zip, end + 16, directoryOffset)) return std::nullopt; + + const auto wanted = Lower(name); + size_t offset = directoryOffset; + for (uint16_t i = 0; i < entries; i++) { + uint32_t signature{}, compressedSize{}, size{}, localOffset{}; + uint16_t method{}, nameLength{}, extraLength{}, commentLength{}; + if (!ReadAt(zip, offset, signature) || signature != CENTRAL_SIGNATURE || !ReadAt(zip, offset + 10, method) || + !ReadAt(zip, offset + 20, compressedSize) || !ReadAt(zip, offset + 24, size) || !ReadAt(zip, offset + 28, nameLength) || + !ReadAt(zip, offset + 30, extraLength) || !ReadAt(zip, offset + 32, commentLength) || !ReadAt(zip, offset + 42, localOffset) || + offset + 46 + nameLength > zip.size()) { + return std::nullopt; + } + const auto entryName = zip.substr(offset + 46, nameLength); + offset += 46 + nameLength + extraLength + commentLength; + if (Lower(entryName) != wanted) continue; + + uint16_t localNameLength{}, localExtraLength{}; + if (!ReadAt(zip, localOffset, signature) || signature != LOCAL_SIGNATURE || !ReadAt(zip, localOffset + 26, localNameLength) || + !ReadAt(zip, localOffset + 28, localExtraLength)) { + return std::nullopt; + } + const size_t dataStart = static_cast(localOffset) + 30 + localNameLength + localExtraLength; + if (dataStart > zip.size() || compressedSize > zip.size() - dataStart) return std::nullopt; + const auto data = zip.substr(dataStart, compressedSize); + if (method == 0) return std::string(data); + if (method == 8) return ZCompression::InflateRaw(data, size); + return std::nullopt; + } + return std::nullopt; + } + + std::optional ResolvePath(const std::filesystem::path& root, std::string_view relative) { + std::string normalized(relative); + std::replace(normalized.begin(), normalized.end(), '\\', '/'); + std::filesystem::path current = root; + std::stringstream parts(normalized); + std::string part; + std::error_code error; + while (std::getline(parts, part, '/')) { + if (part.empty() || part == ".") continue; + if (part == "..") return std::nullopt; + if (std::filesystem::exists(current / part, error)) { + current /= part; + continue; + } + const auto wanted = Lower(part); + bool found = false; + for (std::filesystem::directory_iterator it(current, error), endIt; !error && it != endIt; it.increment(error)) { + if (Lower(it->path().filename().string()) == wanted) { + current = it->path(); + found = true; + break; + } + } + if (!found) return std::nullopt; + } + if (!std::filesystem::is_regular_file(current, error)) return std::nullopt; + return current; + } + + std::optional ReadFile(const std::filesystem::path& path) { + std::ifstream file(path, std::ios::binary); + if (!file) return std::nullopt; + std::ostringstream contents; + contents << file.rdbuf(); + return contents.str(); + } + + BrickLibrary::BrickLibrary(std::filesystem::path res, uint32_t lod) : m_Res(std::move(res)), m_Lod(std::min(lod, 2)) {} + + bool BrickLibrary::LoadMaterials() { + const auto zipPath = ResolvePath(m_Res, "brickdb.zip"); + const auto zip = zipPath ? ReadFile(*zipPath) : std::nullopt; + const auto xml = zip ? ReadZipEntry(*zip, "Materials.xml") : std::nullopt; + if (!xml) return false; + auto materials = ParseMaterials(*xml); + if (materials.empty()) return false; + SetMaterials(std::move(materials)); + return true; + } + + void BrickLibrary::SetMaterials(std::map materials) { + std::lock_guard lock(m_Mutex); + m_Materials = std::move(materials); + } + + Material BrickLibrary::GetMaterial(uint32_t id) const { + // Written once before the workers start, only read after + const auto it = m_Materials.find(id); + return it != m_Materials.end() ? it->second : Material{}; + } + + std::shared_ptr> BrickLibrary::GetDesign(uint32_t design) { + { + std::lock_guard lock(m_Mutex); + if (const auto it = m_Designs.find(design); it != m_Designs.end()) return it->second; + } + // Loaded outside the lock; two threads loading the same design at once is harmless + auto parts = std::make_shared>(); + const auto folder = "brickprimitives/lod" + std::to_string(m_Lod) + "/"; + for (uint32_t index = 0; index < MAX_GEOMETRY_PARTS; index++) { + const auto name = std::to_string(design) + ".g" + (index == 0 ? "" : std::to_string(index)); + const auto path = ResolvePath(m_Res, folder + name); + const auto data = path ? ReadFile(*path) : std::nullopt; + if (!data) break; + auto geometry = ParseGeometry(*data); + if (!geometry) break; + parts->push_back(std::move(*geometry)); + } + std::lock_guard lock(m_Mutex); + return m_Designs.emplace(design, std::move(parts)).first->second; + } +} diff --git a/dUgcServer/UgcBricks.h b/dUgcServer/UgcBricks.h new file mode 100644 index 000000000..2a844b17d --- /dev/null +++ b/dUgcServer/UgcBricks.h @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * The client's brick data the UGC server builds models from: LDD geometry (res/brickprimitives/lod/.g, .g1, + * ...) and the material colors (Materials.xml in res/brickdb.zip). The parsers are pure; BrickLibrary loads and caches + * files and may be used from any thread. + */ +namespace UgcBricks { + // One LDD .g file: a triangle mesh with a normal per vertex + struct Geometry { + std::vector positions; // x, y, z per vertex + std::vector normals; // x, y, z per vertex + std::vector indices; + }; + + // An LDD material: sRGB color and opacity, 0-255 + struct Material { + uint8_t r{ 160 }; + uint8_t g{ 160 }; + uint8_t b{ 160 }; + uint8_t a{ 255 }; + bool Transparent() const { return a < 255; } + }; + + // A .g file ("10GB" magic, counts, positions, normals, texture coordinates for decorated parts, indices) + std::optional ParseGeometry(std::string_view data); + + // Materials.xml: MatID -> color + std::map ParseMaterials(std::string_view xml); + + // A file from a zip archive (stored or deflated), matched without regard to case; nullopt when it isn't there + std::optional ReadZipEntry(std::string_view zip, std::string_view name); + + // `relative` (either slash, any case) under `root`, matching each part without regard to case as the client's + // files are named on Windows; nullopt when there is no such file + std::optional ResolvePath(const std::filesystem::path& root, std::string_view relative); + + // Reads a whole file; nullopt when it can't + std::optional ReadFile(const std::filesystem::path& path); + + class BrickLibrary { + public: + // `res` is the client's res folder; `lod` the brickprimitives level used (0 is the most detailed) + BrickLibrary(std::filesystem::path res, uint32_t lod); + + // Loads the material colors; false when brickdb.zip or its Materials.xml can't be read + bool LoadMaterials(); + + // For tests and callers that have the colors already + void SetMaterials(std::map materials); + + // A material's color (a grey when the id is unknown) + Material GetMaterial(uint32_t id) const; + + // Every geometry file of a design, in order (.g, .g1, ...); empty when the design has none. Loaded once. + std::shared_ptr> GetDesign(uint32_t design); + + const std::filesystem::path& GetResPath() const { return m_Res; } + + private: + std::filesystem::path m_Res; + uint32_t m_Lod; + std::map m_Materials; + std::mutex m_Mutex; + std::map>> m_Designs; + }; +} diff --git a/dUgcServer/UgcCdClient.cpp b/dUgcServer/UgcCdClient.cpp new file mode 100644 index 000000000..4c53ece1c --- /dev/null +++ b/dUgcServer/UgcCdClient.cpp @@ -0,0 +1,58 @@ +#include "UgcCdClient.h" + +#include + +#include "CDClientDatabase.h" +#include "UgcModular.h" + +namespace { + constexpr int32_t MODULE_COMPONENT = 28; + constexpr int32_t RENDER_COMPONENT = 2; +} + +namespace UgcCdClient { + bool GatherModular(const std::string& modules, UgcJobs::ModularInput& input, std::string& error) { + const auto lots = UgcModular::ParseModuleLots(modules); + if (lots.empty()) { + error = "no modules in \"" + modules + "\""; + return false; + } + std::optional buildType; + for (const auto lot : lots) { + auto moduleQuery = CDClientDatabase::CreatePreppedStmt( + "SELECT m.partCode, m.buildType, m.xml FROM ComponentsRegistry cr JOIN ModuleComponent m ON m.id = cr.component_id " + "WHERE cr.id = ? AND cr.component_type = ? LIMIT 1;"); + moduleQuery.bind(1, static_cast(lot)); + moduleQuery.bind(2, MODULE_COMPONENT); + auto moduleRow = moduleQuery.execQuery(); + if (moduleRow.eof()) { + error = "LOT " + std::to_string(lot) + " is not a module"; + return false; + } + UgcJobs::ModuleInput module; + module.lot = lot; + module.partCode = static_cast(moduleRow.getIntField("partCode", 0)); + module.moduleXml = moduleRow.getStringField("xml", ""); + if (!buildType) buildType = moduleRow.getIntField("buildType", 0); + + auto renderQuery = CDClientDatabase::CreatePreppedStmt( + "SELECT rc.render_asset FROM ComponentsRegistry cr JOIN RenderComponent rc ON rc.id = cr.component_id " + "WHERE cr.id = ? AND cr.component_type = ? LIMIT 1;"); + renderQuery.bind(1, static_cast(lot)); + renderQuery.bind(2, RENDER_COMPONENT); + auto renderRow = renderQuery.execQuery(); + if (!renderRow.eof()) module.renderAsset = renderRow.getStringField("render_asset", ""); + input.modules.push_back(std::move(module)); + } + + auto buildQuery = CDClientDatabase::CreatePreppedStmt("SELECT xml FROM ModularBuildComponent WHERE buildType = ? ORDER BY id LIMIT 1;"); + buildQuery.bind(1, *buildType); + auto buildRow = buildQuery.execQuery(); + if (buildRow.eof()) { + error = "no ModularBuildComponent for build type " + std::to_string(*buildType); + return false; + } + input.buildXml = buildRow.getStringField("xml", ""); + return true; + } +} diff --git a/dUgcServer/UgcCdClient.h b/dUgcServer/UgcCdClient.h new file mode 100644 index 000000000..e8cfe56af --- /dev/null +++ b/dUgcServer/UgcCdClient.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +#include "UgcJobs.h" + +// What the UGC server reads from the CDClient (main thread only: the CDClient connection isn't shared with workers) +namespace UgcCdClient { + // A modular build's modules (ugc_modular_build.ldf_config) with their render assets and module data, and the + // ModularBuildComponent of their build type; false and `error` when it can't be put together + bool GatherModular(const std::string& modules, UgcJobs::ModularInput& input, std::string& error); +} diff --git a/dUgcServer/UgcFormats.cpp b/dUgcServer/UgcFormats.cpp new file mode 100644 index 000000000..f6e31530e --- /dev/null +++ b/dUgcServer/UgcFormats.cpp @@ -0,0 +1,311 @@ +#include "UgcFormats.h" + +#include +#include +#include +#include + +#include "MD5.h" +#include "ZCompression.h" + +namespace { + class Writer { + public: + template + void Put(T value) { + char bytes[sizeof(T)]; + std::memcpy(bytes, &value, sizeof(T)); + m_Data.append(bytes, sizeof(T)); + } + void U8(uint8_t value) { Put(value); } + void U16(uint16_t value) { Put(value); } + void U32(uint32_t value) { Put(value); } + void I32(int32_t value) { Put(value); } + void Float(float value) { Put(value); } + void SizedString(const std::string& value) { + U32(static_cast(value.size())); + m_Data += value; + } + void Raw(std::string_view bytes) { m_Data += bytes; } + std::string& Data() { return m_Data; } + + private: + std::string m_Data; + }; + + void PutBigEndian(std::string& out, uint32_t value) { + for (int shift = 24; shift >= 0; shift -= 8) out += static_cast((value >> shift) & 0xFF); + } + + uint32_t Crc32(std::string_view data) { + static const auto table = [] { + std::array values{}; + for (uint32_t i = 0; i < 256; i++) { + uint32_t c = i; + for (int k = 0; k < 8; k++) c = (c & 1) ? 0xEDB88320u ^ (c >> 1) : c >> 1; + values[i] = c; + } + return values; + }(); + uint32_t crc = 0xFFFFFFFFu; + for (const auto byte : data) crc = table[(crc ^ static_cast(byte)) & 0xFF] ^ (crc >> 8); + return crc ^ 0xFFFFFFFFu; + } + + void PngChunk(std::string& out, const char* type, std::string_view data) { + PutBigEndian(out, static_cast(data.size())); + std::string typed(type, 4); + typed += data; + out += typed; + PutBigEndian(out, Crc32(typed)); + } + + // Gamebryo's block writing: a block per call, types and strings collected into the header's tables + class NifBuilder { + public: + int32_t String(const std::string& value) { + if (value.empty()) return -1; + const auto it = std::find(m_Strings.begin(), m_Strings.end(), value); + if (it != m_Strings.end()) return static_cast(it - m_Strings.begin()); + m_Strings.push_back(value); + return static_cast(m_Strings.size() - 1); + } + + int32_t Add(const std::string& type, std::string data) { + auto it = std::find(m_Types.begin(), m_Types.end(), type); + if (it == m_Types.end()) it = m_Types.insert(m_Types.end(), type); + m_BlockTypes.push_back(static_cast(it - m_Types.begin())); + m_Blocks.push_back(std::move(data)); + return static_cast(m_Blocks.size() - 1); + } + + // Reserves a block to fill in later (a parent that lists children made after it) + int32_t Reserve(const std::string& type) { return Add(type, {}); } + void Fill(int32_t block, std::string data) { m_Blocks[block] = std::move(data); } + + std::string Finish(int32_t root) { + Writer out; + out.Raw("Gamebryo File Format, Version 20.3.0.9\n"); + out.U32(0x14030009); + out.U8(1); // little endian + out.U32(0); // user version + out.U32(static_cast(m_Blocks.size())); + out.U16(static_cast(m_Types.size())); + for (const auto& type : m_Types) out.SizedString(type); + for (const auto type : m_BlockTypes) out.U16(type); + for (const auto& block : m_Blocks) out.U32(static_cast(block.size())); + out.U32(static_cast(m_Strings.size())); + size_t longest = 0; + for (const auto& value : m_Strings) longest = std::max(longest, value.size()); + out.U32(static_cast(longest)); + for (const auto& value : m_Strings) out.SizedString(value); + out.U32(0); // groups + for (const auto& block : m_Blocks) out.Raw(block); + out.U32(1); // roots + out.I32(root); + return std::move(out.Data()); + } + + private: + std::vector m_Types; + std::vector m_BlockTypes; + std::vector m_Blocks; + std::vector m_Strings; + }; + + constexpr uint16_t AV_FLAGS = 14; // the usual NiAVObject flags (selective update), not hidden + + void WriteNet(Writer& out, int32_t name) { + out.I32(name); + out.U32(0); // extra data + out.I32(-1); // controller + } + + void WriteAv(Writer& out, int32_t name, const std::vector& properties) { + WriteNet(out, name); + out.U16(AV_FLAGS); + for (int i = 0; i < 3; i++) out.Float(0.0f); // translation + for (int row = 0; row < 3; row++) { + for (int col = 0; col < 3; col++) out.Float(row == col ? 1.0f : 0.0f); + } + out.Float(1.0f); // scale + out.U32(static_cast(properties.size())); + for (const auto property : properties) out.I32(property); + out.I32(-1); // collision object + } + + std::string TriShapeData(const UgcModel::Mesh& mesh) { + Writer out; + const auto count = static_cast(mesh.positions.size()); + out.I32(0); // group ID + out.U16(count); + out.U8(0); // keep flags + out.U8(0); // compress flags + out.U8(1); // has vertices + glm::vec3 min(std::numeric_limits::max()), max(-std::numeric_limits::max()); + for (const auto& p : mesh.positions) { + out.Float(p.x); + out.Float(p.y); + out.Float(p.z); + min = glm::min(min, p); + max = glm::max(max, p); + } + out.U16(0); // data flags: no texture coordinates or tangents + const bool normals = mesh.normals.size() == mesh.positions.size(); + out.U8(normals ? 1 : 0); + if (normals) { + for (const auto& n : mesh.normals) { + out.Float(n.x); + out.Float(n.y); + out.Float(n.z); + } + } + const glm::vec3 center = mesh.positions.empty() ? glm::vec3(0.0f) : (min + max) * 0.5f; + float radius = 0.0f; + for (const auto& p : mesh.positions) radius = std::max(radius, glm::length(p - center)); + out.Float(center.x); + out.Float(center.y); + out.Float(center.z); + out.Float(radius); + const bool colors = mesh.colors.size() == mesh.positions.size(); + out.U8(colors ? 1 : 0); + if (colors) { + for (const auto& c : mesh.colors) { + out.Float(std::clamp(c.r, 0.0f, 1.0f)); + out.Float(std::clamp(c.g, 0.0f, 1.0f)); + out.Float(std::clamp(c.b, 0.0f, 1.0f)); + out.Float(std::clamp(c.a, 0.0f, 1.0f)); + } + } + out.U16(0x4000); // consistency: static + out.I32(-1); // additional data + const auto triangles = static_cast(mesh.indices.size() / 3); + out.U16(triangles); + out.U32(static_cast(triangles) * 3); + out.U8(1); // has triangles + for (size_t i = 0; i < static_cast(triangles) * 3; i++) out.U16(static_cast(mesh.indices[i])); + out.U16(0); // match groups + return std::move(out.Data()); + } +} + +namespace UgcFormats { + std::string WriteNif(const std::string& rootName, const std::vector& shapes) { + NifBuilder nif; + const int32_t root = nif.Reserve("NiNode"); + + // Shared properties + Writer material; + WriteNet(material, -1); + for (int i = 0; i < 3; i++) material.Float(1.0f); // ambient + for (int i = 0; i < 3; i++) material.Float(1.0f); // diffuse + for (int i = 0; i < 3; i++) material.Float(0.0f); // specular + for (int i = 0; i < 3; i++) material.Float(0.0f); // emissive + material.Float(10.0f); // glossiness + material.Float(1.0f); // alpha + const auto materialBlock = nif.Add("NiMaterialProperty", std::move(material.Data())); + + Writer vertexColor; + WriteNet(vertexColor, -1); + vertexColor.U16((2 << 4) | (1 << 3)); // vertex colors are ambient and diffuse; lit + const auto vertexColorBlock = nif.Add("NiVertexColorProperty", std::move(vertexColor.Data())); + + int32_t alphaBlock = -1; + std::vector children; + for (const auto& shape : shapes) { + if (!shape.mesh || shape.mesh->Empty() || shape.mesh->positions.size() > 65535 || shape.mesh->TriangleCount() > 65535) continue; + std::vector properties{ materialBlock, vertexColorBlock }; + if (shape.transparent) { + if (alphaBlock < 0) { + Writer alpha; + WriteNet(alpha, -1); + alpha.U16(1 | (6 << 1) | (7 << 5)); // blend source alpha over one minus source alpha + alpha.U8(0); + alphaBlock = nif.Add("NiAlphaProperty", std::move(alpha.Data())); + } + properties.push_back(alphaBlock); + } + const auto shapeBlock = nif.Reserve("NiTriShape"); + const auto dataBlock = nif.Add("NiTriShapeData", TriShapeData(*shape.mesh)); + Writer tri; + WriteAv(tri, nif.String(shape.name), properties); + tri.I32(dataBlock); + tri.I32(-1); // skin instance + tri.U32(0); // materials + tri.I32(-1); // active material + tri.U8(0); // material needs update + nif.Fill(shapeBlock, std::move(tri.Data())); + children.push_back(shapeBlock); + } + + Writer node; + WriteAv(node, nif.String(rootName), {}); + node.U32(static_cast(children.size())); + for (const auto child : children) node.I32(child); + node.U32(0); // effects + nif.Fill(root, std::move(node.Data())); + return nif.Finish(root); + } + + std::string EncodePng(const UgcRender::Image& image) { + std::string raw; + raw.reserve(static_cast(image.height) * (image.width * 4 + 1)); + for (int y = 0; y < image.height; y++) { + raw += '\0'; // no filter + raw.append(reinterpret_cast(image.rgba.data()) + static_cast(y) * image.width * 4, static_cast(image.width) * 4); + } + std::string compressed(ZCompression::GetMaxCompressedLength(static_cast(raw.size())) + 64, '\0'); + const auto size = ZCompression::Compress(reinterpret_cast(raw.data()), static_cast(raw.size()), + reinterpret_cast(compressed.data()), static_cast(compressed.size())); + if (size <= 0) return {}; + compressed.resize(static_cast(size)); + + std::string out("\x89PNG\r\n\x1a\n", 8); + std::string header; + PutBigEndian(header, static_cast(image.width)); + PutBigEndian(header, static_cast(image.height)); + header += std::string("\x08\x06\x00\x00\x00", 5); // 8 bits, RGBA, deflate, no filter method, no interlace + PngChunk(out, "IHDR", header); + PngChunk(out, "IDAT", compressed); + PngChunk(out, "IEND", {}); + return out; + } + + std::string EncodeDds(const UgcRender::Image& image) { + std::array header{}; + header[0] = 124; + header[1] = 0x1 | 0x2 | 0x4 | 0x8 | 0x1000; // caps, height, width, pitch, pixel format + header[2] = static_cast(image.height); + header[3] = static_cast(image.width); + header[4] = static_cast(image.width) * 4; // pitch + header[18] = 32; // pixel format size + header[19] = 0x41; // RGB with alpha + header[21] = 32; + header[22] = 0x00FF0000; + header[23] = 0x0000FF00; + header[24] = 0x000000FF; + header[25] = 0xFF000000; + header[26] = 0x1000; // texture + std::string out = "DDS "; + out.append(reinterpret_cast(header.data()), header.size() * 4); + out.reserve(out.size() + image.rgba.size()); + for (size_t i = 0; i + 3 < image.rgba.size(); i += 4) { + out += static_cast(image.rgba[i + 2]); + out += static_cast(image.rgba[i + 1]); + out += static_cast(image.rgba[i]); + out += static_cast(image.rgba[i + 3]); + } + return out; + } + + std::string Md5Hex(std::string_view data) { + MD5 md5; + md5.update(reinterpret_cast(data.data()), static_cast(data.size())); + md5.finalize(); + return md5.hexdigest(); + } + + std::string ChecksumXml(std::string_view data) { + return "\n" + Md5Hex(data) + "" + std::to_string(data.size()) + "\n"; + } +} diff --git a/dUgcServer/UgcFormats.h b/dUgcServer/UgcFormats.h new file mode 100644 index 000000000..a07ded93e --- /dev/null +++ b/dUgcServer/UgcFormats.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +#include "UgcModel.h" +#include "UgcRender.h" + +/** + * The files the UGC server writes: the mesh as a Gamebryo .nif the client loads, icons as .png and .dds, and the + * client's .checksum files for its downloads. Pure (data in, bytes out). + */ +namespace UgcFormats { + struct NifShape { + std::string name; + const UgcModel::Mesh* mesh{}; + bool transparent{}; + }; + + /** + * A Gamebryo 20.3.0.9 file (user version 0, as the client's own meshes): a root NiNode named `rootName` with one + * NiTriShape per shape (at most 65535 vertices and triangles each: UgcModel::Split), with vertex colors that are + * the diffuse and ambient color (NiVertexColorProperty), a white NiMaterialProperty, and for transparent shapes an + * NiAlphaProperty blending with the vertex alpha. + */ + std::string WriteNif(const std::string& rootName, const std::vector& shapes); + + // PNG (8-bit RGBA) + std::string EncodePng(const UgcRender::Image& image); + + // DDS, uncompressed 32-bit BGRA without mipmaps + std::string EncodeDds(const UgcRender::Image& image); + + // Lowercase hex MD5 of `data` + std::string Md5Hex(std::string_view data); + + // What the client reads from a UGC file's .checksum: the MD5 and size of the file as it is after inflating it + std::string ChecksumXml(std::string_view data); +} diff --git a/dUgcServer/UgcJobs.cpp b/dUgcServer/UgcJobs.cpp new file mode 100644 index 000000000..4e26d9a41 --- /dev/null +++ b/dUgcServer/UgcJobs.cpp @@ -0,0 +1,130 @@ +#include "UgcJobs.h" + +#include + +#include "Sd0.h" +#include "UgcFormats.h" +#include "UgcModel.h" +#include "UgcModular.h" +#include "ZCompression.h" + +namespace UgcJobs { + void AddDownload(UgcStorage::Files& files, const std::string& name, const std::string& data) { + files[name + ".gz"] = ZCompression::Gzip(data); + files[name + ".checksum"] = UgcFormats::ChecksumXml(data); + } + + std::string LxfmlFromBlob(const std::string& blob) { + if (blob.starts_with(" 0) { + if (!outcome.note.empty()) outcome.note += "; "; + outcome.note += "removed " + std::to_string(optimized.trianglesRemoved) + " of " + std::to_string(optimized.trianglesBefore) + " triangles"; + } + + // Named like LU Toolbox names its shapes: S__ + std::vector> pieces; + for (const bool transparent : { false, true }) { + for (auto& piece : UgcModel::Split(transparent ? model.transparent : model.opaque)) pieces.emplace_back(std::move(piece), transparent); + } + std::vector shapes; + size_t opaqueIndex = 0, transparentIndex = 0; + for (const auto& [piece, transparent] : pieces) { + const auto index = transparent ? transparentIndex++ : opaqueIndex++; + shapes.push_back({ std::string("S01_") + (transparent ? "Alpha" : "Opaque") + "_Model" + (index > 0 ? std::to_string(index) : ""), &piece, transparent }); + } + const auto nif = UgcFormats::WriteNif("SceneNode_Model", shapes); + outcome.files["model.nif"] = nif; + AddDownload(outcome.files, "model.nif", nif); + AddDownload(outcome.files, "model.lxfml", lxfml); + AddIcon(outcome.files, model, settings.icon); + outcome.ok = true; + return outcome; + } + + Outcome ProcessModular(const ModularInput& input, const std::filesystem::path& res, const Settings& settings) { + Outcome outcome; + const auto build = UgcModular::ParseBuild(input.buildXml); + if (!build) { + outcome.error = "the build type has no topology in ModularBuildComponent"; + return outcome; + } + std::vector modules; + for (const auto& moduleInput : input.modules) { + const auto path = UgcBricks::ResolvePath(res, moduleInput.renderAsset); + const auto data = path ? UgcBricks::ReadFile(*path) : std::nullopt; + if (!data) { + outcome.note += "module " + std::to_string(moduleInput.lot) + " has no mesh (" + moduleInput.renderAsset + "); "; + continue; + } + std::string error; + auto nif = NifFile::Parse(*data, 0, error); + if (!nif) { + outcome.note += "module " + std::to_string(moduleInput.lot) + ": " + error + "; "; + continue; + } + modules.push_back({ moduleInput.partCode, std::move(*nif), UgcModular::ParseModuleConnections(moduleInput.moduleXml) }); + } + if (modules.empty()) { + outcome.error = "none of the modules have a mesh"; + if (!outcome.note.empty()) outcome.error += " (" + outcome.note + ")"; + return outcome; + } + const auto model = UgcModular::Assemble(*build, modules, outcome.note); + if (model.Empty()) { + outcome.error = "the modules have no triangles"; + return outcome; + } + auto options = settings.modularIcon; + options.modelRotation = build->additionalRotation * options.modelRotation; + AddIcon(outcome.files, model, options); + outcome.ok = true; + return outcome; + } +} diff --git a/dUgcServer/UgcJobs.h b/dUgcServer/UgcJobs.h new file mode 100644 index 000000000..5b97cd5b4 --- /dev/null +++ b/dUgcServer/UgcJobs.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "UgcBricks.h" +#include "UgcRender.h" +#include "UgcStorage.h" + +/** + * The work a UGC worker thread does for one item: everything from the stored LXFML (or a modular build's modules) to + * the finished files. No database, network or CDClient: the main thread gathers the input and stores the result. + */ +namespace UgcJobs { + struct Settings { + UgcRender::OptimizeOptions optimize; + UgcRender::IconOptions icon; + UgcRender::IconOptions modularIcon; + }; + + struct Outcome { + bool ok{}; + std::string error; // why it failed + std::string note; // what was odd but not fatal (missing bricks, ...) + UgcStorage::Files files; // name -> bytes, when ok + bool aoBaked{}; + }; + + // The client's download of `data` (`name` + ".gz" and ".checksum") added to `files` + void AddDownload(UgcStorage::Files& files, const std::string& name, const std::string& data); + + // The LXFML of a ugc row's lxfml column (an sd0 stream, or plain LXFML); empty when it can't be read + std::string LxfmlFromBlob(const std::string& blob); + + // A player model: the optimized .nif, its icon and its LXFML for download + Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings); + + struct ModuleInput { + uint32_t lot{}; + uint32_t partCode{}; + std::string renderAsset; // RenderComponent.render_asset, relative to the client's res folder + std::string moduleXml; // ModuleComponent.xml + }; + + struct ModularInput { + std::vector modules; + std::string buildXml; // ModularBuildComponent.xml of the modules' build type + }; + + // A modular build's icon, from its modules' meshes put together + Outcome ProcessModular(const ModularInput& input, const std::filesystem::path& res, const Settings& settings); +} diff --git a/dUgcServer/UgcModel.cpp b/dUgcServer/UgcModel.cpp new file mode 100644 index 000000000..8407fcadc --- /dev/null +++ b/dUgcServer/UgcModel.cpp @@ -0,0 +1,275 @@ +#include "UgcModel.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include "NifFile.h" +#include "tinyxml2.h" + +namespace { + std::vector ParseFloats(const char* text) { + std::vector values; + if (!text) return values; + std::stringstream stream(text); + std::string item; + while (std::getline(stream, item, ',')) { + try { + values.push_back(std::stof(item)); + } catch (...) { + values.push_back(0.0f); + } + } + return values; + } + + std::vector ParseMaterials(const char* text) { + std::vector values; + if (!text) return values; + std::stringstream stream(text); + std::string item; + while (std::getline(stream, item, ',')) { + try { + values.push_back(static_cast(std::stoul(item))); + } catch (...) { + values.push_back(0); + } + } + // Material 0 means "the same as the part's first" + for (auto& value : values) { + if (value == 0 && !values.empty()) value = values[0]; + } + return values; + } + + // LXFML 5 bone: a row-major 3x3 rotation followed by a translation + glm::mat4 BoneMatrix(const std::vector& t) { + glm::mat4 matrix(1.0f); + if (t.size() < 12) return matrix; + matrix[0] = glm::vec4(t[0], t[1], t[2], 0.0f); + matrix[1] = glm::vec4(t[3], t[4], t[5], 0.0f); + matrix[2] = glm::vec4(t[6], t[7], t[8], 0.0f); + matrix[3] = glm::vec4(t[9], t[10], t[11], 1.0f); + return matrix; + } + + // LXFML 4: rotate `angle` degrees around (ax, ay, az), then translate + glm::mat4 AxisAngleMatrix(const tinyxml2::XMLElement* element) { + const glm::vec3 axis(element->FloatAttribute("ax"), element->FloatAttribute("ay"), element->FloatAttribute("az")); + const glm::vec3 translation(element->FloatAttribute("tx"), element->FloatAttribute("ty"), element->FloatAttribute("tz")); + glm::mat4 matrix = glm::translate(glm::mat4(1.0f), translation); + if (glm::dot(axis, axis) > 0.0f) matrix = glm::rotate(matrix, glm::radians(element->FloatAttribute("angle")), glm::normalize(axis)); + return matrix; + } + + bool ParseDesign(const tinyxml2::XMLElement* element, uint32_t& design) { + const char* text = element->Attribute("designID"); + if (!text || !*text) return false; + for (const char* c = text; *c; c++) { + if (*c < '0' || *c > '9') return false; + } + try { + design = static_cast(std::stoul(text)); + } catch (...) { + return false; + } + return true; + } + + glm::vec4 ToColor(const UgcBricks::Material& material) { + return glm::vec4(material.r, material.g, material.b, material.a) / 255.0f; + } +} + +namespace UgcModel { + std::vector ParseLxfml(std::string_view lxfml, std::string& error) { + std::vector parts; + tinyxml2::XMLDocument doc; + if (doc.Parse(lxfml.data(), lxfml.size()) != tinyxml2::XML_SUCCESS) { + error = "the LXFML is not valid XML"; + return parts; + } + const auto* root = doc.FirstChildElement("LXFML"); + if (!root) { + error = "no LXFML element"; + return parts; + } + + if (const auto* bricks = root->FirstChildElement("Bricks")) { + for (const auto* brick = bricks->FirstChildElement("Brick"); brick; brick = brick->NextSiblingElement("Brick")) { + for (const auto* partElement = brick->FirstChildElement("Part"); partElement; partElement = partElement->NextSiblingElement("Part")) { + const auto* bone = partElement->FirstChildElement("Bone"); + Part part; + if (!bone || !ParseDesign(partElement, part.designId)) continue; + const char* materials = partElement->Attribute("materials"); + if (!materials) materials = partElement->Attribute("materialID"); + part.materials = ParseMaterials(materials ? materials : "0"); + part.transform = BoneMatrix(ParseFloats(bone->Attribute("transformation"))); + parts.push_back(std::move(part)); + } + } + if (!parts.empty()) return parts; + } + + // LXFML 4: groups (with their own transforms) nest parts + std::function walk = [&](const tinyxml2::XMLElement* element, const glm::mat4& parent) { + for (const auto* child = element->FirstChildElement(); child; child = child->NextSiblingElement()) { + const std::string_view name = child->Name(); + if (name != "Group" && name != "Part") continue; + const auto matrix = parent * AxisAngleMatrix(child); + if (name == "Group") { + walk(child, matrix); + continue; + } + Part part; + if (!ParseDesign(child, part.designId)) continue; + const char* material = child->Attribute("materialID"); + part.materials = ParseMaterials(material ? material : "0"); + part.transform = matrix; + parts.push_back(std::move(part)); + } + }; + if (const auto* scene = root->FirstChildElement("Scene")) { + for (const auto* model = scene->FirstChildElement("Model"); model; model = model->NextSiblingElement("Model")) walk(model, glm::mat4(1.0f)); + } + if (parts.empty()) error = "the LXFML has no bricks"; + return parts; + } + + void Mesh::Append(const Mesh& other) { + const auto base = static_cast(positions.size()); + positions.insert(positions.end(), other.positions.begin(), other.positions.end()); + normals.insert(normals.end(), other.normals.begin(), other.normals.end()); + colors.insert(colors.end(), other.colors.begin(), other.colors.end()); + indices.reserve(indices.size() + other.indices.size()); + for (const auto index : other.indices) indices.push_back(base + index); + } + + void Mesh::Transform(const glm::mat4& transform) { + const glm::mat3 normalMatrix = glm::transpose(glm::inverse(glm::mat3(transform))); + for (auto& position : positions) position = glm::vec3(transform * glm::vec4(position, 1.0f)); + for (auto& normal : normals) { + const auto n = normalMatrix * normal; + const auto length = glm::length(n); + normal = length > 0.0f ? n / length : n; + } + } + + bool Model::Bounds(glm::vec3& min, glm::vec3& max) const { + bool any = false; + for (const auto* mesh : { &opaque, &transparent }) { + for (const auto& position : mesh->positions) { + min = any ? glm::min(min, position) : position; + max = any ? glm::max(max, position) : position; + any = true; + } + } + return any; + } + + Model Build(const std::vector& parts, UgcBricks::BrickLibrary& library) { + Model model; + std::set missing; + for (const auto& part : parts) { + const auto design = library.GetDesign(part.designId); + if (!design || design->empty()) { + missing.insert(part.designId); + continue; + } + model.bricks++; + const glm::mat3 normalMatrix = glm::transpose(glm::inverse(glm::mat3(part.transform))); + for (size_t index = 0; index < design->size(); index++) { + const auto& geometry = (*design)[index]; + const auto materialId = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]); + const auto material = library.GetMaterial(materialId); + auto& mesh = material.Transparent() ? model.transparent : model.opaque; + const auto base = static_cast(mesh.positions.size()); + const auto color = ToColor(material); + const size_t vertexCount = geometry.positions.size() / 3; + for (size_t v = 0; v < vertexCount; v++) { + const glm::vec3 position(geometry.positions[v * 3], geometry.positions[v * 3 + 1], geometry.positions[v * 3 + 2]); + glm::vec3 normal(geometry.normals[v * 3], geometry.normals[v * 3 + 1], geometry.normals[v * 3 + 2]); + normal = normalMatrix * normal; + const auto length = glm::length(normal); + if (length > 0.0f) normal /= length; + mesh.positions.push_back(glm::vec3(part.transform * glm::vec4(position, 1.0f))); + mesh.normals.push_back(normal); + mesh.colors.push_back(color); + } + for (const auto i : geometry.indices) mesh.indices.push_back(base + i); + } + } + model.missingDesigns.assign(missing.begin(), missing.end()); + return model; + } + + Model FromNif(const NifFile::Model& nif) { + Model model; + for (const auto& source : nif.meshes) { + Mesh mesh; + const size_t count = source.positions.size() / 3; + const bool vertexColors = source.material.vertexColorMode == 2 && source.colors.size() == count * 4; + const glm::vec4 materialColor(source.material.diffuse[0], source.material.diffuse[1], source.material.diffuse[2], source.material.alpha); + for (size_t v = 0; v < count; v++) { + mesh.positions.emplace_back(source.positions[v * 3], source.positions[v * 3 + 1], source.positions[v * 3 + 2]); + if (source.normals.size() == count * 3) mesh.normals.emplace_back(source.normals[v * 3], source.normals[v * 3 + 1], source.normals[v * 3 + 2]); + glm::vec4 color = materialColor; + if (vertexColors) color *= glm::vec4(source.colors[v * 4], source.colors[v * 4 + 1], source.colors[v * 4 + 2], source.colors[v * 4 + 3]) / 255.0f; + mesh.colors.push_back(color); + } + mesh.indices.assign(source.indices.begin(), source.indices.end()); + // Normals from the faces when the file has none + if (mesh.normals.size() != count) { + mesh.normals.assign(count, glm::vec3(0.0f)); + for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) { + const auto& a = mesh.positions[mesh.indices[i]]; + const auto face = glm::cross(mesh.positions[mesh.indices[i + 1]] - a, mesh.positions[mesh.indices[i + 2]] - a); + for (int k = 0; k < 3; k++) mesh.normals[mesh.indices[i + k]] += face; + } + for (auto& normal : mesh.normals) { + const auto length = glm::length(normal); + normal = length > 0.0f ? normal / length : glm::vec3(0.0f, 1.0f, 0.0f); + } + } + (source.material.alphaBlend ? model.transparent : model.opaque).Append(mesh); + } + return model; + } + + std::vector Split(const Mesh& mesh, size_t maxVertices, size_t maxTriangles) { + std::vector pieces; + if (mesh.positions.size() <= maxVertices && mesh.TriangleCount() <= maxTriangles) { + if (!mesh.Empty()) pieces.push_back(mesh); + return pieces; + } + Mesh current; + std::unordered_map remap; + const auto flush = [&]() { + if (!current.Empty()) pieces.push_back(std::move(current)); + current = Mesh{}; + remap.clear(); + }; + for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) { + size_t newVertices = 0; + for (int k = 0; k < 3; k++) newVertices += remap.contains(mesh.indices[i + k]) ? 0 : 1; + if (current.positions.size() + newVertices > maxVertices || current.TriangleCount() + 1 > maxTriangles) flush(); + for (int k = 0; k < 3; k++) { + const auto source = mesh.indices[i + k]; + auto [it, added] = remap.try_emplace(source, static_cast(current.positions.size())); + if (added) { + current.positions.push_back(mesh.positions[source]); + if (source < mesh.normals.size()) current.normals.push_back(mesh.normals[source]); + if (source < mesh.colors.size()) current.colors.push_back(mesh.colors[source]); + } + current.indices.push_back(it->second); + } + } + flush(); + return pieces; + } +} diff --git a/dUgcServer/UgcModel.h b/dUgcServer/UgcModel.h new file mode 100644 index 000000000..50016d59b --- /dev/null +++ b/dUgcServer/UgcModel.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "UgcBricks.h" + +namespace NifFile { + struct Model; +} + +/** + * Brick models as triangle meshes: LXFML parts, the mesh built from them (opaque and transparent bricks apart, with + * the material colors as vertex colors) and the mesh of a client .nif. Pure apart from BrickLibrary's file reads. + */ +namespace UgcModel { + // A brick in a model: its primitive, materials (one per geometry part) and where it is + struct Part { + uint32_t designId{}; + std::vector materials; + glm::mat4 transform{ 1.0f }; + }; + + // The parts of an LXFML 5 (Bricks/Brick/Part with bones) or 4 (Scene/Model/Group/Part with axis angles) model. + // Empty with `error` set when it can't be read. + std::vector ParseLxfml(std::string_view lxfml, std::string& error); + + struct Mesh { + std::vector positions; + std::vector normals; + std::vector colors; // sRGB, 0..1, alpha is opacity + std::vector indices; + + size_t TriangleCount() const { return indices.size() / 3; } + bool Empty() const { return indices.empty(); } + void Append(const Mesh& other); + void Transform(const glm::mat4& transform); + }; + + struct Model { + Mesh opaque; + Mesh transparent; + std::vector missingDesigns; // designs without geometry in the client, skipped + size_t bricks{}; + + bool Empty() const { return opaque.Empty() && transparent.Empty(); } + // The bounds of every vertex; false when there are none + bool Bounds(glm::vec3& min, glm::vec3& max) const; + }; + + // The mesh of a model's parts + Model Build(const std::vector& parts, UgcBricks::BrickLibrary& library); + + // A client .nif's meshes as one model (vertex colors times material color; transparent when blended) + Model FromNif(const NifFile::Model& nif); + + // Splits a mesh into pieces the .nif format can hold (at most `maxVertices` vertices and `maxTriangles` triangles) + std::vector Split(const Mesh& mesh, size_t maxVertices = 65535, size_t maxTriangles = 65535); +} diff --git a/dUgcServer/UgcModular.cpp b/dUgcServer/UgcModular.cpp new file mode 100644 index 000000000..53c71fe6b --- /dev/null +++ b/dUgcServer/UgcModular.cpp @@ -0,0 +1,132 @@ +#include "UgcModular.h" + +#include +#include + +#include +#include + +#include "tinyxml2.h" + +namespace { + const tinyxml2::XMLElement* Child(const tinyxml2::XMLElement* element, const char* name) { + return element ? element->FirstChildElement(name) : nullptr; + } +} + +namespace UgcModular { + std::optional ParseBuild(std::string_view xml) { + tinyxml2::XMLDocument doc; + if (doc.Parse(xml.data(), xml.size()) != tinyxml2::XML_SUCCESS) return std::nullopt; + const auto* root = doc.FirstChildElement("ModularBuild"); + const auto* topology = Child(root, "topology"); + if (!topology) return std::nullopt; + BuildInfo build; + if (const auto* rootPart = Child(topology, "rootPart")) build.rootPart = rootPart->UnsignedAttribute("value"); + if (const auto* count = Child(topology, "numberOfParts")) build.parts = count->UnsignedAttribute("value"); + for (const auto* connection = topology->FirstChildElement("connection"); connection; connection = connection->NextSiblingElement("connection")) { + const char* location = connection->Attribute("myLocation"); + build.connections.push_back({ connection->UnsignedAttribute("myPartid"), location ? location : "", connection->UnsignedAttribute("connectingPart") }); + } + if (const auto* rotation = Child(Child(Child(root, "Placement"), "AdditionalModelRotation"), "Rotation")) { + glm::quat q(rotation->FloatAttribute("w", 1.0f), rotation->FloatAttribute("x"), rotation->FloatAttribute("y"), rotation->FloatAttribute("z")); + if (glm::length(q) > 0.0f) build.additionalRotation = glm::mat4_cast(glm::normalize(q)); + } + return build; + } + + std::map ParseModuleConnections(std::string_view xml) { + std::map connections; + tinyxml2::XMLDocument doc; + if (doc.Parse(xml.data(), xml.size()) != tinyxml2::XML_SUCCESS) return connections; + const auto* root = doc.FirstChildElement("ModuleInfo"); + if (!root) return connections; + for (const auto* connection = root->FirstChildElement("connection"); connection; connection = connection->NextSiblingElement("connection")) { + const char* name = connection->Attribute("name"); + const auto* translation = connection->FirstChildElement("translation"); + if (!name || !translation) continue; + connections[name] = glm::vec3(translation->FloatAttribute("x"), translation->FloatAttribute("y"), translation->FloatAttribute("z")); + } + return connections; + } + + std::vector ParseModuleLots(std::string_view ldf) { + std::vector lots; + size_t start = 0; + while (start <= ldf.size()) { + size_t end = ldf.find_first_of("+;,", start); + if (end == std::string_view::npos) end = ldf.size(); + auto item = ldf.substr(start, end - start); + if (const auto colon = item.find(':'); colon != std::string_view::npos) item = item.substr(colon + 1); + uint32_t value = 0; + bool digits = !item.empty(); + for (const char c : item) { + if (c < '0' || c > '9') { + digits = false; + break; + } + value = value * 10 + static_cast(c - '0'); + } + if (digits && value != 0) lots.push_back(value); + start = end + 1; + } + return lots; + } + + UgcModel::Model Assemble(const BuildInfo& build, const std::vector& modules, std::string& warnings) { + UgcModel::Model model; + std::map byPart; + for (const auto& module : modules) byPart.emplace(module.partCode, &module); + + std::map placed; + std::deque pending; + if (byPart.contains(build.rootPart)) { + placed[build.rootPart] = glm::mat4(1.0f); + pending.push_back(build.rootPart); + } else if (!modules.empty()) { + warnings += "no module for the root part " + std::to_string(build.rootPart) + "; "; + placed[modules.front().partCode] = glm::mat4(1.0f); + pending.push_back(modules.front().partCode); + } + + while (!pending.empty()) { + const auto part = pending.front(); + pending.pop_front(); + const auto* parent = byPart.at(part); + for (const auto& connection : build.connections) { + if (connection.part != part || placed.contains(connection.connectingPart)) continue; + const auto childIt = byPart.find(connection.connectingPart); + if (childIt == byPart.end()) continue; // optional parts + glm::vec3 anchor(0.0f); + if (const auto node = parent->nif.nodes.find(connection.location); node != parent->nif.nodes.end()) { + anchor = glm::vec3(node->second.translation[0], node->second.translation[1], node->second.translation[2]); + } else if (const auto offset = parent->connections.find(connection.location); offset != parent->connections.end()) { + anchor = offset->second; + } else { + warnings += "part " + std::to_string(part) + " has no " + connection.location + "; "; + } + const auto& child = *childIt->second; + if (const auto node = child.nif.nodes.find(connection.location); node != child.nif.nodes.end()) { + anchor -= glm::vec3(node->second.translation[0], node->second.translation[1], node->second.translation[2]); + } + placed[connection.connectingPart] = placed[part] * glm::translate(glm::mat4(1.0f), anchor); + pending.push_back(connection.connectingPart); + } + } + + for (const auto& module : modules) { + const auto it = placed.find(module.partCode); + if (it == placed.end()) { + warnings += "part " + std::to_string(module.partCode) + " isn't connected; "; + continue; + } + auto mesh = UgcModel::FromNif(module.nif); + mesh.opaque.Transform(it->second); + mesh.transparent.Transform(it->second); + model.opaque.Append(mesh.opaque); + model.transparent.Append(mesh.transparent); + model.bricks++; + } + return model; + } +} diff --git a/dUgcServer/UgcModular.h b/dUgcServer/UgcModular.h new file mode 100644 index 000000000..7e20bc030 --- /dev/null +++ b/dUgcServer/UgcModular.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include "NifFile.h" +#include "UgcModel.h" + +/** + * Modular builds (cars, rockets) put together from their modules the way the CDClient describes it: + * ModularBuildComponent.xml has the topology (a root part and which part connects to which named location of which + * other part), ModuleComponent.xml each module's connection offsets, and each module's render asset has the named + * attach-point nodes. Pure. + */ +namespace UgcModular { + struct Connection { + uint32_t part{}; // the part the location is on + std::string location; // node name on that part + uint32_t connectingPart{}; // the part placed there + }; + + struct BuildInfo { + uint32_t rootPart{}; + uint32_t parts{}; + std::vector connections; + glm::mat4 additionalRotation{ 1.0f }; // Placement/AdditionalModelRotation + }; + + // ModularBuildComponent.xml; nullopt when it has no topology + std::optional ParseBuild(std::string_view xml); + + // ModuleComponent.xml's connection offsets, by location name + std::map ParseModuleConnections(std::string_view xml); + + // The LOTs in a ugc_modular_build.ldf_config ("1:4713+1:4714+1:4715", also with ';' or ',' between) + std::vector ParseModuleLots(std::string_view ldf); + + struct Module { + uint32_t partCode{}; + NifFile::Model nif; + std::map connections; // from ParseModuleConnections + }; + + /** + * The modules placed per the build's topology, starting at the root part: a connection puts the connecting part + * where the parent's node of the location's name is (or the parent module's connection offset of that name when + * its mesh has no such node), less where the connecting part's own node of that name is, if it has one. + * Parts without a module are skipped; `warnings` says what was. + */ + UgcModel::Model Assemble(const BuildInfo& build, const std::vector& modules, std::string& warnings); +} diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp new file mode 100644 index 000000000..b4ec8865e --- /dev/null +++ b/dUgcServer/UgcProcessor.cpp @@ -0,0 +1,235 @@ +#include "UgcProcessor.h" + +#include "CDClientDatabase.h" +#include "Database.h" +#include "Logger.h" +#include "UgcCdClient.h" +#include "json.hpp" + +namespace { + constexpr size_t MAX_ERROR_LENGTH = 1000; // process_error holds 1024 + constexpr size_t LOG_LENGTH = 50; + constexpr auto EVICTION_INTERVAL = std::chrono::minutes(5); + constexpr auto RECENT_ANSWER_TIME = std::chrono::seconds(10); + + const char* KindName(UgcStorage::Kind kind) { + return kind == UgcStorage::Kind::MODEL ? "model" : "modular"; + } + + int64_t UnixNow() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + } +} + +UgcProcessor::UgcProcessor(Config config, UgcStorage& storage, UgcBricks::BrickLibrary& library, UgcJobs::Settings settings) + : m_Config(config), m_Storage(storage), m_Library(library), m_Settings(std::move(settings)) {} + +UgcProcessor::~UgcProcessor() { + Stop(); +} + +void UgcProcessor::Start() { + for (const auto& entry : m_Storage.List()) m_StoredBytes += entry.bytes; + m_NextEviction = std::chrono::steady_clock::now(); + m_Stopping = false; + for (size_t i = 0; i < std::max(m_Config.threads, 1); i++) m_Threads.emplace_back(&UgcProcessor::Worker, this); + LOG("UGC processing started with %zu worker(s), %llu MB stored", m_Threads.size(), static_cast(m_StoredBytes / (1024 * 1024))); +} + +void UgcProcessor::Stop() { + { + std::lock_guard lock(m_Mutex); + m_Stopping = true; + m_Jobs.clear(); + } + m_Wake.notify_all(); + for (auto& thread : m_Threads) { + if (thread.joinable()) thread.join(); + } + m_Threads.clear(); +} + +void UgcProcessor::Worker() { + while (true) { + Job job; + { + std::unique_lock lock(m_Mutex); + m_Wake.wait(lock, [this] { return m_Stopping || !m_Jobs.empty(); }); + if (m_Stopping) return; + job = std::move(m_Jobs.front()); + m_Jobs.pop_front(); + m_Active++; + } + + const auto start = std::chrono::steady_clock::now(); + Done done{ job.kind, job.id, job.attempts }; + try { + done.outcome = job.kind == Kind::MODEL + ? UgcJobs::ProcessModel(job.blob, m_Library, m_Settings) + : UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), m_Settings); + } catch (const std::exception& ex) { + done.outcome.ok = false; + done.outcome.error = std::string("crashed: ") + ex.what(); + } + if (done.outcome.ok) { + std::string error; + const auto bytes = m_Storage.Write(job.kind, job.id, done.outcome.files, error); + if (bytes) { + done.bytes = *bytes; + } else { + done.outcome.ok = false; + done.outcome.error = error; + } + } + done.outcome.files.clear(); + done.milliseconds = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + + std::lock_guard lock(m_Mutex); + m_Active--; + m_Done.push_back(std::move(done)); + } +} + +void UgcProcessor::Poll() { + size_t queued = 0, active = 0; + { + std::lock_guard lock(m_Mutex); + queued = m_Jobs.size(); + active = m_Active; + } + // Enough to keep every worker busy until the next poll + const size_t wanted = std::max(m_Threads.size() * 2, m_Config.pollBatch); + if (queued + active >= wanted) return; + const auto limit = static_cast(wanted - queued - active + m_InFlight.size()); + + std::vector jobs; + for (auto& model : Database::Get()->GetUgcModelsToProcess(limit)) { + if (m_InFlight.contains({ Kind::MODEL, model.id })) continue; + jobs.push_back(Job{ Kind::MODEL, model.id, model.attempts, std::move(model.lxfml) }); + } + for (auto& build : Database::Get()->GetModularBuildsToProcess(limit)) { + if (m_InFlight.contains({ Kind::MODULAR, build.id })) continue; + Job job{ Kind::MODULAR, build.id, build.attempts }; + std::string error; + if (!UgcCdClient::GatherModular(build.modules, job.modular, error)) { + // Nothing a worker could do: record it right away + Record(Done{ Kind::MODULAR, build.id, build.attempts, UgcJobs::Outcome{ false, error } }); + continue; + } + jobs.push_back(std::move(job)); + } + if (jobs.empty()) return; + { + std::lock_guard lock(m_Mutex); + for (auto& job : jobs) { + m_InFlight.insert({ job.kind, job.id }); + m_Jobs.push_back(std::move(job)); + } + } + m_Wake.notify_all(); +} + +void UgcProcessor::Record(const Done& done) { + auto error = done.outcome.error.substr(0, MAX_ERROR_LENGTH); + const auto attempts = done.attempts + 1; + const auto state = done.outcome.ok ? IUgc::eProcessState::DONE + : attempts >= m_Config.maxAttempts ? IUgc::eProcessState::FAILED : IUgc::eProcessState::PENDING; + if (done.kind == Kind::MODEL) { + Database::Get()->SetUgcModelProcessed(done.id, state, attempts, error, done.outcome.ok && done.outcome.aoBaked); + } else { + Database::Get()->SetModularBuildProcessed(done.id, state, attempts, error); + } + m_Recent.erase({ done.kind, done.id }); + + if (done.outcome.ok) { + m_Made++; + m_StoredBytes += done.bytes; + LOG_DEBUG("Made %s %llu in %.0f ms%s%s", KindName(done.kind), static_cast(done.id), done.milliseconds, + done.outcome.note.empty() ? "" : ": ", done.outcome.note.c_str()); + } else { + m_Failed++; + LOG("Couldn't make %s %llu (attempt %u): %s", KindName(done.kind), static_cast(done.id), attempts, error.c_str()); + } + m_Log.push_back({ done.kind, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? done.outcome.note : error, UnixNow() }); + while (m_Log.size() > LOG_LENGTH) m_Log.pop_front(); +} + +void UgcProcessor::Collect() { + std::deque finished; + { + std::lock_guard lock(m_Mutex); + finished.swap(m_Done); + } + for (const auto& done : finished) { + m_InFlight.erase({ done.kind, done.id }); + Record(done); + } + // Something finished: look for more now rather than at the next interval + if (!finished.empty()) m_NextPoll = std::min(m_NextPoll, std::chrono::steady_clock::now() + std::chrono::milliseconds(100)); +} + +void UgcProcessor::Update() { + Collect(); + const auto now = std::chrono::steady_clock::now(); + if (now >= m_NextPoll) { + m_NextPoll = now + std::chrono::milliseconds(m_Config.pollIntervalMs); + Poll(); + } + if (m_Config.maxStorageBytes > 0 && (m_StoredBytes > m_Config.maxStorageBytes || now >= m_NextEviction)) { + m_NextEviction = now + EVICTION_INTERVAL; + // Deleted files stay marked made: they're made again when someone asks for them (Request) + const auto removed = m_Storage.Evict(m_Config.maxStorageBytes); + m_StoredBytes = 0; + for (const auto& entry : m_Storage.List()) m_StoredBytes += entry.bytes; + if (!removed.empty()) { + m_Evicted += removed.size(); + LOG("Deleted the files of %zu item(s) used longest ago to stay under %llu MB", removed.size(), static_cast(m_Config.maxStorageBytes / (1024 * 1024))); + } + } + // Forget old answers + std::erase_if(m_Recent, [now](const auto& item) { return now - item.second.first > RECENT_ANSWER_TIME; }); +} + +UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) { + if (m_InFlight.contains({ kind, id })) return Availability::QUEUED; + if (m_Storage.File(kind, id, "icon.png")) { + m_Storage.Touch(kind, id); + return Availability::READY; + } + if (const auto it = m_Recent.find({ kind, id }); it != m_Recent.end()) return it->second.second; + + const auto info = kind == Kind::MODEL ? Database::Get()->GetUgcProcessInfo(id) : Database::Get()->GetModularBuildProcessInfo(id); + Availability answer = Availability::UNKNOWN; + if (info && info->state != IUgc::eProcessState::FAILED) { + // Made before but the files are gone (deleted to save space): make them again, first + if (info->state == IUgc::eProcessState::DONE) { + if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false); + else Database::Get()->ResetModularBuildProcessing(id, false); + } + m_NextPoll = std::chrono::steady_clock::now(); + answer = Availability::QUEUED; + } + m_Recent[{ kind, id }] = { std::chrono::steady_clock::now(), answer }; + return answer; +} + +nlohmann::json UgcProcessor::Status() const { + nlohmann::json status; + { + std::lock_guard lock(m_Mutex); + status["queued"] = m_Jobs.size(); + status["active"] = m_Active; + } + status["workers"] = m_Threads.size(); + status["made"] = m_Made; + status["failed"] = m_Failed; + status["evicted"] = m_Evicted; + status["storedBytes"] = m_StoredBytes; + status["maxStorageBytes"] = m_Config.maxStorageBytes; + auto& recent = status["recent"] = nlohmann::json::array(); + for (auto it = m_Log.rbegin(); it != m_Log.rend(); ++it) { + recent.push_back({ { "kind", KindName(it->kind) }, { "id", std::to_string(it->id) }, { "ok", it->ok }, + { "ms", static_cast(it->milliseconds) }, { "message", it->message }, { "time", it->time } }); + } + return status; +} diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h new file mode 100644 index 000000000..b4d9bf727 --- /dev/null +++ b/dUgcServer/UgcProcessor.h @@ -0,0 +1,115 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dCommonVars.h" +#include "json_fwd.hpp" +#include "UgcBricks.h" +#include "UgcJobs.h" +#include "UgcStorage.h" + +/** + * The UGC server's queue. The main thread (Update) takes pending rows from the database, gathers each one's input + * (the stored LXFML, or a modular build's modules from the CDClient) and hands it to the worker threads; they make + * and write the files; the main thread records the outcome in the database. Workers never touch the database, the + * CDClient or the network. + */ +class UgcProcessor { +public: + using Kind = UgcStorage::Kind; + + struct Config { + uint32_t pollIntervalMs{ 2000 }; + uint32_t pollBatch{ 32 }; + uint32_t maxAttempts{ 3 }; + uint64_t maxStorageBytes{}; // 0: no limit + size_t threads{ 2 }; + }; + + UgcProcessor(Config config, UgcStorage& storage, UgcBricks::BrickLibrary& library, UgcJobs::Settings settings); + ~UgcProcessor(); + + void Start(); + // Waits for the jobs that are running; queued ones are dropped (they stay pending in the database) + void Stop(); + + // Main thread: poll, dispatch, record results, keep the storage under its cap + void Update(); + + enum class Availability { + READY, // the files are there + QUEUED, // it exists and is being made (again) + UNKNOWN, // no such item, or it failed + }; + + // Main thread: whether an item's files can be served, queuing it to be made when it exists but they're missing + Availability Request(Kind kind, LWOOBJID id); + + // Main thread: what the server is doing, for /status + nlohmann::json Status() const; + +private: + struct Job { + Kind kind{}; + LWOOBJID id{}; + uint32_t attempts{}; + std::string blob; // models: the stored LXFML + UgcJobs::ModularInput modular; // modular builds + }; + + struct Done { + Kind kind{}; + LWOOBJID id{}; + uint32_t attempts{}; + UgcJobs::Outcome outcome; + uint64_t bytes{}; + double milliseconds{}; + }; + + void Poll(); + void Collect(); + void Record(const Done& done); + void Worker(); + + Config m_Config; + UgcStorage& m_Storage; + UgcBricks::BrickLibrary& m_Library; + UgcJobs::Settings m_Settings; + + mutable std::mutex m_Mutex; + std::condition_variable m_Wake; + std::deque m_Jobs; + std::deque m_Done; + size_t m_Active{}; + bool m_Stopping{}; + std::vector m_Threads; + + // Main thread only + std::set> m_InFlight; + std::chrono::steady_clock::time_point m_NextPoll{}; + std::chrono::steady_clock::time_point m_NextEviction{}; + std::map, std::pair> m_Recent; // answers for missing files + uint64_t m_StoredBytes{}; + uint64_t m_Made{}; + uint64_t m_Failed{}; + uint64_t m_Evicted{}; + struct LogEntry { + Kind kind{}; + LWOOBJID id{}; + bool ok{}; + double milliseconds{}; + std::string message; + int64_t time{}; + }; + std::deque m_Log; // the last results +}; diff --git a/dUgcServer/UgcRender.cpp b/dUgcServer/UgcRender.cpp new file mode 100644 index 000000000..a1dfd27a2 --- /dev/null +++ b/dUgcServer/UgcRender.cpp @@ -0,0 +1,360 @@ +#include "UgcRender.h" + +#include +#include +#include +#include + +#include + +namespace { + constexpr float INF = std::numeric_limits::infinity(); + + float Edge(const glm::vec3& a, const glm::vec3& b, float px, float py) { + return (b.x - a.x) * (py - a.y) - (b.y - a.y) * (px - a.x); + } + + // Calls fragment(x, y, z, w0, w1, w2) for every pixel centre inside the screen-space triangle (x, y in pixels) + template + void Rasterize(int width, int height, const glm::vec3& a, const glm::vec3& b, const glm::vec3& c, Fragment&& fragment) { + const float area = Edge(a, b, c.x, c.y); + if (!(std::abs(area) > 1e-9f)) return; + const float inverse = 1.0f / area; + const int minX = std::max(0, static_cast(std::floor(std::min({ a.x, b.x, c.x })))); + const int maxX = std::min(width - 1, static_cast(std::ceil(std::max({ a.x, b.x, c.x })))); + const int minY = std::max(0, static_cast(std::floor(std::min({ a.y, b.y, c.y })))); + const int maxY = std::min(height - 1, static_cast(std::ceil(std::max({ a.y, b.y, c.y })))); + for (int y = minY; y <= maxY; y++) { + const float py = y + 0.5f; + for (int x = minX; x <= maxX; x++) { + const float px = x + 0.5f; + const float w0 = Edge(b, c, px, py) * inverse; + const float w1 = Edge(c, a, px, py) * inverse; + const float w2 = 1.0f - w0 - w1; + if (w0 < 0.0f || w1 < 0.0f || w2 < 0.0f) continue; + fragment(x, y, w0 * a.z + w1 * b.z + w2 * c.z, w0, w1, w2); + } + } + } + + float ToLinear(float c) { return std::pow(std::clamp(c, 0.0f, 1.0f), 2.2f); } + float ToSrgb(float c) { return std::pow(std::clamp(c, 0.0f, 1.0f), 1.0f / 2.2f); } + + // Looking at a sphere (center, radius) from direction `dir` (towards the viewer), square orthographic view + struct OrthoView { + glm::vec3 center{}; + glm::vec3 dir{}; + glm::vec3 right{}; + glm::vec3 up{}; + float radius{}; + int resolution{}; + + OrthoView(const glm::vec3& center_, float radius_, const glm::vec3& dir_, int resolution_) + : center(center_), dir(dir_), radius(radius_), resolution(resolution_) { + const glm::vec3 worldUp = std::abs(dir.y) < 0.99f ? glm::vec3(0.0f, 1.0f, 0.0f) : glm::vec3(1.0f, 0.0f, 0.0f); + right = glm::normalize(glm::cross(worldUp, dir)); + up = glm::cross(dir, right); + } + + // x, y in pixels; z the distance from the camera plane (smaller is nearer) + glm::vec3 Project(const glm::vec3& p) const { + const auto offset = p - center; + const float scale = 0.5f * resolution / radius; + return { resolution * 0.5f + glm::dot(offset, right) * scale, resolution * 0.5f - glm::dot(offset, up) * scale, radius - glm::dot(offset, dir) }; + } + + float PixelSize() const { return 2.0f * radius / resolution; } + }; + + void Bounds(const UgcModel::Model& model, glm::vec3& center, float& radius) { + glm::vec3 min{}, max{}; + if (!model.Bounds(min, max)) { + center = glm::vec3(0.0f); + radius = 1.0f; + return; + } + center = (min + max) * 0.5f; + radius = std::max(glm::length(max - min) * 0.5f, 0.01f); + } +} + +namespace UgcRender { + std::vector SphereDirections() { + const float t = (1.0f + std::sqrt(5.0f)) / 2.0f; + const std::vector corners = { + { -1, t, 0 }, { 1, t, 0 }, { -1, -t, 0 }, { 1, -t, 0 }, + { 0, -1, t }, { 0, 1, t }, { 0, -1, -t }, { 0, 1, -t }, + { t, 0, -1 }, { t, 0, 1 }, { -t, 0, -1 }, { -t, 0, 1 }, + }; + std::vector directions; + for (const auto& corner : corners) directions.push_back(glm::normalize(corner)); + // Edge centres: the pairs of corners that are neighbours (the shortest distance apart) + const float edge = glm::length(corners[0] - corners[1]); + for (size_t i = 0; i < corners.size(); i++) { + for (size_t j = i + 1; j < corners.size(); j++) { + if (std::abs(glm::length(corners[i] - corners[j]) - edge) < 1e-3f) directions.push_back(glm::normalize(corners[i] + corners[j])); + } + } + return directions; + } + + OptimizeResult Optimize(UgcModel::Model& model, const OptimizeOptions& options) { + OptimizeResult result; + auto& opaque = model.opaque; + result.trianglesBefore = opaque.TriangleCount() + model.transparent.TriangleCount(); + if (model.Empty() || (!options.removeHidden && !options.bakeAo)) return result; + + glm::vec3 center{}; + float radius{}; + Bounds(model, center, radius); + radius *= 1.02f; + const int resolution = std::clamp(options.resolution, 64, 4096); + const size_t pixels = static_cast(resolution) * resolution; + std::vector depth(pixels); + std::vector ids(pixels); + std::vector screen(opaque.positions.size()); + + const size_t triangles = opaque.TriangleCount(); + std::vector visible(triangles, false); + std::vector facing(triangles, true); + std::vector faceNormals(triangles, glm::vec3(0.0f)); + for (size_t t = 0; t < triangles; t++) { + if (opaque.normals.size() != opaque.positions.size()) break; + faceNormals[t] = glm::normalize(opaque.normals[opaque.indices[t * 3]] + opaque.normals[opaque.indices[t * 3 + 1]] + opaque.normals[opaque.indices[t * 3 + 2]] + glm::vec3(1e-6f)); + } + // Without normals nothing is culled + if (opaque.normals.size() != opaque.positions.size()) std::fill(faceNormals.begin(), faceNormals.end(), glm::vec3(0.0f)); + std::vector aoOpaque(opaque.positions.size()), weightOpaque(opaque.positions.size()); + std::vector aoTransparent(model.transparent.positions.size()), weightTransparent(model.transparent.positions.size()); + + for (const auto& direction : SphereDirections()) { + const OrthoView view(center, radius, direction, resolution); + const float bias = view.PixelSize(); + std::fill(depth.begin(), depth.end(), INF); + std::fill(ids.begin(), ids.end(), 0); + for (size_t v = 0; v < opaque.positions.size(); v++) screen[v] = view.Project(opaque.positions[v]); + // Faces turned away can't be seen from here (they're seen from the directions they face); their vertex + // normals say which way they face, which doesn't depend on the files' winding + for (size_t t = 0; t < triangles; t++) facing[t] = glm::dot(faceNormals[t], direction) > -0.1f; + for (size_t t = 0; t < triangles; t++) { + if (!facing[t]) continue; + const auto id = static_cast(t + 1); + Rasterize(resolution, resolution, screen[opaque.indices[t * 3]], screen[opaque.indices[t * 3 + 1]], screen[opaque.indices[t * 3 + 2]], + [&](int x, int y, float z, float, float, float) { + auto& stored = depth[static_cast(y) * resolution + x]; + if (z < stored) { + stored = z; + ids[static_cast(y) * resolution + x] = id; + } + }); + } + for (const auto id : ids) { + if (id != 0) visible[id - 1] = true; + } + + // Whether a point is in front of what was drawn around its pixel + const auto exposed = [&](const glm::vec3& point, float allowance, bool neighbours) { + const auto p = view.Project(point); + const int px = static_cast(std::floor(p.x)), py = static_cast(std::floor(p.y)); + const int reach = neighbours ? 1 : 0; + for (int dy = -reach; dy <= reach; dy++) { + for (int dx = -reach; dx <= reach; dx++) { + const int x = px + dx, y = py + dy; + if (x < 0 || y < 0 || x >= resolution || y >= resolution) return true; + if (p.z <= depth[static_cast(y) * resolution + x] + allowance) return true; + } + } + return false; + }; + + // Triangles too small or thin to cover a pixel centre: kept when their centre isn't behind what was drawn. + // Bigger ones that show would have covered one. + if (options.removeHidden) { + const float smallArea = 2.0f; // pixels + for (size_t t = 0; t < triangles; t++) { + if (visible[t] || !facing[t]) continue; + const auto& a = screen[opaque.indices[t * 3]]; + const auto& b = screen[opaque.indices[t * 3 + 1]]; + const auto& c = screen[opaque.indices[t * 3 + 2]]; + if (std::abs(Edge(a, b, c.x, c.y)) * 0.5f > smallArea) continue; + const auto centre = (a + b + c) / 3.0f; + const int x = static_cast(centre.x), y = static_cast(centre.y); + if (x < 0 || y < 0 || x >= resolution || y >= resolution || centre.z <= depth[static_cast(y) * resolution + x] + bias) visible[t] = true; + } + } + + if (options.bakeAo) { + const auto accumulate = [&](const UgcModel::Mesh& mesh, std::vector& ao, std::vector& weight) { + for (size_t v = 0; v < mesh.positions.size(); v++) { + const float w = v < mesh.normals.size() ? glm::dot(mesh.normals[v], direction) : 1.0f; + if (w <= 0.0f) continue; + weight[v] += w; + const auto normal = v < mesh.normals.size() ? mesh.normals[v] : glm::vec3(0.0f); + if (exposed(mesh.positions[v] + normal * bias, bias, false)) ao[v] += w; + } + }; + accumulate(opaque, aoOpaque, weightOpaque); + accumulate(model.transparent, aoTransparent, weightTransparent); + } + } + + if (options.bakeAo) { + const float strength = std::clamp(options.aoStrength, 0.0f, 1.0f); + const auto apply = [strength](UgcModel::Mesh& mesh, const std::vector& ao, const std::vector& weight) { + for (size_t v = 0; v < mesh.colors.size() && v < ao.size(); v++) { + if (weight[v] <= 0.0f) continue; + const float factor = 1.0f - strength * (1.0f - ao[v] / weight[v]); + auto& color = mesh.colors[v]; + color.r = ToSrgb(ToLinear(color.r) * factor); + color.g = ToSrgb(ToLinear(color.g) * factor); + color.b = ToSrgb(ToLinear(color.b) * factor); + } + }; + apply(opaque, aoOpaque, weightOpaque); + apply(model.transparent, aoTransparent, weightTransparent); + } + + if (options.removeHidden) { + UgcModel::Mesh kept; + std::vector remap(opaque.positions.size(), UINT32_MAX); + for (size_t t = 0; t < triangles; t++) { + if (!visible[t]) { + result.trianglesRemoved++; + continue; + } + for (int k = 0; k < 3; k++) { + const auto source = opaque.indices[t * 3 + k]; + if (remap[source] == UINT32_MAX) { + remap[source] = static_cast(kept.positions.size()); + kept.positions.push_back(opaque.positions[source]); + if (source < opaque.normals.size()) kept.normals.push_back(opaque.normals[source]); + if (source < opaque.colors.size()) kept.colors.push_back(opaque.colors[source]); + } + kept.indices.push_back(remap[source]); + } + } + opaque = std::move(kept); + } + return result; + } + + Image RenderIcon(const UgcModel::Model& source, const IconOptions& options) { + const int size = std::clamp(options.size, 8, 1024); + const int supersample = std::clamp(options.supersample, 1, 8); + const int n = size * supersample; + Image image{ size, size, std::vector(static_cast(size) * size * 4, 0) }; + if (source.Empty()) return image; + + UgcModel::Model model = source; + model.opaque.Transform(options.modelRotation); + model.transparent.Transform(options.modelRotation); + glm::vec3 center{}; + float radius{}; + Bounds(model, center, radius); + + const float yaw = glm::radians(options.yawDegrees), pitch = glm::radians(options.pitchDegrees); + const glm::vec3 dir(std::sin(yaw) * std::cos(pitch), std::sin(pitch), std::cos(yaw) * std::cos(pitch)); + const float fov = glm::radians(std::clamp(options.fovDegrees, 1.0f, 120.0f)); + const float distance = radius / std::sin(fov * 0.5f); + const glm::vec3 eye = center + dir * distance; + const glm::mat4 viewProjection = glm::perspective(fov, 1.0f, std::max(distance - radius * 1.5f, distance * 0.01f), distance + radius * 1.5f) * + glm::lookAt(eye, center, glm::vec3(0.0f, 1.0f, 0.0f)); + + // Frame the model: its projected bounds, scaled to fill the icon less the margin + float minX = INF, minY = INF, maxX = -INF, maxY = -INF; + for (const auto* mesh : { &model.opaque, &model.transparent }) { + for (const auto& position : mesh->positions) { + const auto clip = viewProjection * glm::vec4(position, 1.0f); + if (clip.w <= 0.0f) continue; + minX = std::min(minX, clip.x / clip.w); + maxX = std::max(maxX, clip.x / clip.w); + minY = std::min(minY, clip.y / clip.w); + maxY = std::max(maxY, clip.y / clip.w); + } + } + if (minX > maxX) return image; + const float centerX = (minX + maxX) * 0.5f, centerY = (minY + maxY) * 0.5f; + const float scale = 2.0f / (std::max({ maxX - minX, maxY - minY, 1e-6f }) * std::max(options.margin, 0.1f)); + const auto project = [&](const glm::vec3& position) { + const auto clip = viewProjection * glm::vec4(position, 1.0f); + const float w = clip.w > 1e-6f ? clip.w : 1e-6f; + return glm::vec3((0.5f + (clip.x / w - centerX) * scale * 0.5f) * n, (0.5f - (clip.y / w - centerY) * scale * 0.5f) * n, clip.z / w); + }; + + // Linear, premultiplied + std::vector color(static_cast(n) * n, glm::vec4(0.0f)); + std::vector depth(static_cast(n) * n, INF); + const glm::vec3 light = glm::normalize(dir + glm::vec3(-0.35f, 1.1f, 0.25f)); + const auto shade = [&](const UgcModel::Mesh& mesh, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) { + glm::vec3 normal(0.0f, 1.0f, 0.0f); + if (mesh.normals.size() == mesh.positions.size()) { + normal = mesh.normals[i0] * w0 + mesh.normals[i1] * w1 + mesh.normals[i2] * w2; + const auto length = glm::length(normal); + normal = length > 0.0f ? normal / length : glm::vec3(0.0f, 1.0f, 0.0f); + if (glm::dot(normal, dir) < 0.0f) normal = -normal; // the back of a face + } + glm::vec4 base(0.63f, 0.63f, 0.63f, 1.0f); + if (mesh.colors.size() == mesh.positions.size()) base = mesh.colors[i0] * w0 + mesh.colors[i1] * w1 + mesh.colors[i2] * w2; + const float lighting = 0.55f + 0.6f * std::max(0.0f, glm::dot(normal, light)); + return glm::vec4(ToLinear(base.r) * lighting, ToLinear(base.g) * lighting, ToLinear(base.b) * lighting, std::clamp(base.a, 0.0f, 1.0f)); + }; + + // Opaque first, with the depth buffer + { + const auto& mesh = model.opaque; + std::vector screen(mesh.positions.size()); + for (size_t v = 0; v < mesh.positions.size(); v++) screen[v] = project(mesh.positions[v]); + for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) { + const auto i0 = mesh.indices[i], i1 = mesh.indices[i + 1], i2 = mesh.indices[i + 2]; + Rasterize(n, n, screen[i0], screen[i1], screen[i2], [&](int x, int y, float z, float w0, float w1, float w2) { + const size_t index = static_cast(y) * n + x; + if (z >= depth[index]) return; + depth[index] = z; + const auto shaded = shade(mesh, i0, i1, i2, w0, w1, w2); + color[index] = glm::vec4(glm::vec3(shaded), 1.0f); + }); + } + } + + // Then transparent triangles, farthest first, blended over it + { + const auto& mesh = model.transparent; + std::vector screen(mesh.positions.size()); + for (size_t v = 0; v < mesh.positions.size(); v++) screen[v] = project(mesh.positions[v]); + std::vector order(mesh.indices.size() / 3); + std::iota(order.begin(), order.end(), 0); + const auto depthOf = [&](size_t t) { return screen[mesh.indices[t * 3]].z + screen[mesh.indices[t * 3 + 1]].z + screen[mesh.indices[t * 3 + 2]].z; }; + std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { return depthOf(a) > depthOf(b); }); + for (const auto t : order) { + const auto i0 = mesh.indices[t * 3], i1 = mesh.indices[t * 3 + 1], i2 = mesh.indices[t * 3 + 2]; + Rasterize(n, n, screen[i0], screen[i1], screen[i2], [&](int x, int y, float z, float w0, float w1, float w2) { + const size_t index = static_cast(y) * n + x; + if (z >= depth[index]) return; + const auto shaded = shade(mesh, i0, i1, i2, w0, w1, w2); + const float alpha = shaded.a; + auto& target = color[index]; + target = glm::vec4(glm::vec3(shaded) * alpha + glm::vec3(target) * (1.0f - alpha), alpha + target.a * (1.0f - alpha)); + }); + } + } + + // Box filter down to the icon's size + const float samples = static_cast(supersample * supersample); + for (int y = 0; y < size; y++) { + for (int x = 0; x < size; x++) { + glm::vec4 sum(0.0f); + for (int sy = 0; sy < supersample; sy++) { + for (int sx = 0; sx < supersample; sx++) sum += color[static_cast(y * supersample + sy) * n + (x * supersample + sx)]; + } + sum /= samples; + uint8_t* out = &image.rgba[(static_cast(y) * size + x) * 4]; + if (sum.a <= 0.0f) continue; + out[0] = static_cast(std::lround(ToSrgb(sum.r / sum.a) * 255.0f)); + out[1] = static_cast(std::lround(ToSrgb(sum.g / sum.a) * 255.0f)); + out[2] = static_cast(std::lround(ToSrgb(sum.b / sum.a) * 255.0f)); + out[3] = static_cast(std::lround(std::clamp(sum.a, 0.0f, 1.0f) * 255.0f)); + } + } + return image; + } +} diff --git a/dUgcServer/UgcRender.h b/dUgcServer/UgcRender.h new file mode 100644 index 000000000..222115e4d --- /dev/null +++ b/dUgcServer/UgcRender.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + +#include + +#include "UgcModel.h" + +/** + * A small software rasterizer (no GPU or display needed) for what the UGC server draws: the icons, and the renders + * from many directions that find the faces nobody can see and bake ambient occlusion into the vertex colors. + */ +namespace UgcRender { + // 8-bit RGBA, rows top to bottom, not premultiplied + struct Image { + int width{}; + int height{}; + std::vector rgba; + }; + + struct IconOptions { + int size{ 128 }; + int supersample{ 4 }; + float yawDegrees{ 35.0f }; // camera around the model, from +Z towards +X + float pitchDegrees{ 25.0f }; // camera above the model + float fovDegrees{ 30.0f }; + float margin{ 1.03f }; // 1 fills the icon, more leaves a border + glm::mat4 modelRotation{ 1.0f }; // applied to the model before the camera looks at it + }; + + // The model drawn from the icon's camera, framed to fit, on a transparent background + Image RenderIcon(const UgcModel::Model& model, const IconOptions& options); + + struct OptimizeOptions { + int resolution{ 1024 }; // of each direction's render + bool removeHidden{ true }; + bool bakeAo{ true }; + float aoStrength{ 0.6f }; // 0 leaves the colors, 1 makes fully hidden corners black + }; + + struct OptimizeResult { + size_t trianglesBefore{}; + size_t trianglesRemoved{}; + }; + + /** + * Renders the opaque mesh from 42 directions around it: triangles that show in none of them are removed (with a + * conservative test, so small visible ones stay), and each vertex's share of the directions it can be seen from + * darkens its color (ambient occlusion). Transparent bricks hide nothing but are darkened too. + */ + OptimizeResult Optimize(UgcModel::Model& model, const OptimizeOptions& options); + + // The 42 directions Optimize renders from (an icosahedron's corners and edge centres), unit length + std::vector SphereDirections(); +} diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp new file mode 100644 index 000000000..611cc4f92 --- /dev/null +++ b/dUgcServer/UgcServer.cpp @@ -0,0 +1,360 @@ +// The UGC server: makes the meshes and icons of what players build and serves them, and the models' LXFML, to the +// game client over HTTP. See docs/UgcServer.md. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "BinaryPathFinder.h" +#include "CDClientDatabase.h" +#include "ConfigSync.h" +#include "Database.h" +#include "Diagnostics.h" +#include "Game.h" +#include "GeneralUtils.h" +#include "Logger.h" +#include "Server.h" +#include "ServiceType.h" +#include "Web.h" +#include "dConfig.h" +#include "dServer.h" +#include "eHTTPMethod.h" +#include "json.hpp" + +#include "UgcBricks.h" +#include "UgcCdClient.h" +#include "UgcJobs.h" +#include "UgcModel.h" +#include "UgcProcessor.h" +#include "UgcStorage.h" + +namespace Game { + Logger* logger = nullptr; + dServer* server = nullptr; + dConfig* config = nullptr; + Game::signal_t lastSignal = 0; + std::mt19937 randomEngine; +} + +namespace { + dServer* g_Server = nullptr; + UgcProcessor* g_Processor = nullptr; + UgcStorage* g_Storage = nullptr; + + template + T Setting(const std::string& key, T fallback) { + return GeneralUtils::TryParse(Game::config->GetValue(key)).value_or(fallback); + } + + UgcJobs::Settings ReadSettings() { + UgcJobs::Settings settings; + settings.optimize.removeHidden = Setting("remove_hidden_faces", 1) != 0; + settings.optimize.bakeAo = Setting("bake_ao", 1) != 0; + settings.optimize.aoStrength = Setting("ao_strength", 0.6f); + settings.optimize.resolution = Setting("optimize_resolution", 1024); + settings.icon.size = Setting("icon_size", 128); + settings.icon.yawDegrees = Setting("icon_yaw", 35.0f); + settings.icon.pitchDegrees = Setting("icon_pitch", 25.0f); + settings.icon.margin = Setting("icon_margin", 1.03f); + settings.modularIcon = settings.icon; + settings.modularIcon.yawDegrees = Setting("modular_icon_yaw", 35.0f); + settings.modularIcon.pitchDegrees = Setting("modular_icon_pitch", 20.0f); + return settings; + } + + std::vector Segments(const std::string& path) { + std::vector segments; + size_t start = 0; + while (start < path.size()) { + auto end = path.find('/', start); + if (end == std::string::npos) end = path.size(); + if (end > start) segments.push_back(path.substr(start, end - start)); + start = end + 1; + } + return segments; + } + + std::string Lower(std::string text) { + std::transform(text.begin(), text.end(), text.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + return text; + } + + void NotFound(HTTPReply& reply) { + reply.status = eHTTPStatusCode::NOT_FOUND; + reply.message = "Not Found"; + reply.contentType = eContentType::TEXT_PLAIN; + } + + void ServeFile(HTTPReply& reply, UgcStorage::Kind kind, LWOOBJID id, const std::string& name, eContentType type, bool crossOrigin) { + switch (g_Processor->Request(kind, id)) { + case UgcProcessor::Availability::READY: + break; + case UgcProcessor::Availability::QUEUED: + // The client asks again later on 408 + reply.status = eHTTPStatusCode::REQUEST_TIMEOUT; + reply.message = "Being made"; + reply.contentType = eContentType::TEXT_PLAIN; + reply.headers.push_back("Retry-After: 5"); + return; + case UgcProcessor::Availability::UNKNOWN: + NotFound(reply); + return; + } + const auto path = g_Storage->File(kind, id, name); + if (!path) { + NotFound(reply); + return; + } + reply.status = eHTTPStatusCode::OK; + reply.file = path->string(); + reply.contentType = type; + reply.headers.push_back("Cache-Control: public, max-age=3600"); + if (crossOrigin) reply.headers.push_back("Access-Control-Allow-Origin: *"); + } + + /** + * A game client download (3D services): /UGCC/[3DOPTIMIZED/|IMAGE128DDS/]<.ext>(.gz|.checksum). + * `segments` are the path's parts after client_path. + */ + void ServeClientDownload(HTTPReply& reply, const std::vector& segments) { + NotFound(reply); + if (segments.size() < 2 || segments.size() > 3) return; + const auto folder = Lower(segments[0]); + if (!folder.starts_with("ugcc")) return; + const auto datacenter = folder.substr(4); + const std::string typeFolder = segments.size() == 3 ? Lower(segments[1]) : ""; + auto file = Lower(segments.back()); + + std::string suffix; + if (file.ends_with(".gz")) suffix = ".gz"; + else if (file.ends_with(".checksum")) suffix = ".checksum"; + else return; + file.resize(file.size() - suffix.size()); + const auto dot = file.rfind('.'); + if (dot == std::string::npos) return; + const auto extension = file.substr(dot); + auto number = file.substr(0, dot); + // The file name starts with the datacenter id + if (!number.starts_with(datacenter)) return; + number = number.substr(datacenter.size()); + const auto id = GeneralUtils::TryParse(number); + if (!id || number.empty()) return; + const auto blueprint = static_cast(*id); + + const auto type = suffix == ".gz" ? eContentType::APPLICATION_OCTET_STREAM : eContentType::TEXT_PLAIN; + if (extension == ".lxfml" && typeFolder.empty()) { + ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.lxfml" + suffix, type, false); + } else if (extension == ".nif" && typeFolder == "3doptimized") { + ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.nif" + suffix, type, false); + } else if (extension == ".dds" && typeFolder == "image128dds") { + // A player model's icon, else a modular build's + ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "icon.dds" + suffix, type, false); + if (reply.status == eHTTPStatusCode::NOT_FOUND) ServeFile(reply, UgcStorage::Kind::MODULAR, blueprint, "icon.dds" + suffix, type, false); + } + // .hkx: no physics is made + } + + void RegisterRoutes() { + const auto prefix = Segments(Lower(Game::config->GetValue("client_path").empty() ? "/ugc" : Game::config->GetValue("client_path"))); + std::string base; + for (const auto& segment : prefix) base += "/" + segment; + + const auto clientRoute = [prefixSize = prefix.size()](HTTPReply& reply, const HTTPContext& context) { + auto segments = Segments(context.originalPath); + if (segments.size() <= prefixSize) { + NotFound(reply); + return; + } + segments.erase(segments.begin(), segments.begin() + static_cast(prefixSize)); + ServeClientDownload(reply, segments); + }; + Game::web.RegisterHTTPRoute({ .path = base + "/:folder/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = clientRoute }); + Game::web.RegisterHTTPRoute({ .path = base + "/:folder/:type/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = clientRoute }); + + // Previews for the dashboard + Game::web.RegisterHTTPRoute({ .path = "/files/:kind/:id/:file", .method = eHTTPMethod::GET, .middleware = {}, + .handle = [](HTTPReply& reply, const HTTPContext& context) { + const auto segments = Segments(context.originalPath); + NotFound(reply); + reply.headers.push_back("Access-Control-Allow-Origin: *"); + if (segments.size() != 4) return; + const auto id = GeneralUtils::TryParse(segments[2]); + const auto kind = segments[1] == "model" ? std::optional(UgcStorage::Kind::MODEL) : segments[1] == "modular" ? std::optional(UgcStorage::Kind::MODULAR) : std::nullopt; + const auto& name = segments[3]; + if (!id || !kind || (name != "icon.png" && name != "model.nif")) return; + reply.headers.clear(); + ServeFile(reply, *kind, static_cast(*id), name, name == "icon.png" ? eContentType::IMAGE_PNG : eContentType::APPLICATION_OCTET_STREAM, true); + } }); + + Game::web.RegisterHTTPRoute({ .path = "/status", .method = eHTTPMethod::GET, .middleware = {}, + .handle = [](HTTPReply& reply, const HTTPContext&) { + reply.status = eHTTPStatusCode::OK; + reply.contentType = eContentType::APPLICATION_JSON; + reply.message = g_Processor->Status().dump(); + reply.headers.push_back("Access-Control-Allow-Origin: *"); + reply.headers.push_back("Cache-Control: no-store"); + } }); + LOG("Serving the client's downloads under %s/UGCC/", base.c_str()); + } + + std::filesystem::path ResPath() { + const auto client = Game::config->GetValue("client_location"); + return client.empty() ? std::filesystem::path{} : std::filesystem::path(client) / "res"; + } + + // Command line tools: make one model's or modular build's files into a folder, without a database + int MakeFromCommandLine(const std::string& mode, const std::string& input, const std::filesystem::path& output) { + const auto res = ResPath(); + const auto settings = ReadSettings(); + UgcBricks::BrickLibrary library(res, Setting("brick_lod", 0)); + if (!library.LoadMaterials()) std::cerr << "Couldn't read Materials.xml from " << (res / "brickdb.zip") << "; bricks will be grey\n"; + UgcJobs::Outcome outcome; + const auto start = std::chrono::steady_clock::now(); + if (mode == "--make-model") { + const auto data = UgcBricks::ReadFile(input); + if (!data) { + std::cerr << "Can't read " << input << "\n"; + return EXIT_FAILURE; + } + outcome = UgcJobs::ProcessModel(*data, library, settings); + } else { + CDClientDatabase::Connect((BinaryPathFinder::GetBinaryDir() / "resServer/CDServer.sqlite").string()); + UgcJobs::ModularInput modular; + std::string error; + if (!UgcCdClient::GatherModular(input, modular, error)) { + std::cerr << error << "\n"; + return EXIT_FAILURE; + } + outcome = UgcJobs::ProcessModular(modular, res, settings); + } + const auto ms = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + if (!outcome.ok) { + std::cerr << "Failed: " << outcome.error << "\n"; + return EXIT_FAILURE; + } + std::filesystem::create_directories(output); + for (const auto& [name, data] : outcome.files) { + std::ofstream(output / name, std::ios::binary).write(data.data(), static_cast(data.size())); + } + std::cout << "Made " << outcome.files.size() << " files in " << ms << " ms" << (outcome.note.empty() ? "" : ": " + outcome.note) << "\n"; + return EXIT_SUCCESS; + } +} + +int main(int argc, char** argv) { + Diagnostics::SetProduceMemoryDump(true); + std::signal(SIGINT, Game::OnSignal); + std::signal(SIGTERM, Game::OnSignal); + + Game::config = new dConfig("ugcconfig.ini"); + + // UgcServer --make-model or --make-modular "1:4713+1:4714+1:4715" + if (argc == 4 && (std::string(argv[1]) == "--make-model" || std::string(argv[1]) == "--make-modular")) { + return MakeFromCommandLine(argv[1], argv[2], argv[3]); + } + + Server::SetupLogger("UgcServer"); + if (!Game::logger) return EXIT_FAILURE; + Game::config->LogSettings(); + LOG("Starting UGC Server"); + + const auto res = ResPath(); + if (res.empty() || !std::filesystem::exists(res)) { + LOG("client_location is not set or has no res folder; the UGC server needs the client's brick data"); + return EXIT_FAILURE; + } + + try { + CDClientDatabase::Connect((BinaryPathFinder::GetBinaryDir() / "resServer/CDServer.sqlite").string()); + } catch (std::exception& ex) { + LOG("Failed to connect to CDClient database: %s", ex.what()); + return EXIT_FAILURE; + } + + try { + Database::Connect(); + } catch (std::exception& ex) { + LOG("Failed to connect to the database: %s", ex.what()); + return EXIT_FAILURE; + } + + // Settings edited on the dashboard (server_config table) are layered over the files from here on + Game::config->SetDatabaseSync(ConfigSync::Sync); + + std::string masterIP = "localhost"; + uint32_t masterPort = 1000; + std::string masterPassword; + if (const auto masterInfo = Database::Get()->GetMasterInfo()) { + masterIP = masterInfo->ip; + masterPort = masterInfo->port; + masterPassword = masterInfo->password; + } + + // The master starts it again when this link drops + g_Server = new dServer(masterIP, Setting("net_port", 2012), 0, 16, false, false, Game::logger, masterIP, masterPort, + ServiceType::UGC, Game::config, &Game::lastSignal, masterPassword); + Game::server = g_Server; + + UgcBricks::BrickLibrary library(res, Setting("brick_lod", 0)); + if (!library.LoadMaterials()) LOG("Couldn't read Materials.xml from %s; bricks will be grey", (res / "brickdb.zip").string().c_str()); + + auto outputDir = std::filesystem::path(Game::config->GetValue("ugc_output_dir").empty() ? "ugc" : Game::config->GetValue("ugc_output_dir")); + if (outputDir.is_relative()) outputDir = BinaryPathFinder::GetBinaryDir() / outputDir; + UgcStorage storage(outputDir); + g_Storage = &storage; + + UgcProcessor::Config processorConfig; + processorConfig.pollIntervalMs = std::max(Setting("poll_interval_ms", 2000), 100); + processorConfig.pollBatch = std::max(Setting("poll_batch", 32), 1); + processorConfig.maxAttempts = std::max(Setting("max_attempts", 3), 1); + processorConfig.maxStorageBytes = Setting("ugc_max_storage_mb", 2048) * 1024 * 1024; + const auto threads = Setting("worker_threads", 0); + processorConfig.threads = threads > 0 ? threads : std::max(std::thread::hardware_concurrency() / 2, 1); + UgcProcessor processor(processorConfig, storage, library, ReadSettings()); + g_Processor = &processor; + + const auto listenIp = Game::config->GetValue("listen_ip").empty() ? std::string("0.0.0.0") : Game::config->GetValue("listen_ip"); + const auto port = Setting("port", 2008); + if (!Game::web.Startup(listenIp, port)) { + LOG("Failed to start the web server on %s:%u", listenIp.c_str(), port); + return EXIT_FAILURE; + } + RegisterRoutes(); + processor.Start(); + LOG("UGC Server started on %s:%u, files in %s", listenIp.c_str(), port, outputDir.string().c_str()); + + auto lastTick = std::chrono::steady_clock::now(); + while (!Game::ShouldShutdown()) { + // The poll's wait for network traffic is also this loop's sleep + Game::web.ReceiveRequests(10); + const auto now = std::chrono::steady_clock::now(); + if (now - lastTick < std::chrono::milliseconds(16)) continue; + lastTick = now; + + Packet* packet = g_Server->ReceiveFromMaster(); + while (packet) { + g_Server->DeallocateMasterPacket(packet); + packet = g_Server->ReceiveFromMaster(); + } + processor.Update(); + } + + LOG("Stopping the UGC server"); + processor.Stop(); + Game::web.Shutdown(); + g_Processor = nullptr; + Database::Destroy("UgcServer"); + delete g_Server; + g_Server = nullptr; + Game::server = nullptr; + delete Game::logger; + Game::logger = nullptr; + delete Game::config; + Game::config = nullptr; + return EXIT_SUCCESS; +} diff --git a/dUgcServer/UgcStorage.cpp b/dUgcServer/UgcStorage.cpp new file mode 100644 index 000000000..0aaa4e83f --- /dev/null +++ b/dUgcServer/UgcStorage.cpp @@ -0,0 +1,132 @@ +#include "UgcStorage.h" + +#include +#include +#include +#include + +namespace { + constexpr std::array KNOWN_FILES = { + "model.nif", "model.nif.gz", "model.nif.checksum", + "model.lxfml.gz", "model.lxfml.checksum", + "icon.dds.gz", "icon.dds.checksum", "icon.png", + }; + + const char* KindFolder(UgcStorage::Kind kind) { + return kind == UgcStorage::Kind::MODEL ? "models" : "modular"; + } + + std::string RandomSuffix() { + static thread_local std::mt19937_64 random{ std::random_device{}() }; + return std::to_string(random()); + } +} + +UgcStorage::UgcStorage(std::filesystem::path root) : m_Root(std::move(root)) {} + +bool UgcStorage::IsKnownFile(const std::string& name) { + return std::find(KNOWN_FILES.begin(), KNOWN_FILES.end(), name) != KNOWN_FILES.end(); +} + +std::filesystem::path UgcStorage::Folder(Kind kind, LWOOBJID id) const { + const auto id64 = static_cast(id); + return m_Root / KindFolder(kind) / std::to_string(id64 % 1000) / std::to_string(id64); +} + +std::optional UgcStorage::File(Kind kind, LWOOBJID id, const std::string& name) const { + if (!IsKnownFile(name)) return std::nullopt; + auto path = Folder(kind, id) / name; + std::error_code error; + if (!std::filesystem::is_regular_file(path, error)) return std::nullopt; + return path; +} + +std::optional UgcStorage::Write(Kind kind, LWOOBJID id, const Files& files, std::string& error) const { + const auto folder = Folder(kind, id); + const auto temporary = folder.parent_path() / (".tmp-" + folder.filename().string() + "-" + RandomSuffix()); + std::error_code code; + std::filesystem::create_directories(temporary, code); + if (code) { + error = "could not create " + temporary.string() + ": " + code.message(); + return std::nullopt; + } + uint64_t bytes = 0; + for (const auto& [name, data] : files) { + std::ofstream out(temporary / name, std::ios::binary | std::ios::trunc); + out.write(data.data(), static_cast(data.size())); + if (!out) { + error = "could not write " + (temporary / name).string(); + std::filesystem::remove_all(temporary, code); + return std::nullopt; + } + bytes += data.size(); + } + // Swap the old folder out and the new one in; the old one is deleted after + const auto old = folder.parent_path() / (".old-" + folder.filename().string() + "-" + RandomSuffix()); + const bool hadOld = std::filesystem::exists(folder, code); + if (hadOld) std::filesystem::rename(folder, old, code); + std::filesystem::rename(temporary, folder, code); + if (code) { + error = "could not move the files into " + folder.string() + ": " + code.message(); + std::filesystem::remove_all(temporary, code); + return std::nullopt; + } + if (hadOld) std::filesystem::remove_all(old, code); + return bytes; +} + +void UgcStorage::Remove(Kind kind, LWOOBJID id) const { + std::error_code error; + std::filesystem::remove_all(Folder(kind, id), error); +} + +void UgcStorage::Touch(Kind kind, LWOOBJID id) const { + std::error_code error; + std::filesystem::last_write_time(Folder(kind, id), std::filesystem::file_time_type::clock::now(), error); +} + +std::vector UgcStorage::List() const { + std::vector entries; + std::error_code error; + for (const auto kind : { Kind::MODEL, Kind::MODULAR }) { + const auto base = m_Root / KindFolder(kind); + for (std::filesystem::directory_iterator bucket(base, error), end; !error && bucket != end; bucket.increment(error)) { + if (!bucket->is_directory(error)) continue; + for (std::filesystem::directory_iterator item(bucket->path(), error), itemEnd; !error && item != itemEnd; item.increment(error)) { + const auto name = item->path().filename().string(); + if (name.empty() || name[0] == '.' || !item->is_directory(error)) continue; + Entry entry; + entry.kind = kind; + try { + entry.id = static_cast(std::stoull(name)); + } catch (...) { + continue; + } + entry.used = std::filesystem::last_write_time(item->path(), error); + for (std::filesystem::directory_iterator file(item->path(), error), fileEnd; !error && file != fileEnd; file.increment(error)) { + if (file->is_regular_file(error)) entry.bytes += file->file_size(error); + } + entries.push_back(entry); + } + error.clear(); + } + error.clear(); + } + return entries; +} + +std::vector UgcStorage::Evict(uint64_t maxBytes) const { + auto entries = List(); + uint64_t total = 0; + for (const auto& entry : entries) total += entry.bytes; + std::vector removed; + if (total <= maxBytes) return removed; + std::sort(entries.begin(), entries.end(), [](const Entry& a, const Entry& b) { return a.used < b.used; }); + for (const auto& entry : entries) { + if (total <= maxBytes) break; + Remove(entry.kind, entry.id); + total -= std::min(total, entry.bytes); + removed.push_back(entry); + } + return removed; +} diff --git a/dUgcServer/UgcStorage.h b/dUgcServer/UgcStorage.h new file mode 100644 index 000000000..f3caac4d0 --- /dev/null +++ b/dUgcServer/UgcStorage.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "dCommonVars.h" + +/** + * Where the UGC server keeps what it makes: /models/// for player models and + * /modular/// for modular builds. An item's files are written to a temporary folder and renamed + * into place, so a request never sees half of them. Plain file operations, safe from any thread for different items. + */ +class UgcStorage { +public: + enum class Kind { MODEL, MODULAR }; + + // The files an item has, by name (see FileName) + using Files = std::map; + + explicit UgcStorage(std::filesystem::path root); + + const std::filesystem::path& GetRoot() const { return m_Root; } + + std::filesystem::path Folder(Kind kind, LWOOBJID id) const; + + // A file of an item, if it is there; `name` must be one of the names the server writes + std::optional File(Kind kind, LWOOBJID id, const std::string& name) const; + + // Replaces an item's files with `files`; the bytes written, nullopt (and `error`) on failure + std::optional Write(Kind kind, LWOOBJID id, const Files& files, std::string& error) const; + + // Deletes an item's files + void Remove(Kind kind, LWOOBJID id) const; + + // Marks an item used now, for eviction + void Touch(Kind kind, LWOOBJID id) const; + + struct Entry { + Kind kind{}; + LWOOBJID id{}; + uint64_t bytes{}; + std::filesystem::file_time_type used{}; + }; + + // Everything stored, with sizes and when it was last used + std::vector List() const; + + // Removes the least recently used items until the total is at most `maxBytes`; what was removed + std::vector Evict(uint64_t maxBytes) const; + + // Names the server writes (and serves); anything else is refused + static bool IsKnownFile(const std::string& name); + +private: + std::filesystem::path m_Root; +}; diff --git a/docs/UgcServer.md b/docs/UgcServer.md index 47966fde8..3a48a9f53 100644 --- a/docs/UgcServer.md +++ b/docs/UgcServer.md @@ -77,13 +77,16 @@ Modular builds (`ugc_modular_build` rows, `ldf_config` like `1:4713+1:4714+1:471 Files live under `ugc_output_dir` (default `ugc` next to the server binaries): ``` -ugc/models///model.nif(.gz, .checksum), model.lxfml(.gz, .checksum), icon.dds(.gz, .checksum), icon.png -ugc/modular///icon.dds(.gz, .checksum), icon.png +ugc/models///model.nif, model.nif.gz, model.nif.checksum, model.lxfml.gz, model.lxfml.checksum, + icon.dds.gz, icon.dds.checksum, icon.png +ugc/modular///icon.dds.gz, icon.dds.checksum, icon.png ``` A model's files are written to a temporary folder and renamed into place, so a half written model is never served. `ugc_max_storage_mb` (default 2048, 0 for no limit) caps the folder: when it is over, the models whose files were used -longest ago are deleted and marked unprocessed, and are made again the next time they are asked for. +longest ago are deleted. Their rows stay `is_optimized = 1`; when something asks for their files the server sets +them back to 0 and makes them again (the request answers 408 meanwhile), so deleted files are only made again when +wanted. ## Database state @@ -102,6 +105,15 @@ The database is the queue: the UGC server looks for rows with `is_optimized = 0` `max_attempts` (default 3) times. Reprocessing (dashboard) sets rows back to `is_optimized = 0, process_attempts = 0`. No master messages are needed for any of it. +### Marking models for processing (for code that writes `ugc` rows) + +* A new row needs nothing: `is_optimized` defaults to 0 (`InsertNewUgcModel` writes 0). +* When a model's LXFML changes, `UpdateUgcModelData` sets `is_optimized = 0, process_attempts = 0, process_error = ''` + in the same statement. Anything that writes `ugc.lxfml` another way must do the same, or call + `ResetUgcModelProcessing(id, false)`. +* Deleting the row is enough when a model is deleted; its files are left until the storage cap removes them. +* The same holds for `ugc_modular_build` (`InsertUgcBuild` rows start at 0; `ResetModularBuildProcessing`). + ## Threads The main thread owns RakNet (the master link), mongoose (HTTP) and the database. `worker_threads` (default: half the @@ -119,11 +131,14 @@ files are what every player on a property sees anyway): `/UGCC/IMAGE128DDS/.dds.gz|.checksum` for the client (`client_path`, default `/ugc`, is the client's `UGCSERVERDIR`). HKX answers 404. A model that exists but isn't made yet (or was evicted) is moved to the front of the queue and answers 408 so the client asks again. -* `/files/model//icon.png|model.nif|model.lxfml` and `/files/modular//icon.png` for the dashboard's previews +* `/files/model//icon.png|model.nif` and `/files/modular//icon.png` for the dashboard's previews (with `Access-Control-Allow-Origin: *`). * `/status`: JSON with the queue length, what the workers are doing and totals since start. -Files are sent from disk (mongoose streams them) with `Cache-Control: public, max-age=86400` and an ETag of the MD5. +Files are sent from disk (mongoose streams them, with its own ETag) with `Cache-Control: public, max-age=3600`. + +`UgcServer --make-model ` and `UgcServer --make-modular "1:4713+1:4714+1:4715" ` +make one item's files into a folder without a database, for trying settings. ## Dashboard diff --git a/migrations/dlu/mysql/81_ugc_processing.sql b/migrations/dlu/mysql/81_ugc_processing.sql new file mode 100644 index 000000000..e59d89756 --- /dev/null +++ b/migrations/dlu/mysql/81_ugc_processing.sql @@ -0,0 +1,57 @@ +/* UGC server state (see docs/UgcServer.md). ugc.is_optimized: 0 waiting to be made, 1 made, 2 failed; processed_at is + the last attempt (Unix seconds), process_attempts how many there have been and process_error the last failure. The + same columns on ugc_modular_build for the builds' icons. Columns are only added when they are not there yet. */ +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN processed_at BIGINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'processed_at'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; + +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN process_attempts INT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'process_attempts'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; + +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN process_error VARCHAR(1024) NOT NULL DEFAULT ''''', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'process_error'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; + +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN is_optimized INT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'is_optimized'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; + +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN processed_at BIGINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'processed_at'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; + +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN process_attempts INT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'process_attempts'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; + +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN process_error VARCHAR(1024) NOT NULL DEFAULT ''''', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'process_error'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; + +/* What the UGC server looks for */ +SET @dlu_index = (SELECT IF(COUNT(*) = 0, 'CREATE INDEX ugc_is_optimized ON ugc (is_optimized)', 'DO 0') FROM information_schema.statistics + WHERE table_schema = DATABASE() AND table_name = 'ugc' AND index_name = 'ugc_is_optimized'); +PREPARE dlu_index_stmt FROM @dlu_index; +EXECUTE dlu_index_stmt; +DEALLOCATE PREPARE dlu_index_stmt; + +SET @dlu_index = (SELECT IF(COUNT(*) = 0, 'CREATE INDEX ugc_modular_build_is_optimized ON ugc_modular_build (is_optimized)', 'DO 0') FROM information_schema.statistics + WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND index_name = 'ugc_modular_build_is_optimized'); +PREPARE dlu_index_stmt FROM @dlu_index; +EXECUTE dlu_index_stmt; +DEALLOCATE PREPARE dlu_index_stmt; diff --git a/migrations/dlu/sqlite/64_ugc_processing.sql b/migrations/dlu/sqlite/64_ugc_processing.sql new file mode 100644 index 000000000..9d4f3a42f --- /dev/null +++ b/migrations/dlu/sqlite/64_ugc_processing.sql @@ -0,0 +1,10 @@ +/* UGC server state: see the MySQL migration. */ +ALTER TABLE ugc ADD COLUMN processed_at BIGINT NOT NULL DEFAULT 0; +ALTER TABLE ugc ADD COLUMN process_attempts INTEGER NOT NULL DEFAULT 0; +ALTER TABLE ugc ADD COLUMN process_error TEXT NOT NULL DEFAULT ''; +ALTER TABLE ugc_modular_build ADD COLUMN is_optimized INTEGER NOT NULL DEFAULT 0; +ALTER TABLE ugc_modular_build ADD COLUMN processed_at BIGINT NOT NULL DEFAULT 0; +ALTER TABLE ugc_modular_build ADD COLUMN process_attempts INTEGER NOT NULL DEFAULT 0; +ALTER TABLE ugc_modular_build ADD COLUMN process_error TEXT NOT NULL DEFAULT ''; +CREATE INDEX IF NOT EXISTS ugc_is_optimized ON ugc (is_optimized); +CREATE INDEX IF NOT EXISTS ugc_modular_build_is_optimized ON ugc_modular_build (is_optimized); diff --git a/resources/masterconfig.ini b/resources/masterconfig.ini index 1c65584d4..4bc2fe095 100644 --- a/resources/masterconfig.ini +++ b/resources/masterconfig.ini @@ -15,3 +15,7 @@ master_password=3.25DARKFLAME1 # Enable the web dashboard (0 = disabled, 1 = enabled) # Dashboard settings are configured in dashboardconfig.ini enable_dashboard=0 + +# Start the UGC server, which makes and serves the meshes and icons of what players build (0 = disabled, 1 = enabled) +# Its settings are in ugcconfig.ini; see docs/UgcServer.md for the client's boot.cfg settings +enable_ugc_server=0 diff --git a/resources/ugcconfig.ini b/resources/ugcconfig.ini new file mode 100644 index 000000000..286d82935 --- /dev/null +++ b/resources/ugcconfig.ini @@ -0,0 +1,48 @@ +# UGC server: makes the meshes and icons of what players build and serves them to the game client. +# The master starts it when enable_ugc_server=1 (masterconfig.ini). See docs/UgcServer.md. + +# HTTP port the game client downloads from (the client's UGCSERVERPORT) +port=2008 + +# The IP address to bind to. The game client has to reach it, so the default is every interface. +listen_ip=0.0.0.0 + +# UDP port for the connection to the master server (the next port is used too). It must not clash with another +# server's ports: auth uses 1500-1501, chat 2005-2006, the dashboard 2010-2011, worlds 3000 and up. +net_port=2012 + +# The client's UGCSERVERDIR: the path the downloads are under (the client adds /UGCC/...) +client_path=/ugc + +# Where the made files are kept (relative to the server binaries), and at most how much space they may take in MB. +# When there's more, the files used longest ago are deleted; they're made again when asked for. 0: no limit. +ugc_output_dir=ugc +ugc_max_storage_mb=2048 + +# Worker threads making files (0: half the CPU cores) +worker_threads=0 + +# How often to look for models to make (milliseconds), how many to take at once, and how often a model may fail +# before it's given up on (the dashboard can try again) +poll_interval_ms=2000 +poll_batch=32 +max_attempts=3 + +# Brick detail used for meshes and icons: 0 (most detailed) to 2 +brick_lod=0 + +# Remove faces that can't be seen from anywhere, and darken hidden corners (ambient occlusion baked into the vertex +# colors): 0 or 1. ao_strength: 0 (none) to 1. optimize_resolution: detail of the renders that decide both. +remove_hidden_faces=1 +bake_ao=1 +ao_strength=0.6 +optimize_resolution=1024 + +# Icons: size in pixels, the camera's angle around (yaw, from the model's front) and above (pitch) the model in degrees, +# and the empty border (1 fills the icon). Modular builds (cars, rockets) have their own angles. +icon_size=128 +icon_yaw=35 +icon_pitch=25 +icon_margin=1.03 +modular_icon_yaw=35 +modular_icon_pitch=20 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ce46ac973..4cff02d34 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -9,3 +9,4 @@ add_subdirectory(dCommonTests) add_subdirectory(dGameTests) add_subdirectory(dWebTests) add_subdirectory(dDatabaseTests) +add_subdirectory(dUgcTests) diff --git a/tests/dCommonTests/MessageIdPinTests.cpp b/tests/dCommonTests/MessageIdPinTests.cpp index e32d58342..1f742dbf9 100644 --- a/tests/dCommonTests/MessageIdPinTests.cpp +++ b/tests/dCommonTests/MessageIdPinTests.cpp @@ -1833,6 +1833,7 @@ static_assert(static_cast(ServiceType::WORLD) == 4); static_assert(static_cast(ServiceType::CLIENT) == 5); static_assert(static_cast(ServiceType::MASTER) == 6); static_assert(static_cast(ServiceType::UNKNOWN) == 7); +static_assert(static_cast(ServiceType::UGC) == 8); // The asserts above are compile time; this test only makes the file show up in test runs. TEST(MessageIdPinTests, MessageTypeValuesArePinned) { SUCCEED(); } diff --git a/tests/dDatabaseTests/DatabaseParityTests.cpp b/tests/dDatabaseTests/DatabaseParityTests.cpp index c2d96af44..6867144a3 100644 --- a/tests/dDatabaseTests/DatabaseParityTests.cpp +++ b/tests/dDatabaseTests/DatabaseParityTests.cpp @@ -704,6 +704,50 @@ TEST_F(ParitySeeded, UgcModel) { auto model = db.GetUgcModel(1152921510000500001LL); return model ? json{ model->id, model->modelID, model->lxfmlData.str() } : json(); }); + + // The UGC server's processing state (processed_at is the time of the call, so only whether it's set is compared) + const auto infoJson = [](const std::optional& info) { + return info ? json{ info->id, info->characterId, info->characterName, static_cast(info->state), info->attempts, + info->processedAt > 0, info->error, info->bakeAo, info->details } : json(); + }; + const auto countsJson = [](const std::vector>& counts) { + json out = json::array(); + for (const auto& [state, count] : counts) out.push_back({ static_cast(state), count }); + return out; + }; + Both("GetUgcModelsToProcess", [](GameDatabase& db) { + json out = json::array(); + for (const auto& model : db.GetUgcModelsToProcess(10)) out.push_back({ model.id, model.lxfml, model.attempts }); + return out; + }); + Both("SetUgcModelProcessed failed", [&](GameDatabase& db) { + db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::FAILED, 3, "no bricks", false); + return json{ infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), countsJson(db.GetUgcProcessCounts()), db.GetUgcModelsToProcess(10).size() }; + }); + Both("GetUgcProcessList", [&](GameDatabase& db) { + json out = json::array(); + for (const auto& info : db.GetUgcProcessList(IUgc::eProcessState::FAILED, 0, 10)) out.push_back(infoJson(info)); + out.push_back(db.GetUgcProcessList(std::nullopt, 0, 10).size()); + out.push_back(db.GetUgcProcessList(IUgc::eProcessState::DONE, 0, 10).size()); + return out; + }); + Both("ResetUgcModelProcessing", [&](GameDatabase& db) { + const auto changed = db.ResetUgcModelProcessing(std::nullopt, true); + db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::DONE, 1, "", true); + return json{ changed, infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), infoJson(db.GetUgcProcessInfo(1)) }; + }); + Both("Modular build processing", [&](GameDatabase& db) { + db.InsertUgcBuild("1:4713+1:4714+1:4715", 1152921510000500002LL, CHAR_ALICE); + json out = json::array(); + for (const auto& build : db.GetModularBuildsToProcess(10)) out.push_back({ build.id, build.modules, build.attempts }); + db.SetModularBuildProcessed(1152921510000500002LL, IUgc::eProcessState::PENDING, 1, "no mesh"); + out.push_back(infoJson(db.GetModularBuildProcessInfo(1152921510000500002LL))); + out.push_back(countsJson(db.GetModularBuildProcessCounts())); + out.push_back(db.GetModularBuildProcessList(std::nullopt, 0, 10).size()); + out.push_back(db.ResetModularBuildProcessing(1152921510000500002LL, false)); + out.push_back(infoJson(db.GetModularBuildProcessInfo(1152921510000500002LL))); + return out; + }); } TEST_F(ParitySeeded, LogsAndAudit) { diff --git a/tests/dUgcTests/CMakeLists.txt b/tests/dUgcTests/CMakeLists.txt new file mode 100644 index 000000000..f7cd8925d --- /dev/null +++ b/tests/dUgcTests/CMakeLists.txt @@ -0,0 +1,3 @@ +add_executable(dUgcTests "UgcTests.cpp") +target_link_libraries(dUgcTests ${COMMON_LIBRARIES} dUgc GTest::gtest_main) +gtest_discover_tests(dUgcTests) diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp new file mode 100644 index 000000000..08e23cb1d --- /dev/null +++ b/tests/dUgcTests/UgcTests.cpp @@ -0,0 +1,291 @@ +#include + +#include +#include +#include + +#include + +#include "Game.h" +#include "NifFile.h" +#include "UgcBricks.h" +#include "UgcFormats.h" +#include "UgcModel.h" +#include "UgcModular.h" +#include "UgcRender.h" +#include "UgcStorage.h" +#include "ZCompression.h" + +class Logger; +class dConfig; +namespace Game { + Logger* logger = nullptr; + dConfig* config = nullptr; +} + +namespace { + // A closed box as an LDD .g file: 8 corners, 12 triangles + std::string BoxGeometry(glm::vec3 min, glm::vec3 max) { + std::vector positions, normals; + for (int i = 0; i < 8; i++) { + const glm::vec3 p((i & 1) ? max.x : min.x, (i & 2) ? max.y : min.y, (i & 4) ? max.z : min.z); + const auto n = glm::normalize(p - (min + max) * 0.5f); + positions.insert(positions.end(), { p.x, p.y, p.z }); + normals.insert(normals.end(), { n.x, n.y, n.z }); + } + const std::vector indices = { 0, 2, 1, 1, 2, 3, 4, 5, 6, 5, 7, 6, 0, 1, 4, 1, 5, 4, 2, 6, 3, 3, 6, 7, 0, 4, 2, 2, 4, 6, 1, 3, 5, 3, 7, 5 }; + std::string out; + const int32_t header[4] = { 0x42473031, 8, static_cast(indices.size()), 0 }; + out.append(reinterpret_cast(header), sizeof(header)); + out.append(reinterpret_cast(positions.data()), positions.size() * 4); + out.append(reinterpret_cast(normals.data()), normals.size() * 4); + out.append(reinterpret_cast(indices.data()), indices.size() * 4); + return out; + } + + std::filesystem::path TempFolder(const std::string& name) { + auto path = std::filesystem::temp_directory_path() / ("dlu_ugc_test_" + name + "_" + std::to_string(::testing::UnitTest::GetInstance()->random_seed())); + std::filesystem::remove_all(path); + std::filesystem::create_directories(path); + return path; + } + + // A res folder with brick 3001 (a 1x1x1 box) and brick 3002 (a big box) + std::filesystem::path MakeRes() { + const auto res = TempFolder("res"); + std::filesystem::create_directories(res / "brickprimitives" / "lod0"); + std::ofstream(res / "brickprimitives" / "lod0" / "3001.g", std::ios::binary) << BoxGeometry(glm::vec3(0.0f), glm::vec3(1.0f)); + std::ofstream(res / "brickprimitives" / "lod0" / "3002.g", std::ios::binary) << BoxGeometry(glm::vec3(-4.0f), glm::vec3(4.0f)); + return res; + } + + const char* LXFML5 = R"( + + + + +)"; +} + +TEST(UgcCompression, GzipRoundTrip) { + const std::string data(10000, 'x'); + const auto gz = ZCompression::Gzip(data); + ASSERT_GE(gz.size(), 2u); + EXPECT_EQ(static_cast(gz[0]), 0x1f); + EXPECT_EQ(static_cast(gz[1]), 0x8b); + EXPECT_EQ(ZCompression::Gunzip(gz), data); + EXPECT_FALSE(ZCompression::Gunzip("not gzip")); +} + +TEST(UgcBricks, ParsesGeometryAndRejectsBadData) { + const auto geometry = UgcBricks::ParseGeometry(BoxGeometry(glm::vec3(0.0f), glm::vec3(1.0f))); + ASSERT_TRUE(geometry); + EXPECT_EQ(geometry->positions.size(), 24u); + EXPECT_EQ(geometry->indices.size(), 36u); + EXPECT_FALSE(UgcBricks::ParseGeometry("10GB")); + auto broken = BoxGeometry(glm::vec3(0.0f), glm::vec3(1.0f)); + broken.resize(broken.size() - 4); + EXPECT_FALSE(UgcBricks::ParseGeometry(broken)); +} + +TEST(UgcBricks, ParsesMaterials) { + const auto materials = UgcBricks::ParseMaterials(R"()"); + ASSERT_EQ(materials.size(), 2u); + EXPECT_EQ(materials.at(21).r, 222); + EXPECT_FALSE(materials.at(21).Transparent()); + EXPECT_TRUE(materials.at(40).Transparent()); +} + +TEST(UgcBricks, ReadsStoredZipEntries) { + // A zip with one stored file, "Materials.xml" + const std::string name = "Materials.xml", content = ""; + std::string zip; + const auto u16 = [&zip](uint16_t v) { zip.append(reinterpret_cast(&v), 2); }; + const auto u32 = [&zip](uint32_t v) { zip.append(reinterpret_cast(&v), 4); }; + u32(0x04034b50); u16(20); u16(0); u16(0); u16(0); u16(0); u32(0); u32(content.size()); u32(content.size()); u16(name.size()); u16(0); + zip += name + content; + const auto central = static_cast(zip.size()); + u32(0x02014b50); u16(20); u16(20); u16(0); u16(0); u16(0); u16(0); u32(0); u32(content.size()); u32(content.size()); u16(name.size()); + u16(0); u16(0); u16(0); u16(0); u32(0); u32(0); + zip += name; + const auto centralSize = static_cast(zip.size()) - central; + u32(0x06054b50); u16(0); u16(0); u16(1); u16(1); u32(centralSize); u32(central); u16(0); + EXPECT_EQ(UgcBricks::ReadZipEntry(zip, "materials.XML"), content); + EXPECT_FALSE(UgcBricks::ReadZipEntry(zip, "Other.xml")); +} + +TEST(UgcModel, ParsesLxfml5And4) { + std::string error; + const auto parts = UgcModel::ParseLxfml(LXFML5, error); + ASSERT_EQ(parts.size(), 3u); + EXPECT_EQ(parts[0].designId, 3001u); + EXPECT_EQ(parts[0].materials, (std::vector{ 21, 21 })); // 0: the part's first material + EXPECT_FLOAT_EQ(parts[0].transform[3].x, 10.0f); + + const auto v4 = UgcModel::ParseLxfml(R"( + )", error); + ASSERT_EQ(v4.size(), 1u); + const auto origin = v4[0].transform * glm::vec4(0, 0, 0, 1); + EXPECT_NEAR(origin.x, 1.0f, 1e-5f); + EXPECT_NEAR(origin.y, 2.0f, 1e-5f); + + EXPECT_TRUE(UgcModel::ParseLxfml("{ 9999 }); + EXPECT_EQ(model.opaque.TriangleCount(), 12u); + EXPECT_EQ(model.transparent.TriangleCount(), 12u); + EXPECT_NEAR(model.opaque.colors[0].r, 222.0f / 255.0f, 1e-5f); + EXPECT_NEAR(model.transparent.colors[0].a, 150.0f / 255.0f, 1e-5f); + EXPECT_NEAR(model.opaque.positions[0].x, 10.0f, 1e-5f); +} + +TEST(UgcModel, SplitsBigMeshes) { + UgcModel::Mesh mesh; + for (uint32_t i = 0; i < 30; i++) { + mesh.positions.push_back(glm::vec3(static_cast(i))); + mesh.normals.push_back(glm::vec3(0, 1, 0)); + mesh.colors.push_back(glm::vec4(1.0f)); + } + for (uint32_t i = 0; i + 2 < 30; i += 3) mesh.indices.insert(mesh.indices.end(), { i, i + 1, i + 2 }); + const auto pieces = UgcModel::Split(mesh, 9, 100); + ASSERT_EQ(pieces.size(), 4u); // 10 triangles, 3 fit per piece + size_t triangles = 0; + for (const auto& piece : pieces) { + EXPECT_LE(piece.positions.size(), 9u); + triangles += piece.TriangleCount(); + } + EXPECT_EQ(triangles, 10u); +} + +TEST(UgcRender, RemovesWhatIsInsideAndDrawsIcons) { + UgcBricks::BrickLibrary library(MakeRes(), 0); + library.SetMaterials({ { 21, { 222, 0, 13, 255 } } }); + std::string error; + // A small box inside the big one: its faces can't be seen + const auto parts = UgcModel::ParseLxfml(R"( + + + )", error); + auto model = UgcModel::Build(parts, library); + ASSERT_EQ(model.opaque.TriangleCount(), 24u); + const auto result = UgcRender::Optimize(model, UgcRender::OptimizeOptions{ 256, true, true, 0.6f }); + EXPECT_EQ(result.trianglesRemoved, 12u); + EXPECT_EQ(model.opaque.TriangleCount(), 12u); + EXPECT_EQ(model.opaque.positions.size(), 8u); + + const auto icon = UgcRender::RenderIcon(model, UgcRender::IconOptions{ 32, 2 }); + ASSERT_EQ(icon.rgba.size(), 32u * 32u * 4u); + EXPECT_EQ(icon.rgba[3], 0); // a corner is background + EXPECT_EQ(icon.rgba[(16 * 32 + 16) * 4 + 3], 255); // the middle is the box + EXPECT_GT(icon.rgba[(16 * 32 + 16) * 4], icon.rgba[(16 * 32 + 16) * 4 + 1]); // red + EXPECT_EQ(UgcRender::SphereDirections().size(), 42u); +} + +TEST(UgcFormats, NifReadsBack) { + UgcModel::Mesh opaque, transparent; + opaque.positions = { { 0, 0, 0 }, { 1, 0, 0 }, { 0, 1, 0 } }; + opaque.normals = { { 0, 0, 1 }, { 0, 0, 1 }, { 0, 0, 1 } }; + opaque.colors = { { 1, 0, 0, 1 }, { 1, 0, 0, 1 }, { 1, 0, 0, 1 } }; + opaque.indices = { 0, 1, 2 }; + transparent = opaque; + for (auto& color : transparent.colors) color.a = 0.5f; + const auto nif = UgcFormats::WriteNif("SceneNode_Model", { { "S01_Opaque_Model", &opaque, false }, { "S01_Alpha_Model", &transparent, true } }); + ASSERT_TRUE(nif.starts_with("Gamebryo File Format, Version 20.3.0.9\n")); + std::string error; + const auto model = NifFile::Parse(nif, 0, error); + ASSERT_TRUE(model) << error; + EXPECT_TRUE(model->skipped.empty()); + ASSERT_EQ(model->meshes.size(), 2u); + EXPECT_EQ(model->meshes[0].indices.size(), 3u); + EXPECT_EQ(model->meshes[0].colors[0], 255); + EXPECT_FALSE(model->meshes[0].material.alphaBlend); + EXPECT_TRUE(model->meshes[1].material.alphaBlend); + EXPECT_EQ(model->meshes[1].colors[3], 128); + EXPECT_EQ(model->meshes[0].material.vertexColorMode, 2); + EXPECT_TRUE(model->nodes.contains("SceneNode_Model")); +} + +TEST(UgcFormats, ImagesAndChecksums) { + UgcRender::Image image{ 2, 2, std::vector(16, 0) }; + image.rgba[0] = 10; // red of the first pixel + image.rgba[3] = 255; + const auto png = UgcFormats::EncodePng(image); + EXPECT_TRUE(png.starts_with("\x89PNG\r\n\x1a\n")); + const auto dds = UgcFormats::EncodeDds(image); + ASSERT_EQ(dds.size(), 128u + 16u); + EXPECT_TRUE(dds.starts_with("DDS ")); + EXPECT_EQ(dds[128 + 2], 10); // stored BGRA + EXPECT_EQ(UgcFormats::Md5Hex("abc"), "900150983cd24fb0d6963f7d28e17f72"); + EXPECT_NE(UgcFormats::ChecksumXml("abc").find("900150983cd24fb0d6963f7d28e17f723"), std::string::npos); +} + +TEST(UgcModular, ParsesTheCdClientData) { + EXPECT_EQ(UgcModular::ParseModuleLots("1:4713+1:4714+1:4715"), (std::vector{ 4713, 4714, 4715 })); + EXPECT_EQ(UgcModular::ParseModuleLots("1:8129;1:x;1:8130"), (std::vector{ 8129, 8130 })); + const auto build = UgcModular::ParseBuild(R"( + + )"); + ASSERT_TRUE(build); + EXPECT_EQ(build->rootPart, 2u); + ASSERT_EQ(build->connections.size(), 2u); + EXPECT_EQ(build->connections[1].location, "CP_B2"); + const auto connections = UgcModular::ParseModuleConnections(R"()"); + EXPECT_FLOAT_EQ(connections.at("CP_B2").z, 5.2f); + EXPECT_FALSE(UgcModular::ParseBuild("")); +} + +TEST(UgcModular, PutsPartsOnTheirAttachPoints) { + const auto triangle = [] { + NifFile::Model nif; + NifFile::Mesh mesh; + mesh.positions = { 0, 0, 0, 1, 0, 0, 0, 1, 0 }; + mesh.indices = { 0, 1, 2 }; + nif.meshes.push_back(mesh); + return nif; + }; + UgcModular::BuildInfo build; + build.rootPart = 2; + build.connections = { { 2, "CP_A1", 1 }, { 1, "CP_B2", 0 } }; + std::vector modules(3); + modules[0].partCode = 2; // bottom: node CP_A1 at y 5 + modules[0].nif = triangle(); + modules[0].nif.nodes["CP_A1"].translation = { 0, 5, 0 }; + modules[1].partCode = 1; // middle: no node, a connection offset of y 3 in the CDClient + modules[1].nif = triangle(); + modules[1].connections["CP_B2"] = glm::vec3(0, 3, 0); + modules[2].partCode = 0; // top: its own CP_B2 node at y 1 lines up with the middle's + modules[2].nif = triangle(); + modules[2].nif.nodes["CP_B2"].translation = { 0, 1, 0 }; + std::string warnings; + const auto model = UgcModular::Assemble(build, modules, warnings); + EXPECT_TRUE(warnings.empty()) << warnings; + ASSERT_EQ(model.opaque.positions.size(), 9u); + EXPECT_FLOAT_EQ(model.opaque.positions[0].y, 0.0f); + EXPECT_FLOAT_EQ(model.opaque.positions[3].y, 5.0f); + EXPECT_FLOAT_EQ(model.opaque.positions[6].y, 7.0f); // 5 + 3 - 1 +} + +TEST(UgcStorage, WritesListsAndEvicts) { + UgcStorage storage(TempFolder("storage")); + std::string error; + ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODEL, 1001, { { "icon.png", std::string(100, 'a') } }, error)) << error; + ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODULAR, 2002, { { "icon.png", std::string(100, 'b') } }, error)) << error; + ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODEL, 1001, { { "icon.png", std::string(50, 'c') } }, error)) << error; // replaced + EXPECT_TRUE(storage.File(UgcStorage::Kind::MODEL, 1001, "icon.png")); + EXPECT_FALSE(storage.File(UgcStorage::Kind::MODEL, 1001, "../../etc/passwd")); + EXPECT_EQ(storage.List().size(), 2u); + std::filesystem::last_write_time(storage.Folder(UgcStorage::Kind::MODULAR, 2002), std::filesystem::file_time_type::clock::now() - std::chrono::hours(1)); + const auto removed = storage.Evict(60); + ASSERT_EQ(removed.size(), 1u); + EXPECT_EQ(removed[0].id, 2002); + EXPECT_TRUE(storage.File(UgcStorage::Kind::MODEL, 1001, "icon.png")); + std::filesystem::remove_all(storage.GetRoot()); +} diff --git a/tests/dWebTests/CMakeLists.txt b/tests/dWebTests/CMakeLists.txt index 4246977ed..40b13c1b6 100644 --- a/tests/dWebTests/CMakeLists.txt +++ b/tests/dWebTests/CMakeLists.txt @@ -38,7 +38,6 @@ set(DWEBTESTS_SOURCES "WorkerPoolTests.cpp" "OnceCacheTests.cpp" "${PROJECT_SOURCE_DIR}/dDashboardServer/routes/WorkerPool.cpp" - "${PROJECT_SOURCE_DIR}/dDashboardServer/routes/NifFile.cpp" "BackupFilesTests.cpp" "SecurityFixesTests.cpp" "${PROJECT_SOURCE_DIR}/dDashboardServer/routes/BackupFiles.cpp"