From 79235702b312215eaf95a840184e32d67b0996b0 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sun, 27 Sep 2026 17:46:47 -0500 Subject: [PATCH] feat(ugc): answer the client's UGC manifest requests without 3D services With UGCUSE3DSERVICES=7:0 (the client's default) the client asks its world for a blueprint file's MD5 and size (REQUEST_UGC_MANIFEST_INFO, world 27) and then downloads BrickModels/UserMade//..sd0. Layouts checked in the 1.10.64 client: the request is a u64 blueprint id and a u8 resource type; the answer (UGC_MANIFEST_RESPONSE, client 60) repeats them and adds a u8 valid, the u32 size and the 16 byte MD5 of the inflated file, 37 bytes after the 0x53 exactly or the client drops it. - dNet: WorldPackets::RequestUgcManifestInfo, ClientPackets::UgcManifestResponse (eUgcResourceType), with byte tests against the client's layouts. - Database: ugc_file_checksums (per model or module combination and file) and ugc_modular_build.combination_id (migrations 89 / 72), GetUgcFileChecksum looks a blueprint up as a model, else as a build through its combination. - UGC server: every download is also written as .sd0 (Sd0::Compress); workers hand the checksums back and the main thread stores them; old items get their sd0 icon and checksum, and builds their combination id, once at start-up, a few per tick; serves /BrickModels/UserMade//..sd0 under client_path, //UserBrickModels and the root (.hkx 404). - World: UgcManifest answers on the main thread with one indexed query per request; files not made yet are answered when they are (looked at again every 5 seconds), and a model in its quiet period is made right away. No worker threads, HTTP or file reads in the world. Off by default (ugc_manifest=0): checked in game, the client then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says and logs the player out when it can't connect, so icons need the UGC server on port 80 of each player's machine. docs/UgcServer.md has the details. Co-Authored-By: Claude Opus 5.5 --- dCommon/dEnums/eUgcResourceType.h | 15 +++ dDashboardServer/routes/SettingsCatalog.cpp | 1 + dDatabase/GameDatabase/ITables/IUgc.h | 21 ++++ .../GameDatabase/ITables/IUgcModularBuild.h | 6 + dDatabase/GameDatabase/MySQL/MySQLDatabase.h | 4 + dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp | 13 +++ .../MySQL/Tables/UgcModularBuild.cpp | 13 +++ .../GameDatabase/SQLite/SQLiteDatabase.h | 4 + dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp | 13 +++ .../SQLite/Tables/UgcModularBuild.cpp | 14 +++ .../GameDatabase/TestSQL/TestSQLDatabase.h | 4 + dGame/dUtilities/CMakeLists.txt | 1 + dGame/dUtilities/UgcManifest.cpp | 102 ++++++++++++++++ dGame/dUtilities/UgcManifest.h | 29 +++++ dNet/ClientPackets.cpp | 19 +++ dNet/ClientPackets.h | 19 +++ dNet/WorldPackets.cpp | 11 ++ dNet/WorldPackets.h | 12 ++ dUgcServer/UgcFormats.cpp | 22 ++++ dUgcServer/UgcFormats.h | 4 + dUgcServer/UgcJobs.cpp | 3 + dUgcServer/UgcProcessor.cpp | 78 ++++++++++++- dUgcServer/UgcProcessor.h | 18 +++ dUgcServer/UgcServer.cpp | 63 ++++++++++ dUgcServer/UgcStorage.cpp | 4 +- dWorldServer/WorldServer.cpp | 11 ++ docs/UgcServer.md | 109 ++++++++++++++++-- .../dlu/mysql/89_ugc_file_checksums.sql | 19 +++ .../dlu/sqlite/72_ugc_file_checksums.sql | 11 ++ resources/sharedconfig.ini | 6 + tests/dDatabaseTests/DatabaseParityTests.cpp | 15 +++ .../dNetTests/WorldPacketsTests.cpp | 32 +++++ tests/dUgcTests/UgcTests.cpp | 24 ++++ 33 files changed, 706 insertions(+), 14 deletions(-) create mode 100644 dCommon/dEnums/eUgcResourceType.h create mode 100644 dGame/dUtilities/UgcManifest.cpp create mode 100644 dGame/dUtilities/UgcManifest.h create mode 100644 migrations/dlu/mysql/89_ugc_file_checksums.sql create mode 100644 migrations/dlu/sqlite/72_ugc_file_checksums.sql diff --git a/dCommon/dEnums/eUgcResourceType.h b/dCommon/dEnums/eUgcResourceType.h new file mode 100644 index 000000000..7c0b727aa --- /dev/null +++ b/dCommon/dEnums/eUgcResourceType.h @@ -0,0 +1,15 @@ +#ifndef EUGCRESOURCETYPE_H +#define EUGCRESOURCETYPE_H + +#include + +// The kinds of file the client keeps for a blueprint (a player's model or a car or rocket), as it numbers them in +// REQUEST_UGC_MANIFEST_INFO and UGC_MANIFEST_RESPONSE and in its download paths +enum class eUgcResourceType : uint8_t { + LXFML = 0, + NIF, + HKX, + DDS +}; + +#endif //!EUGCRESOURCETYPE_H diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index 431554f7c..3cc4cd9d9 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -417,6 +417,7 @@ namespace { c.Add(Format(Text(UGC, "listen_ip", "Listen address", "The game client has to reach it.", "0.0.0.0", true), eFormat::HOST)); c.Add(Port(UGC, "net_port", "Master connection port", "UDP; the next port is used too.", "2012")); c.Add(Text(UGC, "client_path", "Download path", "The game client's UGCSERVERDIR.", "/ugc", true)); + c.Add(Bool(SHARED, "ugc_manifest", "Answer clients without 3D services", "Worlds tell game clients with UGCUSE3DSERVICES=7:0 (the default) the checksums of the files the UGC server made, so they download icons of cars, rockets and models. The client then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and one that can't connect there is logged out: only turn on when the UGC server answers there for every player.", false)); c.Add(Format(Text(DASHBOARD, "ugc_internal_url", "UGC server address (internal)", "Where the dashboard itself reaches the UGC server for its status and files. Empty: http://127.0.0.1:2008.", ""), eFormat::URL)); c.Add(Format(Text(DASHBOARD, "ugc_public_url", "UGC server address (public)", "Only for the \"open on the UGC server\" links, e.g. https://ugc.example.com. The dashboard's own pages don't need the browser to reach it.", ""), eFormat::URL)); c.AddSection("UGC processing", "When models are made and how much of the machine the workers may use.", eLayout::ROWS, Condition{ MASTER, "enable_ugc_server", { "1" } }); diff --git a/dDatabase/GameDatabase/ITables/IUgc.h b/dDatabase/GameDatabase/ITables/IUgc.h index bbcc81e08..902719339 100644 --- a/dDatabase/GameDatabase/ITables/IUgc.h +++ b/dDatabase/GameDatabase/ITables/IUgc.h @@ -107,5 +107,26 @@ public: // How many models are in each state virtual std::vector> GetUgcProcessCounts() = 0; + + // ---- Checksums of the files the UGC server made (ugc_file_checksums), for the client's manifest requests ---- + + // What the files are stored under: a player model's ugc id, or a combination of car or rocket modules + enum class eFileOwner : int32_t { + MODEL = 0, + COMBINATION = 1 + }; + + // A made file as the client has it after inflating the download: its MD5 (32 lowercase hex digits) and size + struct FileChecksum { + std::string md5; + uint32_t size{}; + }; + + // Records (or replaces) the checksum of a made file, e.g. "icon.dds" or "model.nif" + virtual void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) = 0; + + // The checksum of a blueprint's file: a player model's (the blueprint is its ugc id), else a car or rocket build's + // (the blueprint is its ugc_modular_build id; the file is its combination's) + virtual std::optional GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) = 0; }; #endif //!__IUGC__H__ diff --git a/dDatabase/GameDatabase/ITables/IUgcModularBuild.h b/dDatabase/GameDatabase/ITables/IUgcModularBuild.h index f48871a27..bb62b9a27 100644 --- a/dDatabase/GameDatabase/ITables/IUgcModularBuild.h +++ b/dDatabase/GameDatabase/ITables/IUgcModularBuild.h @@ -29,6 +29,12 @@ public: virtual std::vector GetModularBuildProcessList(const std::optional state, const std::string_view search, const uint32_t offset, const uint32_t limit) = 0; virtual std::vector> GetModularBuildProcessCounts() = 0; + // The combination of modules a build was made as (UgcModularKey::StorageId of its modules), whose files it shares + virtual void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) = 0; + + // Up to `limit` builds whose combination isn't recorded yet (combination_id 0): their ids and modules + virtual std::vector GetModularBuildsWithoutCombination(const uint32_t limit) = 0; + // How many builds there are of each ldf_config as stored (the same modules may be written differently) virtual std::vector> GetModularBuildConfigCounts() = 0; diff --git a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h index 5469eba1e..78c629bbc 100644 --- a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h +++ b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h @@ -429,6 +429,10 @@ public: void DeleteUgcIconSettings(const std::string_view target) override; void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override; void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override; + void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override; + std::optional GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override; + void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override; + std::vector GetModularBuildsWithoutCombination(const uint32_t limit) override; std::optional GetUgcProcessInfo(const LWOOBJID id) override; uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override; std::vector GetUgcProcessList(const std::optional state, const std::string_view search, const uint32_t offset, const uint32_t limit) override; diff --git a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp index 62edd6e57..a2270e15d 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/Ugc.cpp @@ -181,3 +181,16 @@ void MySQLDatabase::ExpediteUgcModels(const LWOOBJID characterId) { void MySQLDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) { ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ? WHERE id = ?;", bricks, triangles, id); } + +void MySQLDatabase::SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) { + ExecuteInsert("INSERT INTO ugc_file_checksums (kind, storage_id, file, md5, size) VALUES (?, ?, ?, ?, ?) " + "ON DUPLICATE KEY UPDATE md5 = VALUES(md5), size = VALUES(size);", static_cast(owner), storageId, file, md5, size); +} + +std::optional MySQLDatabase::GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) { + auto result = ExecuteSelect("SELECT md5, size FROM ugc_file_checksums WHERE kind = 0 AND storage_id = ? AND file = ? " + "UNION ALL SELECT f.md5, f.size FROM ugc_modular_build b JOIN ugc_file_checksums f ON f.kind = 1 AND f.storage_id = b.combination_id AND f.file = ? " + "WHERE b.ugc_id = ? AND b.combination_id != 0 LIMIT 1;", blueprintId, file, file, blueprintId); + if (!result->next()) return std::nullopt; + return IUgc::FileChecksum{ std::string(result->getString("md5").c_str()), static_cast(result->getUInt("size")) }; +} diff --git a/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp b/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp index 508a221af..d43ad3dc2 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp @@ -106,3 +106,16 @@ void MySQLDatabase::SetUgcIconSettings(const std::string_view target, const std: void MySQLDatabase::DeleteUgcIconSettings(const std::string_view target) { ExecuteDelete("DELETE FROM ugc_icon_settings WHERE target = ?;", target); } + +void MySQLDatabase::SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) { + ExecuteUpdate("UPDATE ugc_modular_build SET combination_id = ? WHERE ugc_id = ?;", combinationId, id); +} + +std::vector MySQLDatabase::GetModularBuildsWithoutCombination(const uint32_t limit) { + auto result = ExecuteSelect("SELECT ugc_id, ldf_config, process_attempts FROM ugc_modular_build WHERE combination_id = 0 LIMIT ?;", limit); + std::vector builds; + while (result->next()) { + builds.push_back({ result->getInt64("ugc_id"), std::string(result->getString("ldf_config").c_str()), static_cast(result->getInt("process_attempts")) }); + } + return builds; +} diff --git a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h index 6aa6bab30..507836b95 100644 --- a/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h +++ b/dDatabase/GameDatabase/SQLite/SQLiteDatabase.h @@ -413,6 +413,10 @@ public: void DeleteUgcIconSettings(const std::string_view target) override; void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override; void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override; + void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override; + std::optional GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override; + void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override; + std::vector GetModularBuildsWithoutCombination(const uint32_t limit) override; std::optional GetUgcProcessInfo(const LWOOBJID id) override; uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override; std::vector GetUgcProcessList(const std::optional state, const std::string_view search, const uint32_t offset, const uint32_t limit) override; diff --git a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp index e11fe4266..0a26438b3 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/Ugc.cpp @@ -184,3 +184,16 @@ void SQLiteDatabase::ExpediteUgcModels(const LWOOBJID characterId) { void SQLiteDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) { ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ? WHERE id = ?;", bricks, triangles, id); } + +void SQLiteDatabase::SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) { + ExecuteInsert("INSERT INTO ugc_file_checksums (kind, storage_id, file, md5, size) VALUES (?, ?, ?, ?, ?) " + "ON CONFLICT(kind, storage_id, file) DO UPDATE SET md5 = excluded.md5, size = excluded.size;", static_cast(owner), storageId, file, md5, size); +} + +std::optional SQLiteDatabase::GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) { + auto [_, result] = ExecuteSelect("SELECT md5, size FROM ugc_file_checksums WHERE kind = 0 AND storage_id = ? AND file = ? " + "UNION ALL SELECT f.md5, f.size FROM ugc_modular_build b JOIN ugc_file_checksums f ON f.kind = 1 AND f.storage_id = b.combination_id AND f.file = ? " + "WHERE b.ugc_id = ? AND b.combination_id != 0 LIMIT 1;", blueprintId, file, file, blueprintId); + if (result.eof()) return std::nullopt; + return IUgc::FileChecksum{ std::string(result.getStringField("md5", "")), static_cast(result.getInt64Field("size")) }; +} diff --git a/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp b/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp index 3a4490fb6..a069d2908 100644 --- a/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp +++ b/dDatabase/GameDatabase/SQLite/Tables/UgcModularBuild.cpp @@ -112,3 +112,17 @@ void SQLiteDatabase::SetUgcIconSettings(const std::string_view target, const std void SQLiteDatabase::DeleteUgcIconSettings(const std::string_view target) { ExecuteDelete("DELETE FROM ugc_icon_settings WHERE target = ?;", target); } + +void SQLiteDatabase::SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) { + ExecuteUpdate("UPDATE ugc_modular_build SET combination_id = ? WHERE ugc_id = ?;", combinationId, id); +} + +std::vector SQLiteDatabase::GetModularBuildsWithoutCombination(const uint32_t limit) { + auto [_, result] = ExecuteSelect("SELECT ugc_id, ldf_config, process_attempts FROM ugc_modular_build WHERE combination_id = 0 LIMIT ?;", limit); + std::vector builds; + while (!result.eof()) { + builds.push_back({ result.getInt64Field("ugc_id"), result.getStringField("ldf_config", ""), static_cast(result.getIntField("process_attempts")) }); + result.nextRow(); + } + return builds; +} diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h index 2b7a79115..6582ac826 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h @@ -411,6 +411,10 @@ class TestSQLDatabase : public GameDatabase { void DeleteUgcIconSettings(const std::string_view target) override {} void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override {} void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override {} + void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override {} + std::optional GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override { return std::nullopt; } + void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override {} + std::vector GetModularBuildsWithoutCombination(const uint32_t limit) override { return {}; } std::optional GetUgcProcessInfo(const LWOOBJID id) override { return {}; } uint64_t ResetUgcModelProcessing(const std::optional id, const bool failedOnly) override { return 0; } std::vector GetUgcProcessList(const std::optional state, const std::string_view search, const uint32_t offset, const uint32_t limit) override { return {}; } diff --git a/dGame/dUtilities/CMakeLists.txt b/dGame/dUtilities/CMakeLists.txt index eb7351e5e..df9c882e9 100644 --- a/dGame/dUtilities/CMakeLists.txt +++ b/dGame/dUtilities/CMakeLists.txt @@ -17,6 +17,7 @@ set(DGAME_DUTILITIES_SOURCES "BrickDatabase.cpp" "PlayerReports.cpp" "Preconditions.cpp" "SlashCommandHandler.cpp" + "UgcManifest.cpp" "VanityUtilities.cpp" "WorldMigration.cpp") diff --git a/dGame/dUtilities/UgcManifest.cpp b/dGame/dUtilities/UgcManifest.cpp new file mode 100644 index 000000000..86da0e097 --- /dev/null +++ b/dGame/dUtilities/UgcManifest.cpp @@ -0,0 +1,102 @@ +#include "UgcManifest.h" + +#include +#include +#include +#include + +#include "ClientPackets.h" +#include "Database.h" +#include "dConfig.h" +#include "Game.h" +#include "Logger.h" + +namespace { + // How often the waiting requests are looked up again, how long they wait at most and how many there can be + constexpr auto RETRY_INTERVAL = std::chrono::seconds(5); + constexpr auto MAX_WAIT = std::chrono::minutes(15); + constexpr size_t MAX_WAITING = 512; + + struct Waiting { + SystemAddress sysAddr; + LWOOBJID blueprintId{}; + eUgcResourceType resourceType{}; + std::chrono::steady_clock::time_point since; + }; + + std::vector g_Waiting; + std::chrono::steady_clock::time_point g_NextRetry{}; + + // The UGC server's name for a blueprint's file of this type; nothing for the ones it doesn't make + std::optional FileName(eUgcResourceType type) { + switch (type) { + case eUgcResourceType::DDS: return "icon.dds"; + case eUgcResourceType::NIF: return "model.nif"; + default: return std::nullopt; + } + } + + bool Enabled() { + // Off unless set: the 1.10.64 client downloads from http://127.0.0.1:80 whatever its boot.cfg says, and one that + // can't connect there is logged out (docs/UgcServer.md) + return Game::config->GetValue("ugc_manifest") == "1"; + } + + int HexDigit(char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; + } + + // Answers when the file's checksum is known; false when it isn't (yet) + bool TryAnswer(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) { + const auto file = FileName(resourceType); + if (!file) return false; + const auto checksum = Database::Get()->GetUgcFileChecksum(blueprintId, *file); + if (!checksum || checksum->md5.size() != 32) return false; + + ClientPackets::UgcManifestResponse response; + response.blueprintId = blueprintId; + response.resourceType = resourceType; + response.fileSize = checksum->size; + for (size_t i = 0; i < response.md5.size(); i++) { + const auto high = HexDigit(checksum->md5[i * 2]); + const auto low = HexDigit(checksum->md5[i * 2 + 1]); + if (high < 0 || low < 0) return false; + response.md5[i] = static_cast(high << 4 | low); + } + response.valid = true; + response.Send(sysAddr); + LOG_DEBUG("Sent the UGC manifest of %llu (type %i): %s, %u bytes", static_cast(blueprintId), + static_cast(resourceType), checksum->md5.c_str(), checksum->size); + return true; + } +} + +void UgcManifest::OnRequest(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) { + if (sysAddr == UNASSIGNED_SYSTEM_ADDRESS || !Enabled() || !FileName(resourceType)) return; + if (TryAnswer(sysAddr, blueprintId, resourceType)) return; + // Not made yet: a client wants it, so a model still in its quiet period after a save is made now (as when a client + // asks the UGC server for it), and the request is answered when it's made + Database::Get()->ExpediteUgcModel(blueprintId); + for (const auto& waiting : g_Waiting) { + if (waiting.sysAddr == sysAddr && waiting.blueprintId == blueprintId && waiting.resourceType == resourceType) return; + } + if (g_Waiting.size() >= MAX_WAITING) g_Waiting.erase(g_Waiting.begin()); + g_Waiting.push_back({ sysAddr, blueprintId, resourceType, std::chrono::steady_clock::now() }); +} + +void UgcManifest::Update() { + if (g_Waiting.empty()) return; + const auto now = std::chrono::steady_clock::now(); + if (now < g_NextRetry) return; + g_NextRetry = now + RETRY_INTERVAL; + std::erase_if(g_Waiting, [now](const Waiting& waiting) { + return now - waiting.since > MAX_WAIT || TryAnswer(waiting.sysAddr, waiting.blueprintId, waiting.resourceType); + }); +} + +void UgcManifest::OnDisconnect(const SystemAddress& sysAddr) { + std::erase_if(g_Waiting, [&sysAddr](const Waiting& waiting) { return waiting.sysAddr == sysAddr; }); +} diff --git a/dGame/dUtilities/UgcManifest.h b/dGame/dUtilities/UgcManifest.h new file mode 100644 index 000000000..fa9508b12 --- /dev/null +++ b/dGame/dUtilities/UgcManifest.h @@ -0,0 +1,29 @@ +#ifndef UGCMANIFEST_H +#define UGCMANIFEST_H + +#include "dCommonVars.h" +#include "eUgcResourceType.h" +#include "RakNetTypes.h" + +/** + * Answers the client's REQUEST_UGC_MANIFEST_INFO (sent when its UGCUSE3DSERVICES is off, the default): the MD5 and + * size of a blueprint's file, which the client needs before it downloads the file from the UGC server + * (BrickModels/UserMade//..sd0). The checksums are the ones the UGC server stored when it made the + * files (ugc_file_checksums); a file it hasn't made yet is answered once it has (Update looks again every few seconds + * for a while). Nothing is answered for files it never makes (LXFML, HKX), or unless `ugc_manifest` is 1: the client + * then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and is logged out when + * it can't connect there. + * + * Main thread only: each lookup is one indexed query and the answers are sent from here. See docs/UgcServer.md. + */ +namespace UgcManifest { + void OnRequest(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType); + + // Answers the waiting requests whose files have been made since, and forgets the ones waited on too long + void Update(); + + // A client left: its waiting requests are dropped + void OnDisconnect(const SystemAddress& sysAddr); +} + +#endif //!UGCMANIFEST_H diff --git a/dNet/ClientPackets.cpp b/dNet/ClientPackets.cpp index 087e15034..5cd6de794 100644 --- a/dNet/ClientPackets.cpp +++ b/dNet/ClientPackets.cpp @@ -484,6 +484,25 @@ bool ClientPackets::BlueprintLoadItemResponse::Deserialize(RakNet::BitStream& bi return true; } +void ClientPackets::UgcManifestResponse::Serialize(RakNet::BitStream& bitStream) const { + bitStream.Write(blueprintId); + bitStream.Write(resourceType); + bitStream.Write(valid ? 1 : 0); + bitStream.Write(fileSize); + for (const auto byte : md5) bitStream.Write(byte); +} + +bool ClientPackets::UgcManifestResponse::Deserialize(RakNet::BitStream& bitStream) { + VALIDATE_READ(bitStream.Read(blueprintId)); + VALIDATE_READ(bitStream.Read(resourceType)); + uint8_t validByte{}; + VALIDATE_READ(bitStream.Read(validByte)); + valid = validByte != 0; + VALIDATE_READ(bitStream.Read(fileSize)); + for (auto& byte : md5) VALIDATE_READ(bitStream.Read(byte)); + return true; +} + namespace { // A zone ID as three fields: map (u16), instance (u16), clone (u32) void WriteZone(RakNet::BitStream& bitStream, const LWOZONEID& zoneID) { diff --git a/dNet/ClientPackets.h b/dNet/ClientPackets.h index 19eb5ffa8..5814378d7 100644 --- a/dNet/ClientPackets.h +++ b/dNet/ClientPackets.h @@ -6,6 +6,7 @@ #ifndef CLIENTPACKETS_H #define CLIENTPACKETS_H +#include #include #include #include @@ -18,6 +19,7 @@ #include "eAddFriendResponseType.h" #include "eAddIgnoreResponse.h" #include "eBlueprintSaveResponseType.h" +#include "eUgcResourceType.h" #include "MessageType/Client.h" #include "MessageType/Game.h" #include "NiPoint3.h" @@ -264,6 +266,23 @@ namespace ClientPackets { LWOOBJID itemId{}; LWOOBJID destItemId{}; }; + + // Server -> client. The answer to REQUEST_UGC_MANIFEST_INFO: the MD5 and size of one of a blueprint's files as the + // client has it after downloading and inflating it (BrickModels/UserMade//..sd0 from the UGC + // server). The client checks a file it has against it and downloads the file when it differs; with valid 0 it uses + // a file it has as it is and downloads it only when it has none. The client ignores the answer unless it is exactly + // this long (37 bytes after the 0x53). + struct UgcManifestResponse : public LUBitStream { + UgcManifestResponse() : LUBitStream(ServiceType::CLIENT, MessageType::Client::UGC_MANIFEST_RESPONSE) {} + void Serialize(RakNet::BitStream& bitStream) const override; + bool Deserialize(RakNet::BitStream& bitStream) override; + + LWOOBJID blueprintId{}; + eUgcResourceType resourceType{}; + bool valid{}; // A byte + uint32_t fileSize{}; // Of the inflated file + std::array md5{}; // Of the inflated file, as bytes + }; }; /** diff --git a/dNet/WorldPackets.cpp b/dNet/WorldPackets.cpp index aafc826fc..7aa101714 100644 --- a/dNet/WorldPackets.cpp +++ b/dNet/WorldPackets.cpp @@ -340,6 +340,17 @@ namespace WorldPackets { return true; } + void RequestUgcManifestInfo::Serialize(RakNet::BitStream& bitStream) const { + bitStream.Write(blueprintId); + bitStream.Write(resourceType); + } + + bool RequestUgcManifestInfo::Deserialize(RakNet::BitStream& bitStream) { + VALIDATE_READ(bitStream.Read(blueprintId)); + VALIDATE_READ(bitStream.Read(resourceType)); + return true; + } + void UIHelpTop5::Serialize(RakNet::BitStream& bitStream) const { bitStream.Write(language); } diff --git a/dNet/WorldPackets.h b/dNet/WorldPackets.h index d37664e78..866025d54 100644 --- a/dNet/WorldPackets.h +++ b/dNet/WorldPackets.h @@ -5,6 +5,7 @@ #include "ChatPackets.h" #include "dCommonVars.h" #include "eFunnessTypes.h" +#include "eUgcResourceType.h" #include "MessageType/Game.h" #include "MessageType/World.h" #include "PositionUpdate.h" @@ -201,6 +202,17 @@ namespace WorldPackets { bool Deserialize(RakNet::BitStream& bitStream) override; }; + // Client -> server, when its UGCUSE3DSERVICES is off (the default): the MD5 and size of one of a blueprint's files + // (a player's model, or a car or rocket's build), answered with ClientPackets::UgcManifestResponse. + struct RequestUgcManifestInfo : public WorldLUBitStream { + LWOOBJID blueprintId{}; + eUgcResourceType resourceType{}; + + RequestUgcManifestInfo() : WorldLUBitStream(MessageType::World::REQUEST_UGC_MANIFEST_INFO) {} + void Serialize(RakNet::BitStream& bitStream) const override; + bool Deserialize(RakNet::BitStream& bitStream) override; + }; + // Client -> server. The help screen wants its top 5 issues. struct UIHelpTop5 : public WorldLUBitStream { int32_t language{}; // 0: en_US, 1: pl_US, 2: de_DE, 3: en_GB diff --git a/dUgcServer/UgcFormats.cpp b/dUgcServer/UgcFormats.cpp index eb5368c48..00a41bfbb 100644 --- a/dUgcServer/UgcFormats.cpp +++ b/dUgcServer/UgcFormats.cpp @@ -372,4 +372,26 @@ namespace UgcFormats { std::string ChecksumXml(std::string_view data) { return "\n" + Md5Hex(data) + "" + std::to_string(data.size()) + "\n"; } + + bool ReadChecksumXml(std::string_view xml, std::string& md5, uint32_t& size) { + const auto between = [xml](std::string_view open, std::string_view close) -> std::string_view { + const auto start = xml.find(open); + if (start == std::string_view::npos) return {}; + const auto end = xml.find(close, start + open.size()); + if (end == std::string_view::npos) return {}; + return xml.substr(start + open.size(), end - start - open.size()); + }; + const auto hash = between("", ""); + const auto length = between("", ""); + if (hash.size() != 32 || length.empty()) return false; + uint64_t parsed = 0; + for (const char c : length) { + if (c < '0' || c > '9') return false; + parsed = parsed * 10 + static_cast(c - '0'); + if (parsed > std::numeric_limits::max()) return false; + } + md5 = hash; + size = static_cast(parsed); + return true; + } } diff --git a/dUgcServer/UgcFormats.h b/dUgcServer/UgcFormats.h index c1f146131..26c77f187 100644 --- a/dUgcServer/UgcFormats.h +++ b/dUgcServer/UgcFormats.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -57,4 +58,7 @@ namespace UgcFormats { // What the client reads from a UGC file's .checksum: the MD5 and size of the file as it is after inflating it std::string ChecksumXml(std::string_view data); + + // The MD5 (lowercase hex) and size a .checksum (ChecksumXml) holds; false when it doesn't hold both + bool ReadChecksumXml(std::string_view xml, std::string& md5, uint32_t& size); } diff --git a/dUgcServer/UgcJobs.cpp b/dUgcServer/UgcJobs.cpp index 55ef61e49..95e80bc76 100644 --- a/dUgcServer/UgcJobs.cpp +++ b/dUgcServer/UgcJobs.cpp @@ -17,6 +17,9 @@ namespace UgcJobs { void AddDownload(UgcStorage::Files& files, const std::string& name, const std::string& data) { files[name + ".gz"] = ZCompression::Gzip(data); files[name + ".checksum"] = UgcFormats::ChecksumXml(data); + // Without 3D services the client downloads it as sd0 (after asking its world for the checksum) + auto sd0 = Sd0::Compress(data); + if (!sd0.empty()) files[name + ".sd0"] = std::move(sd0); } std::string LxfmlFromBlob(const std::string& blob) { diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index 60763f3a6..a3b6ba89f 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -5,7 +5,10 @@ #include "Logger.h" #include "UgcBricks.h" #include "UgcCdClient.h" +#include "Sd0.h" +#include "UgcFormats.h" #include "UgcKeys.h" +#include "ZCompression.h" #include "UgcThrottle.h" #include "json.hpp" @@ -26,6 +29,12 @@ namespace { constexpr size_t LOG_LENGTH = 50; constexpr auto EVICTION_INTERVAL = std::chrono::minutes(5); constexpr auto RECENT_ANSWER_TIME = std::chrono::seconds(10); + // In the storage folder once every item stored has its sd0 icon and checksums (Backfill) + constexpr auto BACKFILL_MARKER = ".checksums-stored"; + constexpr size_t BACKFILL_ITEMS_PER_UPDATE = 16; + constexpr uint32_t BACKFILL_BUILDS_PER_UPDATE = 200; + // Combination id of a build whose modules can't be told (so it isn't looked at again) + constexpr LWOOBJID NO_COMBINATION = -1; const char* KindName(UgcStorage::Kind kind) { return kind == UgcStorage::Kind::MODEL ? "model" : "modular"; @@ -95,7 +104,12 @@ UgcProcessor::~UgcProcessor() { } void UgcProcessor::Start() { - for (const auto& entry : m_Storage.List()) m_StoredBytes += entry.bytes; + const auto stored = m_Storage.List(); + for (const auto& entry : stored) m_StoredBytes += entry.bytes; + // Items made before the checksums were stored get their sd0 icon and checksums once (Backfill) + std::error_code error; + m_BackfillItemsDone = std::filesystem::exists(m_Storage.GetRoot() / BACKFILL_MARKER, error); + if (!m_BackfillItemsDone) m_BackfillItems.assign(stored.begin(), stored.end()); m_NextEviction = std::chrono::steady_clock::now(); m_Stopping = false; for (size_t i = 0; i < std::max(m_Config.threads, 1); i++) m_Threads.emplace_back(&UgcProcessor::Worker, this); @@ -233,6 +247,14 @@ void UgcProcessor::Worker() { const auto bytes = job.iconOnly ? m_Storage.Update(job.kind, job.id, done.outcome.files, error) : m_Storage.Write(job.kind, job.id, done.outcome.files, error); if (bytes) { done.bytes = *bytes; + // The checksums of what the client downloads as sd0, for the main thread to store (the worlds answer + // the clients' manifest requests with them) + for (const std::string name : { "icon.dds", "model.nif" }) { + const auto checksum = done.outcome.files.find(name + ".checksum"); + if (checksum == done.outcome.files.end() || !done.outcome.files.contains(name + ".sd0")) continue; + Checksum parsed{ name }; + if (UgcFormats::ReadChecksumXml(checksum->second, parsed.md5, parsed.size)) done.checksums.push_back(std::move(parsed)); + } } else { done.outcome.ok = false; done.outcome.error = error; @@ -355,6 +377,8 @@ void UgcProcessor::Record(const Done& done) { } } else { Database::Get()->SetModularBuildProcessed(done.id, state, attempts, error); + // Which combination's files it shares, for the worlds' manifest answers + if (const auto combo = m_ComboOf.find(done.id); combo != m_ComboOf.end()) Database::Get()->SetModularBuildCombination(done.id, combo->second); } m_Recent.erase({ done.kind, done.id }); @@ -381,6 +405,8 @@ void UgcProcessor::Collect() { finished.swap(m_Done); } for (const auto& done : finished) { + // done.id is the model, or the combination + if (done.outcome.ok) StoreChecksums(done.kind, done.id, done.checksums); if (done.kind == Kind::MODEL) { m_InFlight.erase({ done.kind, done.id }); if (!done.iconOnly) { @@ -413,8 +439,58 @@ void UgcProcessor::Collect() { if (!finished.empty()) m_NextPoll = std::min(m_NextPoll, std::chrono::steady_clock::now() + std::chrono::milliseconds(100)); } +void UgcProcessor::StoreChecksums(Kind kind, LWOOBJID storageId, const std::vector& checksums) { + const auto owner = kind == Kind::MODEL ? IUgc::eFileOwner::MODEL : IUgc::eFileOwner::COMBINATION; + for (const auto& checksum : checksums) Database::Get()->SetUgcFileChecksum(owner, storageId, checksum.file, checksum.md5, checksum.size); +} + +void UgcProcessor::Backfill() { + if (!m_BackfillBuildsDone) { + const auto builds = Database::Get()->GetModularBuildsWithoutCombination(BACKFILL_BUILDS_PER_UPDATE); + for (const auto& build : builds) { + const auto key = UgcModularKey::Normalize(build.modules); + const auto combo = key.empty() ? NO_COMBINATION : UgcModularKey::StorageId(key); + m_ComboOf[build.id] = combo; + Database::Get()->SetModularBuildCombination(build.id, combo); + } + if (builds.size() < BACKFILL_BUILDS_PER_UPDATE) m_BackfillBuildsDone = true; + } + if (m_BackfillItemsDone) return; + const auto read = [](const std::filesystem::path& path) { + std::ifstream in(path, std::ios::binary); + return std::string(std::istreambuf_iterator(in), {}); + }; + for (size_t i = 0; i < BACKFILL_ITEMS_PER_UPDATE && !m_BackfillItems.empty(); i++) { + const auto entry = m_BackfillItems.front(); + m_BackfillItems.pop_front(); + if (m_InFlight.contains({ entry.kind, entry.id }) || (entry.kind == Kind::MODULAR && m_ComboJobs.contains(entry.id))) continue; + // Only the icon: it is small (a model's .nif isn't asked for without 3D services and gets its sd0 when made again) + const auto checksumFile = m_Storage.File(entry.kind, entry.id, "icon.dds.checksum"); + if (!checksumFile) continue; + Checksum checksum{ "icon.dds" }; + if (!UgcFormats::ReadChecksumXml(read(*checksumFile), checksum.md5, checksum.size)) continue; + if (!m_Storage.File(entry.kind, entry.id, "icon.dds.sd0")) { + const auto packed = m_Storage.File(entry.kind, entry.id, "icon.dds.gz"); + const auto icon = packed ? ZCompression::Gunzip(read(*packed)) : std::nullopt; + if (!icon || UgcFormats::Md5Hex(*icon) != checksum.md5) continue; + std::string error; + if (!m_Storage.Update(entry.kind, entry.id, { { "icon.dds.sd0", Sd0::Compress(*icon) } }, error)) { + LOG("Couldn't write the sd0 icon of %llu: %s", static_cast(entry.id), error.c_str()); + continue; + } + } + StoreChecksums(entry.kind, entry.id, { checksum }); + } + if (m_BackfillItems.empty()) { + m_BackfillItemsDone = true; + std::ofstream(m_Storage.GetRoot() / BACKFILL_MARKER) << "1\n"; + LOG("Stored the checksums of the icons made before"); + } +} + void UgcProcessor::Update() { Collect(); + Backfill(); const auto now = std::chrono::steady_clock::now(); if (now - m_CpuSampled >= std::chrono::seconds(2)) SampleUsage(); if (now >= m_NextPoll) { diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h index 9eab69cba..14f84b789 100644 --- a/dUgcServer/UgcProcessor.h +++ b/dUgcServer/UgcProcessor.h @@ -155,6 +155,13 @@ private: UgcIconParams::Values iconValues; // models: the preset and override (UgcIconParams) }; + // A written file's MD5 and size as the client has it after inflating it (from its .checksum) + struct Checksum { + std::string file; // icon.dds, model.nif + std::string md5; + uint32_t size{}; + }; + struct Done { Kind kind{}; LWOOBJID id{}; @@ -163,8 +170,19 @@ private: uint64_t bytes{}; double milliseconds{}; bool iconOnly{}; + std::vector checksums; // of the files written that the client downloads as sd0 }; + // Main thread: records the checksums of an item's files (ugc_file_checksums) + void StoreChecksums(Kind kind, LWOOBJID storageId, const std::vector& checksums); + + // Main thread: fills in what was made before the worlds answered manifest requests, a little per Update: the + // icons' sd0 files and checksums of the items stored, and the builds' combination ids + void Backfill(); + std::deque m_BackfillItems; + bool m_BackfillItemsDone{ true }; + bool m_BackfillBuildsDone{ false }; + void Poll(); // Main thread: the icon values for a kind and an item (the kind's preset, then the item's override) UgcIconParams::Values IconValues(const std::string& kind, const std::string& itemTarget); diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index 5c032f0db..a524d5cda 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -34,6 +34,7 @@ #include "UgcCdClient.h" #include "UgcIconParams.h" #include "UgcJobs.h" +#include "Sd0.h" #include "UgcFormats.h" #include "ZCompression.h" #include "UgcModel.h" @@ -228,6 +229,49 @@ namespace { // .hkx: no physics is made } + /** + * A game client download without 3D services (UGCUSE3DSERVICES=7:0, the client's default): + * BrickModels/UserMade//<.lxfml|.nif|.hkx|.dds>.sd0 under its UGCSERVERDIR. The + * client asked its world for the file's checksum first (UGC_MANIFEST_RESPONSE) and checks the inflated download + * against it. + */ + void ServeClientSd0(HTTPReply& reply, const std::string& bucket, const std::string& name) { + NotFound(reply); + auto file = Lower(name); + if (!file.ends_with(".sd0")) return; + file.resize(file.size() - 4); + const auto dot = file.find('.'); + if (dot == std::string::npos) return; + const auto extension = file.substr(dot); + const auto number = file.substr(0, dot); + const auto id = GeneralUtils::TryParse(number); + if (!id || number.empty() || GeneralUtils::TryParse(bucket) != *id % 1000) return; + const auto blueprint = static_cast(*id); + if (extension == ".lxfml") { + // Straight from the ugc row, like the 3D services download; the last few are kept + static std::map cache; // main thread only + auto it = cache.find(blueprint); + if (it == cache.end()) { + const auto model = Database::Get()->GetUgcModel(blueprint); + const auto lxfml = model ? UgcJobs::LxfmlFromBlob(model->lxfmlData.str()) : std::string(); + if (lxfml.empty()) return; + if (cache.size() >= 64) cache.erase(cache.begin()); + it = cache.emplace(blueprint, Sd0::Compress(lxfml)).first; + } + reply.status = eHTTPStatusCode::OK; + reply.contentType = eContentType::APPLICATION_OCTET_STREAM; + reply.message = it->second; + reply.headers.push_back("Cache-Control: public, max-age=60"); + } else if (extension == ".nif") { + ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.nif.sd0", eContentType::APPLICATION_OCTET_STREAM, false); + } else if (extension == ".dds") { + // A player model's icon, else a car or rocket's (its combination's) + ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "icon.dds.sd0", eContentType::APPLICATION_OCTET_STREAM, false); + if (reply.status == eHTTPStatusCode::NOT_FOUND) ServeFile(reply, UgcStorage::Kind::MODULAR, blueprint, "icon.dds.sd0", eContentType::APPLICATION_OCTET_STREAM, false); + } + // .hkx: no physics is made, the client makes its own + } + void RegisterRoutes() { // The configured path, and the one the 1.10.64 client uses whatever its boot.cfg says (its built-in patch server // folder, lwoclient/UserBrickModels; see docs/UgcServer.md) @@ -253,6 +297,25 @@ namespace { LOG("Serving the client's downloads under %s/UGCC/", base.c_str()); } + // Without 3D services: /BrickModels/UserMade//.sd0, where UGCSERVERDIR is client_path + // when boot.cfg sets it, else the client's default /UserBrickModels (any patch folder), or empty + std::set sd0Bases = { "/:patchdir/userbrickmodels", "" }; + { + std::string base; + for (const auto& segment : Segments(Lower(Game::config->GetValue("client_path").empty() ? std::string("/ugc") : Game::config->GetValue("client_path")))) base += "/" + segment; + sd0Bases.insert(base); + } + for (const auto& base : sd0Bases) { + Game::web.RegisterHTTPRoute({ .path = base + "/brickmodels/usermade/:bucket/:file", .method = eHTTPMethod::GET, .middleware = {}, + .handle = [](HTTPReply& reply, const HTTPContext& context) { + const auto segments = Segments(context.originalPath); + if (segments.size() < 2) return NotFound(reply); + ServeClientSd0(reply, segments[segments.size() - 2], segments.back()); + LOG("Client download %s -> %i%s", context.originalPath.c_str(), static_cast(reply.status), reply.file.empty() ? "" : " (file)"); + } }); + LOG("Serving the client's sd0 downloads under %s/BrickModels/UserMade/", base.c_str()); + } + // Previews for the dashboard Game::web.RegisterHTTPRoute({ .path = "/files/:kind/:id/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = [](HTTPReply& reply, const HTTPContext& context) { diff --git a/dUgcServer/UgcStorage.cpp b/dUgcServer/UgcStorage.cpp index bd9312f9a..c49e07038 100644 --- a/dUgcServer/UgcStorage.cpp +++ b/dUgcServer/UgcStorage.cpp @@ -9,9 +9,9 @@ namespace { constexpr std::array KNOWN_FILES = { - "model.nif", "model.nif.gz", "model.nif.checksum", + "model.nif", "model.nif.gz", "model.nif.checksum", "model.nif.sd0", "model.lxfml.gz", "model.lxfml.checksum", - "icon.dds.gz", "icon.dds.checksum", "icon.png", + "icon.dds.gz", "icon.dds.checksum", "icon.dds.sd0", "icon.png", "model.noao.nif", "model.noao.nif.gz", "stats.json", "combo.json", "previous.icon.png", "previous.model.nif.gz", "previous.stats.json", }; diff --git a/dWorldServer/WorldServer.cpp b/dWorldServer/WorldServer.cpp index 24d4ad7b0..f2b3c864b 100644 --- a/dWorldServer/WorldServer.cpp +++ b/dWorldServer/WorldServer.cpp @@ -10,6 +10,7 @@ #include "PropertyRent.h" #include "PropertyReputation.h" #include "LiveEvents.h" +#include "UgcManifest.h" #include #include #include @@ -474,6 +475,7 @@ int main(int argc, char** argv) { DashboardNotify::SendPlayerPositions(g_InstanceID); MessageInspector::Update(); LiveEvents::Update(); + UgcManifest::Update(); Metrics::StartMeasurement(MetricVariable::UpdateSpawners); Game::zoneManager->Update(deltaTime); @@ -1104,6 +1106,7 @@ void CleanupDisconnectedUser(const SystemAddress& sysAddr) { // other instance has saved since const bool savedByMigration = WorldMigration::IsLeaving(sysAddr); WorldMigration::OnDisconnected(sysAddr); + UgcManifest::OnDisconnect(sysAddr); auto user = UserManager::Instance()->GetUser(sysAddr); if (!user) return; @@ -1604,6 +1607,13 @@ namespace { } }; + struct RequestUgcManifestInfoPacket final : public WorldPackets::RequestUgcManifestInfo { + void Handle() override { + if (!UserManager::Instance()->GetUser(sysAddr)) return; + UgcManifest::OnRequest(sysAddr, blueprintId, resourceType); + } + }; + template std::unique_ptr Create() { return std::make_unique(); } @@ -1623,6 +1633,7 @@ namespace { { MessageType::World::GENERAL_CHAT_MESSAGE, Create }, { MessageType::World::HANDLE_FUNNESS, Create }, { MessageType::World::UI_HELP_TOP_5, Create }, + { MessageType::World::REQUEST_UGC_MANIFEST_INFO, Create }, }; } diff --git a/docs/UgcServer.md b/docs/UgcServer.md index 270b5b2c0..fec1ba86a 100644 --- a/docs/UgcServer.md +++ b/docs/UgcServer.md @@ -21,9 +21,34 @@ Resource types are 0 LXFML, 1 NIF, 2 HKX, 3 DDS. * the same name with `.checksum` instead of `.gz`: `32 hex digitsn`, the MD5 and size of the uncompressed file. The client checks the download against it. * HTTP 408 makes the client try again later; other errors are logged as failed downloads. -* Without 3D services the client asks the world server for each file's MD5 (`REQUEST_UGC_MANIFEST_INFO`, answered - with `UGC_MANIFEST_RESPONSE`) and downloads `BrickModels/UserMade//.sd0`. - That needs the world to answer the manifest requests, which it doesn't yet, so it is not supported. +* `UGCUSE3DSERVICES=7:0` (the client's default, without 3D services). The client asks its world for each file's MD5 + and size (`REQUEST_UGC_MANIFEST_INFO`, answered with `UGC_MANIFEST_RESPONSE`, below) and downloads + `UGCSERVERDIR/BrickModels/UserMade//<.lxfml|.nif|.hkx|.dds>.sd0`: the file as + an sd0 stream (the bytes `s` `d` `0` 0x01 0xff, then chunks of a u32 size and zlib data, each inflating to at most + 256 KiB). It inflates it, saves it as `res/BrickModels/UserMade//` and checks its MD5 against + the world's answer; a file it has already is only downloaded again when its MD5 differs. HTTP 408 makes it try + again later. The worlds answer from checksums the UGC server stores (`ugc_file_checksums`), and the UGC server serves + the `.sd0` files; see "Without 3D services" below for what keeps this off by default. + +### The manifest packets (checked in the client) + +Both are plain packets (not game messages); offsets are after the 0x53 byte and the header (connection type, message +id, a padding byte). + +| Packet | Layout | +| --- | --- | +| `REQUEST_UGC_MANIFEST_INFO` (world message 27), client to world | u64 blueprint id, u8 resource type (16 bytes after the 0x53). Sent by `SendRequestUGCManifestInfoPacket` when the client needs a blueprint's file and 3D services are off. | +| `UGC_MANIFEST_RESPONSE` (client message 60), world to client | u64 blueprint id, u8 resource type, then 21 bytes of manifest info: u8 valid, u32 size of the inflated file, 16 bytes MD5 of it. The client ignores an answer that isn't exactly 37 bytes after the 0x53 (`PacketHandler_MSG_CLIENT_UGC_MANIFEST_RESPONSE`). | + +The answer is cached per blueprint and type (the same message the 3D services `.checksum` download produces). With +it, the client uses a file it has when its MD5 matches and downloads it otherwise; with valid 0 it uses a file it has as +it is and downloads it only when it has none. Structs: `WorldPackets::RequestUgcManifestInfo`, +`ClientPackets::UgcManifestResponse` (`eUgcResourceType`). + +The blueprint id of an inventory item's icon (`LWOInventoryComponent_Client::LoadBlueprintIcon`) is the item's +`blueprintid` config when it has one (Brick-by-Brick models, LOT 6662), else its subkey (cars and rockets: their +`ugc_modular_build` id, which DLU gives them as subkey when they are built). Cars and rockets from before builds were +stored have subkey 0 and no build row, so the client never asks for their icons. A model's render component uses the downloaded NIF when the model's spawn data has `renderUserGen=1` with its `blueprintid`; `NotifyClientUGCModelReady` (game message 909) makes the client fetch the blueprint's NIF and HKX @@ -50,8 +75,18 @@ DATACENTERID=1:150, then don't show. So 3D services only work when the UGC server answers on port 80 of that address; it serves `/lwoclient/UserBrickModels/...` as well as `client_path` for that. * With `UGCUSE3DSERVICES=7:0` (the client's own default) the client builds the models itself from the LXFML the world - sends, and property models load. It also asks the world for manifests (`REQUEST_UGC_MANIFEST_INFO`), which the world - doesn't answer; that doesn't stop the models from loading. Use this until the UGC server can be reached as above. + sends, and property models load. Opening the backpack's models asks the world for each car's, rocket's and model's + icon manifest. With `ugc_manifest=1` the world answers (checked: the answers are read and the client goes on to + download each icon), and the client then downloads from the same built-in address as with 3D services, whatever its + `boot.cfg` says (`UGCSERVERPORT=1:14108`, `UGCSERVERDIR=0:/ugc`, `PATCHSERVERDIR=0:luclient` in the client tested): + its log shows `Download Info: Host 'http://127.0.0.1/'`, `UGC site Info: Sharing main download connection` and + requests for `lwoclient/UserBrickModels/BrickModels/UserMade//.dds.sd0`. In the client the download + thread reads `PATCHSERVERIP`/`PORT`/`DIR` and `UGCSERVERIP`/`PORT`/`DIR` through the config interface + (`LWOResMgr2Interface::DownloadThread_Run`, defaults 127.0.0.1, 80 and `lwoclient`, the UGC ones defaulting to the + patch ones), but the values in `boot.cfg` don't reach it. A download that can't connect (HTTP status 0) counts as a + UGC connection failure, and the client logs the player out for it ("connection failed downloading UGC assets", + `MainThread_LogoutDueToConnectionFailures`) unless a client mod stops that. So, as with 3D services, the icons only + arrive when the UGC server answers on port 80 of the player's own machine; see "Without 3D services". * A NIF the UGC server makes, put in place of one of the game's own models and spawned, renders in the client with its colors: the NIFs are written like the game's `res/BrickModels/ndmade` files (nif.xml's 20.3.0.9, user version 0; every shape with a material, alpha blending by the vertex alpha, specular off and vertex colors, in that order). @@ -183,11 +218,20 @@ shared files. `combo.json` in the combination's folder says its LOTs and build t Files live under `ugc_output_dir` (default `ugc` next to the server binaries): ``` -ugc/models///model.nif.gz, model.nif.checksum, icon.dds.gz, icon.dds.checksum, icon.png, - model.noao.nif.gz, stats.json, previous.icon.png, previous.model.nif.gz, previous.stats.json -ugc/modular///icon.dds.gz, icon.dds.checksum, icon.png, combo.json +ugc/models///model.nif.gz, model.nif.checksum, model.nif.sd0, icon.dds.gz, icon.dds.checksum, + icon.dds.sd0, icon.png, model.noao.nif.gz, stats.json, previous.icon.png, + previous.model.nif.gz, previous.stats.json +ugc/modular///icon.dds.gz, icon.dds.checksum, icon.dds.sd0, icon.png, combo.json +ugc/.checksums-stored ``` +Every file the client downloads is written three ways: `.gz` and `.checksum` for 3D services and `.sd0` (dCommon's +`Sd0::Compress`) without them. When a worker writes an item, the checksums of its `.sd0` files (MD5 and size of the +inflated file, from the `.checksum`) go back to the main thread, which stores them in `ugc_file_checksums` for the +worlds. Items made before this get their `icon.dds.sd0` (from `icon.dds.gz`) and their icon's checksum once, a few per +tick on the main thread when the server starts; `.checksums-stored` marks that done. Their `.nif` gets its `.sd0` when +the model is made again (the client doesn't ask for it without 3D services, see "Not done yet"). + A model's files are written to a temporary folder and renamed into place, so a half written model is never served. Meshes are only stored compressed (a model took about 18 MB when the .nif was kept uncompressed beside its .gz, so the 2 GB cap held about 110 models and the server kept evicting and remaking them); the dashboard's copies are inflated @@ -228,6 +272,39 @@ Migrations `dlu/mysql/86_ugc_debounce_icon_settings.sql` and `dlu/sqlite/69_ugc_ `ugc.process_after` and the table `ugc_icon_settings` (`target`: `kind:`, `model:` or `combo:`; `params` JSON; `updated_at`). +Migrations `dlu/mysql/89_ugc_file_checksums.sql` and `dlu/sqlite/72_ugc_file_checksums.sql`: the table +`ugc_file_checksums` (`kind` 0 a player model, 1 a combination of car or rocket modules; `storage_id` the model's ugc +id or the combination's id; `file` `icon.dds` or `model.nif`; `md5`, 32 lowercase hex digits, and `size` of the inflated +file), written by the UGC server whenever it writes a file, and `ugc_modular_build.combination_id`, the combination a +build shares its files with (0 until the UGC server has seen the build: it fills them in for existing builds when it +starts, -1 when the modules can't be told). `IUgc::GetUgcFileChecksum(blueprint, file)` looks a blueprint up as a model +first, then as a build through its combination. + +## Without 3D services (`UGCUSE3DSERVICES=7:0`) + +`UgcManifest` (dGame/dUtilities) answers `REQUEST_UGC_MANIFEST_INFO` when `ugc_manifest=1` (`sharedconfig.ini`, +default 0): + +* The world's main thread handles the packet (`g_WorldHandlers`), looks the checksum up with one indexed query and sends + `UGC_MANIFEST_RESPONSE` to that client in the same tick. Only DDS (`icon.dds`) and NIF (`model.nif`) are answered; + LXFML and HKX aren't (the client builds those itself, as today). +* A file that isn't made yet isn't answered: the request waits (at most 512, for 15 minutes) and `UgcManifest::Update` + (world tick) looks again every 5 seconds, answering once the UGC server has stored the checksum. A model still in its + quiet period after a save is made right away (`ExpediteUgcModel`), as when a client asks the UGC server directly. + Waiting requests are dropped when the client disconnects. +* No worker threads, HTTP or file reads in the world: the UGC server precomputes everything into the database. + +The UGC server serves `BrickModels/UserMade//..sd0` under `client_path`, under +`//UserBrickModels` (the client's default `UGCSERVERDIR`, `lwoclient/UserBrickModels` with its built-in +patch folder) and at the root; `.dds` is a model's icon or a car or rocket build's combination icon, `.nif` a model's +mesh, `.lxfml` the model's LXFML from the database, `.hkx` 404. + +It is off by default because the 1.10.64 client downloads from `http://127.0.0.1:80/lwoclient/UserBrickModels/` +whatever its `boot.cfg` says (checked in game, see "What the 1.10.64 client does in practice"), and logs the player +out when it can't connect there. Turning it on only helps when the UGC server answers on port 80 of every player's own +machine, which normal players don't have; until a way is found to point the client at the server, the icons don't show +in 7:0. + ### Waiting while the owner is still building A saved model isn't made right away: `InsertNewUgcModel` stores `process_after` = now + `ugc_debounce_seconds` @@ -255,8 +332,9 @@ No master messages are needed for any of it. The main thread owns RakNet (the master link), mongoose (HTTP) and the database. `worker_threads` (default: half the CPUs) workers do only pure work: parse LXFML, build and write meshes and icons, compress files. They get their input (LXFML text, module data) from the main thread and hand results back through a queue the main thread drains, which -then updates the database. Brick geometry and materials are loaded once and shared read-only (the cache has its own -lock). +then updates the database (including the files' checksums, parsed by the worker from the `.checksum` it wrote). +Brick geometry and materials are loaded once and shared read-only (the cache has its own lock). The start-up backfill +of old items' `.sd0` icons and checksums and of builds' combination ids runs on the main thread, a few items per tick. ## CPU and memory @@ -297,6 +375,9 @@ files are what every player on a property sees anyway): the client's `UGCSERVERDIR`), and the same under `/lwoclient/UserBrickModels` (the path the client really uses, see above). HKX answers 404. The LXFML comes from the database. A model that exists but isn't made yet (or was evicted) is moved to the front of the queue and answers 408 so the client asks again. +* `/BrickModels/UserMade//..sd0`, the same under + `//UserBrickModels` and at the root, for clients without 3D services (see "Without 3D services"); 408 and 404 + as above. * `/files/model//` and `/files/modular//` for the dashboard (`icon.png`, `model.nif` and `model.noao.nif` (inflated), `stats.json`, `combo.json` and `previous.` versions), not cached by browsers. * `/admin/preview`, `/admin/assembly`, `/admin/regenerate-icons`, `/admin/delete` (POST, JSON) for the dashboard only: @@ -424,4 +505,10 @@ pages show the item's own icon. keeps building its own meshes; switching them to the served NIFs (and sending `NotifyClientUGCModelReady` when a model is made) is the next step, and needs checking in game. * HKX (physics) is not generated. -* The non-3D-services download path (world manifest packets) is not answered. +* Icons in game: both of the client's modes download from its built-in `http://127.0.0.1:80/lwoclient/...` whatever + its `boot.cfg` says, so neither reaches the UGC server for normal players. Worlds answer the 7:0 manifest requests + (`ugc_manifest=1`) and the UGC server serves the `.sd0` files, but it stays off until the client can be pointed at + the server (why the client ignores its `UGCSERVER*`/`PATCHSERVER*` lines is not found yet). +* Cars and rockets built before builds were stored (subkey 0, no `ugc_modular_build` row) never get an icon: the client + has no blueprint id to ask for. +* A model's `.nif` made before `.sd0` files were written has none until the model is made again. diff --git a/migrations/dlu/mysql/89_ugc_file_checksums.sql b/migrations/dlu/mysql/89_ugc_file_checksums.sql new file mode 100644 index 000000000..b80eb98d2 --- /dev/null +++ b/migrations/dlu/mysql/89_ugc_file_checksums.sql @@ -0,0 +1,19 @@ +/* UGC server: ugc_file_checksums holds the MD5 (lowercase hex) and size of each file it made, as the game client has + it after inflating the download (kind 0: a player model, storage_id its ugc id; kind 1: a combination of car or + rocket modules, storage_id the combination's id). ugc_modular_build.combination_id is the combination a build was + made as (0 until the UGC server has seen it). Worlds read both to answer the client's REQUEST_UGC_MANIFEST_INFO. + See docs/UgcServer.md. */ +CREATE TABLE IF NOT EXISTS ugc_file_checksums ( + kind INT NOT NULL, + storage_id BIGINT NOT NULL, + file VARCHAR(32) NOT NULL, + md5 CHAR(32) NOT NULL, + size INT UNSIGNED NOT NULL, + PRIMARY KEY (kind, storage_id, file) +); + +SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN combination_id BIGINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'combination_id'); +PREPARE dlu_column_stmt FROM @dlu_column; +EXECUTE dlu_column_stmt; +DEALLOCATE PREPARE dlu_column_stmt; diff --git a/migrations/dlu/sqlite/72_ugc_file_checksums.sql b/migrations/dlu/sqlite/72_ugc_file_checksums.sql new file mode 100644 index 000000000..d4f57f3d9 --- /dev/null +++ b/migrations/dlu/sqlite/72_ugc_file_checksums.sql @@ -0,0 +1,11 @@ +/* UGC server: the checksums of the files it made and each car or rocket build's combination. See the MySQL migration. */ +CREATE TABLE IF NOT EXISTS ugc_file_checksums ( + kind INTEGER NOT NULL, + storage_id BIGINT NOT NULL, + file TEXT NOT NULL, + md5 TEXT NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (kind, storage_id, file) +); + +ALTER TABLE ugc_modular_build ADD COLUMN combination_id BIGINT NOT NULL DEFAULT 0; diff --git a/resources/sharedconfig.ini b/resources/sharedconfig.ini index 5eb93115b..9c14d0f7e 100644 --- a/resources/sharedconfig.ini +++ b/resources/sharedconfig.ini @@ -93,3 +93,9 @@ mute_auto_reject_names=1 # again for all their waiting models), or as soon as a game client asks for it or the owner leaves the world. 0: right # away. See docs/UgcServer.md. ugc_debounce_seconds=120 + +# 0 or 1: worlds answer game clients without 3D services (UGCUSE3DSERVICES=7:0, the default) with the checksums of the +# files the UGC server made, so they download the icons of cars, rockets and models. The 1.10.64 client then downloads +# from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and a client that can't connect there +# is logged out, so only set 1 when the UGC server answers there for every player. See docs/UgcServer.md. +ugc_manifest=0 diff --git a/tests/dDatabaseTests/DatabaseParityTests.cpp b/tests/dDatabaseTests/DatabaseParityTests.cpp index 5f2cf1f70..3c8dd8afe 100644 --- a/tests/dDatabaseTests/DatabaseParityTests.cpp +++ b/tests/dDatabaseTests/DatabaseParityTests.cpp @@ -774,6 +774,21 @@ TEST_F(ParitySeeded, UgcModel) { out.push_back(db.GetUgcIconSettings("combo:4713-4714-4715").value_or("")); db.DeleteUgcIconSettings("combo:4713-4714-4715"); out.push_back(db.GetUgcIconSettings("combo:4713-4714-4715").has_value()); + // The files' checksums the worlds answer manifest requests with: a build's are its combination's + const auto checksumJson = [](const std::optional& checksum) { + return checksum ? json{ checksum->md5, checksum->size } : json(nullptr); + }; + for (const auto& build : db.GetModularBuildsWithoutCombination(10)) out.push_back(build.id); + out.push_back(checksumJson(db.GetUgcFileChecksum(1152921510000500002LL, "icon.dds"))); + db.SetModularBuildCombination(1152921510000500002LL, 77); + db.SetUgcFileChecksum(IUgc::eFileOwner::COMBINATION, 77, "icon.dds", "900150983cd24fb0d6963f7d28e17f72", 3); + db.SetUgcFileChecksum(IUgc::eFileOwner::COMBINATION, 77, "icon.dds", "0123456789abcdef0123456789abcdef", 65664); + out.push_back(db.GetModularBuildsWithoutCombination(10).size()); + out.push_back(checksumJson(db.GetUgcFileChecksum(1152921510000500002LL, "icon.dds"))); + out.push_back(checksumJson(db.GetUgcFileChecksum(1152921510000500002LL, "model.nif"))); + db.SetUgcFileChecksum(IUgc::eFileOwner::MODEL, 1152921510000500009LL, "icon.dds", "11111111111111111111111111111111", 10); + out.push_back(checksumJson(db.GetUgcFileChecksum(1152921510000500009LL, "icon.dds"))); + out.push_back(checksumJson(db.GetUgcFileChecksum(77, "icon.dds"))); return out; }); } diff --git a/tests/dGameTests/dNetTests/WorldPacketsTests.cpp b/tests/dGameTests/dNetTests/WorldPacketsTests.cpp index bef02e965..3c650505b 100644 --- a/tests/dGameTests/dNetTests/WorldPacketsTests.cpp +++ b/tests/dGameTests/dNetTests/WorldPacketsTests.cpp @@ -717,3 +717,35 @@ TEST_F(WorldPacketsTests, SmallRequestsMatchLegacy) { ASSERT_TRUE(mailRead.Deserialize(mailBytes)); EXPECT_PACKET_EQ(FromBitStream(mail.data), FromBitStream(mailRead.data)); } + +// The layouts the 1.10.64 client uses without 3D services (UGCUSE3DSERVICES=7:0): it sends 16 bytes after the 0x53 +// (SendRequestUGCManifestInfoPacket) and only reads an answer that is exactly 37 bytes after it +// (PacketHandler_MSG_CLIENT_UGC_MANIFEST_RESPONSE: 21 bytes of manifest info after the blueprint and type). +TEST_F(WorldPacketsTests, UgcManifestPacketsMatchTheClient) { + WorldPackets::RequestUgcManifestInfo request; + request.blueprintId = 0x0102030405060708; + request.resourceType = eUgcResourceType::DDS; + RakNet::BitStream requestBytes; + request.WritePacket(requestBytes); + EXPECT_PACKET_EQ(FromHex("53 04 00 1b 00 00 00 00 08 07 06 05 04 03 02 01 03"), FromBitStream(requestBytes)); + EXPECT_EQ(requestBytes.GetNumberOfBytesUsed() - 1, 16); + EXPECT_EQ(RoundTrip(request).blueprintId, request.blueprintId); + ExpectTruncatedFails(request); + + ClientPackets::UgcManifestResponse response; + response.blueprintId = 0x0102030405060708; + response.resourceType = eUgcResourceType::DDS; + response.valid = true; + response.fileSize = 65664; + for (size_t i = 0; i < response.md5.size(); i++) response.md5[i] = static_cast(0xa0 + i); + RakNet::BitStream responseBytes; + response.WritePacket(responseBytes); + EXPECT_PACKET_EQ(FromHex("53 05 00 3c 00 00 00 00 08 07 06 05 04 03 02 01 03 01 80 00 01 00 " + "a0 a1 a2 a3 a4 a5 a6 a7 a8 a9 aa ab ac ad ae af"), FromBitStream(responseBytes)); + EXPECT_EQ(responseBytes.GetNumberOfBytesUsed() - 1, 37); + const auto read = RoundTrip(response); + EXPECT_TRUE(read.valid); + EXPECT_EQ(read.fileSize, 65664u); + EXPECT_EQ(read.md5, response.md5); + ExpectTruncatedFails(response); +} diff --git a/tests/dUgcTests/UgcTests.cpp b/tests/dUgcTests/UgcTests.cpp index fcd6d3049..9c4f88088 100644 --- a/tests/dUgcTests/UgcTests.cpp +++ b/tests/dUgcTests/UgcTests.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -22,6 +23,7 @@ #include "UgcRender.h" #include "UgcStorage.h" #include "UgcThrottle.h" +#include "Sd0.h" #include "ZCompression.h" #include "json.hpp" @@ -269,6 +271,28 @@ TEST(UgcFormats, ImagesAndChecksums) { EXPECT_EQ(dds[128 + 2], 10); // stored BGRA EXPECT_EQ(UgcFormats::Md5Hex("abc"), "900150983cd24fb0d6963f7d28e17f72"); EXPECT_NE(UgcFormats::ChecksumXml("abc").find("900150983cd24fb0d6963f7d28e17f723"), std::string::npos); + std::string md5; + uint32_t size{}; + ASSERT_TRUE(UgcFormats::ReadChecksumXml(UgcFormats::ChecksumXml("abc"), md5, size)); + EXPECT_EQ(md5, "900150983cd24fb0d6963f7d28e17f72"); + EXPECT_EQ(size, 3u); + EXPECT_FALSE(UgcFormats::ReadChecksumXml("abc3", md5, size)); + EXPECT_FALSE(UgcFormats::ReadChecksumXml("900150983cd24fb0d6963f7d28e17f72x", md5, size)); +} + +// A download is written for both of the client's modes: .gz and .checksum (3D services) and .sd0 (without), all +// holding the same file +TEST(UgcJobs, AddsTheDownloadForBothClientModes) { + UgcStorage::Files files; + UgcJobs::AddDownload(files, "icon.dds", "abc"); + EXPECT_EQ(ZCompression::Gunzip(files.at("icon.dds.gz")).value_or(""), "abc"); + std::istringstream sd0(files.at("icon.dds.sd0")); + EXPECT_EQ(Sd0(sd0).GetAsStringUncompressed(), "abc"); + std::string md5; + uint32_t size{}; + ASSERT_TRUE(UgcFormats::ReadChecksumXml(files.at("icon.dds.checksum"), md5, size)); + EXPECT_EQ(md5, "900150983cd24fb0d6963f7d28e17f72"); + EXPECT_EQ(size, 3u); } TEST(UgcModular, ParsesTheCdClientData) {