From 9d76fe9550c53d219c20d20d0c83ade2542ac79b Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sun, 27 Sep 2026 19:12:48 -0500 Subject: [PATCH] feat(ugc): placed models from the UGC server without 3D services A placed model's client (1.10.64, UGCUSE3DSERVICES=7:0) always loads its blueprint's NIF and HKX (its BlueprintComponent sets renderUserGen and physicsUserGen itself) and its LXFML, asks the world for each file's manifest first and waits for the answer with no timeout. With ugc_manifest=1 and the new ugc_manifest_models=1 (default 0) the world: - leaves the models whose mesh the UGC server made out of the LXFML it sends when a property loads, - answers their NIF with the UGC server's checksum, their LXFML with the stored LXFML's (worked out once and kept) and their HKX as not known, - sends a model that isn't made its LXFML (once for the three requests), so the client builds it itself and no model is left waiting. When the UGC server writes a model's mesh with a new checksum it sends UGC_MODELS_MADE (a new master message, appended) to the master, which passes it to every world; a world with that model placed sends its players the new NIF checksum and NotifyClientUGCModelReady (game message 909, the blueprint id), so clients switch to the served mesh. Co-Authored-By: Claude Opus 5.5 --- dCommon/dEnums/MessageType/Master.h | 4 + dDashboardServer/routes/SettingsCatalog.cpp | 3 +- dGame/dGameMessages/BuildingMessages.cpp | 9 + dGame/dGameMessages/BuildingMessages.h | 13 ++ dGame/dUtilities/UgcManifest.cpp | 208 +++++++++++++++--- dGame/dUtilities/UgcManifest.h | 50 ++++- dMasterServer/MasterServer.cpp | 11 + dNet/MasterPackets.h | 4 +- dNet/master/UgcModelsMade.h | 42 ++++ dUgcServer/UgcProcessor.cpp | 9 +- dUgcServer/UgcProcessor.h | 5 + dUgcServer/UgcServer.cpp | 9 + dWorldServer/WorldServer.cpp | 14 +- resources/sharedconfig.ini | 12 +- tests/dGameTests/CMakeLists.txt | 1 + tests/dGameTests/UgcManifestTests.cpp | 121 ++++++++++ .../BuildingMessagesTests.cpp | 16 ++ .../dNetTests/MasterPacketsTests.cpp | 21 ++ 18 files changed, 508 insertions(+), 44 deletions(-) create mode 100644 dNet/master/UgcModelsMade.h create mode 100644 tests/dGameTests/UgcManifestTests.cpp diff --git a/dCommon/dEnums/MessageType/Master.h b/dCommon/dEnums/MessageType/Master.h index c0c531551..cc803ab07 100644 --- a/dCommon/dEnums/MessageType/Master.h +++ b/dCommon/dEnums/MessageType/Master.h @@ -66,5 +66,9 @@ namespace MessageType { // Any server -> master -> dashboard: traffic counters of the last few seconds (see ServerTraffic.h) SERVER_TRAFFIC, + + // UGC server -> master -> every world: player models whose mesh (model.nif) was just made or made again (see + // UgcModelsMade.h), so worlds showing them tell their clients (NotifyClientUGCModelReady) + UGC_MODELS_MADE, }; } diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index 57ce91b3f..360ac096c 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -422,7 +422,8 @@ namespace { c.Add(Format(Text(UGC, "listen_ip", "Listen address", "The game client has to reach it.", "0.0.0.0", true), eFormat::HOST)); c.Add(Port(UGC, "net_port", "Master connection port", "UDP; the next port is used too.", "2012")); c.Add(Text(UGC, "client_path", "Download path", "The game client's UGCSERVERDIR.", "/ugc", true)); - c.Add(Bool(SHARED, "ugc_manifest", "Answer clients without 3D services", "Worlds tell game clients with UGCUSE3DSERVICES=7:0 (the default) the checksums of the files the UGC server made, so they download icons of cars, rockets and models. The client then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and one that can't connect there is logged out: only turn on when the UGC server answers there for every player.", false)); + c.Add(Bool(SHARED, "ugc_manifest", "Answer clients without 3D services", "Worlds tell game clients with UGCUSE3DSERVICES=7:0 (the default) the checksums of the files the UGC server made, so they download icons of cars, rockets and models from UGCSERVERIP, UGCSERVERPORT and UGCSERVERDIR in their boot.cfg (default: the patch server's). A client that can't connect there is logged out, so only turn on when every player's boot.cfg points at the UGC server.", false)); + c.Add(When(Bool(SHARED, "ugc_manifest_models", "Placed models from the UGC server", "With the answer to clients without 3D services on: placed player models the UGC server has made are downloaded from it (its mesh) instead of built by each client from the LXFML the world sends. It makes no physics: a client that never built a model itself has no collision for it. Models not made yet are still sent as LXFML.", false), SHARED, "ugc_manifest", { "1" })); c.Add(Format(Text(DASHBOARD, "ugc_internal_url", "UGC server address (internal)", "Where the dashboard itself reaches the UGC server for its status and files. Empty: http://127.0.0.1:2008.", ""), eFormat::URL)); c.Add(Format(Text(DASHBOARD, "ugc_public_url", "UGC server address (public)", "Only for the \"open on the UGC server\" links, e.g. https://ugc.example.com. The dashboard's own pages don't need the browser to reach it.", ""), eFormat::URL)); c.AddSection("UGC processing", "When models are made and how much of the machine the workers may use.", eLayout::ROWS, Condition{ MASTER, "enable_ugc_server", { "1" } }); diff --git a/dGame/dGameMessages/BuildingMessages.cpp b/dGame/dGameMessages/BuildingMessages.cpp index e11f2e451..76e0c6d24 100644 --- a/dGame/dGameMessages/BuildingMessages.cpp +++ b/dGame/dGameMessages/BuildingMessages.cpp @@ -603,6 +603,15 @@ namespace GameMessages { return true; } + void NotifyClientUGCModelReady::Serialize(RakNet::BitStream& bitStream) const { + bitStream.Write(blueprintID); + } + + bool NotifyClientUGCModelReady::Deserialize(RakNet::BitStream& bitStream) { + VALIDATE_READ(bitStream.Read(blueprintID)); + return true; + } + void UGObjectMetadata::Serialize(RakNet::BitStream& bitStream) const { bitStream.Write(userModelID); bitStream.Write(blueprintID); diff --git a/dGame/dGameMessages/BuildingMessages.h b/dGame/dGameMessages/BuildingMessages.h index 3417e8960..b4a1907e1 100644 --- a/dGame/dGameMessages/BuildingMessages.h +++ b/dGame/dGameMessages/BuildingMessages.h @@ -242,6 +242,19 @@ namespace GameMessages { int32_t count{}; }; + // Server -> client, to a placed model. Its blueprint's files were made (again) on the UGC server: the client's + // BlueprintComponent with that blueprint flushes its cached NIF, HKX and LXFML and requests the NIF and HKX again + // (LWOBlueprintComponent::OnNotifyClientUGCModelReady, 0x00ca6430; the payload is the blueprint id only, + // GameMessage::NotifyClientUGCIconReady::Serialize 0x00daae30 is shared with it). The client keeps using its + // cached manifest entry, so the new checksum is sent first (UgcManifestResponse). See docs/UgcServer.md. + struct NotifyClientUGCModelReady : public NetGameMsg { + NotifyClientUGCModelReady() : NetGameMsg(MessageType::Game::NOTIFY_CLIENT_UGC_MODEL_READY) {} + void Serialize(RakNet::BitStream& bitStream) const override; + bool Deserialize(RakNet::BitStream& bitStream) override; + + LWOOBJID blueprintID{}; + }; + // A model's user generated data (UGObjectMetadata::Serialize, 0x00f5f590) struct UGObjectMetadata { LWOOBJID userModelID{}; diff --git a/dGame/dUtilities/UgcManifest.cpp b/dGame/dUtilities/UgcManifest.cpp index 86da0e097..509180b4b 100644 --- a/dGame/dUtilities/UgcManifest.cpp +++ b/dGame/dUtilities/UgcManifest.cpp @@ -1,21 +1,34 @@ #include "UgcManifest.h" #include -#include +#include +#include #include #include +#include "BrickByBrick.h" +#include "BuildingMessages.h" #include "ClientPackets.h" #include "Database.h" #include "dConfig.h" +#include "eBlueprintSaveResponseType.h" +#include "Entity.h" +#include "EntityManager.h" #include "Game.h" #include "Logger.h" +#include "MD5.h" +#include "PlayerManager.h" +#include "Sd0.h" namespace { // How often the waiting requests are looked up again, how long they wait at most and how many there can be constexpr auto RETRY_INTERVAL = std::chrono::seconds(5); constexpr auto MAX_WAIT = std::chrono::minutes(15); constexpr size_t MAX_WAITING = 512; + // A client asks for a model's NIF, HKX and LXFML at once: its LXFML is sent once for all three + constexpr auto LXFML_RESEND_AFTER = std::chrono::seconds(10); + // LXFML checksums worked out, kept for the next clients (a model's LXFML only changes with a new save) + constexpr size_t MAX_LXFML_CHECKSUMS = 4096; struct Waiting { SystemAddress sysAddr; @@ -26,20 +39,17 @@ namespace { std::vector g_Waiting; std::chrono::steady_clock::time_point g_NextRetry{}; + std::map, std::chrono::steady_clock::time_point> g_LxfmlSent; + std::map g_LxfmlChecksums; - // The UGC server's name for a blueprint's file of this type; nothing for the ones it doesn't make - std::optional FileName(eUgcResourceType type) { - switch (type) { - case eUgcResourceType::DDS: return "icon.dds"; - case eUgcResourceType::NIF: return "model.nif"; - default: return std::nullopt; - } + bool ManifestOn() { + // Off unless set: a client whose boot.cfg doesn't point it at the UGC server downloads from its built-in + // http://127.0.0.1:80/lwoclient and is logged out when it can't connect there (docs/UgcServer.md) + return Game::config && Game::config->GetValue("ugc_manifest") == "1"; } - bool Enabled() { - // Off unless set: the 1.10.64 client downloads from http://127.0.0.1:80 whatever its boot.cfg says, and one that - // can't connect there is logged out (docs/UgcServer.md) - return Game::config->GetValue("ugc_manifest") == "1"; + bool ModelsOn() { + return Game::config && Game::config->GetValue("ugc_manifest_models") == "1"; } int HexDigit(char c) { @@ -49,34 +59,143 @@ namespace { return -1; } - // Answers when the file's checksum is known; false when it isn't (yet) - bool TryAnswer(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) { - const auto file = FileName(resourceType); - if (!file) return false; - const auto checksum = Database::Get()->GetUgcFileChecksum(blueprintId, *file); - if (!checksum || checksum->md5.size() != 32) return false; + bool MeshMade(LWOOBJID blueprintId) { + const auto checksum = Database::Get()->GetUgcFileChecksum(blueprintId, "model.nif"); + return checksum && checksum->md5.size() == 32; + } + void Send(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType, const std::optional& checksum) { ClientPackets::UgcManifestResponse response; response.blueprintId = blueprintId; response.resourceType = resourceType; - response.fileSize = checksum->size; - for (size_t i = 0; i < response.md5.size(); i++) { - const auto high = HexDigit(checksum->md5[i * 2]); - const auto low = HexDigit(checksum->md5[i * 2 + 1]); - if (high < 0 || low < 0) return false; - response.md5[i] = static_cast(high << 4 | low); + if (checksum && checksum->md5.size() == 32) { + bool valid = true; + for (size_t i = 0; i < response.md5.size(); i++) { + const auto high = HexDigit(checksum->md5[i * 2]); + const auto low = HexDigit(checksum->md5[i * 2 + 1]); + if (high < 0 || low < 0) valid = false; + response.md5[i] = static_cast((high << 4) | low); + } + if (valid) { + response.valid = true; + response.fileSize = checksum->size; + } else { + response.md5 = {}; + } } - response.valid = true; response.Send(sysAddr); LOG_DEBUG("Sent the UGC manifest of %llu (type %i): %s, %u bytes", static_cast(blueprintId), - static_cast(resourceType), checksum->md5.c_str(), checksum->size); + static_cast(resourceType), response.valid ? checksum->md5.c_str() : "not known", response.fileSize); + } + + // Answers an icon's request when its checksum is known; false when it isn't (yet) + bool TryAnswerIcon(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) { + const auto checksum = Database::Get()->GetUgcFileChecksum(blueprintId, "icon.dds"); + if (!checksum || checksum->md5.size() != 32) return false; + Send(sysAddr, blueprintId, resourceType, checksum); + return true; + } + + std::optional StoredLxfmlChecksum(LWOOBJID blueprintId) { + if (const auto known = g_LxfmlChecksums.find(blueprintId); known != g_LxfmlChecksums.end()) return known->second; + const auto model = Database::Get()->GetUgcModel(blueprintId); + if (!model) return std::nullopt; + auto checksum = UgcManifest::LxfmlChecksum(model->lxfmlData.str()); + if (!checksum) return std::nullopt; + if (g_LxfmlChecksums.size() >= MAX_LXFML_CHECKSUMS) g_LxfmlChecksums.clear(); + g_LxfmlChecksums.emplace(blueprintId, *checksum); + return checksum; + } + + // The model's LXFML to one client, which builds the model from it (as when a property loads); false when there is + // no such model + bool SendLxfml(const SystemAddress& sysAddr, LWOOBJID blueprintId) { + const auto now = std::chrono::steady_clock::now(); + std::erase_if(g_LxfmlSent, [now](const auto& sent) { return now - sent.second > LXFML_RESEND_AFTER; }); + if (g_LxfmlSent.contains({ sysAddr, blueprintId })) return true; + + auto model = Database::Get()->GetUgcModel(blueprintId); + if (!model) return false; + ClientPackets::BlueprintSaveResponse response; + response.localId = LWOOBJID_EMPTY; // zero, like the LXFML sent when a property loads + response.reasonCode = eBlueprintSaveResponseType::EverythingWorked; + response.models.push_back({ blueprintId, model->lxfmlData.str() }); + response.Send(sysAddr); + g_LxfmlSent[{ sysAddr, blueprintId }] = now; + // Its mesh is wanted: a model still in its quiet period after a save is made now + Database::Get()->ExpediteUgcModel(blueprintId); + LOG_DEBUG("Sent the LXFML of %llu for the client to build", static_cast(blueprintId)); return true; } } +UgcManifest::eAction UgcManifest::Decide(const eUgcResourceType type, const bool manifestOn, const bool modelsOn, const bool meshMade) { + if (!manifestOn) return eAction::NONE; + switch (type) { + case eUgcResourceType::DDS: return eAction::WAIT; // answered at once when the icon is made + case eUgcResourceType::NIF: + case eUgcResourceType::LXFML: + return modelsOn && meshMade ? eAction::ANSWER : eAction::SEND_LXFML; + case eUgcResourceType::HKX: + // The UGC server makes no physics: the client uses an HKX it built before, else it asks and gets 404 + return modelsOn && meshMade ? eAction::ANSWER_UNKNOWN : eAction::SEND_LXFML; + default: return eAction::NONE; + } +} + +std::optional UgcManifest::LxfmlChecksum(const std::string& stored) { + std::string lxfml; + if (stored.starts_with("(lxfml.data()), static_cast(lxfml.size())); + md5.finalize(); + return IUgc::FileChecksum{ md5.hexdigest(), static_cast(lxfml.size()) }; +} + +bool UgcManifest::ServesModels() { + return ManifestOn() && ModelsOn(); +} + +bool UgcManifest::ClientBuildsModel(const LWOOBJID blueprintId) { + return !ServesModels() || !MeshMade(blueprintId); +} + void UgcManifest::OnRequest(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) { - if (sysAddr == UNASSIGNED_SYSTEM_ADDRESS || !Enabled() || !FileName(resourceType)) return; - if (TryAnswer(sysAddr, blueprintId, resourceType)) return; + if (sysAddr == UNASSIGNED_SYSTEM_ADDRESS) return; + const auto manifestOn = ManifestOn(); + const auto modelsOn = ModelsOn(); + const bool isModelFile = resourceType == eUgcResourceType::NIF || resourceType == eUgcResourceType::HKX || resourceType == eUgcResourceType::LXFML; + const auto meshMade = manifestOn && modelsOn && isModelFile && MeshMade(blueprintId); + + switch (Decide(resourceType, manifestOn, modelsOn, meshMade)) { + case eAction::NONE: return; + case eAction::ANSWER: + Send(sysAddr, blueprintId, resourceType, resourceType == eUgcResourceType::LXFML + ? StoredLxfmlChecksum(blueprintId) : Database::Get()->GetUgcFileChecksum(blueprintId, "model.nif")); + return; + case eAction::ANSWER_UNKNOWN: + Send(sysAddr, blueprintId, resourceType, std::nullopt); + return; + case eAction::SEND_LXFML: + // Not a player model (or gone): answered as not known, so the client doesn't wait for ever + if (!SendLxfml(sysAddr, blueprintId)) Send(sysAddr, blueprintId, resourceType, std::nullopt); + return; + case eAction::WAIT: + break; + } + + if (TryAnswerIcon(sysAddr, blueprintId, resourceType)) return; // Not made yet: a client wants it, so a model still in its quiet period after a save is made now (as when a client // asks the UGC server for it), and the request is answered when it's made Database::Get()->ExpediteUgcModel(blueprintId); @@ -93,10 +212,41 @@ void UgcManifest::Update() { if (now < g_NextRetry) return; g_NextRetry = now + RETRY_INTERVAL; std::erase_if(g_Waiting, [now](const Waiting& waiting) { - return now - waiting.since > MAX_WAIT || TryAnswer(waiting.sysAddr, waiting.blueprintId, waiting.resourceType); + return now - waiting.since > MAX_WAIT || TryAnswerIcon(waiting.sysAddr, waiting.blueprintId, waiting.resourceType); }); } void UgcManifest::OnDisconnect(const SystemAddress& sysAddr) { std::erase_if(g_Waiting, [&sysAddr](const Waiting& waiting) { return waiting.sysAddr == sysAddr; }); + std::erase_if(g_LxfmlSent, [&sysAddr](const auto& sent) { return sent.first.first == sysAddr; }); +} + +void UgcManifest::OnModelsMade(const std::vector& blueprintIds) { + for (const auto id : blueprintIds) g_LxfmlChecksums.erase(id); // made again after its LXFML changed, maybe + if (!ServesModels() || !Game::entityManager) return; + + const auto& players = PlayerManager::GetAllPlayers(); + if (players.empty()) return; + const auto models = Game::entityManager->GetEntitiesByLOT(BrickByBrick::MODEL_OBJECT_LOT); + for (const auto id : blueprintIds) { + std::vector shown; + for (auto* const model : models) { + if (model && model->GetVar(u"blueprintid") == id) shown.push_back(model); + } + if (shown.empty()) continue; + const auto checksum = Database::Get()->GetUgcFileChecksum(id, "model.nif"); + if (!checksum || checksum->md5.size() != 32) continue; + + // The new checksum first: the client keeps its cached one otherwise, and would load the file it has + for (auto* const player : players) { + if (player) Send(player->GetSystemAddress(), id, eUgcResourceType::NIF, checksum); + } + for (auto* const model : shown) { + GameMessages::NotifyClientUGCModelReady ready; + ready.target = model->GetObjectID(); + ready.blueprintID = id; + ready.Send(UNASSIGNED_SYSTEM_ADDRESS); + } + LOG("The UGC server made model %llu again: told %zu client(s) about %zu placed model(s)", static_cast(id), players.size(), shown.size()); + } } diff --git a/dGame/dUtilities/UgcManifest.h b/dGame/dUtilities/UgcManifest.h index fa9508b12..26fdb7c60 100644 --- a/dGame/dUtilities/UgcManifest.h +++ b/dGame/dUtilities/UgcManifest.h @@ -1,22 +1,56 @@ #ifndef UGCMANIFEST_H #define UGCMANIFEST_H +#include +#include +#include + #include "dCommonVars.h" #include "eUgcResourceType.h" +#include "IUgc.h" #include "RakNetTypes.h" /** * Answers the client's REQUEST_UGC_MANIFEST_INFO (sent when its UGCUSE3DSERVICES is off, the default): the MD5 and - * size of a blueprint's file, which the client needs before it downloads the file from the UGC server - * (BrickModels/UserMade//..sd0). The checksums are the ones the UGC server stored when it made the - * files (ugc_file_checksums); a file it hasn't made yet is answered once it has (Update looks again every few seconds - * for a while). Nothing is answered for files it never makes (LXFML, HKX), or unless `ugc_manifest` is 1: the client - * then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and is logged out when - * it can't connect there. + * size of a blueprint's file, which the client needs before it uses or downloads the file from the UGC server + * (BrickModels/UserMade//..sd0). Nothing is answered unless `ugc_manifest` is 1. + * + * * Icons (DDS): the checksum the UGC server stored when it made the icon (ugc_file_checksums); one it hasn't made yet + * is answered once it has (Update looks again every few seconds for a while). + * * Player models (NIF, HKX, LXFML): a placed model's client always asks for all three (its BlueprintComponent loads the + * blueprint's NIF and HKX, its ModelBehaviorComponent the LXFML) and waits for the answers without a timeout. With + * `ugc_manifest_models` 1 and the model's mesh made, the NIF is answered with the UGC server's checksum (the client + * downloads the served mesh), the LXFML with the stored LXFML's and the HKX as not known (valid 0: the client uses + * one it has, else asks the UGC server, which answers 404). Otherwise the world sends that client the model's LXFML + * (BlueprintSaveResponse), the client builds the NIF and HKX itself and caches their checksums, which answers its + * own requests: a model is never left waiting. * * Main thread only: each lookup is one indexed query and the answers are sent from here. See docs/UgcServer.md. */ namespace UgcManifest { + // What a request gets + enum class eAction { + NONE, // not answered (ugc_manifest off, or not a UGC type) + ANSWER, // the stored (or, for LXFML, computed) checksum + ANSWER_UNKNOWN,// valid 0: the client uses the file it has, or downloads it + SEND_LXFML, // the model's LXFML, for the client to build the model itself + WAIT, // icons not made yet: answered once they are + }; + + // The rule, for a request of `type` when the UGC server has (`meshMade`) or hasn't made the blueprint's mesh + eAction Decide(eUgcResourceType type, bool manifestOn, bool modelsOn, bool meshMade); + + // The MD5 (lowercase hex) and size of a stored LXFML (ugc.lxfml: sd0, or plain XML) as the client has it after + // downloading and inflating it; nothing when it can't be read + std::optional LxfmlChecksum(const std::string& stored); + + // ugc_manifest=1 and ugc_manifest_models=1: placed models' meshes come from the UGC server when made + bool ServesModels(); + + // Whether the client should get this model's LXFML when a property loads (it builds the model itself): not when + // ServesModels() and the UGC server made its mesh + bool ClientBuildsModel(LWOOBJID blueprintId); + void OnRequest(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType); // Answers the waiting requests whose files have been made since, and forgets the ones waited on too long @@ -24,6 +58,10 @@ namespace UgcManifest { // A client left: its waiting requests are dropped void OnDisconnect(const SystemAddress& sysAddr); + + // The UGC server made these models' meshes (again, with a new checksum): each one placed in this world is sent the + // new checksum and NotifyClientUGCModelReady, so clients load the served mesh + void OnModelsMade(const std::vector& blueprintIds); } #endif //!UGCMANIFEST_H diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index c46734851..b7b4e1829 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -52,6 +52,7 @@ #include "master/MessageCapture.h" #include "master/InstanceMigration.h" #include "master/ServerTraffic.h" +#include "master/UgcModelsMade.h" #ifdef DARKFLAME_PLATFORM_UNIX @@ -925,6 +926,15 @@ namespace { LOG("Dashboard announcement sent to %u world(s)", worlds); } + // The UGC server made models' meshes: every world hears of it, and those showing the models tell their clients + void OnUgcModelsMade(const UgcModelsMade& made, const SystemAddress& sysAddr) { + if (sysAddr != ugcServerMasterPeerSysAddr) return; + for (const auto& instance : Game::im->GetInstances()) { + if (!instance || !instance->GetIsReady() || instance->GetIsShuttingDown()) continue; + MasterPackets::SendTo(instance->GetSysAddr(), made); + } + } + void OnConfigReload(const ConfigReload& reload, const SystemAddress& sysAddr) { if (sysAddr != dashboardServerMasterPeerSysAddr) { LOG("Ignoring config reload from a server that is not the dashboard"); @@ -1017,6 +1027,7 @@ namespace { handlers.On(Master::MESSAGE_CAPTURE_DATA, ForwardWorldToDashboard); handlers.On(Master::REQUEST_SERVER_LIST, OnRequestServerList); handlers.On(Master::SERVER_TRAFFIC, OnServerTraffic); + handlers.On(Master::UGC_MODELS_MADE, OnUgcModelsMade); return handlers; }(); return handlers; diff --git a/dNet/MasterPackets.h b/dNet/MasterPackets.h index 3637508e9..61ae4ea73 100644 --- a/dNet/MasterPackets.h +++ b/dNet/MasterPackets.h @@ -15,6 +15,7 @@ #include "master/InstanceMigration.h" #include "master/MessageCapture.h" #include "master/PlayerAction.h" +#include "master/UgcModelsMade.h" class dServer; @@ -22,7 +23,8 @@ class dServer; * Packets between master and the other servers (MessageType::Master). None of them reach a client. * * The dashboard's messages and instance migration are MASTER packets too; they live in per-topic headers under - * dNet/master/ (PlayerAction.h, DataChanged.h, DashboardMessages.h, MessageCapture.h, InstanceMigration.h), which + * dNet/master/ (PlayerAction.h, DataChanged.h, DashboardMessages.h, MessageCapture.h, InstanceMigration.h, + * UgcModelsMade.h), which * this header includes, so including MasterPackets.h gives every MASTER struct. */ namespace MasterPackets { diff --git a/dNet/master/UgcModelsMade.h b/dNet/master/UgcModelsMade.h new file mode 100644 index 000000000..711d52185 --- /dev/null +++ b/dNet/master/UgcModelsMade.h @@ -0,0 +1,42 @@ +#ifndef __UGCMODELSMADE__H__ +#define __UGCMODELSMADE__H__ + +#include +#include +#include + +#include "BitStream.h" +#include "BitStreamUtils.h" +#include "MessageType/Master.h" +#include "dCommonVars.h" + +/** + * UGC_MODELS_MADE payload (UGC server -> master -> every world): the player models (their ugc id, which is the + * blueprint id) whose mesh the UGC server just made or made again. A world showing one of them tells its clients the + * new checksum and sends NotifyClientUGCModelReady, so they load the served mesh (docs/UgcServer.md). + */ +struct UgcModelsMade : public LUBitStream { + UgcModelsMade() : LUBitStream(ServiceType::MASTER, MessageType::Master::UGC_MODELS_MADE) {} + + static constexpr size_t MAX_MODELS = 4096; + + std::vector blueprintIds; + + void Serialize(RakNet::BitStream& stream) const override { + const auto count = static_cast(std::min(blueprintIds.size(), MAX_MODELS)); + stream.Write(count); + for (size_t i = 0; i < count; i++) stream.Write(blueprintIds[i]); + } + + bool Deserialize(RakNet::BitStream& stream) override { + uint16_t count{}; + if (!stream.Read(count) || count > MAX_MODELS) return false; + blueprintIds.resize(count); + for (auto& id : blueprintIds) { + if (!stream.Read(id)) return false; + } + return true; + } +}; + +#endif //!__UGCMODELSMADE__H__ diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index a3b6ba89f..d4dd045c4 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -441,7 +441,14 @@ void UgcProcessor::Collect() { void UgcProcessor::StoreChecksums(Kind kind, LWOOBJID storageId, const std::vector& checksums) { const auto owner = kind == Kind::MODEL ? IUgc::eFileOwner::MODEL : IUgc::eFileOwner::COMBINATION; - for (const auto& checksum : checksums) Database::Get()->SetUgcFileChecksum(owner, storageId, checksum.file, checksum.md5, checksum.size); + for (const auto& checksum : checksums) { + // A player model's mesh that changed: the worlds showing it tell their clients (UGC_MODELS_MADE) + if (kind == Kind::MODEL && checksum.file == "model.nif") { + const auto before = Database::Get()->GetUgcFileChecksum(storageId, checksum.file); + if (!before || before->md5 != checksum.md5 || before->size != checksum.size) m_ChangedMeshes.push_back(storageId); + } + Database::Get()->SetUgcFileChecksum(owner, storageId, checksum.file, checksum.md5, checksum.size); + } } void UgcProcessor::Backfill() { diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h index 14f84b789..23582c2e2 100644 --- a/dUgcServer/UgcProcessor.h +++ b/dUgcServer/UgcProcessor.h @@ -140,6 +140,10 @@ public: uint64_t StoredBytes() const { return m_StoredBytes; } uint64_t MaxStorageBytes() const { return m_Config.maxStorageBytes; } + // Main thread: the models whose mesh (model.nif) was written with a different checksum than before since the last + // call, for the worlds (UGC_MODELS_MADE); a model made again unchanged (e.g. after eviction) isn't listed + std::vector TakeChangedMeshes() { std::vector ids; ids.swap(m_ChangedMeshes); return ids; } + private: struct Job { Kind kind{}; @@ -223,6 +227,7 @@ private: std::chrono::steady_clock::time_point m_NextEviction{}; std::map, std::pair> m_Recent; // answers for missing files uint64_t m_StoredBytes{}; + std::vector m_ChangedMeshes; // TakeChangedMeshes uint64_t m_Made{}; uint64_t m_Failed{}; uint64_t m_Evicted{}; diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index a524d5cda..c5f18b060 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -21,6 +21,7 @@ #include "Game.h" #include "GeneralUtils.h" #include "Logger.h" +#include "MasterPackets.h" #include "Server.h" #include "ServiceType.h" #include "Web.h" @@ -642,6 +643,14 @@ int main(int argc, char** argv) { packet = g_Server->ReceiveFromMaster(); } processor.Update(); + // Worlds showing a model whose mesh changed tell their clients (docs/UgcServer.md, "Models without 3D services") + if (auto changed = processor.TakeChangedMeshes(); !changed.empty()) { + for (size_t start = 0; start < changed.size(); start += UgcModelsMade::MAX_MODELS) { + UgcModelsMade made; + made.blueprintIds.assign(changed.begin() + start, changed.begin() + std::min(changed.size(), start + UgcModelsMade::MAX_MODELS)); + MasterPackets::SendToMaster(made, g_Server); + } + } // Settings the dashboard changed arrive as a config reload; pick them up if (now - lastConfigure >= std::chrono::seconds(5)) { lastConfigure = now; diff --git a/dWorldServer/WorldServer.cpp b/dWorldServer/WorldServer.cpp index f2b3c864b..231bcbd8d 100644 --- a/dWorldServer/WorldServer.cpp +++ b/dWorldServer/WorldServer.cpp @@ -836,6 +836,7 @@ namespace { handlers.On(Master::MESSAGE_CAPTURE_CONTROL, [](const MessageCaptureControl& control, const SystemAddress&) { MessageInspector::Control(control); }); handlers.On(Master::ANNOUNCE, [](const Announcement& announcement, const SystemAddress&) { DashboardNotify::Announce(announcement.title, announcement.message); }); handlers.On(Master::NEW_SESSION_ALERT, OnNewSessionAlert); + handlers.On(Master::UGC_MODELS_MADE, [](const UgcModelsMade& made, const SystemAddress&) { UgcManifest::OnModelsMade(made.blueprintIds); }); return handlers; }(); return handlers; @@ -1031,8 +1032,9 @@ void LoadPlayer(const SystemAddress& sysAddr) { goto noBBB; } - // Workaround for not having a UGC server to get model LXFML onto the client so it - // can generate the physics and nif for the object. + // The models' LXFML, for the client to build each model's NIF and HKX itself. With ugc_manifest_models + // the models whose mesh the UGC server made are left out: the client asks for their files and downloads + // the mesh (UgcManifest, docs/UgcServer.md). auto bbbModels = Database::Get()->GetUgcModels(propertyId); if (bbbModels.empty()) { @@ -1043,7 +1045,12 @@ void LoadPlayer(const SystemAddress& sysAddr) { ClientPackets::BlueprintSaveResponse response; response.localId = LWOOBJID_EMPTY; //always zero so that a check on the client passes response.reasonCode = eBlueprintSaveResponseType::EverythingWorked; + size_t served = 0; for (auto& bbbModel : bbbModels) { + if (!UgcManifest::ClientBuildsModel(bbbModel.id)) { + served++; + continue; + } LOG("Getting lxfml ugcID: %llu", bbbModel.id); bbbModel.lxfmlData.seekg(0, std::ios::end); @@ -1055,7 +1062,8 @@ void LoadPlayer(const SystemAddress& sysAddr) { model.blueprintId = bbbModel.id; model.data = bbbModel.lxfmlData.str().substr(0, lxfmlSize); } - response.Send(sysAddr); + if (served > 0) LOG("%zu of the property's %zu models come from the UGC server", served, bbbModels.size()); + if (!response.models.empty()) response.Send(sysAddr); } noBBB: diff --git a/resources/sharedconfig.ini b/resources/sharedconfig.ini index 9c14d0f7e..a9152cf25 100644 --- a/resources/sharedconfig.ini +++ b/resources/sharedconfig.ini @@ -95,7 +95,13 @@ mute_auto_reject_names=1 ugc_debounce_seconds=120 # 0 or 1: worlds answer game clients without 3D services (UGCUSE3DSERVICES=7:0, the default) with the checksums of the -# files the UGC server made, so they download the icons of cars, rockets and models. The 1.10.64 client then downloads -# from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and a client that can't connect there -# is logged out, so only set 1 when the UGC server answers there for every player. See docs/UgcServer.md. +# files the UGC server made, so they download the icons of cars, rockets and models from the UGCSERVERIP, UGCSERVERPORT +# and UGCSERVERDIR of their boot.cfg (default: the PATCHSERVER* values). A client that can't connect there is logged +# out, so only set 1 when every player's boot.cfg points at the UGC server. See docs/UgcServer.md. ugc_manifest=0 + +# 0 or 1, with ugc_manifest=1: placed player models the UGC server has made are downloaded from it (their mesh) instead +# of being built by each client from the LXFML the world sends; models not made yet are still sent as LXFML. The UGC +# server makes no physics (.hkx): a client that never built a model itself has no collision for it. See +# docs/UgcServer.md. +ugc_manifest_models=0 diff --git a/tests/dGameTests/CMakeLists.txt b/tests/dGameTests/CMakeLists.txt index ff6fd5316..c263a03bd 100644 --- a/tests/dGameTests/CMakeLists.txt +++ b/tests/dGameTests/CMakeLists.txt @@ -13,6 +13,7 @@ set(DGAMETEST_SOURCES "PowerUpStatisticTests.cpp" "BrickByBrickTests.cpp" "ModularBuildTableTests.cpp" + "UgcManifestTests.cpp" ) add_subdirectory(dComponentsTests) diff --git a/tests/dGameTests/UgcManifestTests.cpp b/tests/dGameTests/UgcManifestTests.cpp new file mode 100644 index 000000000..cf2e26505 --- /dev/null +++ b/tests/dGameTests/UgcManifestTests.cpp @@ -0,0 +1,121 @@ +#include "UgcManifest.h" + +#include "ClientPackets.h" +#include "dConfig.h" +#include "GameDependencies.h" +#include "MD5.h" +#include "PacketTestUtils.h" +#include "Sd0.h" + +#include + +using namespace PacketTestUtils; +using UgcManifest::eAction; + +namespace { + std::string Md5Hex(const std::string& data) { + MD5 md5; + md5.update(reinterpret_cast(data.data()), static_cast(data.size())); + md5.finalize(); + return md5.hexdigest(); + } +} + +// What each request gets. A placed model's client asks for its NIF, HKX and LXFML and waits for every answer without a +// timeout (LWOResMgr2Interface::RequestBlueprintManifestThenLoad, 0x0105a910), so model files are always answered: +// from the UGC server when it made the mesh and models are served, else with the LXFML for the client to build. +TEST(UgcManifestTests, DecideCoversEveryType) { + using T = eUgcResourceType; + for (const auto type : { T::LXFML, T::NIF, T::HKX, T::DDS }) { + EXPECT_EQ(UgcManifest::Decide(type, false, true, true), eAction::NONE); + } + EXPECT_EQ(UgcManifest::Decide(T::DDS, true, false, false), eAction::WAIT); + EXPECT_EQ(UgcManifest::Decide(T::DDS, true, true, true), eAction::WAIT); + + EXPECT_EQ(UgcManifest::Decide(T::NIF, true, true, true), eAction::ANSWER); + EXPECT_EQ(UgcManifest::Decide(T::LXFML, true, true, true), eAction::ANSWER); + EXPECT_EQ(UgcManifest::Decide(T::HKX, true, true, true), eAction::ANSWER_UNKNOWN); + + for (const auto type : { T::LXFML, T::NIF, T::HKX }) { + EXPECT_EQ(UgcManifest::Decide(type, true, true, false), eAction::SEND_LXFML); // not made yet + EXPECT_EQ(UgcManifest::Decide(type, true, false, true), eAction::SEND_LXFML); // models not served + EXPECT_EQ(UgcManifest::Decide(type, true, false, false), eAction::SEND_LXFML); + } + EXPECT_EQ(UgcManifest::Decide(static_cast(7), true, true, true), eAction::NONE); +} + +class UgcManifestRequestTests : public GameDependenciesTest { +protected: + void SetUp() override { SetUpDependencies(); } + void TearDown() override { + UgcManifest::OnDisconnect(Client()); + TearDownDependencies(); + } + + static SystemAddress Client() { + SystemAddress address; + address.binaryAddress = 0x0100007f; + address.port = 15001; + return address; + } + + void Settings(const std::string& manifest, const std::string& models) { + Game::config->SetDatabaseValues({ { "ugc_manifest", manifest }, { "ugc_manifest_models", models } }, {}); + } +}; + +// Off (the default): nothing is answered, as before +TEST_F(UgcManifestRequestTests, NothingWhenOff) { + Settings("0", "1"); + const auto sent = Capture([] { UgcManifest::OnRequest(Client(), 1234, eUgcResourceType::NIF); }); + EXPECT_TRUE(sent.empty()); + EXPECT_FALSE(UgcManifest::ServesModels()); + EXPECT_TRUE(UgcManifest::ClientBuildsModel(1234)); +} + +// A model file of a blueprint that isn't a player model: answered as not known (37 bytes, valid 0), never left waiting +TEST_F(UgcManifestRequestTests, UnknownModelFilesAreAnsweredNotKnown) { + Settings("1", "1"); + EXPECT_TRUE(UgcManifest::ServesModels()); + EXPECT_TRUE(UgcManifest::ClientBuildsModel(1234)); // its mesh isn't made + for (const auto type : { eUgcResourceType::NIF, eUgcResourceType::HKX, eUgcResourceType::LXFML }) { + const auto sent = Capture([type] { UgcManifest::OnRequest(Client(), 0x0102030405060708, type); }); + ASSERT_EQ(sent.size(), 1); + EXPECT_EQ(sent[0].sysAddr, Client()); + EXPECT_FALSE(sent[0].broadcast); + EXPECT_PACKET_EQ(FromHex("53 05 00 3c 00 00 00 00 08 07 06 05 04 03 02 01 0" + std::to_string(static_cast(type)) + + " 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00"), FromCapture(sent[0])); + } +} + +// An icon that isn't made yet waits (answered by Update once the UGC server stored its checksum) +TEST_F(UgcManifestRequestTests, IconsWait) { + Settings("1", "0"); + const auto sent = Capture([] { UgcManifest::OnRequest(Client(), 1234, eUgcResourceType::DDS); }); + EXPECT_TRUE(sent.empty()); +} + +// Models made while nobody is on a property with them: nothing to tell +TEST_F(UgcManifestRequestTests, ModelsMadeWithNobodyToTell) { + Settings("1", "1"); + const auto sent = Capture([] { UgcManifest::OnModelsMade({ 1234, 5678 }); }); + EXPECT_TRUE(sent.empty()); +} + +// The client checks the MD5 and size of the LXFML as it has it after inflating the UGC server's .lxfml.sd0, which is +// the stored LXFML (sd0, or plain XML in old rows) inflated +TEST_F(UgcManifestRequestTests, LxfmlChecksumIsOfTheInflatedLxfml) { + const std::string lxfml = "\n\n"; + const auto plain = UgcManifest::LxfmlChecksum(lxfml); + ASSERT_TRUE(plain.has_value()); + EXPECT_EQ(plain->md5, Md5Hex(lxfml)); + EXPECT_EQ(plain->size, lxfml.size()); + + const auto compressed = UgcManifest::LxfmlChecksum(Sd0::Compress(lxfml)); + ASSERT_TRUE(compressed.has_value()); + EXPECT_EQ(compressed->md5, plain->md5); + EXPECT_EQ(compressed->size, plain->size); + + EXPECT_FALSE(UgcManifest::LxfmlChecksum("").has_value()); + EXPECT_FALSE(UgcManifest::LxfmlChecksum("not an sd0 stream").has_value()); +} diff --git a/tests/dGameTests/dGameMessagesTests/BuildingMessagesTests.cpp b/tests/dGameTests/dGameMessagesTests/BuildingMessagesTests.cpp index bf32b4f85..f95c2eba9 100644 --- a/tests/dGameTests/dGameMessagesTests/BuildingMessagesTests.cpp +++ b/tests/dGameTests/dGameMessagesTests/BuildingMessagesTests.cpp @@ -490,3 +490,19 @@ TEST_F(BuildingMessagesTests, FetchModelMetadataWire) { EXPECT_TRUE(copy.bpData.neverFalseIfPresentInCaps); ExpectTruncatedFails(full); } + +// NotifyClientUGCModelReady (909): the blueprint id only (GameMessage::NotifyClientUGCIconReady::Serialize, 0x00daae30, +// which the client shares with it; read by LWOBlueprintComponent::OnNotifyClientUGCModelReady, 0x00ca6430) +TEST_F(BuildingMessagesTests, NotifyClientUGCModelReadyWire) { + GameMessages::NotifyClientUGCModelReady ready; + ready.target = 0x1122334455667788; + ready.blueprintID = 0x0102030405060708; + EXPECT_PACKET_EQ(FromHex("08 07 06 05 04 03 02 01"), Payload(ready)); + EXPECT_EQ(RoundTrip(ready).blueprintID, ready.blueprintID); + ExpectTruncatedFails(ready); + + // The whole packet: the model's object id, then the message id 909 + RakNet::BitStream bitStream; + ready.WritePacket(bitStream); + EXPECT_PACKET_EQ(FromHex("53 05 00 0c 00 00 00 00 88 77 66 55 44 33 22 11 8d 03 08 07 06 05 04 03 02 01"), FromBitStream(bitStream)); +} diff --git a/tests/dGameTests/dNetTests/MasterPacketsTests.cpp b/tests/dGameTests/dNetTests/MasterPacketsTests.cpp index 3b0ebef07..8eab5cde3 100644 --- a/tests/dGameTests/dNetTests/MasterPacketsTests.cpp +++ b/tests/dGameTests/dNetTests/MasterPacketsTests.cpp @@ -403,6 +403,10 @@ TEST(MasterPacketsTests, DashboardAndMigrationStructsKeepTheirBytes) { changed.entries.push_back({ "characters", 42 }); ExpectHeaderThenSerialize(changed, MessageType::Master::DATA_CHANGED); + UgcModelsMade made; + made.blueprintIds = { 1152921510436831123, 42 }; + ExpectHeaderThenSerialize(made, MessageType::Master::UGC_MODELS_MADE); + PlayerPositions positions; positions.zoneId = 1100; positions.players.push_back({ 42, 1.0f, 2.0f, 3.0f }); @@ -485,3 +489,20 @@ TEST(MasterPacketsTests, DispatcherRoutesAndDropsBadPackets) { delete Game::logger; Game::logger = previousLogger; } + +// UGC server -> master -> worlds: a u16 count, then each blueprint id +TEST(MasterPacketsTests, UgcModelsMadeBytes) { + UgcModelsMade made; + made.blueprintIds = { 0x0102030405060708, 9 }; + EXPECT_PACKET_EQ(FromHex("53 06 00 25 00 00 00 00 02 00 08 07 06 05 04 03 02 01 09 00 00 00 00 00 00 00"), StructPacket(made)); + + RakNet::BitStream tooMany; + tooMany.Write(UgcModelsMade::MAX_MODELS + 1); + UgcModelsMade read; + EXPECT_FALSE(read.Deserialize(tooMany)); + + RakNet::BitStream truncated; + truncated.Write(2); + truncated.Write(1); + EXPECT_FALSE(read.Deserialize(truncated)); +}