diff --git a/dCommon/dEnums/MessageType/Master.h b/dCommon/dEnums/MessageType/Master.h index c0c531551..cc803ab07 100644 --- a/dCommon/dEnums/MessageType/Master.h +++ b/dCommon/dEnums/MessageType/Master.h @@ -66,5 +66,9 @@ namespace MessageType { // Any server -> master -> dashboard: traffic counters of the last few seconds (see ServerTraffic.h) SERVER_TRAFFIC, + + // UGC server -> master -> every world: player models whose mesh (model.nif) was just made or made again (see + // UgcModelsMade.h), so worlds showing them tell their clients (NotifyClientUGCModelReady) + UGC_MODELS_MADE, }; } diff --git a/dDashboardServer/routes/SettingsCatalog.cpp b/dDashboardServer/routes/SettingsCatalog.cpp index 57ce91b3f..360ac096c 100644 --- a/dDashboardServer/routes/SettingsCatalog.cpp +++ b/dDashboardServer/routes/SettingsCatalog.cpp @@ -422,7 +422,8 @@ namespace { c.Add(Format(Text(UGC, "listen_ip", "Listen address", "The game client has to reach it.", "0.0.0.0", true), eFormat::HOST)); c.Add(Port(UGC, "net_port", "Master connection port", "UDP; the next port is used too.", "2012")); c.Add(Text(UGC, "client_path", "Download path", "The game client's UGCSERVERDIR.", "/ugc", true)); - c.Add(Bool(SHARED, "ugc_manifest", "Answer clients without 3D services", "Worlds tell game clients with UGCUSE3DSERVICES=7:0 (the default) the checksums of the files the UGC server made, so they download icons of cars, rockets and models. The client then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and one that can't connect there is logged out: only turn on when the UGC server answers there for every player.", false)); + c.Add(Bool(SHARED, "ugc_manifest", "Answer clients without 3D services", "Worlds tell game clients with UGCUSE3DSERVICES=7:0 (the default) the checksums of the files the UGC server made, so they download icons of cars, rockets and models from UGCSERVERIP, UGCSERVERPORT and UGCSERVERDIR in their boot.cfg (default: the patch server's). A client that can't connect there is logged out, so only turn on when every player's boot.cfg points at the UGC server.", false)); + c.Add(When(Bool(SHARED, "ugc_manifest_models", "Placed models from the UGC server", "With the answer to clients without 3D services on: placed player models the UGC server has made are downloaded from it (its mesh) instead of built by each client from the LXFML the world sends. It makes no physics: a client that never built a model itself has no collision for it. Models not made yet are still sent as LXFML.", false), SHARED, "ugc_manifest", { "1" })); c.Add(Format(Text(DASHBOARD, "ugc_internal_url", "UGC server address (internal)", "Where the dashboard itself reaches the UGC server for its status and files. Empty: http://127.0.0.1:2008.", ""), eFormat::URL)); c.Add(Format(Text(DASHBOARD, "ugc_public_url", "UGC server address (public)", "Only for the \"open on the UGC server\" links, e.g. https://ugc.example.com. The dashboard's own pages don't need the browser to reach it.", ""), eFormat::URL)); c.AddSection("UGC processing", "When models are made and how much of the machine the workers may use.", eLayout::ROWS, Condition{ MASTER, "enable_ugc_server", { "1" } }); diff --git a/dGame/dGameMessages/BuildingMessages.cpp b/dGame/dGameMessages/BuildingMessages.cpp index e11f2e451..76e0c6d24 100644 --- a/dGame/dGameMessages/BuildingMessages.cpp +++ b/dGame/dGameMessages/BuildingMessages.cpp @@ -603,6 +603,15 @@ namespace GameMessages { return true; } + void NotifyClientUGCModelReady::Serialize(RakNet::BitStream& bitStream) const { + bitStream.Write(blueprintID); + } + + bool NotifyClientUGCModelReady::Deserialize(RakNet::BitStream& bitStream) { + VALIDATE_READ(bitStream.Read(blueprintID)); + return true; + } + void UGObjectMetadata::Serialize(RakNet::BitStream& bitStream) const { bitStream.Write(userModelID); bitStream.Write(blueprintID); diff --git a/dGame/dGameMessages/BuildingMessages.h b/dGame/dGameMessages/BuildingMessages.h index 3417e8960..b4a1907e1 100644 --- a/dGame/dGameMessages/BuildingMessages.h +++ b/dGame/dGameMessages/BuildingMessages.h @@ -242,6 +242,19 @@ namespace GameMessages { int32_t count{}; }; + // Server -> client, to a placed model. Its blueprint's files were made (again) on the UGC server: the client's + // BlueprintComponent with that blueprint flushes its cached NIF, HKX and LXFML and requests the NIF and HKX again + // (LWOBlueprintComponent::OnNotifyClientUGCModelReady, 0x00ca6430; the payload is the blueprint id only, + // GameMessage::NotifyClientUGCIconReady::Serialize 0x00daae30 is shared with it). The client keeps using its + // cached manifest entry, so the new checksum is sent first (UgcManifestResponse). See docs/UgcServer.md. + struct NotifyClientUGCModelReady : public NetGameMsg { + NotifyClientUGCModelReady() : NetGameMsg(MessageType::Game::NOTIFY_CLIENT_UGC_MODEL_READY) {} + void Serialize(RakNet::BitStream& bitStream) const override; + bool Deserialize(RakNet::BitStream& bitStream) override; + + LWOOBJID blueprintID{}; + }; + // A model's user generated data (UGObjectMetadata::Serialize, 0x00f5f590) struct UGObjectMetadata { LWOOBJID userModelID{}; diff --git a/dGame/dUtilities/UgcManifest.cpp b/dGame/dUtilities/UgcManifest.cpp index 86da0e097..509180b4b 100644 --- a/dGame/dUtilities/UgcManifest.cpp +++ b/dGame/dUtilities/UgcManifest.cpp @@ -1,21 +1,34 @@ #include "UgcManifest.h" #include -#include +#include +#include #include #include +#include "BrickByBrick.h" +#include "BuildingMessages.h" #include "ClientPackets.h" #include "Database.h" #include "dConfig.h" +#include "eBlueprintSaveResponseType.h" +#include "Entity.h" +#include "EntityManager.h" #include "Game.h" #include "Logger.h" +#include "MD5.h" +#include "PlayerManager.h" +#include "Sd0.h" namespace { // How often the waiting requests are looked up again, how long they wait at most and how many there can be constexpr auto RETRY_INTERVAL = std::chrono::seconds(5); constexpr auto MAX_WAIT = std::chrono::minutes(15); constexpr size_t MAX_WAITING = 512; + // A client asks for a model's NIF, HKX and LXFML at once: its LXFML is sent once for all three + constexpr auto LXFML_RESEND_AFTER = std::chrono::seconds(10); + // LXFML checksums worked out, kept for the next clients (a model's LXFML only changes with a new save) + constexpr size_t MAX_LXFML_CHECKSUMS = 4096; struct Waiting { SystemAddress sysAddr; @@ -26,20 +39,17 @@ namespace { std::vector g_Waiting; std::chrono::steady_clock::time_point g_NextRetry{}; + std::map, std::chrono::steady_clock::time_point> g_LxfmlSent; + std::map g_LxfmlChecksums; - // The UGC server's name for a blueprint's file of this type; nothing for the ones it doesn't make - std::optional FileName(eUgcResourceType type) { - switch (type) { - case eUgcResourceType::DDS: return "icon.dds"; - case eUgcResourceType::NIF: return "model.nif"; - default: return std::nullopt; - } + bool ManifestOn() { + // Off unless set: a client whose boot.cfg doesn't point it at the UGC server downloads from its built-in + // http://127.0.0.1:80/lwoclient and is logged out when it can't connect there (docs/UgcServer.md) + return Game::config && Game::config->GetValue("ugc_manifest") == "1"; } - bool Enabled() { - // Off unless set: the 1.10.64 client downloads from http://127.0.0.1:80 whatever its boot.cfg says, and one that - // can't connect there is logged out (docs/UgcServer.md) - return Game::config->GetValue("ugc_manifest") == "1"; + bool ModelsOn() { + return Game::config && Game::config->GetValue("ugc_manifest_models") == "1"; } int HexDigit(char c) { @@ -49,34 +59,143 @@ namespace { return -1; } - // Answers when the file's checksum is known; false when it isn't (yet) - bool TryAnswer(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) { - const auto file = FileName(resourceType); - if (!file) return false; - const auto checksum = Database::Get()->GetUgcFileChecksum(blueprintId, *file); - if (!checksum || checksum->md5.size() != 32) return false; + bool MeshMade(LWOOBJID blueprintId) { + const auto checksum = Database::Get()->GetUgcFileChecksum(blueprintId, "model.nif"); + return checksum && checksum->md5.size() == 32; + } + void Send(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType, const std::optional& checksum) { ClientPackets::UgcManifestResponse response; response.blueprintId = blueprintId; response.resourceType = resourceType; - response.fileSize = checksum->size; - for (size_t i = 0; i < response.md5.size(); i++) { - const auto high = HexDigit(checksum->md5[i * 2]); - const auto low = HexDigit(checksum->md5[i * 2 + 1]); - if (high < 0 || low < 0) return false; - response.md5[i] = static_cast(high << 4 | low); + if (checksum && checksum->md5.size() == 32) { + bool valid = true; + for (size_t i = 0; i < response.md5.size(); i++) { + const auto high = HexDigit(checksum->md5[i * 2]); + const auto low = HexDigit(checksum->md5[i * 2 + 1]); + if (high < 0 || low < 0) valid = false; + response.md5[i] = static_cast((high << 4) | low); + } + if (valid) { + response.valid = true; + response.fileSize = checksum->size; + } else { + response.md5 = {}; + } } - response.valid = true; response.Send(sysAddr); LOG_DEBUG("Sent the UGC manifest of %llu (type %i): %s, %u bytes", static_cast(blueprintId), - static_cast(resourceType), checksum->md5.c_str(), checksum->size); + static_cast(resourceType), response.valid ? checksum->md5.c_str() : "not known", response.fileSize); + } + + // Answers an icon's request when its checksum is known; false when it isn't (yet) + bool TryAnswerIcon(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) { + const auto checksum = Database::Get()->GetUgcFileChecksum(blueprintId, "icon.dds"); + if (!checksum || checksum->md5.size() != 32) return false; + Send(sysAddr, blueprintId, resourceType, checksum); + return true; + } + + std::optional StoredLxfmlChecksum(LWOOBJID blueprintId) { + if (const auto known = g_LxfmlChecksums.find(blueprintId); known != g_LxfmlChecksums.end()) return known->second; + const auto model = Database::Get()->GetUgcModel(blueprintId); + if (!model) return std::nullopt; + auto checksum = UgcManifest::LxfmlChecksum(model->lxfmlData.str()); + if (!checksum) return std::nullopt; + if (g_LxfmlChecksums.size() >= MAX_LXFML_CHECKSUMS) g_LxfmlChecksums.clear(); + g_LxfmlChecksums.emplace(blueprintId, *checksum); + return checksum; + } + + // The model's LXFML to one client, which builds the model from it (as when a property loads); false when there is + // no such model + bool SendLxfml(const SystemAddress& sysAddr, LWOOBJID blueprintId) { + const auto now = std::chrono::steady_clock::now(); + std::erase_if(g_LxfmlSent, [now](const auto& sent) { return now - sent.second > LXFML_RESEND_AFTER; }); + if (g_LxfmlSent.contains({ sysAddr, blueprintId })) return true; + + auto model = Database::Get()->GetUgcModel(blueprintId); + if (!model) return false; + ClientPackets::BlueprintSaveResponse response; + response.localId = LWOOBJID_EMPTY; // zero, like the LXFML sent when a property loads + response.reasonCode = eBlueprintSaveResponseType::EverythingWorked; + response.models.push_back({ blueprintId, model->lxfmlData.str() }); + response.Send(sysAddr); + g_LxfmlSent[{ sysAddr, blueprintId }] = now; + // Its mesh is wanted: a model still in its quiet period after a save is made now + Database::Get()->ExpediteUgcModel(blueprintId); + LOG_DEBUG("Sent the LXFML of %llu for the client to build", static_cast(blueprintId)); return true; } } +UgcManifest::eAction UgcManifest::Decide(const eUgcResourceType type, const bool manifestOn, const bool modelsOn, const bool meshMade) { + if (!manifestOn) return eAction::NONE; + switch (type) { + case eUgcResourceType::DDS: return eAction::WAIT; // answered at once when the icon is made + case eUgcResourceType::NIF: + case eUgcResourceType::LXFML: + return modelsOn && meshMade ? eAction::ANSWER : eAction::SEND_LXFML; + case eUgcResourceType::HKX: + // The UGC server makes no physics: the client uses an HKX it built before, else it asks and gets 404 + return modelsOn && meshMade ? eAction::ANSWER_UNKNOWN : eAction::SEND_LXFML; + default: return eAction::NONE; + } +} + +std::optional UgcManifest::LxfmlChecksum(const std::string& stored) { + std::string lxfml; + if (stored.starts_with("(lxfml.data()), static_cast(lxfml.size())); + md5.finalize(); + return IUgc::FileChecksum{ md5.hexdigest(), static_cast(lxfml.size()) }; +} + +bool UgcManifest::ServesModels() { + return ManifestOn() && ModelsOn(); +} + +bool UgcManifest::ClientBuildsModel(const LWOOBJID blueprintId) { + return !ServesModels() || !MeshMade(blueprintId); +} + void UgcManifest::OnRequest(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) { - if (sysAddr == UNASSIGNED_SYSTEM_ADDRESS || !Enabled() || !FileName(resourceType)) return; - if (TryAnswer(sysAddr, blueprintId, resourceType)) return; + if (sysAddr == UNASSIGNED_SYSTEM_ADDRESS) return; + const auto manifestOn = ManifestOn(); + const auto modelsOn = ModelsOn(); + const bool isModelFile = resourceType == eUgcResourceType::NIF || resourceType == eUgcResourceType::HKX || resourceType == eUgcResourceType::LXFML; + const auto meshMade = manifestOn && modelsOn && isModelFile && MeshMade(blueprintId); + + switch (Decide(resourceType, manifestOn, modelsOn, meshMade)) { + case eAction::NONE: return; + case eAction::ANSWER: + Send(sysAddr, blueprintId, resourceType, resourceType == eUgcResourceType::LXFML + ? StoredLxfmlChecksum(blueprintId) : Database::Get()->GetUgcFileChecksum(blueprintId, "model.nif")); + return; + case eAction::ANSWER_UNKNOWN: + Send(sysAddr, blueprintId, resourceType, std::nullopt); + return; + case eAction::SEND_LXFML: + // Not a player model (or gone): answered as not known, so the client doesn't wait for ever + if (!SendLxfml(sysAddr, blueprintId)) Send(sysAddr, blueprintId, resourceType, std::nullopt); + return; + case eAction::WAIT: + break; + } + + if (TryAnswerIcon(sysAddr, blueprintId, resourceType)) return; // Not made yet: a client wants it, so a model still in its quiet period after a save is made now (as when a client // asks the UGC server for it), and the request is answered when it's made Database::Get()->ExpediteUgcModel(blueprintId); @@ -93,10 +212,41 @@ void UgcManifest::Update() { if (now < g_NextRetry) return; g_NextRetry = now + RETRY_INTERVAL; std::erase_if(g_Waiting, [now](const Waiting& waiting) { - return now - waiting.since > MAX_WAIT || TryAnswer(waiting.sysAddr, waiting.blueprintId, waiting.resourceType); + return now - waiting.since > MAX_WAIT || TryAnswerIcon(waiting.sysAddr, waiting.blueprintId, waiting.resourceType); }); } void UgcManifest::OnDisconnect(const SystemAddress& sysAddr) { std::erase_if(g_Waiting, [&sysAddr](const Waiting& waiting) { return waiting.sysAddr == sysAddr; }); + std::erase_if(g_LxfmlSent, [&sysAddr](const auto& sent) { return sent.first.first == sysAddr; }); +} + +void UgcManifest::OnModelsMade(const std::vector& blueprintIds) { + for (const auto id : blueprintIds) g_LxfmlChecksums.erase(id); // made again after its LXFML changed, maybe + if (!ServesModels() || !Game::entityManager) return; + + const auto& players = PlayerManager::GetAllPlayers(); + if (players.empty()) return; + const auto models = Game::entityManager->GetEntitiesByLOT(BrickByBrick::MODEL_OBJECT_LOT); + for (const auto id : blueprintIds) { + std::vector shown; + for (auto* const model : models) { + if (model && model->GetVar(u"blueprintid") == id) shown.push_back(model); + } + if (shown.empty()) continue; + const auto checksum = Database::Get()->GetUgcFileChecksum(id, "model.nif"); + if (!checksum || checksum->md5.size() != 32) continue; + + // The new checksum first: the client keeps its cached one otherwise, and would load the file it has + for (auto* const player : players) { + if (player) Send(player->GetSystemAddress(), id, eUgcResourceType::NIF, checksum); + } + for (auto* const model : shown) { + GameMessages::NotifyClientUGCModelReady ready; + ready.target = model->GetObjectID(); + ready.blueprintID = id; + ready.Send(UNASSIGNED_SYSTEM_ADDRESS); + } + LOG("The UGC server made model %llu again: told %zu client(s) about %zu placed model(s)", static_cast(id), players.size(), shown.size()); + } } diff --git a/dGame/dUtilities/UgcManifest.h b/dGame/dUtilities/UgcManifest.h index fa9508b12..26fdb7c60 100644 --- a/dGame/dUtilities/UgcManifest.h +++ b/dGame/dUtilities/UgcManifest.h @@ -1,22 +1,56 @@ #ifndef UGCMANIFEST_H #define UGCMANIFEST_H +#include +#include +#include + #include "dCommonVars.h" #include "eUgcResourceType.h" +#include "IUgc.h" #include "RakNetTypes.h" /** * Answers the client's REQUEST_UGC_MANIFEST_INFO (sent when its UGCUSE3DSERVICES is off, the default): the MD5 and - * size of a blueprint's file, which the client needs before it downloads the file from the UGC server - * (BrickModels/UserMade//..sd0). The checksums are the ones the UGC server stored when it made the - * files (ugc_file_checksums); a file it hasn't made yet is answered once it has (Update looks again every few seconds - * for a while). Nothing is answered for files it never makes (LXFML, HKX), or unless `ugc_manifest` is 1: the client - * then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and is logged out when - * it can't connect there. + * size of a blueprint's file, which the client needs before it uses or downloads the file from the UGC server + * (BrickModels/UserMade//..sd0). Nothing is answered unless `ugc_manifest` is 1. + * + * * Icons (DDS): the checksum the UGC server stored when it made the icon (ugc_file_checksums); one it hasn't made yet + * is answered once it has (Update looks again every few seconds for a while). + * * Player models (NIF, HKX, LXFML): a placed model's client always asks for all three (its BlueprintComponent loads the + * blueprint's NIF and HKX, its ModelBehaviorComponent the LXFML) and waits for the answers without a timeout. With + * `ugc_manifest_models` 1 and the model's mesh made, the NIF is answered with the UGC server's checksum (the client + * downloads the served mesh), the LXFML with the stored LXFML's and the HKX as not known (valid 0: the client uses + * one it has, else asks the UGC server, which answers 404). Otherwise the world sends that client the model's LXFML + * (BlueprintSaveResponse), the client builds the NIF and HKX itself and caches their checksums, which answers its + * own requests: a model is never left waiting. * * Main thread only: each lookup is one indexed query and the answers are sent from here. See docs/UgcServer.md. */ namespace UgcManifest { + // What a request gets + enum class eAction { + NONE, // not answered (ugc_manifest off, or not a UGC type) + ANSWER, // the stored (or, for LXFML, computed) checksum + ANSWER_UNKNOWN,// valid 0: the client uses the file it has, or downloads it + SEND_LXFML, // the model's LXFML, for the client to build the model itself + WAIT, // icons not made yet: answered once they are + }; + + // The rule, for a request of `type` when the UGC server has (`meshMade`) or hasn't made the blueprint's mesh + eAction Decide(eUgcResourceType type, bool manifestOn, bool modelsOn, bool meshMade); + + // The MD5 (lowercase hex) and size of a stored LXFML (ugc.lxfml: sd0, or plain XML) as the client has it after + // downloading and inflating it; nothing when it can't be read + std::optional LxfmlChecksum(const std::string& stored); + + // ugc_manifest=1 and ugc_manifest_models=1: placed models' meshes come from the UGC server when made + bool ServesModels(); + + // Whether the client should get this model's LXFML when a property loads (it builds the model itself): not when + // ServesModels() and the UGC server made its mesh + bool ClientBuildsModel(LWOOBJID blueprintId); + void OnRequest(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType); // Answers the waiting requests whose files have been made since, and forgets the ones waited on too long @@ -24,6 +58,10 @@ namespace UgcManifest { // A client left: its waiting requests are dropped void OnDisconnect(const SystemAddress& sysAddr); + + // The UGC server made these models' meshes (again, with a new checksum): each one placed in this world is sent the + // new checksum and NotifyClientUGCModelReady, so clients load the served mesh + void OnModelsMade(const std::vector& blueprintIds); } #endif //!UGCMANIFEST_H diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index c46734851..b7b4e1829 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -52,6 +52,7 @@ #include "master/MessageCapture.h" #include "master/InstanceMigration.h" #include "master/ServerTraffic.h" +#include "master/UgcModelsMade.h" #ifdef DARKFLAME_PLATFORM_UNIX @@ -925,6 +926,15 @@ namespace { LOG("Dashboard announcement sent to %u world(s)", worlds); } + // The UGC server made models' meshes: every world hears of it, and those showing the models tell their clients + void OnUgcModelsMade(const UgcModelsMade& made, const SystemAddress& sysAddr) { + if (sysAddr != ugcServerMasterPeerSysAddr) return; + for (const auto& instance : Game::im->GetInstances()) { + if (!instance || !instance->GetIsReady() || instance->GetIsShuttingDown()) continue; + MasterPackets::SendTo(instance->GetSysAddr(), made); + } + } + void OnConfigReload(const ConfigReload& reload, const SystemAddress& sysAddr) { if (sysAddr != dashboardServerMasterPeerSysAddr) { LOG("Ignoring config reload from a server that is not the dashboard"); @@ -1017,6 +1027,7 @@ namespace { handlers.On(Master::MESSAGE_CAPTURE_DATA, ForwardWorldToDashboard); handlers.On(Master::REQUEST_SERVER_LIST, OnRequestServerList); handlers.On(Master::SERVER_TRAFFIC, OnServerTraffic); + handlers.On(Master::UGC_MODELS_MADE, OnUgcModelsMade); return handlers; }(); return handlers; diff --git a/dNet/MasterPackets.h b/dNet/MasterPackets.h index 3637508e9..61ae4ea73 100644 --- a/dNet/MasterPackets.h +++ b/dNet/MasterPackets.h @@ -15,6 +15,7 @@ #include "master/InstanceMigration.h" #include "master/MessageCapture.h" #include "master/PlayerAction.h" +#include "master/UgcModelsMade.h" class dServer; @@ -22,7 +23,8 @@ class dServer; * Packets between master and the other servers (MessageType::Master). None of them reach a client. * * The dashboard's messages and instance migration are MASTER packets too; they live in per-topic headers under - * dNet/master/ (PlayerAction.h, DataChanged.h, DashboardMessages.h, MessageCapture.h, InstanceMigration.h), which + * dNet/master/ (PlayerAction.h, DataChanged.h, DashboardMessages.h, MessageCapture.h, InstanceMigration.h, + * UgcModelsMade.h), which * this header includes, so including MasterPackets.h gives every MASTER struct. */ namespace MasterPackets { diff --git a/dNet/master/UgcModelsMade.h b/dNet/master/UgcModelsMade.h new file mode 100644 index 000000000..711d52185 --- /dev/null +++ b/dNet/master/UgcModelsMade.h @@ -0,0 +1,42 @@ +#ifndef __UGCMODELSMADE__H__ +#define __UGCMODELSMADE__H__ + +#include +#include +#include + +#include "BitStream.h" +#include "BitStreamUtils.h" +#include "MessageType/Master.h" +#include "dCommonVars.h" + +/** + * UGC_MODELS_MADE payload (UGC server -> master -> every world): the player models (their ugc id, which is the + * blueprint id) whose mesh the UGC server just made or made again. A world showing one of them tells its clients the + * new checksum and sends NotifyClientUGCModelReady, so they load the served mesh (docs/UgcServer.md). + */ +struct UgcModelsMade : public LUBitStream { + UgcModelsMade() : LUBitStream(ServiceType::MASTER, MessageType::Master::UGC_MODELS_MADE) {} + + static constexpr size_t MAX_MODELS = 4096; + + std::vector blueprintIds; + + void Serialize(RakNet::BitStream& stream) const override { + const auto count = static_cast(std::min(blueprintIds.size(), MAX_MODELS)); + stream.Write(count); + for (size_t i = 0; i < count; i++) stream.Write(blueprintIds[i]); + } + + bool Deserialize(RakNet::BitStream& stream) override { + uint16_t count{}; + if (!stream.Read(count) || count > MAX_MODELS) return false; + blueprintIds.resize(count); + for (auto& id : blueprintIds) { + if (!stream.Read(id)) return false; + } + return true; + } +}; + +#endif //!__UGCMODELSMADE__H__ diff --git a/dUgcServer/UgcProcessor.cpp b/dUgcServer/UgcProcessor.cpp index a3b6ba89f..d4dd045c4 100644 --- a/dUgcServer/UgcProcessor.cpp +++ b/dUgcServer/UgcProcessor.cpp @@ -441,7 +441,14 @@ void UgcProcessor::Collect() { void UgcProcessor::StoreChecksums(Kind kind, LWOOBJID storageId, const std::vector& checksums) { const auto owner = kind == Kind::MODEL ? IUgc::eFileOwner::MODEL : IUgc::eFileOwner::COMBINATION; - for (const auto& checksum : checksums) Database::Get()->SetUgcFileChecksum(owner, storageId, checksum.file, checksum.md5, checksum.size); + for (const auto& checksum : checksums) { + // A player model's mesh that changed: the worlds showing it tell their clients (UGC_MODELS_MADE) + if (kind == Kind::MODEL && checksum.file == "model.nif") { + const auto before = Database::Get()->GetUgcFileChecksum(storageId, checksum.file); + if (!before || before->md5 != checksum.md5 || before->size != checksum.size) m_ChangedMeshes.push_back(storageId); + } + Database::Get()->SetUgcFileChecksum(owner, storageId, checksum.file, checksum.md5, checksum.size); + } } void UgcProcessor::Backfill() { diff --git a/dUgcServer/UgcProcessor.h b/dUgcServer/UgcProcessor.h index 14f84b789..23582c2e2 100644 --- a/dUgcServer/UgcProcessor.h +++ b/dUgcServer/UgcProcessor.h @@ -140,6 +140,10 @@ public: uint64_t StoredBytes() const { return m_StoredBytes; } uint64_t MaxStorageBytes() const { return m_Config.maxStorageBytes; } + // Main thread: the models whose mesh (model.nif) was written with a different checksum than before since the last + // call, for the worlds (UGC_MODELS_MADE); a model made again unchanged (e.g. after eviction) isn't listed + std::vector TakeChangedMeshes() { std::vector ids; ids.swap(m_ChangedMeshes); return ids; } + private: struct Job { Kind kind{}; @@ -223,6 +227,7 @@ private: std::chrono::steady_clock::time_point m_NextEviction{}; std::map, std::pair> m_Recent; // answers for missing files uint64_t m_StoredBytes{}; + std::vector m_ChangedMeshes; // TakeChangedMeshes uint64_t m_Made{}; uint64_t m_Failed{}; uint64_t m_Evicted{}; diff --git a/dUgcServer/UgcServer.cpp b/dUgcServer/UgcServer.cpp index a524d5cda..c5f18b060 100644 --- a/dUgcServer/UgcServer.cpp +++ b/dUgcServer/UgcServer.cpp @@ -21,6 +21,7 @@ #include "Game.h" #include "GeneralUtils.h" #include "Logger.h" +#include "MasterPackets.h" #include "Server.h" #include "ServiceType.h" #include "Web.h" @@ -642,6 +643,14 @@ int main(int argc, char** argv) { packet = g_Server->ReceiveFromMaster(); } processor.Update(); + // Worlds showing a model whose mesh changed tell their clients (docs/UgcServer.md, "Models without 3D services") + if (auto changed = processor.TakeChangedMeshes(); !changed.empty()) { + for (size_t start = 0; start < changed.size(); start += UgcModelsMade::MAX_MODELS) { + UgcModelsMade made; + made.blueprintIds.assign(changed.begin() + start, changed.begin() + std::min(changed.size(), start + UgcModelsMade::MAX_MODELS)); + MasterPackets::SendToMaster(made, g_Server); + } + } // Settings the dashboard changed arrive as a config reload; pick them up if (now - lastConfigure >= std::chrono::seconds(5)) { lastConfigure = now; diff --git a/dWorldServer/WorldServer.cpp b/dWorldServer/WorldServer.cpp index f2b3c864b..231bcbd8d 100644 --- a/dWorldServer/WorldServer.cpp +++ b/dWorldServer/WorldServer.cpp @@ -836,6 +836,7 @@ namespace { handlers.On(Master::MESSAGE_CAPTURE_CONTROL, [](const MessageCaptureControl& control, const SystemAddress&) { MessageInspector::Control(control); }); handlers.On(Master::ANNOUNCE, [](const Announcement& announcement, const SystemAddress&) { DashboardNotify::Announce(announcement.title, announcement.message); }); handlers.On(Master::NEW_SESSION_ALERT, OnNewSessionAlert); + handlers.On(Master::UGC_MODELS_MADE, [](const UgcModelsMade& made, const SystemAddress&) { UgcManifest::OnModelsMade(made.blueprintIds); }); return handlers; }(); return handlers; @@ -1031,8 +1032,9 @@ void LoadPlayer(const SystemAddress& sysAddr) { goto noBBB; } - // Workaround for not having a UGC server to get model LXFML onto the client so it - // can generate the physics and nif for the object. + // The models' LXFML, for the client to build each model's NIF and HKX itself. With ugc_manifest_models + // the models whose mesh the UGC server made are left out: the client asks for their files and downloads + // the mesh (UgcManifest, docs/UgcServer.md). auto bbbModels = Database::Get()->GetUgcModels(propertyId); if (bbbModels.empty()) { @@ -1043,7 +1045,12 @@ void LoadPlayer(const SystemAddress& sysAddr) { ClientPackets::BlueprintSaveResponse response; response.localId = LWOOBJID_EMPTY; //always zero so that a check on the client passes response.reasonCode = eBlueprintSaveResponseType::EverythingWorked; + size_t served = 0; for (auto& bbbModel : bbbModels) { + if (!UgcManifest::ClientBuildsModel(bbbModel.id)) { + served++; + continue; + } LOG("Getting lxfml ugcID: %llu", bbbModel.id); bbbModel.lxfmlData.seekg(0, std::ios::end); @@ -1055,7 +1062,8 @@ void LoadPlayer(const SystemAddress& sysAddr) { model.blueprintId = bbbModel.id; model.data = bbbModel.lxfmlData.str().substr(0, lxfmlSize); } - response.Send(sysAddr); + if (served > 0) LOG("%zu of the property's %zu models come from the UGC server", served, bbbModels.size()); + if (!response.models.empty()) response.Send(sysAddr); } noBBB: diff --git a/resources/sharedconfig.ini b/resources/sharedconfig.ini index 9c14d0f7e..a9152cf25 100644 --- a/resources/sharedconfig.ini +++ b/resources/sharedconfig.ini @@ -95,7 +95,13 @@ mute_auto_reject_names=1 ugc_debounce_seconds=120 # 0 or 1: worlds answer game clients without 3D services (UGCUSE3DSERVICES=7:0, the default) with the checksums of the -# files the UGC server made, so they download the icons of cars, rockets and models. The 1.10.64 client then downloads -# from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and a client that can't connect there -# is logged out, so only set 1 when the UGC server answers there for every player. See docs/UgcServer.md. +# files the UGC server made, so they download the icons of cars, rockets and models from the UGCSERVERIP, UGCSERVERPORT +# and UGCSERVERDIR of their boot.cfg (default: the PATCHSERVER* values). A client that can't connect there is logged +# out, so only set 1 when every player's boot.cfg points at the UGC server. See docs/UgcServer.md. ugc_manifest=0 + +# 0 or 1, with ugc_manifest=1: placed player models the UGC server has made are downloaded from it (their mesh) instead +# of being built by each client from the LXFML the world sends; models not made yet are still sent as LXFML. The UGC +# server makes no physics (.hkx): a client that never built a model itself has no collision for it. See +# docs/UgcServer.md. +ugc_manifest_models=0 diff --git a/tests/dGameTests/CMakeLists.txt b/tests/dGameTests/CMakeLists.txt index ff6fd5316..c263a03bd 100644 --- a/tests/dGameTests/CMakeLists.txt +++ b/tests/dGameTests/CMakeLists.txt @@ -13,6 +13,7 @@ set(DGAMETEST_SOURCES "PowerUpStatisticTests.cpp" "BrickByBrickTests.cpp" "ModularBuildTableTests.cpp" + "UgcManifestTests.cpp" ) add_subdirectory(dComponentsTests) diff --git a/tests/dGameTests/UgcManifestTests.cpp b/tests/dGameTests/UgcManifestTests.cpp new file mode 100644 index 000000000..cf2e26505 --- /dev/null +++ b/tests/dGameTests/UgcManifestTests.cpp @@ -0,0 +1,121 @@ +#include "UgcManifest.h" + +#include "ClientPackets.h" +#include "dConfig.h" +#include "GameDependencies.h" +#include "MD5.h" +#include "PacketTestUtils.h" +#include "Sd0.h" + +#include + +using namespace PacketTestUtils; +using UgcManifest::eAction; + +namespace { + std::string Md5Hex(const std::string& data) { + MD5 md5; + md5.update(reinterpret_cast(data.data()), static_cast(data.size())); + md5.finalize(); + return md5.hexdigest(); + } +} + +// What each request gets. A placed model's client asks for its NIF, HKX and LXFML and waits for every answer without a +// timeout (LWOResMgr2Interface::RequestBlueprintManifestThenLoad, 0x0105a910), so model files are always answered: +// from the UGC server when it made the mesh and models are served, else with the LXFML for the client to build. +TEST(UgcManifestTests, DecideCoversEveryType) { + using T = eUgcResourceType; + for (const auto type : { T::LXFML, T::NIF, T::HKX, T::DDS }) { + EXPECT_EQ(UgcManifest::Decide(type, false, true, true), eAction::NONE); + } + EXPECT_EQ(UgcManifest::Decide(T::DDS, true, false, false), eAction::WAIT); + EXPECT_EQ(UgcManifest::Decide(T::DDS, true, true, true), eAction::WAIT); + + EXPECT_EQ(UgcManifest::Decide(T::NIF, true, true, true), eAction::ANSWER); + EXPECT_EQ(UgcManifest::Decide(T::LXFML, true, true, true), eAction::ANSWER); + EXPECT_EQ(UgcManifest::Decide(T::HKX, true, true, true), eAction::ANSWER_UNKNOWN); + + for (const auto type : { T::LXFML, T::NIF, T::HKX }) { + EXPECT_EQ(UgcManifest::Decide(type, true, true, false), eAction::SEND_LXFML); // not made yet + EXPECT_EQ(UgcManifest::Decide(type, true, false, true), eAction::SEND_LXFML); // models not served + EXPECT_EQ(UgcManifest::Decide(type, true, false, false), eAction::SEND_LXFML); + } + EXPECT_EQ(UgcManifest::Decide(static_cast(7), true, true, true), eAction::NONE); +} + +class UgcManifestRequestTests : public GameDependenciesTest { +protected: + void SetUp() override { SetUpDependencies(); } + void TearDown() override { + UgcManifest::OnDisconnect(Client()); + TearDownDependencies(); + } + + static SystemAddress Client() { + SystemAddress address; + address.binaryAddress = 0x0100007f; + address.port = 15001; + return address; + } + + void Settings(const std::string& manifest, const std::string& models) { + Game::config->SetDatabaseValues({ { "ugc_manifest", manifest }, { "ugc_manifest_models", models } }, {}); + } +}; + +// Off (the default): nothing is answered, as before +TEST_F(UgcManifestRequestTests, NothingWhenOff) { + Settings("0", "1"); + const auto sent = Capture([] { UgcManifest::OnRequest(Client(), 1234, eUgcResourceType::NIF); }); + EXPECT_TRUE(sent.empty()); + EXPECT_FALSE(UgcManifest::ServesModels()); + EXPECT_TRUE(UgcManifest::ClientBuildsModel(1234)); +} + +// A model file of a blueprint that isn't a player model: answered as not known (37 bytes, valid 0), never left waiting +TEST_F(UgcManifestRequestTests, UnknownModelFilesAreAnsweredNotKnown) { + Settings("1", "1"); + EXPECT_TRUE(UgcManifest::ServesModels()); + EXPECT_TRUE(UgcManifest::ClientBuildsModel(1234)); // its mesh isn't made + for (const auto type : { eUgcResourceType::NIF, eUgcResourceType::HKX, eUgcResourceType::LXFML }) { + const auto sent = Capture([type] { UgcManifest::OnRequest(Client(), 0x0102030405060708, type); }); + ASSERT_EQ(sent.size(), 1); + EXPECT_EQ(sent[0].sysAddr, Client()); + EXPECT_FALSE(sent[0].broadcast); + EXPECT_PACKET_EQ(FromHex("53 05 00 3c 00 00 00 00 08 07 06 05 04 03 02 01 0" + std::to_string(static_cast(type)) + + " 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00"), FromCapture(sent[0])); + } +} + +// An icon that isn't made yet waits (answered by Update once the UGC server stored its checksum) +TEST_F(UgcManifestRequestTests, IconsWait) { + Settings("1", "0"); + const auto sent = Capture([] { UgcManifest::OnRequest(Client(), 1234, eUgcResourceType::DDS); }); + EXPECT_TRUE(sent.empty()); +} + +// Models made while nobody is on a property with them: nothing to tell +TEST_F(UgcManifestRequestTests, ModelsMadeWithNobodyToTell) { + Settings("1", "1"); + const auto sent = Capture([] { UgcManifest::OnModelsMade({ 1234, 5678 }); }); + EXPECT_TRUE(sent.empty()); +} + +// The client checks the MD5 and size of the LXFML as it has it after inflating the UGC server's .lxfml.sd0, which is +// the stored LXFML (sd0, or plain XML in old rows) inflated +TEST_F(UgcManifestRequestTests, LxfmlChecksumIsOfTheInflatedLxfml) { + const std::string lxfml = "\n\n"; + const auto plain = UgcManifest::LxfmlChecksum(lxfml); + ASSERT_TRUE(plain.has_value()); + EXPECT_EQ(plain->md5, Md5Hex(lxfml)); + EXPECT_EQ(plain->size, lxfml.size()); + + const auto compressed = UgcManifest::LxfmlChecksum(Sd0::Compress(lxfml)); + ASSERT_TRUE(compressed.has_value()); + EXPECT_EQ(compressed->md5, plain->md5); + EXPECT_EQ(compressed->size, plain->size); + + EXPECT_FALSE(UgcManifest::LxfmlChecksum("").has_value()); + EXPECT_FALSE(UgcManifest::LxfmlChecksum("not an sd0 stream").has_value()); +} diff --git a/tests/dGameTests/dGameMessagesTests/BuildingMessagesTests.cpp b/tests/dGameTests/dGameMessagesTests/BuildingMessagesTests.cpp index bf32b4f85..f95c2eba9 100644 --- a/tests/dGameTests/dGameMessagesTests/BuildingMessagesTests.cpp +++ b/tests/dGameTests/dGameMessagesTests/BuildingMessagesTests.cpp @@ -490,3 +490,19 @@ TEST_F(BuildingMessagesTests, FetchModelMetadataWire) { EXPECT_TRUE(copy.bpData.neverFalseIfPresentInCaps); ExpectTruncatedFails(full); } + +// NotifyClientUGCModelReady (909): the blueprint id only (GameMessage::NotifyClientUGCIconReady::Serialize, 0x00daae30, +// which the client shares with it; read by LWOBlueprintComponent::OnNotifyClientUGCModelReady, 0x00ca6430) +TEST_F(BuildingMessagesTests, NotifyClientUGCModelReadyWire) { + GameMessages::NotifyClientUGCModelReady ready; + ready.target = 0x1122334455667788; + ready.blueprintID = 0x0102030405060708; + EXPECT_PACKET_EQ(FromHex("08 07 06 05 04 03 02 01"), Payload(ready)); + EXPECT_EQ(RoundTrip(ready).blueprintID, ready.blueprintID); + ExpectTruncatedFails(ready); + + // The whole packet: the model's object id, then the message id 909 + RakNet::BitStream bitStream; + ready.WritePacket(bitStream); + EXPECT_PACKET_EQ(FromHex("53 05 00 0c 00 00 00 00 88 77 66 55 44 33 22 11 8d 03 08 07 06 05 04 03 02 01"), FromBitStream(bitStream)); +} diff --git a/tests/dGameTests/dNetTests/MasterPacketsTests.cpp b/tests/dGameTests/dNetTests/MasterPacketsTests.cpp index 3b0ebef07..8eab5cde3 100644 --- a/tests/dGameTests/dNetTests/MasterPacketsTests.cpp +++ b/tests/dGameTests/dNetTests/MasterPacketsTests.cpp @@ -403,6 +403,10 @@ TEST(MasterPacketsTests, DashboardAndMigrationStructsKeepTheirBytes) { changed.entries.push_back({ "characters", 42 }); ExpectHeaderThenSerialize(changed, MessageType::Master::DATA_CHANGED); + UgcModelsMade made; + made.blueprintIds = { 1152921510436831123, 42 }; + ExpectHeaderThenSerialize(made, MessageType::Master::UGC_MODELS_MADE); + PlayerPositions positions; positions.zoneId = 1100; positions.players.push_back({ 42, 1.0f, 2.0f, 3.0f }); @@ -485,3 +489,20 @@ TEST(MasterPacketsTests, DispatcherRoutesAndDropsBadPackets) { delete Game::logger; Game::logger = previousLogger; } + +// UGC server -> master -> worlds: a u16 count, then each blueprint id +TEST(MasterPacketsTests, UgcModelsMadeBytes) { + UgcModelsMade made; + made.blueprintIds = { 0x0102030405060708, 9 }; + EXPECT_PACKET_EQ(FromHex("53 06 00 25 00 00 00 00 02 00 08 07 06 05 04 03 02 01 09 00 00 00 00 00 00 00"), StructPacket(made)); + + RakNet::BitStream tooMany; + tooMany.Write(UgcModelsMade::MAX_MODELS + 1); + UgcModelsMade read; + EXPECT_FALSE(read.Deserialize(tooMany)); + + RakNet::BitStream truncated; + truncated.Write(2); + truncated.Write(1); + EXPECT_FALSE(read.Deserialize(truncated)); +}