Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-02 10:53:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ugc): placed models from the UGC server without 3D services

Browse Source 
A placed model's client (1.10.64, UGCUSE3DSERVICES=7:0) always loads its
blueprint's NIF and HKX (its BlueprintComponent sets renderUserGen and
physicsUserGen itself) and its LXFML, asks the world for each file's
manifest first and waits for the answer with no timeout.

With ugc_manifest=1 and the new ugc_manifest_models=1 (default 0) the world:
- leaves the models whose mesh the UGC server made out of the LXFML it
  sends when a property loads,
- answers their NIF with the UGC server's checksum, their LXFML with the
  stored LXFML's (worked out once and kept) and their HKX as not known,
- sends a model that isn't made its LXFML (once for the three requests),
  so the client builds it itself and no model is left waiting.

When the UGC server writes a model's mesh with a new checksum it sends
UGC_MODELS_MADE (a new master message, appended) to the master, which
passes it to every world; a world with that model placed sends its
players the new NIF checksum and NotifyClientUGCModelReady (game message
909, the blueprint id), so clients switch to the served mesh.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-27 19:12:48 -05:00
parent dce2037d9d
commit 9d76fe9550
18 changed files with 508 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
dCommon/dEnums/MessageType/Master.h
View File

@@ -66,5 +66,9 @@ namespace MessageType {
// Any server -> master -> dashboard: traffic counters of the last few seconds (see ServerTraffic.h)
SERVER_TRAFFIC,
// UGC server -> master -> every world: player models whose mesh (model.nif) was just made or made again (see
// UgcModelsMade.h), so worlds showing them tell their clients (NotifyClientUGCModelReady)
UGC_MODELS_MADE,
};
}

3
dDashboardServer/routes/SettingsCatalog.cpp
View File

@@ -422,7 +422,8 @@ namespace {
c.Add(Format(Text(UGC, "listen_ip", "Listen address", "The game client has to reach it.", "0.0.0.0", true), eFormat::HOST));
c.Add(Port(UGC, "net_port", "Master connection port", "UDP; the next port is used too.", "2012"));
c.Add(Text(UGC, "client_path", "Download path", "The game client's UGCSERVERDIR.", "/ugc", true));
c.Add(Bool(SHARED, "ugc_manifest", "Answer clients without 3D services", "Worlds tell game clients with UGCUSE3DSERVICES=7:0 (the default) the checksums of the files the UGC server made, so they download icons of cars, rockets and models. The client then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and one that can't connect there is logged out: only turn on when the UGC server answers there for every player.", false));
c.Add(Bool(SHARED, "ugc_manifest", "Answer clients without 3D services", "Worlds tell game clients with UGCUSE3DSERVICES=7:0 (the default) the checksums of the files the UGC server made, so they download icons of cars, rockets and models from UGCSERVERIP, UGCSERVERPORT and UGCSERVERDIR in their boot.cfg (default: the patch server's). A client that can't connect there is logged out, so only turn on when every player's boot.cfg points at the UGC server.", false));
c.Add(When(Bool(SHARED, "ugc_manifest_models", "Placed models from the UGC server", "With the answer to clients without 3D services on: placed player models the UGC server has made are downloaded from it (its mesh) instead of built by each client from the LXFML the world sends. It makes no physics: a client that never built a model itself has no collision for it. Models not made yet are still sent as LXFML.", false), SHARED, "ugc_manifest", { "1" }));
c.Add(Format(Text(DASHBOARD, "ugc_internal_url", "UGC server address (internal)", "Where the dashboard itself reaches the UGC server for its status and files. Empty: http://127.0.0.1:2008.", ""), eFormat::URL));
c.Add(Format(Text(DASHBOARD, "ugc_public_url", "UGC server address (public)", "Only for the \"open on the UGC server\" links, e.g. https://ugc.example.com. The dashboard's own pages don't need the browser to reach it.", ""), eFormat::URL));
c.AddSection("UGC processing", "When models are made and how much of the machine the workers may use.", eLayout::ROWS, Condition{ MASTER, "enable_ugc_server", { "1" } });

9
dGame/dGameMessages/BuildingMessages.cpp
View File

@@ -603,6 +603,15 @@ namespace GameMessages {
return true;
}
void NotifyClientUGCModelReady::Serialize(RakNet::BitStream& bitStream) const {
bitStream.Write(blueprintID);
}
bool NotifyClientUGCModelReady::Deserialize(RakNet::BitStream& bitStream) {
VALIDATE_READ(bitStream.Read(blueprintID));
return true;
}
void UGObjectMetadata::Serialize(RakNet::BitStream& bitStream) const {
bitStream.Write(userModelID);
bitStream.Write(blueprintID);

13
dGame/dGameMessages/BuildingMessages.h
View File

@@ -242,6 +242,19 @@ namespace GameMessages {
int32_t count{};
};
// Server -> client, to a placed model. Its blueprint's files were made (again) on the UGC server: the client's
// BlueprintComponent with that blueprint flushes its cached NIF, HKX and LXFML and requests the NIF and HKX again
// (LWOBlueprintComponent::OnNotifyClientUGCModelReady, 0x00ca6430; the payload is the blueprint id only,
// GameMessage::NotifyClientUGCIconReady::Serialize 0x00daae30 is shared with it). The client keeps using its
// cached manifest entry, so the new checksum is sent first (UgcManifestResponse). See docs/UgcServer.md.
struct NotifyClientUGCModelReady : public NetGameMsg {
NotifyClientUGCModelReady() : NetGameMsg(MessageType::Game::NOTIFY_CLIENT_UGC_MODEL_READY) {}
void Serialize(RakNet::BitStream& bitStream) const override;
bool Deserialize(RakNet::BitStream& bitStream) override;
LWOOBJID blueprintID{};
};
// A model's user generated data (UGObjectMetadata::Serialize, 0x00f5f590)
struct UGObjectMetadata {
LWOOBJID userModelID{};

208
dGame/dUtilities/UgcManifest.cpp
View File

@@ -1,21 +1,34 @@
#include "UgcManifest.h"
#include <chrono>
#include <optional>
#include <map>
#include <sstream>
#include <string_view>
#include <vector>
#include "BrickByBrick.h"
#include "BuildingMessages.h"
#include "ClientPackets.h"
#include "Database.h"
#include "dConfig.h"
#include "eBlueprintSaveResponseType.h"
#include "Entity.h"
#include "EntityManager.h"
#include "Game.h"
#include "Logger.h"
#include "MD5.h"
#include "PlayerManager.h"
#include "Sd0.h"
namespace {
// How often the waiting requests are looked up again, how long they wait at most and how many there can be
constexpr auto RETRY_INTERVAL = std::chrono::seconds(5);
constexpr auto MAX_WAIT = std::chrono::minutes(15);
constexpr size_t MAX_WAITING = 512;
// A client asks for a model's NIF, HKX and LXFML at once: its LXFML is sent once for all three
constexpr auto LXFML_RESEND_AFTER = std::chrono::seconds(10);
// LXFML checksums worked out, kept for the next clients (a model's LXFML only changes with a new save)
constexpr size_t MAX_LXFML_CHECKSUMS = 4096;
struct Waiting {
SystemAddress sysAddr;
@@ -26,20 +39,17 @@ namespace {
std::vector<Waiting> g_Waiting;
std::chrono::steady_clock::time_point g_NextRetry{};
std::map<std::pair<SystemAddress, LWOOBJID>, std::chrono::steady_clock::time_point> g_LxfmlSent;
std::map<LWOOBJID, IUgc::FileChecksum> g_LxfmlChecksums;
// The UGC server's name for a blueprint's file of this type; nothing for the ones it doesn't make
std::optional<std::string_view> FileName(eUgcResourceType type) {
switch (type) {
case eUgcResourceType::DDS: return "icon.dds";
case eUgcResourceType::NIF: return "model.nif";
default: return std::nullopt;
}
bool ManifestOn() {
// Off unless set: a client whose boot.cfg doesn't point it at the UGC server downloads from its built-in
// http://127.0.0.1:80/lwoclient and is logged out when it can't connect there (docs/UgcServer.md)
return Game::config && Game::config->GetValue("ugc_manifest") == "1";
}
bool Enabled() {
// Off unless set: the 1.10.64 client downloads from http://127.0.0.1:80 whatever its boot.cfg says, and one that
// can't connect there is logged out (docs/UgcServer.md)
return Game::config->GetValue("ugc_manifest") == "1";
bool ModelsOn() {
return Game::config && Game::config->GetValue("ugc_manifest_models") == "1";
}
int HexDigit(char c) {
@@ -49,34 +59,143 @@ namespace {
return -1;
}
// Answers when the file's checksum is known; false when it isn't (yet)
bool TryAnswer(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) {
const auto file = FileName(resourceType);
if (!file) return false;
const auto checksum = Database::Get()->GetUgcFileChecksum(blueprintId, *file);
if (!checksum || checksum->md5.size() != 32) return false;
bool MeshMade(LWOOBJID blueprintId) {
const auto checksum = Database::Get()->GetUgcFileChecksum(blueprintId, "model.nif");
return checksum && checksum->md5.size() == 32;
}
void Send(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType, const std::optional<IUgc::FileChecksum>& checksum) {
ClientPackets::UgcManifestResponse response;
response.blueprintId = blueprintId;
response.resourceType = resourceType;
response.fileSize = checksum->size;
for (size_t i = 0; i < response.md5.size(); i++) {
const auto high = HexDigit(checksum->md5[i * 2]);
const auto low = HexDigit(checksum->md5[i * 2 + 1]);
if (high < 0 || low < 0) return false;
response.md5[i] = static_cast<uint8_t>(high << 4 | low);
if (checksum && checksum->md5.size() == 32) {
bool valid = true;
for (size_t i = 0; i < response.md5.size(); i++) {
const auto high = HexDigit(checksum->md5[i * 2]);
const auto low = HexDigit(checksum->md5[i * 2 + 1]);
if (high < 0 || low < 0) valid = false;
response.md5[i] = static_cast<uint8_t>((high << 4) | low);
}
if (valid) {
response.valid = true;
response.fileSize = checksum->size;
} else {
response.md5 = {};
}
}
response.valid = true;
response.Send(sysAddr);
LOG_DEBUG("Sent the UGC manifest of %llu (type %i): %s, %u bytes", static_cast<unsigned long long>(blueprintId),
static_cast<int>(resourceType), checksum->md5.c_str(), checksum->size);
static_cast<int>(resourceType), response.valid ? checksum->md5.c_str() : "not known", response.fileSize);
}
// Answers an icon's request when its checksum is known; false when it isn't (yet)
bool TryAnswerIcon(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) {
const auto checksum = Database::Get()->GetUgcFileChecksum(blueprintId, "icon.dds");
if (!checksum || checksum->md5.size() != 32) return false;
Send(sysAddr, blueprintId, resourceType, checksum);
return true;
}
std::optional<IUgc::FileChecksum> StoredLxfmlChecksum(LWOOBJID blueprintId) {
if (const auto known = g_LxfmlChecksums.find(blueprintId); known != g_LxfmlChecksums.end()) return known->second;
const auto model = Database::Get()->GetUgcModel(blueprintId);
if (!model) return std::nullopt;
auto checksum = UgcManifest::LxfmlChecksum(model->lxfmlData.str());
if (!checksum) return std::nullopt;
if (g_LxfmlChecksums.size() >= MAX_LXFML_CHECKSUMS) g_LxfmlChecksums.clear();
g_LxfmlChecksums.emplace(blueprintId, *checksum);
return checksum;
}
// The model's LXFML to one client, which builds the model from it (as when a property loads); false when there is
// no such model
bool SendLxfml(const SystemAddress& sysAddr, LWOOBJID blueprintId) {
const auto now = std::chrono::steady_clock::now();
std::erase_if(g_LxfmlSent, [now](const auto& sent) { return now - sent.second > LXFML_RESEND_AFTER; });
if (g_LxfmlSent.contains({ sysAddr, blueprintId })) return true;
auto model = Database::Get()->GetUgcModel(blueprintId);
if (!model) return false;
ClientPackets::BlueprintSaveResponse response;
response.localId = LWOOBJID_EMPTY; // zero, like the LXFML sent when a property loads
response.reasonCode = eBlueprintSaveResponseType::EverythingWorked;
response.models.push_back({ blueprintId, model->lxfmlData.str() });
response.Send(sysAddr);
g_LxfmlSent[{ sysAddr, blueprintId }] = now;
// Its mesh is wanted: a model still in its quiet period after a save is made now
Database::Get()->ExpediteUgcModel(blueprintId);
LOG_DEBUG("Sent the LXFML of %llu for the client to build", static_cast<unsigned long long>(blueprintId));
return true;
}
}
UgcManifest::eAction UgcManifest::Decide(const eUgcResourceType type, const bool manifestOn, const bool modelsOn, const bool meshMade) {
if (!manifestOn) return eAction::NONE;
switch (type) {
case eUgcResourceType::DDS: return eAction::WAIT; // answered at once when the icon is made
case eUgcResourceType::NIF:
case eUgcResourceType::LXFML:
return modelsOn && meshMade ? eAction::ANSWER : eAction::SEND_LXFML;
case eUgcResourceType::HKX:
// The UGC server makes no physics: the client uses an HKX it built before, else it asks and gets 404
return modelsOn && meshMade ? eAction::ANSWER_UNKNOWN : eAction::SEND_LXFML;
default: return eAction::NONE;
}
}
std::optional<IUgc::FileChecksum> UgcManifest::LxfmlChecksum(const std::string& stored) {
std::string lxfml;
if (stored.starts_with("<?xml") || stored.starts_with("<LXFML")) {
lxfml = stored;
} else {
try {
std::stringstream stream(stored);
Sd0 sd0(stream);
lxfml = sd0.GetAsStringUncompressed();
} catch (...) {
return std::nullopt;
}
}
if (lxfml.empty()) return std::nullopt;
MD5 md5;
md5.update(reinterpret_cast<const unsigned char*>(lxfml.data()), static_cast<MD5::size_type>(lxfml.size()));
md5.finalize();
return IUgc::FileChecksum{ md5.hexdigest(), static_cast<uint32_t>(lxfml.size()) };
}
bool UgcManifest::ServesModels() {
return ManifestOn() && ModelsOn();
}
bool UgcManifest::ClientBuildsModel(const LWOOBJID blueprintId) {
return !ServesModels() || !MeshMade(blueprintId);
}
void UgcManifest::OnRequest(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) {
if (sysAddr == UNASSIGNED_SYSTEM_ADDRESS || !Enabled() || !FileName(resourceType)) return;
if (TryAnswer(sysAddr, blueprintId, resourceType)) return;
if (sysAddr == UNASSIGNED_SYSTEM_ADDRESS) return;
const auto manifestOn = ManifestOn();
const auto modelsOn = ModelsOn();
const bool isModelFile = resourceType == eUgcResourceType::NIF || resourceType == eUgcResourceType::HKX || resourceType == eUgcResourceType::LXFML;
const auto meshMade = manifestOn && modelsOn && isModelFile && MeshMade(blueprintId);
switch (Decide(resourceType, manifestOn, modelsOn, meshMade)) {
case eAction::NONE: return;
case eAction::ANSWER:
Send(sysAddr, blueprintId, resourceType, resourceType == eUgcResourceType::LXFML
? StoredLxfmlChecksum(blueprintId) : Database::Get()->GetUgcFileChecksum(blueprintId, "model.nif"));
return;
case eAction::ANSWER_UNKNOWN:
Send(sysAddr, blueprintId, resourceType, std::nullopt);
return;
case eAction::SEND_LXFML:
// Not a player model (or gone): answered as not known, so the client doesn't wait for ever
if (!SendLxfml(sysAddr, blueprintId)) Send(sysAddr, blueprintId, resourceType, std::nullopt);
return;
case eAction::WAIT:
break;
}
if (TryAnswerIcon(sysAddr, blueprintId, resourceType)) return;
// Not made yet: a client wants it, so a model still in its quiet period after a save is made now (as when a client
// asks the UGC server for it), and the request is answered when it's made
Database::Get()->ExpediteUgcModel(blueprintId);
@@ -93,10 +212,41 @@ void UgcManifest::Update() {
if (now < g_NextRetry) return;
g_NextRetry = now + RETRY_INTERVAL;
std::erase_if(g_Waiting, [now](const Waiting& waiting) {
return now - waiting.since > MAX_WAIT || TryAnswer(waiting.sysAddr, waiting.blueprintId, waiting.resourceType);
return now - waiting.since > MAX_WAIT || TryAnswerIcon(waiting.sysAddr, waiting.blueprintId, waiting.resourceType);
});
}
void UgcManifest::OnDisconnect(const SystemAddress& sysAddr) {
std::erase_if(g_Waiting, [&sysAddr](const Waiting& waiting) { return waiting.sysAddr == sysAddr; });
std::erase_if(g_LxfmlSent, [&sysAddr](const auto& sent) { return sent.first.first == sysAddr; });
}
void UgcManifest::OnModelsMade(const std::vector<LWOOBJID>& blueprintIds) {
for (const auto id : blueprintIds) g_LxfmlChecksums.erase(id); // made again after its LXFML changed, maybe
if (!ServesModels() || !Game::entityManager) return;
const auto& players = PlayerManager::GetAllPlayers();
if (players.empty()) return;
const auto models = Game::entityManager->GetEntitiesByLOT(BrickByBrick::MODEL_OBJECT_LOT);
for (const auto id : blueprintIds) {
std::vector<Entity*> shown;
for (auto* const model : models) {
if (model && model->GetVar<LWOOBJID>(u"blueprintid") == id) shown.push_back(model);
}
if (shown.empty()) continue;
const auto checksum = Database::Get()->GetUgcFileChecksum(id, "model.nif");
if (!checksum || checksum->md5.size() != 32) continue;
// The new checksum first: the client keeps its cached one otherwise, and would load the file it has
for (auto* const player : players) {
if (player) Send(player->GetSystemAddress(), id, eUgcResourceType::NIF, checksum);
}
for (auto* const model : shown) {
GameMessages::NotifyClientUGCModelReady ready;
ready.target = model->GetObjectID();
ready.blueprintID = id;
ready.Send(UNASSIGNED_SYSTEM_ADDRESS);
}
LOG("The UGC server made model %llu again: told %zu client(s) about %zu placed model(s)", static_cast<unsigned long long>(id), players.size(), shown.size());
}
}

50
dGame/dUtilities/UgcManifest.h
View File

@@ -1,22 +1,56 @@
#ifndef UGCMANIFEST_H
#define UGCMANIFEST_H
#include <optional>
#include <string>
#include <vector>
#include "dCommonVars.h"
#include "eUgcResourceType.h"
#include "IUgc.h"
#include "RakNetTypes.h"
/**
* Answers the client's REQUEST_UGC_MANIFEST_INFO (sent when its UGCUSE3DSERVICES is off, the default): the MD5 and
* size of a blueprint's file, which the client needs before it downloads the file from the UGC server
* (BrickModels/UserMade/<id % 1000>/<id>.<ext>.sd0). The checksums are the ones the UGC server stored when it made the
* files (ugc_file_checksums); a file it hasn't made yet is answered once it has (Update looks again every few seconds
* for a while). Nothing is answered for files it never makes (LXFML, HKX), or unless `ugc_manifest` is 1: the client
* then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and is logged out when
* it can't connect there.
* size of a blueprint's file, which the client needs before it uses or downloads the file from the UGC server
* (BrickModels/UserMade/<id % 1000>/<id>.<ext>.sd0). Nothing is answered unless `ugc_manifest` is 1.
*
* * Icons (DDS): the checksum the UGC server stored when it made the icon (ugc_file_checksums); one it hasn't made yet
* is answered once it has (Update looks again every few seconds for a while).
* * Player models (NIF, HKX, LXFML): a placed model's client always asks for all three (its BlueprintComponent loads the
* blueprint's NIF and HKX, its ModelBehaviorComponent the LXFML) and waits for the answers without a timeout. With
* `ugc_manifest_models` 1 and the model's mesh made, the NIF is answered with the UGC server's checksum (the client
* downloads the served mesh), the LXFML with the stored LXFML's and the HKX as not known (valid 0: the client uses
* one it has, else asks the UGC server, which answers 404). Otherwise the world sends that client the model's LXFML
* (BlueprintSaveResponse), the client builds the NIF and HKX itself and caches their checksums, which answers its
* own requests: a model is never left waiting.
*
* Main thread only: each lookup is one indexed query and the answers are sent from here. See docs/UgcServer.md.
*/
namespace UgcManifest {
// What a request gets
enum class eAction {
NONE, // not answered (ugc_manifest off, or not a UGC type)
ANSWER, // the stored (or, for LXFML, computed) checksum
ANSWER_UNKNOWN,// valid 0: the client uses the file it has, or downloads it
SEND_LXFML, // the model's LXFML, for the client to build the model itself
WAIT, // icons not made yet: answered once they are
};
// The rule, for a request of `type` when the UGC server has (`meshMade`) or hasn't made the blueprint's mesh
eAction Decide(eUgcResourceType type, bool manifestOn, bool modelsOn, bool meshMade);
// The MD5 (lowercase hex) and size of a stored LXFML (ugc.lxfml: sd0, or plain XML) as the client has it after
// downloading and inflating it; nothing when it can't be read
std::optional<IUgc::FileChecksum> LxfmlChecksum(const std::string& stored);
// ugc_manifest=1 and ugc_manifest_models=1: placed models' meshes come from the UGC server when made
bool ServesModels();
// Whether the client should get this model's LXFML when a property loads (it builds the model itself): not when
// ServesModels() and the UGC server made its mesh
bool ClientBuildsModel(LWOOBJID blueprintId);
void OnRequest(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType);
// Answers the waiting requests whose files have been made since, and forgets the ones waited on too long
@@ -24,6 +58,10 @@ namespace UgcManifest {
// A client left: its waiting requests are dropped
void OnDisconnect(const SystemAddress& sysAddr);
// The UGC server made these models' meshes (again, with a new checksum): each one placed in this world is sent the
// new checksum and NotifyClientUGCModelReady, so clients load the served mesh
void OnModelsMade(const std::vector<LWOOBJID>& blueprintIds);
}
#endif //!UGCMANIFEST_H

11
dMasterServer/MasterServer.cpp
View File

@@ -52,6 +52,7 @@
#include "master/MessageCapture.h"
#include "master/InstanceMigration.h"
#include "master/ServerTraffic.h"
#include "master/UgcModelsMade.h"
#ifdef DARKFLAME_PLATFORM_UNIX
@@ -925,6 +926,15 @@ namespace {
LOG("Dashboard announcement sent to %u world(s)", worlds);
}
// The UGC server made models' meshes: every world hears of it, and those showing the models tell their clients
void OnUgcModelsMade(const UgcModelsMade& made, const SystemAddress& sysAddr) {
if (sysAddr != ugcServerMasterPeerSysAddr) return;
for (const auto& instance : Game::im->GetInstances()) {
if (!instance || !instance->GetIsReady() || instance->GetIsShuttingDown()) continue;
MasterPackets::SendTo(instance->GetSysAddr(), made);
}
}
void OnConfigReload(const ConfigReload& reload, const SystemAddress& sysAddr) {
if (sysAddr != dashboardServerMasterPeerSysAddr) {
LOG("Ignoring config reload from a server that is not the dashboard");
@@ -1017,6 +1027,7 @@ namespace {
handlers.On<MessageCaptureData>(Master::MESSAGE_CAPTURE_DATA, ForwardWorldToDashboard<MessageCaptureData>);
handlers.On<RequestServerList>(Master::REQUEST_SERVER_LIST, OnRequestServerList);
handlers.On<ServerTraffic>(Master::SERVER_TRAFFIC, OnServerTraffic);
handlers.On<UgcModelsMade>(Master::UGC_MODELS_MADE, OnUgcModelsMade);
return handlers;
}();
return handlers;

4
dNet/MasterPackets.h
View File

@@ -15,6 +15,7 @@
#include "master/InstanceMigration.h"
#include "master/MessageCapture.h"
#include "master/PlayerAction.h"
#include "master/UgcModelsMade.h"
class dServer;
@@ -22,7 +23,8 @@ class dServer;
* Packets between master and the other servers (MessageType::Master). None of them reach a client.
*
* The dashboard's messages and instance migration are MASTER packets too; they live in per-topic headers under
* dNet/master/ (PlayerAction.h, DataChanged.h, DashboardMessages.h, MessageCapture.h, InstanceMigration.h), which
* dNet/master/ (PlayerAction.h, DataChanged.h, DashboardMessages.h, MessageCapture.h, InstanceMigration.h,
* UgcModelsMade.h), which
* this header includes, so including MasterPackets.h gives every MASTER struct.
*/
namespace MasterPackets {

42
dNet/master/UgcModelsMade.h Normal file
View File

@@ -0,0 +1,42 @@
#ifndef __UGCMODELSMADE__H__
#define __UGCMODELSMADE__H__
#include <algorithm>
#include <cstdint>
#include <vector>
#include "BitStream.h"
#include "BitStreamUtils.h"
#include "MessageType/Master.h"
#include "dCommonVars.h"
/**
* UGC_MODELS_MADE payload (UGC server -> master -> every world): the player models (their ugc id, which is the
* blueprint id) whose mesh the UGC server just made or made again. A world showing one of them tells its clients the
* new checksum and sends NotifyClientUGCModelReady, so they load the served mesh (docs/UgcServer.md).
*/
struct UgcModelsMade : public LUBitStream {
UgcModelsMade() : LUBitStream(ServiceType::MASTER, MessageType::Master::UGC_MODELS_MADE) {}
static constexpr size_t MAX_MODELS = 4096;
std::vector<LWOOBJID> blueprintIds;
void Serialize(RakNet::BitStream& stream) const override {
const auto count = static_cast<uint16_t>(std::min(blueprintIds.size(), MAX_MODELS));
stream.Write(count);
for (size_t i = 0; i < count; i++) stream.Write(blueprintIds[i]);
}
bool Deserialize(RakNet::BitStream& stream) override {
uint16_t count{};
if (!stream.Read(count) || count > MAX_MODELS) return false;
blueprintIds.resize(count);
for (auto& id : blueprintIds) {
if (!stream.Read(id)) return false;
}
return true;
}
};
#endif //!__UGCMODELSMADE__H__

9
dUgcServer/UgcProcessor.cpp
View File

@@ -441,7 +441,14 @@ void UgcProcessor::Collect() {
void UgcProcessor::StoreChecksums(Kind kind, LWOOBJID storageId, const std::vector<Checksum>& checksums) {
const auto owner = kind == Kind::MODEL ? IUgc::eFileOwner::MODEL : IUgc::eFileOwner::COMBINATION;
for (const auto& checksum : checksums) Database::Get()->SetUgcFileChecksum(owner, storageId, checksum.file, checksum.md5, checksum.size);
for (const auto& checksum : checksums) {
// A player model's mesh that changed: the worlds showing it tell their clients (UGC_MODELS_MADE)
if (kind == Kind::MODEL && checksum.file == "model.nif") {
const auto before = Database::Get()->GetUgcFileChecksum(storageId, checksum.file);
if (!before || before->md5 != checksum.md5 || before->size != checksum.size) m_ChangedMeshes.push_back(storageId);
}
Database::Get()->SetUgcFileChecksum(owner, storageId, checksum.file, checksum.md5, checksum.size);
}
}
void UgcProcessor::Backfill() {

5
dUgcServer/UgcProcessor.h
View File

@@ -140,6 +140,10 @@ public:
uint64_t StoredBytes() const { return m_StoredBytes; }
uint64_t MaxStorageBytes() const { return m_Config.maxStorageBytes; }
// Main thread: the models whose mesh (model.nif) was written with a different checksum than before since the last
// call, for the worlds (UGC_MODELS_MADE); a model made again unchanged (e.g. after eviction) isn't listed
std::vector<LWOOBJID> TakeChangedMeshes() { std::vector<LWOOBJID> ids; ids.swap(m_ChangedMeshes); return ids; }
private:
struct Job {
Kind kind{};
@@ -223,6 +227,7 @@ private:
std::chrono::steady_clock::time_point m_NextEviction{};
std::map<std::pair<Kind, LWOOBJID>, std::pair<std::chrono::steady_clock::time_point, Availability>> m_Recent; // answers for missing files
uint64_t m_StoredBytes{};
std::vector<LWOOBJID> m_ChangedMeshes; // TakeChangedMeshes
uint64_t m_Made{};
uint64_t m_Failed{};
uint64_t m_Evicted{};

9
dUgcServer/UgcServer.cpp
View File

@@ -21,6 +21,7 @@
#include "Game.h"
#include "GeneralUtils.h"
#include "Logger.h"
#include "MasterPackets.h"
#include "Server.h"
#include "ServiceType.h"
#include "Web.h"
@@ -642,6 +643,14 @@ int main(int argc, char** argv) {
packet = g_Server->ReceiveFromMaster();
}
processor.Update();
// Worlds showing a model whose mesh changed tell their clients (docs/UgcServer.md, "Models without 3D services")
if (auto changed = processor.TakeChangedMeshes(); !changed.empty()) {
for (size_t start = 0; start < changed.size(); start += UgcModelsMade::MAX_MODELS) {
UgcModelsMade made;
made.blueprintIds.assign(changed.begin() + start, changed.begin() + std::min(changed.size(), start + UgcModelsMade::MAX_MODELS));
MasterPackets::SendToMaster(made, g_Server);
}
}
// Settings the dashboard changed arrive as a config reload; pick them up
if (now - lastConfigure >= std::chrono::seconds(5)) {
lastConfigure = now;

14
dWorldServer/WorldServer.cpp
View File

@@ -836,6 +836,7 @@ namespace {
handlers.On<MessageCaptureControl>(Master::MESSAGE_CAPTURE_CONTROL, [](const MessageCaptureControl& control, const SystemAddress&) { MessageInspector::Control(control); });
handlers.On<Announcement>(Master::ANNOUNCE, [](const Announcement& announcement, const SystemAddress&) { DashboardNotify::Announce(announcement.title, announcement.message); });
handlers.On<MasterPackets::NewSessionAlert>(Master::NEW_SESSION_ALERT, OnNewSessionAlert);
handlers.On<UgcModelsMade>(Master::UGC_MODELS_MADE, [](const UgcModelsMade& made, const SystemAddress&) { UgcManifest::OnModelsMade(made.blueprintIds); });
return handlers;
}();
return handlers;
@@ -1031,8 +1032,9 @@ void LoadPlayer(const SystemAddress& sysAddr) {
goto noBBB;
}
// Workaround for not having a UGC server to get model LXFML onto the client so it
// can generate the physics and nif for the object.
// The models' LXFML, for the client to build each model's NIF and HKX itself. With ugc_manifest_models
// the models whose mesh the UGC server made are left out: the client asks for their files and downloads
// the mesh (UgcManifest, docs/UgcServer.md).
auto bbbModels = Database::Get()->GetUgcModels(propertyId);
if (bbbModels.empty()) {
@@ -1043,7 +1045,12 @@ void LoadPlayer(const SystemAddress& sysAddr) {
ClientPackets::BlueprintSaveResponse response;
response.localId = LWOOBJID_EMPTY; //always zero so that a check on the client passes
response.reasonCode = eBlueprintSaveResponseType::EverythingWorked;
size_t served = 0;
for (auto& bbbModel : bbbModels) {
if (!UgcManifest::ClientBuildsModel(bbbModel.id)) {
served++;
continue;
}
LOG("Getting lxfml ugcID: %llu", bbbModel.id);
bbbModel.lxfmlData.seekg(0, std::ios::end);
@@ -1055,7 +1062,8 @@ void LoadPlayer(const SystemAddress& sysAddr) {
model.blueprintId = bbbModel.id;
model.data = bbbModel.lxfmlData.str().substr(0, lxfmlSize);
}
response.Send(sysAddr);
if (served > 0) LOG("%zu of the property's %zu models come from the UGC server", served, bbbModels.size());
if (!response.models.empty()) response.Send(sysAddr);
}
noBBB:

12
resources/sharedconfig.ini
View File

@@ -95,7 +95,13 @@ mute_auto_reject_names=1
ugc_debounce_seconds=120
# 0 or 1: worlds answer game clients without 3D services (UGCUSE3DSERVICES=7:0, the default) with the checksums of the
# files the UGC server made, so they download the icons of cars, rockets and models. The 1.10.64 client then downloads
# from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and a client that can't connect there
# is logged out, so only set 1 when the UGC server answers there for every player. See docs/UgcServer.md.
# files the UGC server made, so they download the icons of cars, rockets and models from the UGCSERVERIP, UGCSERVERPORT
# and UGCSERVERDIR of their boot.cfg (default: the PATCHSERVER* values). A client that can't connect there is logged
# out, so only set 1 when every player's boot.cfg points at the UGC server. See docs/UgcServer.md.
ugc_manifest=0
# 0 or 1, with ugc_manifest=1: placed player models the UGC server has made are downloaded from it (their mesh) instead
# of being built by each client from the LXFML the world sends; models not made yet are still sent as LXFML. The UGC
# server makes no physics (.hkx): a client that never built a model itself has no collision for it. See
# docs/UgcServer.md.
ugc_manifest_models=0

1
tests/dGameTests/CMakeLists.txt
View File

@@ -13,6 +13,7 @@ set(DGAMETEST_SOURCES
"PowerUpStatisticTests.cpp"
"BrickByBrickTests.cpp"
"ModularBuildTableTests.cpp"
"UgcManifestTests.cpp"
)
add_subdirectory(dComponentsTests)

121
tests/dGameTests/UgcManifestTests.cpp Normal file
View File

@@ -0,0 +1,121 @@
#include "UgcManifest.h"
#include "ClientPackets.h"
#include "dConfig.h"
#include "GameDependencies.h"
#include "MD5.h"
#include "PacketTestUtils.h"
#include "Sd0.h"
#include <gtest/gtest.h>
using namespace PacketTestUtils;
using UgcManifest::eAction;
namespace {
std::string Md5Hex(const std::string& data) {
MD5 md5;
md5.update(reinterpret_cast<const unsigned char*>(data.data()), static_cast<MD5::size_type>(data.size()));
md5.finalize();
return md5.hexdigest();
}
}
// What each request gets. A placed model's client asks for its NIF, HKX and LXFML and waits for every answer without a
// timeout (LWOResMgr2Interface::RequestBlueprintManifestThenLoad, 0x0105a910), so model files are always answered:
// from the UGC server when it made the mesh and models are served, else with the LXFML for the client to build.
TEST(UgcManifestTests, DecideCoversEveryType) {
using T = eUgcResourceType;
for (const auto type : { T::LXFML, T::NIF, T::HKX, T::DDS }) {
EXPECT_EQ(UgcManifest::Decide(type, false, true, true), eAction::NONE);
}
EXPECT_EQ(UgcManifest::Decide(T::DDS, true, false, false), eAction::WAIT);
EXPECT_EQ(UgcManifest::Decide(T::DDS, true, true, true), eAction::WAIT);
EXPECT_EQ(UgcManifest::Decide(T::NIF, true, true, true), eAction::ANSWER);
EXPECT_EQ(UgcManifest::Decide(T::LXFML, true, true, true), eAction::ANSWER);
EXPECT_EQ(UgcManifest::Decide(T::HKX, true, true, true), eAction::ANSWER_UNKNOWN);
for (const auto type : { T::LXFML, T::NIF, T::HKX }) {
EXPECT_EQ(UgcManifest::Decide(type, true, true, false), eAction::SEND_LXFML); // not made yet
EXPECT_EQ(UgcManifest::Decide(type, true, false, true), eAction::SEND_LXFML); // models not served
EXPECT_EQ(UgcManifest::Decide(type, true, false, false), eAction::SEND_LXFML);
}
EXPECT_EQ(UgcManifest::Decide(static_cast<T>(7), true, true, true), eAction::NONE);
}
class UgcManifestRequestTests : public GameDependenciesTest {
protected:
void SetUp() override { SetUpDependencies(); }
void TearDown() override {
UgcManifest::OnDisconnect(Client());
TearDownDependencies();
}
static SystemAddress Client() {
SystemAddress address;
address.binaryAddress = 0x0100007f;
address.port = 15001;
return address;
}
void Settings(const std::string& manifest, const std::string& models) {
Game::config->SetDatabaseValues({ { "ugc_manifest", manifest }, { "ugc_manifest_models", models } }, {});
}
};
// Off (the default): nothing is answered, as before
TEST_F(UgcManifestRequestTests, NothingWhenOff) {
Settings("0", "1");
const auto sent = Capture([] { UgcManifest::OnRequest(Client(), 1234, eUgcResourceType::NIF); });
EXPECT_TRUE(sent.empty());
EXPECT_FALSE(UgcManifest::ServesModels());
EXPECT_TRUE(UgcManifest::ClientBuildsModel(1234));
}
// A model file of a blueprint that isn't a player model: answered as not known (37 bytes, valid 0), never left waiting
TEST_F(UgcManifestRequestTests, UnknownModelFilesAreAnsweredNotKnown) {
Settings("1", "1");
EXPECT_TRUE(UgcManifest::ServesModels());
EXPECT_TRUE(UgcManifest::ClientBuildsModel(1234)); // its mesh isn't made
for (const auto type : { eUgcResourceType::NIF, eUgcResourceType::HKX, eUgcResourceType::LXFML }) {
const auto sent = Capture([type] { UgcManifest::OnRequest(Client(), 0x0102030405060708, type); });
ASSERT_EQ(sent.size(), 1);
EXPECT_EQ(sent[0].sysAddr, Client());
EXPECT_FALSE(sent[0].broadcast);
EXPECT_PACKET_EQ(FromHex("53 05 00 3c 00 00 00 00 08 07 06 05 04 03 02 01 0" + std::to_string(static_cast<int>(type)) +
" 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00"), FromCapture(sent[0]));
}
}
// An icon that isn't made yet waits (answered by Update once the UGC server stored its checksum)
TEST_F(UgcManifestRequestTests, IconsWait) {
Settings("1", "0");
const auto sent = Capture([] { UgcManifest::OnRequest(Client(), 1234, eUgcResourceType::DDS); });
EXPECT_TRUE(sent.empty());
}
// Models made while nobody is on a property with them: nothing to tell
TEST_F(UgcManifestRequestTests, ModelsMadeWithNobodyToTell) {
Settings("1", "1");
const auto sent = Capture([] { UgcManifest::OnModelsMade({ 1234, 5678 }); });
EXPECT_TRUE(sent.empty());
}
// The client checks the MD5 and size of the LXFML as it has it after inflating the UGC server's .lxfml.sd0, which is
// the stored LXFML (sd0, or plain XML in old rows) inflated
TEST_F(UgcManifestRequestTests, LxfmlChecksumIsOfTheInflatedLxfml) {
const std::string lxfml = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\" ?>\n<LXFML versionMajor=\"5\"></LXFML>\n";
const auto plain = UgcManifest::LxfmlChecksum(lxfml);
ASSERT_TRUE(plain.has_value());
EXPECT_EQ(plain->md5, Md5Hex(lxfml));
EXPECT_EQ(plain->size, lxfml.size());
const auto compressed = UgcManifest::LxfmlChecksum(Sd0::Compress(lxfml));
ASSERT_TRUE(compressed.has_value());
EXPECT_EQ(compressed->md5, plain->md5);
EXPECT_EQ(compressed->size, plain->size);
EXPECT_FALSE(UgcManifest::LxfmlChecksum("").has_value());
EXPECT_FALSE(UgcManifest::LxfmlChecksum("not an sd0 stream").has_value());
}

16
tests/dGameTests/dGameMessagesTests/BuildingMessagesTests.cpp
View File

@@ -490,3 +490,19 @@ TEST_F(BuildingMessagesTests, FetchModelMetadataWire) {
EXPECT_TRUE(copy.bpData.neverFalseIfPresentInCaps);
ExpectTruncatedFails(full);
}
// NotifyClientUGCModelReady (909): the blueprint id only (GameMessage::NotifyClientUGCIconReady::Serialize, 0x00daae30,
// which the client shares with it; read by LWOBlueprintComponent::OnNotifyClientUGCModelReady, 0x00ca6430)
TEST_F(BuildingMessagesTests, NotifyClientUGCModelReadyWire) {
GameMessages::NotifyClientUGCModelReady ready;
ready.target = 0x1122334455667788;
ready.blueprintID = 0x0102030405060708;
EXPECT_PACKET_EQ(FromHex("08 07 06 05 04 03 02 01"), Payload(ready));
EXPECT_EQ(RoundTrip(ready).blueprintID, ready.blueprintID);
ExpectTruncatedFails(ready);
// The whole packet: the model's object id, then the message id 909
RakNet::BitStream bitStream;
ready.WritePacket(bitStream);
EXPECT_PACKET_EQ(FromHex("53 05 00 0c 00 00 00 00 88 77 66 55 44 33 22 11 8d 03 08 07 06 05 04 03 02 01"), FromBitStream(bitStream));
}

21
tests/dGameTests/dNetTests/MasterPacketsTests.cpp
View File

@@ -403,6 +403,10 @@ TEST(MasterPacketsTests, DashboardAndMigrationStructsKeepTheirBytes) {
changed.entries.push_back({ "characters", 42 });
ExpectHeaderThenSerialize(changed, MessageType::Master::DATA_CHANGED);
UgcModelsMade made;
made.blueprintIds = { 1152921510436831123, 42 };
ExpectHeaderThenSerialize(made, MessageType::Master::UGC_MODELS_MADE);
PlayerPositions positions;
positions.zoneId = 1100;
positions.players.push_back({ 42, 1.0f, 2.0f, 3.0f });
@@ -485,3 +489,20 @@ TEST(MasterPacketsTests, DispatcherRoutesAndDropsBadPackets) {
delete Game::logger;
Game::logger = previousLogger;
}
// UGC server -> master -> worlds: a u16 count, then each blueprint id
TEST(MasterPacketsTests, UgcModelsMadeBytes) {
UgcModelsMade made;
made.blueprintIds = { 0x0102030405060708, 9 };
EXPECT_PACKET_EQ(FromHex("53 06 00 25 00 00 00 00 02 00 08 07 06 05 04 03 02 01 09 00 00 00 00 00 00 00"), StructPacket(made));
RakNet::BitStream tooMany;
tooMany.Write<uint16_t>(UgcModelsMade::MAX_MODELS + 1);
UgcModelsMade read;
EXPECT_FALSE(read.Deserialize(tooMany));
RakNet::BitStream truncated;
truncated.Write<uint16_t>(2);
truncated.Write<LWOOBJID>(1);
EXPECT_FALSE(read.Deserialize(truncated));
}