feat(ugc): answer the client's UGC manifest requests without 3D services

With UGCUSE3DSERVICES=7:0 (the client's default) the client asks its world for
a blueprint file's MD5 and size (REQUEST_UGC_MANIFEST_INFO, world 27) and then
downloads BrickModels/UserMade/<id % 1000>/<id>.<ext>.sd0. Layouts checked in
the 1.10.64 client: the request is a u64 blueprint id and a u8 resource type;
the answer (UGC_MANIFEST_RESPONSE, client 60) repeats them and adds a u8 valid,
the u32 size and the 16 byte MD5 of the inflated file, 37 bytes after the 0x53
exactly or the client drops it.

- dNet: WorldPackets::RequestUgcManifestInfo, ClientPackets::UgcManifestResponse
  (eUgcResourceType), with byte tests against the client's layouts.
- Database: ugc_file_checksums (per model or module combination and file) and
  ugc_modular_build.combination_id (migrations 89 / 72), GetUgcFileChecksum
  looks a blueprint up as a model, else as a build through its combination.
- UGC server: every download is also written as .sd0 (Sd0::Compress); workers
  hand the checksums back and the main thread stores them; old items get their
  sd0 icon and checksum, and builds their combination id, once at start-up, a
  few per tick; serves <dir>/BrickModels/UserMade/<bucket>/<id>.<ext>.sd0 under
  client_path, /<folder>/UserBrickModels and the root (.hkx 404).
- World: UgcManifest answers on the main thread with one indexed query per
  request; files not made yet are answered when they are (looked at again
  every 5 seconds), and a model in its quiet period is made right away. No
  worker threads, HTTP or file reads in the world.

Off by default (ugc_manifest=0): checked in game, the client then downloads
from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says
and logs the player out when it can't connect, so icons need the UGC server on
port 80 of each player's machine. docs/UgcServer.md has the details.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-27 17:46:47 -05:00
parent ceec638e52
commit 79235702b3
33 changed files with 706 additions and 14 deletions

View File

@@ -0,0 +1,15 @@
#ifndef EUGCRESOURCETYPE_H
#define EUGCRESOURCETYPE_H
#include <cstdint>
// The kinds of file the client keeps for a blueprint (a player's model or a car or rocket), as it numbers them in
// REQUEST_UGC_MANIFEST_INFO and UGC_MANIFEST_RESPONSE and in its download paths
enum class eUgcResourceType : uint8_t {
LXFML = 0,
NIF,
HKX,
DDS
};
#endif //!EUGCRESOURCETYPE_H

View File

@@ -417,6 +417,7 @@ namespace {
c.Add(Format(Text(UGC, "listen_ip", "Listen address", "The game client has to reach it.", "0.0.0.0", true), eFormat::HOST));
c.Add(Port(UGC, "net_port", "Master connection port", "UDP; the next port is used too.", "2012"));
c.Add(Text(UGC, "client_path", "Download path", "The game client's UGCSERVERDIR.", "/ugc", true));
c.Add(Bool(SHARED, "ugc_manifest", "Answer clients without 3D services", "Worlds tell game clients with UGCUSE3DSERVICES=7:0 (the default) the checksums of the files the UGC server made, so they download icons of cars, rockets and models. The client then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and one that can't connect there is logged out: only turn on when the UGC server answers there for every player.", false));
c.Add(Format(Text(DASHBOARD, "ugc_internal_url", "UGC server address (internal)", "Where the dashboard itself reaches the UGC server for its status and files. Empty: http://127.0.0.1:2008.", ""), eFormat::URL));
c.Add(Format(Text(DASHBOARD, "ugc_public_url", "UGC server address (public)", "Only for the \"open on the UGC server\" links, e.g. https://ugc.example.com. The dashboard's own pages don't need the browser to reach it.", ""), eFormat::URL));
c.AddSection("UGC processing", "When models are made and how much of the machine the workers may use.", eLayout::ROWS, Condition{ MASTER, "enable_ugc_server", { "1" } });

View File

@@ -107,5 +107,26 @@ public:
// How many models are in each state
virtual std::vector<std::pair<eProcessState, uint64_t>> GetUgcProcessCounts() = 0;
// ---- Checksums of the files the UGC server made (ugc_file_checksums), for the client's manifest requests ----
// What the files are stored under: a player model's ugc id, or a combination of car or rocket modules
enum class eFileOwner : int32_t {
MODEL = 0,
COMBINATION = 1
};
// A made file as the client has it after inflating the download: its MD5 (32 lowercase hex digits) and size
struct FileChecksum {
std::string md5;
uint32_t size{};
};
// Records (or replaces) the checksum of a made file, e.g. "icon.dds" or "model.nif"
virtual void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) = 0;
// The checksum of a blueprint's file: a player model's (the blueprint is its ugc id), else a car or rocket build's
// (the blueprint is its ugc_modular_build id; the file is its combination's)
virtual std::optional<FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) = 0;
};
#endif //!__IUGC__H__

View File

@@ -29,6 +29,12 @@ public:
virtual std::vector<IUgc::ProcessInfo> GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) = 0;
virtual std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetModularBuildProcessCounts() = 0;
// The combination of modules a build was made as (UgcModularKey::StorageId of its modules), whose files it shares
virtual void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) = 0;
// Up to `limit` builds whose combination isn't recorded yet (combination_id 0): their ids and modules
virtual std::vector<PendingBuild> GetModularBuildsWithoutCombination(const uint32_t limit) = 0;
// How many builds there are of each ldf_config as stored (the same modules may be written differently)
virtual std::vector<std::pair<std::string, uint64_t>> GetModularBuildConfigCounts() = 0;

View File

@@ -429,6 +429,10 @@ public:
void DeleteUgcIconSettings(const std::string_view target) override;
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override;
void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override;
void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override;
std::optional<IUgc::FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override;
void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override;
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsWithoutCombination(const uint32_t limit) override;
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override;
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) override;

View File

@@ -181,3 +181,16 @@ void MySQLDatabase::ExpediteUgcModels(const LWOOBJID characterId) {
void MySQLDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) {
ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ? WHERE id = ?;", bricks, triangles, id);
}
void MySQLDatabase::SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) {
ExecuteInsert("INSERT INTO ugc_file_checksums (kind, storage_id, file, md5, size) VALUES (?, ?, ?, ?, ?) "
"ON DUPLICATE KEY UPDATE md5 = VALUES(md5), size = VALUES(size);", static_cast<int32_t>(owner), storageId, file, md5, size);
}
std::optional<IUgc::FileChecksum> MySQLDatabase::GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) {
auto result = ExecuteSelect("SELECT md5, size FROM ugc_file_checksums WHERE kind = 0 AND storage_id = ? AND file = ? "
"UNION ALL SELECT f.md5, f.size FROM ugc_modular_build b JOIN ugc_file_checksums f ON f.kind = 1 AND f.storage_id = b.combination_id AND f.file = ? "
"WHERE b.ugc_id = ? AND b.combination_id != 0 LIMIT 1;", blueprintId, file, file, blueprintId);
if (!result->next()) return std::nullopt;
return IUgc::FileChecksum{ std::string(result->getString("md5").c_str()), static_cast<uint32_t>(result->getUInt("size")) };
}

View File

@@ -106,3 +106,16 @@ void MySQLDatabase::SetUgcIconSettings(const std::string_view target, const std:
void MySQLDatabase::DeleteUgcIconSettings(const std::string_view target) {
ExecuteDelete("DELETE FROM ugc_icon_settings WHERE target = ?;", target);
}
void MySQLDatabase::SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) {
ExecuteUpdate("UPDATE ugc_modular_build SET combination_id = ? WHERE ugc_id = ?;", combinationId, id);
}
std::vector<IUgcModularBuild::PendingBuild> MySQLDatabase::GetModularBuildsWithoutCombination(const uint32_t limit) {
auto result = ExecuteSelect("SELECT ugc_id, ldf_config, process_attempts FROM ugc_modular_build WHERE combination_id = 0 LIMIT ?;", limit);
std::vector<PendingBuild> builds;
while (result->next()) {
builds.push_back({ result->getInt64("ugc_id"), std::string(result->getString("ldf_config").c_str()), static_cast<uint32_t>(result->getInt("process_attempts")) });
}
return builds;
}

View File

@@ -413,6 +413,10 @@ public:
void DeleteUgcIconSettings(const std::string_view target) override;
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override;
void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override;
void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override;
std::optional<IUgc::FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override;
void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override;
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsWithoutCombination(const uint32_t limit) override;
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override;
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) override;

View File

@@ -184,3 +184,16 @@ void SQLiteDatabase::ExpediteUgcModels(const LWOOBJID characterId) {
void SQLiteDatabase::SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) {
ExecuteUpdate("UPDATE ugc SET brick_count = ?, triangle_count = ? WHERE id = ?;", bricks, triangles, id);
}
void SQLiteDatabase::SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) {
ExecuteInsert("INSERT INTO ugc_file_checksums (kind, storage_id, file, md5, size) VALUES (?, ?, ?, ?, ?) "
"ON CONFLICT(kind, storage_id, file) DO UPDATE SET md5 = excluded.md5, size = excluded.size;", static_cast<int32_t>(owner), storageId, file, md5, size);
}
std::optional<IUgc::FileChecksum> SQLiteDatabase::GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) {
auto [_, result] = ExecuteSelect("SELECT md5, size FROM ugc_file_checksums WHERE kind = 0 AND storage_id = ? AND file = ? "
"UNION ALL SELECT f.md5, f.size FROM ugc_modular_build b JOIN ugc_file_checksums f ON f.kind = 1 AND f.storage_id = b.combination_id AND f.file = ? "
"WHERE b.ugc_id = ? AND b.combination_id != 0 LIMIT 1;", blueprintId, file, file, blueprintId);
if (result.eof()) return std::nullopt;
return IUgc::FileChecksum{ std::string(result.getStringField("md5", "")), static_cast<uint32_t>(result.getInt64Field("size")) };
}

View File

@@ -112,3 +112,17 @@ void SQLiteDatabase::SetUgcIconSettings(const std::string_view target, const std
void SQLiteDatabase::DeleteUgcIconSettings(const std::string_view target) {
ExecuteDelete("DELETE FROM ugc_icon_settings WHERE target = ?;", target);
}
void SQLiteDatabase::SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) {
ExecuteUpdate("UPDATE ugc_modular_build SET combination_id = ? WHERE ugc_id = ?;", combinationId, id);
}
std::vector<IUgcModularBuild::PendingBuild> SQLiteDatabase::GetModularBuildsWithoutCombination(const uint32_t limit) {
auto [_, result] = ExecuteSelect("SELECT ugc_id, ldf_config, process_attempts FROM ugc_modular_build WHERE combination_id = 0 LIMIT ?;", limit);
std::vector<PendingBuild> builds;
while (!result.eof()) {
builds.push_back({ result.getInt64Field("ugc_id"), result.getStringField("ldf_config", ""), static_cast<uint32_t>(result.getIntField("process_attempts")) });
result.nextRow();
}
return builds;
}

View File

@@ -411,6 +411,10 @@ class TestSQLDatabase : public GameDatabase {
void DeleteUgcIconSettings(const std::string_view target) override {}
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override {}
void SetUgcModelStats(const LWOOBJID id, const uint32_t bricks, const uint32_t triangles) override {}
void SetUgcFileChecksum(const eFileOwner owner, const LWOOBJID storageId, const std::string_view file, const std::string_view md5, const uint32_t size) override {}
std::optional<IUgc::FileChecksum> GetUgcFileChecksum(const LWOOBJID blueprintId, const std::string_view file) override { return std::nullopt; }
void SetModularBuildCombination(const LWOOBJID id, const LWOOBJID combinationId) override {}
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsWithoutCombination(const uint32_t limit) override { return {}; }
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override { return {}; }
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override { return 0; }
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const std::string_view search, const uint32_t offset, const uint32_t limit) override { return {}; }

View File

@@ -17,6 +17,7 @@ set(DGAME_DUTILITIES_SOURCES "BrickDatabase.cpp"
"PlayerReports.cpp"
"Preconditions.cpp"
"SlashCommandHandler.cpp"
"UgcManifest.cpp"
"VanityUtilities.cpp"
"WorldMigration.cpp")

View File

@@ -0,0 +1,102 @@
#include "UgcManifest.h"
#include <chrono>
#include <optional>
#include <string_view>
#include <vector>
#include "ClientPackets.h"
#include "Database.h"
#include "dConfig.h"
#include "Game.h"
#include "Logger.h"
namespace {
// How often the waiting requests are looked up again, how long they wait at most and how many there can be
constexpr auto RETRY_INTERVAL = std::chrono::seconds(5);
constexpr auto MAX_WAIT = std::chrono::minutes(15);
constexpr size_t MAX_WAITING = 512;
struct Waiting {
SystemAddress sysAddr;
LWOOBJID blueprintId{};
eUgcResourceType resourceType{};
std::chrono::steady_clock::time_point since;
};
std::vector<Waiting> g_Waiting;
std::chrono::steady_clock::time_point g_NextRetry{};
// The UGC server's name for a blueprint's file of this type; nothing for the ones it doesn't make
std::optional<std::string_view> FileName(eUgcResourceType type) {
switch (type) {
case eUgcResourceType::DDS: return "icon.dds";
case eUgcResourceType::NIF: return "model.nif";
default: return std::nullopt;
}
}
bool Enabled() {
// Off unless set: the 1.10.64 client downloads from http://127.0.0.1:80 whatever its boot.cfg says, and one that
// can't connect there is logged out (docs/UgcServer.md)
return Game::config->GetValue("ugc_manifest") == "1";
}
int HexDigit(char c) {
if (c >= '0' && c <= '9') return c - '0';
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
return -1;
}
// Answers when the file's checksum is known; false when it isn't (yet)
bool TryAnswer(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) {
const auto file = FileName(resourceType);
if (!file) return false;
const auto checksum = Database::Get()->GetUgcFileChecksum(blueprintId, *file);
if (!checksum || checksum->md5.size() != 32) return false;
ClientPackets::UgcManifestResponse response;
response.blueprintId = blueprintId;
response.resourceType = resourceType;
response.fileSize = checksum->size;
for (size_t i = 0; i < response.md5.size(); i++) {
const auto high = HexDigit(checksum->md5[i * 2]);
const auto low = HexDigit(checksum->md5[i * 2 + 1]);
if (high < 0 || low < 0) return false;
response.md5[i] = static_cast<uint8_t>(high << 4 | low);
}
response.valid = true;
response.Send(sysAddr);
LOG_DEBUG("Sent the UGC manifest of %llu (type %i): %s, %u bytes", static_cast<unsigned long long>(blueprintId),
static_cast<int>(resourceType), checksum->md5.c_str(), checksum->size);
return true;
}
}
void UgcManifest::OnRequest(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType) {
if (sysAddr == UNASSIGNED_SYSTEM_ADDRESS || !Enabled() || !FileName(resourceType)) return;
if (TryAnswer(sysAddr, blueprintId, resourceType)) return;
// Not made yet: a client wants it, so a model still in its quiet period after a save is made now (as when a client
// asks the UGC server for it), and the request is answered when it's made
Database::Get()->ExpediteUgcModel(blueprintId);
for (const auto& waiting : g_Waiting) {
if (waiting.sysAddr == sysAddr && waiting.blueprintId == blueprintId && waiting.resourceType == resourceType) return;
}
if (g_Waiting.size() >= MAX_WAITING) g_Waiting.erase(g_Waiting.begin());
g_Waiting.push_back({ sysAddr, blueprintId, resourceType, std::chrono::steady_clock::now() });
}
void UgcManifest::Update() {
if (g_Waiting.empty()) return;
const auto now = std::chrono::steady_clock::now();
if (now < g_NextRetry) return;
g_NextRetry = now + RETRY_INTERVAL;
std::erase_if(g_Waiting, [now](const Waiting& waiting) {
return now - waiting.since > MAX_WAIT || TryAnswer(waiting.sysAddr, waiting.blueprintId, waiting.resourceType);
});
}
void UgcManifest::OnDisconnect(const SystemAddress& sysAddr) {
std::erase_if(g_Waiting, [&sysAddr](const Waiting& waiting) { return waiting.sysAddr == sysAddr; });
}

View File

@@ -0,0 +1,29 @@
#ifndef UGCMANIFEST_H
#define UGCMANIFEST_H
#include "dCommonVars.h"
#include "eUgcResourceType.h"
#include "RakNetTypes.h"
/**
* Answers the client's REQUEST_UGC_MANIFEST_INFO (sent when its UGCUSE3DSERVICES is off, the default): the MD5 and
* size of a blueprint's file, which the client needs before it downloads the file from the UGC server
* (BrickModels/UserMade/<id % 1000>/<id>.<ext>.sd0). The checksums are the ones the UGC server stored when it made the
* files (ugc_file_checksums); a file it hasn't made yet is answered once it has (Update looks again every few seconds
* for a while). Nothing is answered for files it never makes (LXFML, HKX), or unless `ugc_manifest` is 1: the client
* then downloads from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and is logged out when
* it can't connect there.
*
* Main thread only: each lookup is one indexed query and the answers are sent from here. See docs/UgcServer.md.
*/
namespace UgcManifest {
void OnRequest(const SystemAddress& sysAddr, LWOOBJID blueprintId, eUgcResourceType resourceType);
// Answers the waiting requests whose files have been made since, and forgets the ones waited on too long
void Update();
// A client left: its waiting requests are dropped
void OnDisconnect(const SystemAddress& sysAddr);
}
#endif //!UGCMANIFEST_H

View File

@@ -484,6 +484,25 @@ bool ClientPackets::BlueprintLoadItemResponse::Deserialize(RakNet::BitStream& bi
return true;
}
void ClientPackets::UgcManifestResponse::Serialize(RakNet::BitStream& bitStream) const {
bitStream.Write(blueprintId);
bitStream.Write(resourceType);
bitStream.Write<uint8_t>(valid ? 1 : 0);
bitStream.Write(fileSize);
for (const auto byte : md5) bitStream.Write(byte);
}
bool ClientPackets::UgcManifestResponse::Deserialize(RakNet::BitStream& bitStream) {
VALIDATE_READ(bitStream.Read(blueprintId));
VALIDATE_READ(bitStream.Read(resourceType));
uint8_t validByte{};
VALIDATE_READ(bitStream.Read(validByte));
valid = validByte != 0;
VALIDATE_READ(bitStream.Read(fileSize));
for (auto& byte : md5) VALIDATE_READ(bitStream.Read(byte));
return true;
}
namespace {
// A zone ID as three fields: map (u16), instance (u16), clone (u32)
void WriteZone(RakNet::BitStream& bitStream, const LWOZONEID& zoneID) {

View File

@@ -6,6 +6,7 @@
#ifndef CLIENTPACKETS_H
#define CLIENTPACKETS_H
#include <array>
#include <cstdint>
#include <ctime>
#include <set>
@@ -18,6 +19,7 @@
#include "eAddFriendResponseType.h"
#include "eAddIgnoreResponse.h"
#include "eBlueprintSaveResponseType.h"
#include "eUgcResourceType.h"
#include "MessageType/Client.h"
#include "MessageType/Game.h"
#include "NiPoint3.h"
@@ -264,6 +266,23 @@ namespace ClientPackets {
LWOOBJID itemId{};
LWOOBJID destItemId{};
};
// Server -> client. The answer to REQUEST_UGC_MANIFEST_INFO: the MD5 and size of one of a blueprint's files as the
// client has it after downloading and inflating it (BrickModels/UserMade/<id % 1000>/<id>.<ext>.sd0 from the UGC
// server). The client checks a file it has against it and downloads the file when it differs; with valid 0 it uses
// a file it has as it is and downloads it only when it has none. The client ignores the answer unless it is exactly
// this long (37 bytes after the 0x53).
struct UgcManifestResponse : public LUBitStream {
UgcManifestResponse() : LUBitStream(ServiceType::CLIENT, MessageType::Client::UGC_MANIFEST_RESPONSE) {}
void Serialize(RakNet::BitStream& bitStream) const override;
bool Deserialize(RakNet::BitStream& bitStream) override;
LWOOBJID blueprintId{};
eUgcResourceType resourceType{};
bool valid{}; // A byte
uint32_t fileSize{}; // Of the inflated file
std::array<uint8_t, 16> md5{}; // Of the inflated file, as bytes
};
};
/**

View File

@@ -340,6 +340,17 @@ namespace WorldPackets {
return true;
}
void RequestUgcManifestInfo::Serialize(RakNet::BitStream& bitStream) const {
bitStream.Write(blueprintId);
bitStream.Write(resourceType);
}
bool RequestUgcManifestInfo::Deserialize(RakNet::BitStream& bitStream) {
VALIDATE_READ(bitStream.Read(blueprintId));
VALIDATE_READ(bitStream.Read(resourceType));
return true;
}
void UIHelpTop5::Serialize(RakNet::BitStream& bitStream) const {
bitStream.Write(language);
}

View File

@@ -5,6 +5,7 @@
#include "ChatPackets.h"
#include "dCommonVars.h"
#include "eFunnessTypes.h"
#include "eUgcResourceType.h"
#include "MessageType/Game.h"
#include "MessageType/World.h"
#include "PositionUpdate.h"
@@ -201,6 +202,17 @@ namespace WorldPackets {
bool Deserialize(RakNet::BitStream& bitStream) override;
};
// Client -> server, when its UGCUSE3DSERVICES is off (the default): the MD5 and size of one of a blueprint's files
// (a player's model, or a car or rocket's build), answered with ClientPackets::UgcManifestResponse.
struct RequestUgcManifestInfo : public WorldLUBitStream {
LWOOBJID blueprintId{};
eUgcResourceType resourceType{};
RequestUgcManifestInfo() : WorldLUBitStream(MessageType::World::REQUEST_UGC_MANIFEST_INFO) {}
void Serialize(RakNet::BitStream& bitStream) const override;
bool Deserialize(RakNet::BitStream& bitStream) override;
};
// Client -> server. The help screen wants its top 5 issues.
struct UIHelpTop5 : public WorldLUBitStream {
int32_t language{}; // 0: en_US, 1: pl_US, 2: de_DE, 3: en_GB

View File

@@ -372,4 +372,26 @@ namespace UgcFormats {
std::string ChecksumXml(std::string_view data) {
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Checksum><MD5>" + Md5Hex(data) + "</MD5><Filesize>" + std::to_string(data.size()) + "</Filesize></Checksum>\n";
}
bool ReadChecksumXml(std::string_view xml, std::string& md5, uint32_t& size) {
const auto between = [xml](std::string_view open, std::string_view close) -> std::string_view {
const auto start = xml.find(open);
if (start == std::string_view::npos) return {};
const auto end = xml.find(close, start + open.size());
if (end == std::string_view::npos) return {};
return xml.substr(start + open.size(), end - start - open.size());
};
const auto hash = between("<MD5>", "</MD5>");
const auto length = between("<Filesize>", "</Filesize>");
if (hash.size() != 32 || length.empty()) return false;
uint64_t parsed = 0;
for (const char c : length) {
if (c < '0' || c > '9') return false;
parsed = parsed * 10 + static_cast<uint64_t>(c - '0');
if (parsed > std::numeric_limits<uint32_t>::max()) return false;
}
md5 = hash;
size = static_cast<uint32_t>(parsed);
return true;
}
}

View File

@@ -1,5 +1,6 @@
#pragma once
#include <cstdint>
#include <string>
#include <string_view>
#include <vector>
@@ -57,4 +58,7 @@ namespace UgcFormats {
// What the client reads from a UGC file's .checksum: the MD5 and size of the file as it is after inflating it
std::string ChecksumXml(std::string_view data);
// The MD5 (lowercase hex) and size a .checksum (ChecksumXml) holds; false when it doesn't hold both
bool ReadChecksumXml(std::string_view xml, std::string& md5, uint32_t& size);
}

View File

@@ -17,6 +17,9 @@ namespace UgcJobs {
void AddDownload(UgcStorage::Files& files, const std::string& name, const std::string& data) {
files[name + ".gz"] = ZCompression::Gzip(data);
files[name + ".checksum"] = UgcFormats::ChecksumXml(data);
// Without 3D services the client downloads it as sd0 (after asking its world for the checksum)
auto sd0 = Sd0::Compress(data);
if (!sd0.empty()) files[name + ".sd0"] = std::move(sd0);
}
std::string LxfmlFromBlob(const std::string& blob) {

View File

@@ -5,7 +5,10 @@
#include "Logger.h"
#include "UgcBricks.h"
#include "UgcCdClient.h"
#include "Sd0.h"
#include "UgcFormats.h"
#include "UgcKeys.h"
#include "ZCompression.h"
#include "UgcThrottle.h"
#include "json.hpp"
@@ -26,6 +29,12 @@ namespace {
constexpr size_t LOG_LENGTH = 50;
constexpr auto EVICTION_INTERVAL = std::chrono::minutes(5);
constexpr auto RECENT_ANSWER_TIME = std::chrono::seconds(10);
// In the storage folder once every item stored has its sd0 icon and checksums (Backfill)
constexpr auto BACKFILL_MARKER = ".checksums-stored";
constexpr size_t BACKFILL_ITEMS_PER_UPDATE = 16;
constexpr uint32_t BACKFILL_BUILDS_PER_UPDATE = 200;
// Combination id of a build whose modules can't be told (so it isn't looked at again)
constexpr LWOOBJID NO_COMBINATION = -1;
const char* KindName(UgcStorage::Kind kind) {
return kind == UgcStorage::Kind::MODEL ? "model" : "modular";
@@ -95,7 +104,12 @@ UgcProcessor::~UgcProcessor() {
}
void UgcProcessor::Start() {
for (const auto& entry : m_Storage.List()) m_StoredBytes += entry.bytes;
const auto stored = m_Storage.List();
for (const auto& entry : stored) m_StoredBytes += entry.bytes;
// Items made before the checksums were stored get their sd0 icon and checksums once (Backfill)
std::error_code error;
m_BackfillItemsDone = std::filesystem::exists(m_Storage.GetRoot() / BACKFILL_MARKER, error);
if (!m_BackfillItemsDone) m_BackfillItems.assign(stored.begin(), stored.end());
m_NextEviction = std::chrono::steady_clock::now();
m_Stopping = false;
for (size_t i = 0; i < std::max<size_t>(m_Config.threads, 1); i++) m_Threads.emplace_back(&UgcProcessor::Worker, this);
@@ -233,6 +247,14 @@ void UgcProcessor::Worker() {
const auto bytes = job.iconOnly ? m_Storage.Update(job.kind, job.id, done.outcome.files, error) : m_Storage.Write(job.kind, job.id, done.outcome.files, error);
if (bytes) {
done.bytes = *bytes;
// The checksums of what the client downloads as sd0, for the main thread to store (the worlds answer
// the clients' manifest requests with them)
for (const std::string name : { "icon.dds", "model.nif" }) {
const auto checksum = done.outcome.files.find(name + ".checksum");
if (checksum == done.outcome.files.end() || !done.outcome.files.contains(name + ".sd0")) continue;
Checksum parsed{ name };
if (UgcFormats::ReadChecksumXml(checksum->second, parsed.md5, parsed.size)) done.checksums.push_back(std::move(parsed));
}
} else {
done.outcome.ok = false;
done.outcome.error = error;
@@ -355,6 +377,8 @@ void UgcProcessor::Record(const Done& done) {
}
} else {
Database::Get()->SetModularBuildProcessed(done.id, state, attempts, error);
// Which combination's files it shares, for the worlds' manifest answers
if (const auto combo = m_ComboOf.find(done.id); combo != m_ComboOf.end()) Database::Get()->SetModularBuildCombination(done.id, combo->second);
}
m_Recent.erase({ done.kind, done.id });
@@ -381,6 +405,8 @@ void UgcProcessor::Collect() {
finished.swap(m_Done);
}
for (const auto& done : finished) {
// done.id is the model, or the combination
if (done.outcome.ok) StoreChecksums(done.kind, done.id, done.checksums);
if (done.kind == Kind::MODEL) {
m_InFlight.erase({ done.kind, done.id });
if (!done.iconOnly) {
@@ -413,8 +439,58 @@ void UgcProcessor::Collect() {
if (!finished.empty()) m_NextPoll = std::min(m_NextPoll, std::chrono::steady_clock::now() + std::chrono::milliseconds(100));
}
void UgcProcessor::StoreChecksums(Kind kind, LWOOBJID storageId, const std::vector<Checksum>& checksums) {
const auto owner = kind == Kind::MODEL ? IUgc::eFileOwner::MODEL : IUgc::eFileOwner::COMBINATION;
for (const auto& checksum : checksums) Database::Get()->SetUgcFileChecksum(owner, storageId, checksum.file, checksum.md5, checksum.size);
}
void UgcProcessor::Backfill() {
if (!m_BackfillBuildsDone) {
const auto builds = Database::Get()->GetModularBuildsWithoutCombination(BACKFILL_BUILDS_PER_UPDATE);
for (const auto& build : builds) {
const auto key = UgcModularKey::Normalize(build.modules);
const auto combo = key.empty() ? NO_COMBINATION : UgcModularKey::StorageId(key);
m_ComboOf[build.id] = combo;
Database::Get()->SetModularBuildCombination(build.id, combo);
}
if (builds.size() < BACKFILL_BUILDS_PER_UPDATE) m_BackfillBuildsDone = true;
}
if (m_BackfillItemsDone) return;
const auto read = [](const std::filesystem::path& path) {
std::ifstream in(path, std::ios::binary);
return std::string(std::istreambuf_iterator<char>(in), {});
};
for (size_t i = 0; i < BACKFILL_ITEMS_PER_UPDATE && !m_BackfillItems.empty(); i++) {
const auto entry = m_BackfillItems.front();
m_BackfillItems.pop_front();
if (m_InFlight.contains({ entry.kind, entry.id }) || (entry.kind == Kind::MODULAR && m_ComboJobs.contains(entry.id))) continue;
// Only the icon: it is small (a model's .nif isn't asked for without 3D services and gets its sd0 when made again)
const auto checksumFile = m_Storage.File(entry.kind, entry.id, "icon.dds.checksum");
if (!checksumFile) continue;
Checksum checksum{ "icon.dds" };
if (!UgcFormats::ReadChecksumXml(read(*checksumFile), checksum.md5, checksum.size)) continue;
if (!m_Storage.File(entry.kind, entry.id, "icon.dds.sd0")) {
const auto packed = m_Storage.File(entry.kind, entry.id, "icon.dds.gz");
const auto icon = packed ? ZCompression::Gunzip(read(*packed)) : std::nullopt;
if (!icon || UgcFormats::Md5Hex(*icon) != checksum.md5) continue;
std::string error;
if (!m_Storage.Update(entry.kind, entry.id, { { "icon.dds.sd0", Sd0::Compress(*icon) } }, error)) {
LOG("Couldn't write the sd0 icon of %llu: %s", static_cast<unsigned long long>(entry.id), error.c_str());
continue;
}
}
StoreChecksums(entry.kind, entry.id, { checksum });
}
if (m_BackfillItems.empty()) {
m_BackfillItemsDone = true;
std::ofstream(m_Storage.GetRoot() / BACKFILL_MARKER) << "1\n";
LOG("Stored the checksums of the icons made before");
}
}
void UgcProcessor::Update() {
Collect();
Backfill();
const auto now = std::chrono::steady_clock::now();
if (now - m_CpuSampled >= std::chrono::seconds(2)) SampleUsage();
if (now >= m_NextPoll) {

View File

@@ -155,6 +155,13 @@ private:
UgcIconParams::Values iconValues; // models: the preset and override (UgcIconParams)
};
// A written file's MD5 and size as the client has it after inflating it (from its .checksum)
struct Checksum {
std::string file; // icon.dds, model.nif
std::string md5;
uint32_t size{};
};
struct Done {
Kind kind{};
LWOOBJID id{};
@@ -163,8 +170,19 @@ private:
uint64_t bytes{};
double milliseconds{};
bool iconOnly{};
std::vector<Checksum> checksums; // of the files written that the client downloads as sd0
};
// Main thread: records the checksums of an item's files (ugc_file_checksums)
void StoreChecksums(Kind kind, LWOOBJID storageId, const std::vector<Checksum>& checksums);
// Main thread: fills in what was made before the worlds answered manifest requests, a little per Update: the
// icons' sd0 files and checksums of the items stored, and the builds' combination ids
void Backfill();
std::deque<UgcStorage::Entry> m_BackfillItems;
bool m_BackfillItemsDone{ true };
bool m_BackfillBuildsDone{ false };
void Poll();
// Main thread: the icon values for a kind and an item (the kind's preset, then the item's override)
UgcIconParams::Values IconValues(const std::string& kind, const std::string& itemTarget);

View File

@@ -34,6 +34,7 @@
#include "UgcCdClient.h"
#include "UgcIconParams.h"
#include "UgcJobs.h"
#include "Sd0.h"
#include "UgcFormats.h"
#include "ZCompression.h"
#include "UgcModel.h"
@@ -228,6 +229,49 @@ namespace {
// .hkx: no physics is made
}
/**
* A game client download without 3D services (UGCUSE3DSERVICES=7:0, the client's default):
* BrickModels/UserMade/<id % 1000, 3 digits>/<id, 20 digits><.lxfml|.nif|.hkx|.dds>.sd0 under its UGCSERVERDIR. The
* client asked its world for the file's checksum first (UGC_MANIFEST_RESPONSE) and checks the inflated download
* against it.
*/
void ServeClientSd0(HTTPReply& reply, const std::string& bucket, const std::string& name) {
NotFound(reply);
auto file = Lower(name);
if (!file.ends_with(".sd0")) return;
file.resize(file.size() - 4);
const auto dot = file.find('.');
if (dot == std::string::npos) return;
const auto extension = file.substr(dot);
const auto number = file.substr(0, dot);
const auto id = GeneralUtils::TryParse<uint64_t>(number);
if (!id || number.empty() || GeneralUtils::TryParse<uint64_t>(bucket) != *id % 1000) return;
const auto blueprint = static_cast<LWOOBJID>(*id);
if (extension == ".lxfml") {
// Straight from the ugc row, like the 3D services download; the last few are kept
static std::map<LWOOBJID, std::string> cache; // main thread only
auto it = cache.find(blueprint);
if (it == cache.end()) {
const auto model = Database::Get()->GetUgcModel(blueprint);
const auto lxfml = model ? UgcJobs::LxfmlFromBlob(model->lxfmlData.str()) : std::string();
if (lxfml.empty()) return;
if (cache.size() >= 64) cache.erase(cache.begin());
it = cache.emplace(blueprint, Sd0::Compress(lxfml)).first;
}
reply.status = eHTTPStatusCode::OK;
reply.contentType = eContentType::APPLICATION_OCTET_STREAM;
reply.message = it->second;
reply.headers.push_back("Cache-Control: public, max-age=60");
} else if (extension == ".nif") {
ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.nif.sd0", eContentType::APPLICATION_OCTET_STREAM, false);
} else if (extension == ".dds") {
// A player model's icon, else a car or rocket's (its combination's)
ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "icon.dds.sd0", eContentType::APPLICATION_OCTET_STREAM, false);
if (reply.status == eHTTPStatusCode::NOT_FOUND) ServeFile(reply, UgcStorage::Kind::MODULAR, blueprint, "icon.dds.sd0", eContentType::APPLICATION_OCTET_STREAM, false);
}
// .hkx: no physics is made, the client makes its own
}
void RegisterRoutes() {
// The configured path, and the one the 1.10.64 client uses whatever its boot.cfg says (its built-in patch server
// folder, lwoclient/UserBrickModels; see docs/UgcServer.md)
@@ -253,6 +297,25 @@ namespace {
LOG("Serving the client's downloads under %s/UGCC<datacenter>/", base.c_str());
}
// Without 3D services: <UGCSERVERDIR>/BrickModels/UserMade/<bucket>/<file>.sd0, where UGCSERVERDIR is client_path
// when boot.cfg sets it, else the client's default <PATCHSERVERDIR>/UserBrickModels (any patch folder), or empty
std::set<std::string> sd0Bases = { "/:patchdir/userbrickmodels", "" };
{
std::string base;
for (const auto& segment : Segments(Lower(Game::config->GetValue("client_path").empty() ? std::string("/ugc") : Game::config->GetValue("client_path")))) base += "/" + segment;
sd0Bases.insert(base);
}
for (const auto& base : sd0Bases) {
Game::web.RegisterHTTPRoute({ .path = base + "/brickmodels/usermade/:bucket/:file", .method = eHTTPMethod::GET, .middleware = {},
.handle = [](HTTPReply& reply, const HTTPContext& context) {
const auto segments = Segments(context.originalPath);
if (segments.size() < 2) return NotFound(reply);
ServeClientSd0(reply, segments[segments.size() - 2], segments.back());
LOG("Client download %s -> %i%s", context.originalPath.c_str(), static_cast<int>(reply.status), reply.file.empty() ? "" : " (file)");
} });
LOG("Serving the client's sd0 downloads under %s/BrickModels/UserMade/", base.c_str());
}
// Previews for the dashboard
Game::web.RegisterHTTPRoute({ .path = "/files/:kind/:id/:file", .method = eHTTPMethod::GET, .middleware = {},
.handle = [](HTTPReply& reply, const HTTPContext& context) {

View File

@@ -9,9 +9,9 @@
namespace {
constexpr std::array KNOWN_FILES = {
"model.nif", "model.nif.gz", "model.nif.checksum",
"model.nif", "model.nif.gz", "model.nif.checksum", "model.nif.sd0",
"model.lxfml.gz", "model.lxfml.checksum",
"icon.dds.gz", "icon.dds.checksum", "icon.png",
"icon.dds.gz", "icon.dds.checksum", "icon.dds.sd0", "icon.png",
"model.noao.nif", "model.noao.nif.gz", "stats.json", "combo.json",
"previous.icon.png", "previous.model.nif.gz", "previous.stats.json",
};

View File

@@ -10,6 +10,7 @@
#include "PropertyRent.h"
#include "PropertyReputation.h"
#include "LiveEvents.h"
#include "UgcManifest.h"
#include <iostream>
#include <string>
#include <ctime>
@@ -474,6 +475,7 @@ int main(int argc, char** argv) {
DashboardNotify::SendPlayerPositions(g_InstanceID);
MessageInspector::Update();
LiveEvents::Update();
UgcManifest::Update();
Metrics::StartMeasurement(MetricVariable::UpdateSpawners);
Game::zoneManager->Update(deltaTime);
@@ -1104,6 +1106,7 @@ void CleanupDisconnectedUser(const SystemAddress& sysAddr) {
// other instance has saved since
const bool savedByMigration = WorldMigration::IsLeaving(sysAddr);
WorldMigration::OnDisconnected(sysAddr);
UgcManifest::OnDisconnect(sysAddr);
auto user = UserManager::Instance()->GetUser(sysAddr);
if (!user) return;
@@ -1604,6 +1607,13 @@ namespace {
}
};
struct RequestUgcManifestInfoPacket final : public WorldPackets::RequestUgcManifestInfo {
void Handle() override {
if (!UserManager::Instance()->GetUser(sysAddr)) return;
UgcManifest::OnRequest(sysAddr, blueprintId, resourceType);
}
};
template<typename T>
std::unique_ptr<WorldPackets::WorldLUBitStream> Create() { return std::make_unique<T>(); }
@@ -1623,6 +1633,7 @@ namespace {
{ MessageType::World::GENERAL_CHAT_MESSAGE, Create<GeneralChatMessagePacket> },
{ MessageType::World::HANDLE_FUNNESS, Create<HandleFunnessPacket> },
{ MessageType::World::UI_HELP_TOP_5, Create<UIHelpTop5Packet> },
{ MessageType::World::REQUEST_UGC_MANIFEST_INFO, Create<RequestUgcManifestInfoPacket> },
};
}

View File

@@ -21,9 +21,34 @@ Resource types are 0 LXFML, 1 NIF, 2 HKX, 3 DDS.
* the same name with `.checksum` instead of `.gz`: `<Checksum><MD5>32 hex digits</MD5><Filesize>n</Filesize></Checksum>`,
the MD5 and size of the uncompressed file. The client checks the download against it.
* HTTP 408 makes the client try again later; other errors are logged as failed downloads.
* Without 3D services the client asks the world server for each file's MD5 (`REQUEST_UGC_MANIFEST_INFO`, answered
with `UGC_MANIFEST_RESPONSE`) and downloads `BrickModels/UserMade/<id % 1000, 3 digits>/<id, 20 digits><ext>.sd0`.
That needs the world to answer the manifest requests, which it doesn't yet, so it is not supported.
* `UGCUSE3DSERVICES=7:0` (the client's default, without 3D services). The client asks its world for each file's MD5
and size (`REQUEST_UGC_MANIFEST_INFO`, answered with `UGC_MANIFEST_RESPONSE`, below) and downloads
`UGCSERVERDIR/BrickModels/UserMade/<id % 1000, 3 digits>/<id, 20 digits><.lxfml|.nif|.hkx|.dds>.sd0`: the file as
an sd0 stream (the bytes `s` `d` `0` 0x01 0xff, then chunks of a u32 size and zlib data, each inflating to at most
256 KiB). It inflates it, saves it as `res/BrickModels/UserMade/<id % 1000>/<id><ext>` and checks its MD5 against
the world's answer; a file it has already is only downloaded again when its MD5 differs. HTTP 408 makes it try
again later. The worlds answer from checksums the UGC server stores (`ugc_file_checksums`), and the UGC server serves
the `.sd0` files; see "Without 3D services" below for what keeps this off by default.
### The manifest packets (checked in the client)
Both are plain packets (not game messages); offsets are after the 0x53 byte and the header (connection type, message
id, a padding byte).
| Packet | Layout |
| --- | --- |
| `REQUEST_UGC_MANIFEST_INFO` (world message 27), client to world | u64 blueprint id, u8 resource type (16 bytes after the 0x53). Sent by `SendRequestUGCManifestInfoPacket` when the client needs a blueprint's file and 3D services are off. |
| `UGC_MANIFEST_RESPONSE` (client message 60), world to client | u64 blueprint id, u8 resource type, then 21 bytes of manifest info: u8 valid, u32 size of the inflated file, 16 bytes MD5 of it. The client ignores an answer that isn't exactly 37 bytes after the 0x53 (`PacketHandler_MSG_CLIENT_UGC_MANIFEST_RESPONSE`). |
The answer is cached per blueprint and type (the same message the 3D services `.checksum` download produces). With
it, the client uses a file it has when its MD5 matches and downloads it otherwise; with valid 0 it uses a file it has as
it is and downloads it only when it has none. Structs: `WorldPackets::RequestUgcManifestInfo`,
`ClientPackets::UgcManifestResponse` (`eUgcResourceType`).
The blueprint id of an inventory item's icon (`LWOInventoryComponent_Client::LoadBlueprintIcon`) is the item's
`blueprintid` config when it has one (Brick-by-Brick models, LOT 6662), else its subkey (cars and rockets: their
`ugc_modular_build` id, which DLU gives them as subkey when they are built). Cars and rockets from before builds were
stored have subkey 0 and no build row, so the client never asks for their icons.
A model's render component uses the downloaded NIF when the model's spawn data has `renderUserGen=1` with its
`blueprintid`; `NotifyClientUGCModelReady` (game message 909) makes the client fetch the blueprint's NIF and HKX
@@ -50,8 +75,18 @@ DATACENTERID=1:150,
then don't show. So 3D services only work when the UGC server answers on port 80 of that address; it serves
`/lwoclient/UserBrickModels/...` as well as `client_path` for that.
* With `UGCUSE3DSERVICES=7:0` (the client's own default) the client builds the models itself from the LXFML the world
sends, and property models load. It also asks the world for manifests (`REQUEST_UGC_MANIFEST_INFO`), which the world
doesn't answer; that doesn't stop the models from loading. Use this until the UGC server can be reached as above.
sends, and property models load. Opening the backpack's models asks the world for each car's, rocket's and model's
icon manifest. With `ugc_manifest=1` the world answers (checked: the answers are read and the client goes on to
download each icon), and the client then downloads from the same built-in address as with 3D services, whatever its
`boot.cfg` says (`UGCSERVERPORT=1:14108`, `UGCSERVERDIR=0:/ugc`, `PATCHSERVERDIR=0:luclient` in the client tested):
its log shows `Download Info: Host 'http://127.0.0.1/'`, `UGC site Info: Sharing main download connection` and
requests for `lwoclient/UserBrickModels/BrickModels/UserMade/<bucket>/<id>.dds.sd0`. In the client the download
thread reads `PATCHSERVERIP`/`PORT`/`DIR` and `UGCSERVERIP`/`PORT`/`DIR` through the config interface
(`LWOResMgr2Interface::DownloadThread_Run`, defaults 127.0.0.1, 80 and `lwoclient`, the UGC ones defaulting to the
patch ones), but the values in `boot.cfg` don't reach it. A download that can't connect (HTTP status 0) counts as a
UGC connection failure, and the client logs the player out for it ("connection failed downloading UGC assets",
`MainThread_LogoutDueToConnectionFailures`) unless a client mod stops that. So, as with 3D services, the icons only
arrive when the UGC server answers on port 80 of the player's own machine; see "Without 3D services".
* A NIF the UGC server makes, put in place of one of the game's own models and spawned, renders in the client with its
colors: the NIFs are written like the game's `res/BrickModels/ndmade` files (nif.xml's 20.3.0.9, user version 0;
every shape with a material, alpha blending by the vertex alpha, specular off and vertex colors, in that order).
@@ -183,11 +218,20 @@ shared files. `combo.json` in the combination's folder says its LOTs and build t
Files live under `ugc_output_dir` (default `ugc` next to the server binaries):
```
ugc/models/<id % 1000>/<id>/model.nif.gz, model.nif.checksum, icon.dds.gz, icon.dds.checksum, icon.png,
model.noao.nif.gz, stats.json, previous.icon.png, previous.model.nif.gz, previous.stats.json
ugc/modular/<combination id % 1000>/<combination id>/icon.dds.gz, icon.dds.checksum, icon.png, combo.json
ugc/models/<id % 1000>/<id>/model.nif.gz, model.nif.checksum, model.nif.sd0, icon.dds.gz, icon.dds.checksum,
icon.dds.sd0, icon.png, model.noao.nif.gz, stats.json, previous.icon.png,
previous.model.nif.gz, previous.stats.json
ugc/modular/<combination id % 1000>/<combination id>/icon.dds.gz, icon.dds.checksum, icon.dds.sd0, icon.png, combo.json
ugc/.checksums-stored
```
Every file the client downloads is written three ways: `.gz` and `.checksum` for 3D services and `.sd0` (dCommon's
`Sd0::Compress`) without them. When a worker writes an item, the checksums of its `.sd0` files (MD5 and size of the
inflated file, from the `.checksum`) go back to the main thread, which stores them in `ugc_file_checksums` for the
worlds. Items made before this get their `icon.dds.sd0` (from `icon.dds.gz`) and their icon's checksum once, a few per
tick on the main thread when the server starts; `.checksums-stored` marks that done. Their `.nif` gets its `.sd0` when
the model is made again (the client doesn't ask for it without 3D services, see "Not done yet").
A model's files are written to a temporary folder and renamed into place, so a half written model is never served.
Meshes are only stored compressed (a model took about 18 MB when the .nif was kept uncompressed beside its .gz, so the
2 GB cap held about 110 models and the server kept evicting and remaking them); the dashboard's copies are inflated
@@ -228,6 +272,39 @@ Migrations `dlu/mysql/86_ugc_debounce_icon_settings.sql` and `dlu/sqlite/69_ugc_
`ugc.process_after` and the table `ugc_icon_settings` (`target`: `kind:<kind>`, `model:<id>` or `combo:<key>`; `params`
JSON; `updated_at`).
Migrations `dlu/mysql/89_ugc_file_checksums.sql` and `dlu/sqlite/72_ugc_file_checksums.sql`: the table
`ugc_file_checksums` (`kind` 0 a player model, 1 a combination of car or rocket modules; `storage_id` the model's ugc
id or the combination's id; `file` `icon.dds` or `model.nif`; `md5`, 32 lowercase hex digits, and `size` of the inflated
file), written by the UGC server whenever it writes a file, and `ugc_modular_build.combination_id`, the combination a
build shares its files with (0 until the UGC server has seen the build: it fills them in for existing builds when it
starts, -1 when the modules can't be told). `IUgc::GetUgcFileChecksum(blueprint, file)` looks a blueprint up as a model
first, then as a build through its combination.
## Without 3D services (`UGCUSE3DSERVICES=7:0`)
`UgcManifest` (dGame/dUtilities) answers `REQUEST_UGC_MANIFEST_INFO` when `ugc_manifest=1` (`sharedconfig.ini`,
default 0):
* The world's main thread handles the packet (`g_WorldHandlers`), looks the checksum up with one indexed query and sends
`UGC_MANIFEST_RESPONSE` to that client in the same tick. Only DDS (`icon.dds`) and NIF (`model.nif`) are answered;
LXFML and HKX aren't (the client builds those itself, as today).
* A file that isn't made yet isn't answered: the request waits (at most 512, for 15 minutes) and `UgcManifest::Update`
(world tick) looks again every 5 seconds, answering once the UGC server has stored the checksum. A model still in its
quiet period after a save is made right away (`ExpediteUgcModel`), as when a client asks the UGC server directly.
Waiting requests are dropped when the client disconnects.
* No worker threads, HTTP or file reads in the world: the UGC server precomputes everything into the database.
The UGC server serves `BrickModels/UserMade/<bucket>/<id>.<ext>.sd0` under `client_path`, under
`/<any folder>/UserBrickModels` (the client's default `UGCSERVERDIR`, `lwoclient/UserBrickModels` with its built-in
patch folder) and at the root; `.dds` is a model's icon or a car or rocket build's combination icon, `.nif` a model's
mesh, `.lxfml` the model's LXFML from the database, `.hkx` 404.
It is off by default because the 1.10.64 client downloads from `http://127.0.0.1:80/lwoclient/UserBrickModels/`
whatever its `boot.cfg` says (checked in game, see "What the 1.10.64 client does in practice"), and logs the player
out when it can't connect there. Turning it on only helps when the UGC server answers on port 80 of every player's own
machine, which normal players don't have; until a way is found to point the client at the server, the icons don't show
in 7:0.
### Waiting while the owner is still building
A saved model isn't made right away: `InsertNewUgcModel` stores `process_after` = now + `ugc_debounce_seconds`
@@ -255,8 +332,9 @@ No master messages are needed for any of it.
The main thread owns RakNet (the master link), mongoose (HTTP) and the database. `worker_threads` (default: half the
CPUs) workers do only pure work: parse LXFML, build and write meshes and icons, compress files. They get their input
(LXFML text, module data) from the main thread and hand results back through a queue the main thread drains, which
then updates the database. Brick geometry and materials are loaded once and shared read-only (the cache has its own
lock).
then updates the database (including the files' checksums, parsed by the worker from the `.checksum` it wrote).
Brick geometry and materials are loaded once and shared read-only (the cache has its own lock). The start-up backfill
of old items' `.sd0` icons and checksums and of builds' combination ids runs on the main thread, a few items per tick.
## CPU and memory
@@ -297,6 +375,9 @@ files are what every player on a property sees anyway):
the client's `UGCSERVERDIR`), and the same under `/lwoclient/UserBrickModels` (the path the client really uses, see
above). HKX answers 404. The LXFML comes from the database. A model that exists but isn't made yet (or was evicted) is moved to
the front of the queue and answers 408 so the client asks again.
* `<client_path>/BrickModels/UserMade/<id % 1000>/<id, 20 digits>.<lxfml|nif|hkx|dds>.sd0`, the same under
`/<folder>/UserBrickModels` and at the root, for clients without 3D services (see "Without 3D services"); 408 and 404
as above.
* `/files/model/<id>/<file>` and `/files/modular/<id>/<file>` for the dashboard (`icon.png`, `model.nif` and
`model.noao.nif` (inflated), `stats.json`, `combo.json` and `previous.` versions), not cached by browsers.
* `/admin/preview`, `/admin/assembly`, `/admin/regenerate-icons`, `/admin/delete` (POST, JSON) for the dashboard only:
@@ -424,4 +505,10 @@ pages show the item's own icon.
keeps building its own meshes; switching them to the served NIFs (and sending `NotifyClientUGCModelReady` when a
model is made) is the next step, and needs checking in game.
* HKX (physics) is not generated.
* The non-3D-services download path (world manifest packets) is not answered.
* Icons in game: both of the client's modes download from its built-in `http://127.0.0.1:80/lwoclient/...` whatever
its `boot.cfg` says, so neither reaches the UGC server for normal players. Worlds answer the 7:0 manifest requests
(`ugc_manifest=1`) and the UGC server serves the `.sd0` files, but it stays off until the client can be pointed at
the server (why the client ignores its `UGCSERVER*`/`PATCHSERVER*` lines is not found yet).
* Cars and rockets built before builds were stored (subkey 0, no `ugc_modular_build` row) never get an icon: the client
has no blueprint id to ask for.
* A model's `.nif` made before `.sd0` files were written has none until the model is made again.

View File

@@ -0,0 +1,19 @@
/* UGC server: ugc_file_checksums holds the MD5 (lowercase hex) and size of each file it made, as the game client has
it after inflating the download (kind 0: a player model, storage_id its ugc id; kind 1: a combination of car or
rocket modules, storage_id the combination's id). ugc_modular_build.combination_id is the combination a build was
made as (0 until the UGC server has seen it). Worlds read both to answer the client's REQUEST_UGC_MANIFEST_INFO.
See docs/UgcServer.md. */
CREATE TABLE IF NOT EXISTS ugc_file_checksums (
kind INT NOT NULL,
storage_id BIGINT NOT NULL,
file VARCHAR(32) NOT NULL,
md5 CHAR(32) NOT NULL,
size INT UNSIGNED NOT NULL,
PRIMARY KEY (kind, storage_id, file)
);
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN combination_id BIGINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'combination_id');
PREPARE dlu_column_stmt FROM @dlu_column;
EXECUTE dlu_column_stmt;
DEALLOCATE PREPARE dlu_column_stmt;

View File

@@ -0,0 +1,11 @@
/* UGC server: the checksums of the files it made and each car or rocket build's combination. See the MySQL migration. */
CREATE TABLE IF NOT EXISTS ugc_file_checksums (
kind INTEGER NOT NULL,
storage_id BIGINT NOT NULL,
file TEXT NOT NULL,
md5 TEXT NOT NULL,
size INTEGER NOT NULL,
PRIMARY KEY (kind, storage_id, file)
);
ALTER TABLE ugc_modular_build ADD COLUMN combination_id BIGINT NOT NULL DEFAULT 0;

View File

@@ -93,3 +93,9 @@ mute_auto_reject_names=1
# again for all their waiting models), or as soon as a game client asks for it or the owner leaves the world. 0: right
# away. See docs/UgcServer.md.
ugc_debounce_seconds=120
# 0 or 1: worlds answer game clients without 3D services (UGCUSE3DSERVICES=7:0, the default) with the checksums of the
# files the UGC server made, so they download the icons of cars, rockets and models. The 1.10.64 client then downloads
# from http://127.0.0.1:80/lwoclient/UserBrickModels/ whatever its boot.cfg says, and a client that can't connect there
# is logged out, so only set 1 when the UGC server answers there for every player. See docs/UgcServer.md.
ugc_manifest=0

View File

@@ -774,6 +774,21 @@ TEST_F(ParitySeeded, UgcModel) {
out.push_back(db.GetUgcIconSettings("combo:4713-4714-4715").value_or(""));
db.DeleteUgcIconSettings("combo:4713-4714-4715");
out.push_back(db.GetUgcIconSettings("combo:4713-4714-4715").has_value());
// The files' checksums the worlds answer manifest requests with: a build's are its combination's
const auto checksumJson = [](const std::optional<IUgc::FileChecksum>& checksum) {
return checksum ? json{ checksum->md5, checksum->size } : json(nullptr);
};
for (const auto& build : db.GetModularBuildsWithoutCombination(10)) out.push_back(build.id);
out.push_back(checksumJson(db.GetUgcFileChecksum(1152921510000500002LL, "icon.dds")));
db.SetModularBuildCombination(1152921510000500002LL, 77);
db.SetUgcFileChecksum(IUgc::eFileOwner::COMBINATION, 77, "icon.dds", "900150983cd24fb0d6963f7d28e17f72", 3);
db.SetUgcFileChecksum(IUgc::eFileOwner::COMBINATION, 77, "icon.dds", "0123456789abcdef0123456789abcdef", 65664);
out.push_back(db.GetModularBuildsWithoutCombination(10).size());
out.push_back(checksumJson(db.GetUgcFileChecksum(1152921510000500002LL, "icon.dds")));
out.push_back(checksumJson(db.GetUgcFileChecksum(1152921510000500002LL, "model.nif")));
db.SetUgcFileChecksum(IUgc::eFileOwner::MODEL, 1152921510000500009LL, "icon.dds", "11111111111111111111111111111111", 10);
out.push_back(checksumJson(db.GetUgcFileChecksum(1152921510000500009LL, "icon.dds")));
out.push_back(checksumJson(db.GetUgcFileChecksum(77, "icon.dds")));
return out;
});
}

View File

@@ -717,3 +717,35 @@ TEST_F(WorldPacketsTests, SmallRequestsMatchLegacy) {
ASSERT_TRUE(mailRead.Deserialize(mailBytes));
EXPECT_PACKET_EQ(FromBitStream(mail.data), FromBitStream(mailRead.data));
}
// The layouts the 1.10.64 client uses without 3D services (UGCUSE3DSERVICES=7:0): it sends 16 bytes after the 0x53
// (SendRequestUGCManifestInfoPacket) and only reads an answer that is exactly 37 bytes after it
// (PacketHandler_MSG_CLIENT_UGC_MANIFEST_RESPONSE: 21 bytes of manifest info after the blueprint and type).
TEST_F(WorldPacketsTests, UgcManifestPacketsMatchTheClient) {
WorldPackets::RequestUgcManifestInfo request;
request.blueprintId = 0x0102030405060708;
request.resourceType = eUgcResourceType::DDS;
RakNet::BitStream requestBytes;
request.WritePacket(requestBytes);
EXPECT_PACKET_EQ(FromHex("53 04 00 1b 00 00 00 00 08 07 06 05 04 03 02 01 03"), FromBitStream(requestBytes));
EXPECT_EQ(requestBytes.GetNumberOfBytesUsed() - 1, 16);
EXPECT_EQ(RoundTrip(request).blueprintId, request.blueprintId);
ExpectTruncatedFails(request);
ClientPackets::UgcManifestResponse response;
response.blueprintId = 0x0102030405060708;
response.resourceType = eUgcResourceType::DDS;
response.valid = true;
response.fileSize = 65664;
for (size_t i = 0; i < response.md5.size(); i++) response.md5[i] = static_cast<uint8_t>(0xa0 + i);
RakNet::BitStream responseBytes;
response.WritePacket(responseBytes);
EXPECT_PACKET_EQ(FromHex("53 05 00 3c 00 00 00 00 08 07 06 05 04 03 02 01 03 01 80 00 01 00 "
"a0 a1 a2 a3 a4 a5 a6 a7 a8 a9 aa ab ac ad ae af"), FromBitStream(responseBytes));
EXPECT_EQ(responseBytes.GetNumberOfBytesUsed() - 1, 37);
const auto read = RoundTrip(response);
EXPECT_TRUE(read.valid);
EXPECT_EQ(read.fileSize, 65664u);
EXPECT_EQ(read.md5, response.md5);
ExpectTruncatedFails(response);
}

View File

@@ -3,6 +3,7 @@
#include <cstring>
#include <filesystem>
#include <fstream>
#include <sstream>
#include <unistd.h>
#include <glm/gtc/matrix_transform.hpp>
@@ -22,6 +23,7 @@
#include "UgcRender.h"
#include "UgcStorage.h"
#include "UgcThrottle.h"
#include "Sd0.h"
#include "ZCompression.h"
#include "json.hpp"
@@ -269,6 +271,28 @@ TEST(UgcFormats, ImagesAndChecksums) {
EXPECT_EQ(dds[128 + 2], 10); // stored BGRA
EXPECT_EQ(UgcFormats::Md5Hex("abc"), "900150983cd24fb0d6963f7d28e17f72");
EXPECT_NE(UgcFormats::ChecksumXml("abc").find("<Checksum><MD5>900150983cd24fb0d6963f7d28e17f72</MD5><Filesize>3</Filesize></Checksum>"), std::string::npos);
std::string md5;
uint32_t size{};
ASSERT_TRUE(UgcFormats::ReadChecksumXml(UgcFormats::ChecksumXml("abc"), md5, size));
EXPECT_EQ(md5, "900150983cd24fb0d6963f7d28e17f72");
EXPECT_EQ(size, 3u);
EXPECT_FALSE(UgcFormats::ReadChecksumXml("<Checksum><MD5>abc</MD5><Filesize>3</Filesize></Checksum>", md5, size));
EXPECT_FALSE(UgcFormats::ReadChecksumXml("<Checksum><MD5>900150983cd24fb0d6963f7d28e17f72</MD5><Filesize>x</Filesize></Checksum>", md5, size));
}
// A download is written for both of the client's modes: .gz and .checksum (3D services) and .sd0 (without), all
// holding the same file
TEST(UgcJobs, AddsTheDownloadForBothClientModes) {
UgcStorage::Files files;
UgcJobs::AddDownload(files, "icon.dds", "abc");
EXPECT_EQ(ZCompression::Gunzip(files.at("icon.dds.gz")).value_or(""), "abc");
std::istringstream sd0(files.at("icon.dds.sd0"));
EXPECT_EQ(Sd0(sd0).GetAsStringUncompressed(), "abc");
std::string md5;
uint32_t size{};
ASSERT_TRUE(UgcFormats::ReadChecksumXml(files.at("icon.dds.checksum"), md5, size));
EXPECT_EQ(md5, "900150983cd24fb0d6963f7d28e17f72");
EXPECT_EQ(size, 3u);
}
TEST(UgcModular, ParsesTheCdClientData) {