feat: UGC server that makes and serves player models' meshes and icons

A new server (dUgcServer, started by master with enable_ugc_server=1) that
takes unprocessed ugc and ugc_modular_build rows from the database and makes
what the client downloads with UGCUSE3DSERVICES: an optimized NIF (hidden
faces removed, ambient occlusion baked into vertex colors) and a 128px DDS
icon for player models, rendered by a software rasterizer from the client's
LDD brick primitives, and icons for cars and rockets assembled from their
modules per ModularBuildComponent/ModuleComponent. It serves them, with the
models' LXFML, over HTTP in the client's UGCC<dc>/3DOPTIMIZED and
IMAGE128DDS layout with .gz and .checksum files, and keeps its folder under
a size cap.

Processing state lives in ugc.is_optimized plus new processed_at,
process_attempts and process_error columns (and the same on
ugc_modular_build). ServiceType::UGC is appended. NifFile moves to dCommon
and records named node transforms for the modules' attach points.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-27 03:52:22 -05:00
parent 28b57afe26
commit ea8070c1ad
52 changed files with 3869 additions and 14 deletions

View File

@@ -133,7 +133,7 @@ endif()
message(STATUS "Variable: DLU_CONFIG_DIR = ${DLU_CONFIG_DIR}")
# Copy resource files on first build
set(RESOURCE_FILES "sharedconfig.ini" "authconfig.ini" "chatconfig.ini" "worldconfig.ini" "masterconfig.ini" "dashboardconfig.ini" "blocklist.dcf")
set(RESOURCE_FILES "sharedconfig.ini" "authconfig.ini" "chatconfig.ini" "worldconfig.ini" "masterconfig.ini" "dashboardconfig.ini" "ugcconfig.ini" "blocklist.dcf")
message(STATUS "Checking resource file integrity")
include(Utils)
@@ -330,6 +330,7 @@ add_subdirectory(dWorldServer)
add_subdirectory(dAuthServer)
add_subdirectory(dChatServer)
add_subdirectory(dDashboardServer)
add_subdirectory(dUgcServer)
add_subdirectory(dMasterServer) # Add MasterServer last so it can rely on the other binaries
target_precompile_headers(

View File

@@ -31,6 +31,7 @@ set(DCOMMON_SOURCES
"Raw.cpp"
"Process.cpp"
"Permissions.cpp"
"NifFile.cpp"
)
# Workaround for compiler bug where the optimized code could result in a memcpy of 0 bytes, even though that isnt possible.

View File

@@ -383,9 +383,17 @@ namespace {
const auto children = reader.Array<int32_t>(childCount);
const auto effectCount = reader.U32();
reader.Skip(static_cast<uint64_t>(effectCount) * 4);
if (!reader.Ok() || (av.flags & APP_CULLED)) return;
if (!reader.Ok()) return;
const auto world = parent.Then(av.transform);
// Recorded even when hidden: attach points often are
if (!av.name.empty() && !m_Model.nodes.contains(av.name)) {
NifFile::NodeTransform node;
for (int i = 0; i < 9; i++) node.rotation[i] = world.r[i] * world.s;
node.translation = world.t;
m_Model.nodes.emplace(av.name, node);
}
if (av.flags & APP_CULLED) return;
properties = Inherit(properties, av.properties);
if (const auto tag = NifFile::ShaderTag(av.name); tag >= 0) properties.shaderTag = tag;

View File

@@ -72,9 +72,16 @@ namespace NifFile {
// What a texture's alpha does under a shader (mapShaders.gameValue); -1 is fixed function (opacity)
eTextureAlpha TextureAlphaFor(int32_t shader);
// Where a node is in the model's space: p' = rotation * p + translation (row-major, scale folded in)
struct NodeTransform {
std::array<float, 9> rotation{ 1, 0, 0, 0, 1, 0, 0, 0, 1 };
std::array<float, 3> translation{};
};
struct Model {
uint32_t version{};
std::vector<Mesh> meshes;
std::map<std::string, NodeTransform> nodes; // named nodes that are drawn (the first of each name), e.g. attach points
std::array<float, 3> min{};
std::array<float, 3> max{};
std::map<std::string, uint32_t> skipped; // block types in the file that aren't drawn, with how many

View File

@@ -46,5 +46,55 @@ namespace ZCompression {
inflateEnd(&zInfo); // zlib function
return(nRet);
}
}
namespace {
// Inflate everything in `data` with zlib's `windowBits` (31: gzip, -15: raw); nullopt on any error
std::optional<std::string> Inflate(std::string_view data, int windowBits, size_t sizeHint) {
z_stream stream{};
if (inflateInit2(&stream, windowBits) != Z_OK) return std::nullopt;
std::string out;
out.reserve(sizeHint);
stream.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(data.data()));
stream.avail_in = static_cast<uInt>(data.size());
char buffer[65536];
int result = Z_OK;
while (result == Z_OK) {
stream.next_out = reinterpret_cast<Bytef*>(buffer);
stream.avail_out = sizeof(buffer);
result = inflate(&stream, Z_NO_FLUSH);
if (result != Z_OK && result != Z_STREAM_END) break;
out.append(buffer, sizeof(buffer) - stream.avail_out);
if (result == Z_OK && stream.avail_in == 0 && stream.avail_out != 0) break; // truncated
}
inflateEnd(&stream);
if (result != Z_STREAM_END) return std::nullopt;
return out;
}
}
std::string Gzip(std::string_view data) {
z_stream stream{};
// 31: a gzip header and trailer instead of zlib's
if (deflateInit2(&stream, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY) != Z_OK) return {};
std::string out;
out.resize(deflateBound(&stream, static_cast<uLong>(data.size())) + 32);
stream.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(data.data()));
stream.avail_in = static_cast<uInt>(data.size());
stream.next_out = reinterpret_cast<Bytef*>(out.data());
stream.avail_out = static_cast<uInt>(out.size());
const auto result = deflate(&stream, Z_FINISH);
out.resize(stream.total_out);
deflateEnd(&stream);
return result == Z_STREAM_END ? out : std::string{};
}
std::optional<std::string> Gunzip(std::string_view data) {
return Inflate(data, 31, data.size() * 4);
}
std::optional<std::string> InflateRaw(std::string_view data, size_t size) {
auto out = Inflate(data, -15, size);
if (!out || out->size() != size) return std::nullopt;
return out;
}
}

View File

@@ -1,6 +1,9 @@
#pragma once
#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
namespace ZCompression {
uint32_t GetMaxCompressedLength(uint32_t nLenSrc);
@@ -8,5 +11,14 @@ namespace ZCompression {
int32_t Compress(const uint8_t* abSrc, uint32_t nLenSrc, uint8_t* abDst, uint32_t nLenDst);
int32_t Decompress(const uint8_t* abSrc, uint32_t nLenSrc, uint8_t* abDst, uint32_t nLenDst, int32_t& nErr);
// `data` as a gzip file (RFC 1952), what HTTP clients and the game client's UGC downloads inflate
std::string Gzip(std::string_view data);
// A gzip file's contents; nullopt when it isn't one or is damaged
std::optional<std::string> Gunzip(std::string_view data);
// Raw deflate data (no zlib or gzip header, as in zip files) that inflates to `size` bytes; nullopt when it doesn't
std::optional<std::string> InflateRaw(std::string_view data, size_t size);
}

View File

@@ -9,7 +9,8 @@ enum class ServiceType : uint16_t {
WORLD,
CLIENT,
MASTER,
UNKNOWN
UNKNOWN,
UGC // the UGC server (dUgcServer): appended, the values above are on the wire
};
#endif //!__SERVICETYPE__H__

View File

@@ -40,7 +40,6 @@ set(DASHBOARDROUTES_SOURCES
"ChallengeRoutes.cpp"
"InstanceLoad.cpp"
"WorldView.cpp"
"NifFile.cpp"
"Scenery.cpp"
"WorkerPool.cpp"
"Workers.cpp"

View File

@@ -16,6 +16,7 @@ namespace {
const std::string CHAT = "chatconfig.ini";
const std::string WORLD = "worldconfig.ini";
const std::string DASHBOARD = "dashboardconfig.ini";
const std::string UGC = "ugcconfig.ini";
constexpr double PORT_MIN = 1;
constexpr double PORT_MAX = 65535;
@@ -127,6 +128,7 @@ namespace {
c.AddSection("Startup");
c.Add(Bool(MASTER, "prestart_servers", "Start auth, chat and char servers", "Master starts the other servers itself.", true, true));
c.Add(Bool(MASTER, "enable_dashboard", "Start the web dashboard", "", false, true));
c.Add(Bool(MASTER, "enable_ugc_server", "Start the UGC server", "Makes and serves the meshes and icons of what players build (docs/UgcServer.md).", false, true));
c.Add(Bool(SHARED, "skip_account_creation", "Skip the first-account prompt", "For non-interactive setups: master doesn't ask for an account when there are none.", false, true));
c.AddSection("Game client");
@@ -400,6 +402,30 @@ namespace {
c.AddSection("Character history");
c.Add(Days("snapshot_days", "Snapshots", "Older snapshots are deleted, but each character keeps its newest few (below).", "90", 1));
c.Add(Unit(Int(DASHBOARD, "snapshot_keep", "Always keep per character", "", "10", 1, 1000), "snapshots"));
c.AddSection("UGC server", "Makes the meshes and icons of what players build and serves them to the game client. See docs/UgcServer.md.",
eLayout::ROWS, Condition{ MASTER, "enable_ugc_server", { "1" } });
c.Add(Port(UGC, "port", "Download port", "The game client's UGCSERVERPORT.", "2008"));
c.Add(Format(Text(UGC, "listen_ip", "Listen address", "The game client has to reach it.", "0.0.0.0", true), eFormat::HOST));
c.Add(Port(UGC, "net_port", "Master connection port", "UDP; the next port is used too.", "2012"));
c.Add(Text(UGC, "client_path", "Download path", "The game client's UGCSERVERDIR.", "/ugc", true));
c.Add(Format(Text(UGC, "ugc_output_dir", "Files folder", "Relative to the server binaries.", "ugc", true), eFormat::PATH));
c.Add(Unit(Int(UGC, "ugc_max_storage_mb", "Most space for files", "The files used longest ago are deleted past this and made again when asked for. 0: no limit.", "2048", 0, std::nullopt, true), "MB"));
c.Add(Int(UGC, "worker_threads", "Worker threads", "0: half the CPU cores.", "0", 0, 64, true));
c.Add(Unit(Int(UGC, "poll_interval_ms", "Look for new models every", "", "2000", 100, 600000, true), "ms"));
c.Add(Int(UGC, "poll_batch", "Models taken at once", "", "32", 1, 1000, true));
c.Add(Int(UGC, "max_attempts", "Attempts before giving up", "", "3", 1, 100, true));
c.Add(Int(UGC, "brick_lod", "Brick detail", "0 is the most detailed, 2 the least.", "0", 0, 2, true));
c.Add(Bool(UGC, "remove_hidden_faces", "Remove faces nobody can see", "", true, true));
c.Add(Bool(UGC, "bake_ao", "Darken hidden corners", "Ambient occlusion baked into the vertex colors.", true, true));
c.Add(Float(UGC, "ao_strength", "Darkening strength", "0 to 1.", "0.6", 0, 1));
c.Add(Unit(Int(UGC, "optimize_resolution", "Detail of the visibility renders", "", "1024", 64, 4096, true), "pixels"));
c.Add(Unit(Int(UGC, "icon_size", "Icon size", "", "128", 16, 1024, true), "pixels"));
c.Add(Unit(Float(UGC, "icon_yaw", "Icon angle around", "", "35", -360, 360), "degrees"));
c.Add(Unit(Float(UGC, "icon_pitch", "Icon angle above", "", "25", -90, 90), "degrees"));
c.Add(Float(UGC, "icon_margin", "Icon border", "1 fills the icon.", "1.03", 0.1, 10));
c.Add(Unit(Float(UGC, "modular_icon_yaw", "Car and rocket icon angle around", "", "35", -360, 360), "degrees"));
c.Add(Unit(Float(UGC, "modular_icon_pitch", "Car and rocket icon angle above", "", "20", -90, 90), "degrees"));
return c;
}
@@ -456,7 +482,7 @@ namespace SettingsCatalog {
const std::vector<std::pair<std::string, std::string>>& Files() {
static const std::vector<std::pair<std::string, std::string>> files{
{ SHARED, "Shared" }, { WORLD, "World" }, { AUTH, "Auth" }, { CHAT, "Chat" }, { MASTER, "Master" }, { DASHBOARD, "Dashboard" }
{ SHARED, "Shared" }, { WORLD, "World" }, { AUTH, "Auth" }, { CHAT, "Chat" }, { MASTER, "Master" }, { DASHBOARD, "Dashboard" }, { UGC, "UGC" }
};
return files;
}

View File

@@ -6,6 +6,7 @@
#include <optional>
#include <string>
#include <string_view>
#include <vector>
class IUgc {
public:
@@ -31,5 +32,53 @@ public:
virtual void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) = 0;
virtual std::optional<IUgc::Model> GetUgcModel(const LWOOBJID ugcId) = 0;
// ---- Processing by the UGC server (is_optimized, processed_at, process_attempts, process_error) ----
// is_optimized's values
enum class eProcessState : int32_t {
PENDING = 0, // not made yet (or to be made again)
DONE = 1, // the UGC server's files are made
FAILED = 2, // gave up after the allowed attempts
};
// A row's processing state, for the UGC server and the dashboard
struct ProcessInfo {
LWOOBJID id{};
LWOOBJID characterId{};
std::string characterName; // empty when the character is gone
eProcessState state{};
uint32_t attempts{};
int64_t processedAt{}; // Unix seconds of the last attempt, 0 for none
std::string error;
bool bakeAo{};
std::string details; // modular builds: the modules (ldf_config)
};
// A model waiting to be made: its id, stored LXFML (sd0) and the attempts so far
struct PendingModel {
LWOOBJID id{};
std::string lxfml;
uint32_t attempts{};
};
// Up to `limit` pending models, the least tried and then the newest first
virtual std::vector<PendingModel> GetUgcModelsToProcess(const uint32_t limit) = 0;
// Records an attempt: the new state, how many attempts there have been, why it failed (empty when it didn't) and
// whether lighting was baked in; processed_at becomes now
virtual void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) = 0;
virtual std::optional<ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) = 0;
// Sets models back to pending with no attempts: one (`id`), or all of them (`id` nullopt; only the failed ones
// with `failedOnly`). Returns how many rows changed.
virtual uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) = 0;
// A page of models (all, or those in `state`), the newest first
virtual std::vector<ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const uint32_t offset, const uint32_t limit) = 0;
// How many models are in each state
virtual std::vector<std::pair<eProcessState, uint64_t>> GetUgcProcessCounts() = 0;
};
#endif //!__IUGC__H__

View File

@@ -4,11 +4,30 @@
#include <cstdint>
#include <optional>
#include <string>
#include <vector>
#include "IUgc.h"
class IUgcModularBuild {
public:
virtual void InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional<LWOOBJID> characterId) = 0;
virtual void DeleteUgcBuild(const LWOOBJID bigId) = 0;
// ---- Icons made by the UGC server; the same columns and meanings as IUgc's processing ----
// A modular build waiting for its icon: its id, modules (ldf_config) and the attempts so far
struct PendingBuild {
LWOOBJID id{};
std::string modules;
uint32_t attempts{};
};
virtual std::vector<PendingBuild> GetModularBuildsToProcess(const uint32_t limit) = 0;
virtual void SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) = 0;
virtual std::optional<IUgc::ProcessInfo> GetModularBuildProcessInfo(const LWOOBJID id) = 0;
virtual uint64_t ResetModularBuildProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) = 0;
virtual std::vector<IUgc::ProcessInfo> GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const uint32_t offset, const uint32_t limit) = 0;
virtual std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetModularBuildProcessCounts() = 0;
};
#endif //!IUGCMODULARBUILD_H

View File

@@ -393,6 +393,18 @@ public:
nlohmann::json GetCharacterById(const LWOOBJID charId) override;
std::optional<IPropertyContents::Model> GetModel(const LWOOBJID modelID) override;
std::optional<IUgc::Model> GetUgcModel(const LWOOBJID ugcId) override;
std::vector<IUgc::PendingModel> GetUgcModelsToProcess(const uint32_t limit) override;
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override;
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override;
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const uint32_t offset, const uint32_t limit) override;
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetUgcProcessCounts() override;
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsToProcess(const uint32_t limit) override;
void SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) override;
std::optional<IUgc::ProcessInfo> GetModularBuildProcessInfo(const LWOOBJID id) override;
uint64_t ResetModularBuildProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
std::vector<IUgc::ProcessInfo> GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const uint32_t offset, const uint32_t limit) override;
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetModularBuildProcessCounts() override;
std::optional<IProperty::Info> GetPropertyInfo(const LWOOBJID id) override;
std::string GetPropertiesTable(uint32_t start, uint32_t length, const std::string_view search = "", uint32_t orderColumn = 0, bool orderAsc = true, bool pendingOnly = false) override;
uint32_t GetPropertyCount() override;

View File

@@ -1,5 +1,27 @@
#include "MySQLDatabase.h"
#include <chrono>
namespace {
IUgc::ProcessInfo ReadUgcProcessInfo(PreparedStmtResultSet& result, bool modular) {
IUgc::ProcessInfo info;
info.id = result->getInt64("id");
info.characterId = result->getInt64("character_id");
info.characterName = std::string(result->getString("character_name").c_str());
info.state = static_cast<IUgc::eProcessState>(result->getInt("is_optimized"));
info.attempts = static_cast<uint32_t>(result->getInt("process_attempts"));
info.processedAt = result->getInt64("processed_at");
info.error = std::string(result->getString("process_error").c_str());
if (modular) info.details = std::string(result->getString("ldf_config").c_str());
else info.bakeAo = result->getInt("bake_ao") != 0;
return info;
}
int64_t UnixNow() {
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
}
}
IUgc::Model ReadModel(PreparedStmtResultSet& result) {
IUgc::Model model;
@@ -67,7 +89,8 @@ void MySQLDatabase::DeleteUgcModelData(const LWOOBJID& modelId) {
void MySQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) {
const std::istream stream(lxfml.rdbuf());
ExecuteUpdate("UPDATE ugc SET lxfml = ? WHERE id = ?;", &stream, modelId);
// The UGC server makes the model's files again
ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", &stream, modelId);
}
std::optional<IUgc::Model> MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) {
@@ -81,3 +104,66 @@ std::optional<IUgc::Model> MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) {
return toReturn;
}
std::vector<IUgc::PendingModel> MySQLDatabase::GetUgcModelsToProcess(const uint32_t limit) {
auto result = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 ORDER BY process_attempts ASC, id DESC LIMIT ?;", limit);
std::vector<IUgc::PendingModel> models;
while (result->next()) {
auto& model = models.emplace_back();
model.id = result->getInt64("id");
std::unique_ptr<std::istream> blob(result->getBlob("lxfml"));
std::stringstream contents;
contents << blob->rdbuf();
model.lxfml = contents.str();
model.attempts = static_cast<uint32_t>(result->getInt("process_attempts"));
}
return models;
}
void MySQLDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) {
ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ? WHERE id = ?;",
static_cast<int32_t>(state), attempts, error, bakeAo, UnixNow(), id);
}
std::optional<IUgc::ProcessInfo> MySQLDatabase::GetUgcProcessInfo(const LWOOBJID id) {
auto result = ExecuteSelect(
"SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao "
"FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id WHERE u.id = ? LIMIT 1;", id);
if (!result->next()) return std::nullopt;
return ReadUgcProcessInfo(result, false);
}
uint64_t MySQLDatabase::ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) {
if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", *id);
if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;");
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '';");
}
std::vector<IUgc::ProcessInfo> MySQLDatabase::GetUgcProcessList(const std::optional<eProcessState> state, const uint32_t offset, const uint32_t limit) {
const std::string select =
"SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao "
"FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id ";
std::vector<IUgc::ProcessInfo> list;
auto read = [&list](PreparedStmtResultSet& result) {
while (result->next()) {
list.push_back(ReadUgcProcessInfo(result, false));
}
};
if (state) {
auto result = ExecuteSelect(select + "WHERE u.is_optimized = ? ORDER BY u.id DESC LIMIT ? OFFSET ?;", static_cast<int32_t>(*state), limit, offset);
read(result);
} else {
auto result = ExecuteSelect(select + "ORDER BY u.id DESC LIMIT ? OFFSET ?;", limit, offset);
read(result);
}
return list;
}
std::vector<std::pair<IUgc::eProcessState, uint64_t>> MySQLDatabase::GetUgcProcessCounts() {
auto result = ExecuteSelect("SELECT is_optimized, COUNT(*) AS count FROM ugc GROUP BY is_optimized;");
std::vector<std::pair<eProcessState, uint64_t>> counts;
while (result->next()) {
counts.emplace_back(static_cast<eProcessState>(result->getInt("is_optimized")), static_cast<uint64_t>(result->getInt64("count")));
}
return counts;
}

View File

@@ -1,5 +1,27 @@
#include "MySQLDatabase.h"
#include <chrono>
namespace {
IUgc::ProcessInfo ReadModularProcessInfo(PreparedStmtResultSet& result, bool modular) {
IUgc::ProcessInfo info;
info.id = result->getInt64("id");
info.characterId = result->getInt64("character_id");
info.characterName = std::string(result->getString("character_name").c_str());
info.state = static_cast<IUgc::eProcessState>(result->getInt("is_optimized"));
info.attempts = static_cast<uint32_t>(result->getInt("process_attempts"));
info.processedAt = result->getInt64("processed_at");
info.error = std::string(result->getString("process_error").c_str());
if (modular) info.details = std::string(result->getString("ldf_config").c_str());
else info.bakeAo = result->getInt("bake_ao") != 0;
return info;
}
int64_t UnixNow() {
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
}
}
void MySQLDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional<LWOOBJID> characterId) {
ExecuteInsert("INSERT INTO ugc_modular_build (ugc_id, ldf_config, character_id) VALUES (?,?,?)", bigId, modules, characterId);
}
@@ -7,3 +29,61 @@ void MySQLDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID bi
void MySQLDatabase::DeleteUgcBuild(const LWOOBJID bigId) {
ExecuteDelete("DELETE FROM ugc_modular_build WHERE ugc_id = ?;", bigId);
}
std::vector<IUgcModularBuild::PendingBuild> MySQLDatabase::GetModularBuildsToProcess(const uint32_t limit) {
auto result = ExecuteSelect("SELECT ugc_id, ldf_config, process_attempts FROM ugc_modular_build WHERE is_optimized = 0 ORDER BY process_attempts ASC, ugc_id DESC LIMIT ?;", limit);
std::vector<PendingBuild> builds;
while (result->next()) {
builds.push_back({ result->getInt64("ugc_id"), std::string(result->getString("ldf_config").c_str()), static_cast<uint32_t>(result->getInt("process_attempts")) });
}
return builds;
}
void MySQLDatabase::SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) {
ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = ?, process_attempts = ?, process_error = ?, processed_at = ? WHERE ugc_id = ?;",
static_cast<int32_t>(state), attempts, error, UnixNow(), id);
}
namespace {
const std::string MODULAR_SELECT =
"SELECT b.ugc_id AS id, b.character_id, c.name AS character_name, b.is_optimized, b.process_attempts, b.processed_at, b.process_error, b.ldf_config "
"FROM ugc_modular_build AS b LEFT JOIN charinfo AS c ON c.id = b.character_id ";
}
std::optional<IUgc::ProcessInfo> MySQLDatabase::GetModularBuildProcessInfo(const LWOOBJID id) {
auto result = ExecuteSelect(MODULAR_SELECT + "WHERE b.ugc_id = ? LIMIT 1;", id);
if (!result->next()) return std::nullopt;
return ReadModularProcessInfo(result, true);
}
uint64_t MySQLDatabase::ResetModularBuildProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) {
if (id) return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE ugc_id = ?;", *id);
if (failedOnly) return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;");
return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '';");
}
std::vector<IUgc::ProcessInfo> MySQLDatabase::GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const uint32_t offset, const uint32_t limit) {
std::vector<IUgc::ProcessInfo> list;
auto read = [&list](PreparedStmtResultSet& result) {
while (result->next()) {
list.push_back(ReadModularProcessInfo(result, true));
}
};
if (state) {
auto result = ExecuteSelect(MODULAR_SELECT + "WHERE b.is_optimized = ? ORDER BY b.ugc_id DESC LIMIT ? OFFSET ?;", static_cast<int32_t>(*state), limit, offset);
read(result);
} else {
auto result = ExecuteSelect(MODULAR_SELECT + "ORDER BY b.ugc_id DESC LIMIT ? OFFSET ?;", limit, offset);
read(result);
}
return list;
}
std::vector<std::pair<IUgc::eProcessState, uint64_t>> MySQLDatabase::GetModularBuildProcessCounts() {
auto result = ExecuteSelect("SELECT is_optimized, COUNT(*) AS count FROM ugc_modular_build GROUP BY is_optimized;");
std::vector<std::pair<IUgc::eProcessState, uint64_t>> counts;
while (result->next()) {
counts.emplace_back(static_cast<IUgc::eProcessState>(result->getInt("is_optimized")), static_cast<uint64_t>(result->getInt64("count")));
}
return counts;
}

View File

@@ -377,6 +377,18 @@ public:
nlohmann::json GetCharacterById(const LWOOBJID charId) override;
std::optional<IPropertyContents::Model> GetModel(const LWOOBJID modelID) override;
std::optional<IUgc::Model> GetUgcModel(const LWOOBJID ugcId) override;
std::vector<IUgc::PendingModel> GetUgcModelsToProcess(const uint32_t limit) override;
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override;
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override;
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const uint32_t offset, const uint32_t limit) override;
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetUgcProcessCounts() override;
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsToProcess(const uint32_t limit) override;
void SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) override;
std::optional<IUgc::ProcessInfo> GetModularBuildProcessInfo(const LWOOBJID id) override;
uint64_t ResetModularBuildProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
std::vector<IUgc::ProcessInfo> GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const uint32_t offset, const uint32_t limit) override;
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetModularBuildProcessCounts() override;
std::optional<IProperty::Info> GetPropertyInfo(const LWOOBJID id) override;
std::string GetPropertiesTable(uint32_t start, uint32_t length, const std::string_view search = "", uint32_t orderColumn = 0, bool orderAsc = true, bool pendingOnly = false) override;
uint32_t GetPropertyCount() override;

View File

@@ -1,5 +1,27 @@
#include "SQLiteDatabase.h"
#include <chrono>
namespace {
IUgc::ProcessInfo ReadUgcProcessInfo(CppSQLite3Query& result, bool modular) {
IUgc::ProcessInfo info;
info.id = result.getInt64Field("id");
info.characterId = result.getInt64Field("character_id");
info.characterName = result.getStringField("character_name", "");
info.state = static_cast<IUgc::eProcessState>(result.getIntField("is_optimized"));
info.attempts = static_cast<uint32_t>(result.getIntField("process_attempts"));
info.processedAt = result.getInt64Field("processed_at");
info.error = result.getStringField("process_error", "");
if (modular) info.details = result.getStringField("ldf_config", "");
else info.bakeAo = result.getIntField("bake_ao") != 0;
return info;
}
int64_t UnixNow() {
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
}
}
IUgc::Model ReadModel(CppSQLite3Query& result) {
IUgc::Model model;
@@ -68,7 +90,8 @@ void SQLiteDatabase::DeleteUgcModelData(const LWOOBJID& modelId) {
void SQLiteDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) {
const std::istream stream(lxfml.rdbuf());
ExecuteUpdate("UPDATE ugc SET lxfml = ? WHERE id = ?;", &stream, modelId);
// The UGC server makes the model's files again
ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", &stream, modelId);
}
std::optional<IUgc::Model> SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) {
@@ -82,3 +105,68 @@ std::optional<IUgc::Model> SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) {
return toReturn;
}
std::vector<IUgc::PendingModel> SQLiteDatabase::GetUgcModelsToProcess(const uint32_t limit) {
auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 ORDER BY process_attempts ASC, id DESC LIMIT ?;", limit);
std::vector<IUgc::PendingModel> models;
while (!result.eof()) {
auto& model = models.emplace_back();
model.id = result.getInt64Field("id");
int blobSize{};
const auto* blob = result.getBlobField("lxfml", blobSize);
model.lxfml.assign(reinterpret_cast<const char*>(blob), blobSize);
model.attempts = static_cast<uint32_t>(result.getIntField("process_attempts"));
result.nextRow();
}
return models;
}
void SQLiteDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) {
ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ? WHERE id = ?;",
static_cast<int32_t>(state), attempts, error, bakeAo, UnixNow(), id);
}
std::optional<IUgc::ProcessInfo> SQLiteDatabase::GetUgcProcessInfo(const LWOOBJID id) {
auto [_, result] = ExecuteSelect(
"SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao "
"FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id WHERE u.id = ? LIMIT 1;", id);
if (result.eof()) return std::nullopt;
return ReadUgcProcessInfo(result, false);
}
uint64_t SQLiteDatabase::ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) {
if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", *id);
if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;");
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '';");
}
std::vector<IUgc::ProcessInfo> SQLiteDatabase::GetUgcProcessList(const std::optional<eProcessState> state, const uint32_t offset, const uint32_t limit) {
const std::string select =
"SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao "
"FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id ";
std::vector<IUgc::ProcessInfo> list;
auto read = [&list](CppSQLite3Query& result) {
while (!result.eof()) {
list.push_back(ReadUgcProcessInfo(result, false));
result.nextRow();
}
};
if (state) {
auto [_, result] = ExecuteSelect(select + "WHERE u.is_optimized = ? ORDER BY u.id DESC LIMIT ? OFFSET ?;", static_cast<int32_t>(*state), limit, offset);
read(result);
} else {
auto [_, result] = ExecuteSelect(select + "ORDER BY u.id DESC LIMIT ? OFFSET ?;", limit, offset);
read(result);
}
return list;
}
std::vector<std::pair<IUgc::eProcessState, uint64_t>> SQLiteDatabase::GetUgcProcessCounts() {
auto [_, result] = ExecuteSelect("SELECT is_optimized, COUNT(*) AS count FROM ugc GROUP BY is_optimized;");
std::vector<std::pair<eProcessState, uint64_t>> counts;
while (!result.eof()) {
counts.emplace_back(static_cast<eProcessState>(result.getIntField("is_optimized")), static_cast<uint64_t>(result.getInt64Field("count")));
result.nextRow();
}
return counts;
}

View File

@@ -1,5 +1,27 @@
#include "SQLiteDatabase.h"
#include <chrono>
namespace {
IUgc::ProcessInfo ReadModularProcessInfo(CppSQLite3Query& result, bool modular) {
IUgc::ProcessInfo info;
info.id = result.getInt64Field("id");
info.characterId = result.getInt64Field("character_id");
info.characterName = result.getStringField("character_name", "");
info.state = static_cast<IUgc::eProcessState>(result.getIntField("is_optimized"));
info.attempts = static_cast<uint32_t>(result.getIntField("process_attempts"));
info.processedAt = result.getInt64Field("processed_at");
info.error = result.getStringField("process_error", "");
if (modular) info.details = result.getStringField("ldf_config", "");
else info.bakeAo = result.getIntField("bake_ao") != 0;
return info;
}
int64_t UnixNow() {
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
}
}
void SQLiteDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional<LWOOBJID> characterId) {
ExecuteInsert("INSERT INTO ugc_modular_build (ugc_id, ldf_config, character_id) VALUES (?,?,?)", bigId, modules, characterId);
}
@@ -7,3 +29,64 @@ void SQLiteDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID b
void SQLiteDatabase::DeleteUgcBuild(const LWOOBJID bigId) {
ExecuteDelete("DELETE FROM ugc_modular_build WHERE ugc_id = ?;", bigId);
}
std::vector<IUgcModularBuild::PendingBuild> SQLiteDatabase::GetModularBuildsToProcess(const uint32_t limit) {
auto [_, result] = ExecuteSelect("SELECT ugc_id, ldf_config, process_attempts FROM ugc_modular_build WHERE is_optimized = 0 ORDER BY process_attempts ASC, ugc_id DESC LIMIT ?;", limit);
std::vector<PendingBuild> builds;
while (!result.eof()) {
builds.push_back({ result.getInt64Field("ugc_id"), result.getStringField("ldf_config", ""), static_cast<uint32_t>(result.getIntField("process_attempts")) });
result.nextRow();
}
return builds;
}
void SQLiteDatabase::SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) {
ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = ?, process_attempts = ?, process_error = ?, processed_at = ? WHERE ugc_id = ?;",
static_cast<int32_t>(state), attempts, error, UnixNow(), id);
}
namespace {
const std::string MODULAR_SELECT =
"SELECT b.ugc_id AS id, b.character_id, c.name AS character_name, b.is_optimized, b.process_attempts, b.processed_at, b.process_error, b.ldf_config "
"FROM ugc_modular_build AS b LEFT JOIN charinfo AS c ON c.id = b.character_id ";
}
std::optional<IUgc::ProcessInfo> SQLiteDatabase::GetModularBuildProcessInfo(const LWOOBJID id) {
auto [_, result] = ExecuteSelect(MODULAR_SELECT + "WHERE b.ugc_id = ? LIMIT 1;", id);
if (result.eof()) return std::nullopt;
return ReadModularProcessInfo(result, true);
}
uint64_t SQLiteDatabase::ResetModularBuildProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) {
if (id) return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE ugc_id = ?;", *id);
if (failedOnly) return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;");
return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '';");
}
std::vector<IUgc::ProcessInfo> SQLiteDatabase::GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const uint32_t offset, const uint32_t limit) {
std::vector<IUgc::ProcessInfo> list;
auto read = [&list](CppSQLite3Query& result) {
while (!result.eof()) {
list.push_back(ReadModularProcessInfo(result, true));
result.nextRow();
}
};
if (state) {
auto [_, result] = ExecuteSelect(MODULAR_SELECT + "WHERE b.is_optimized = ? ORDER BY b.ugc_id DESC LIMIT ? OFFSET ?;", static_cast<int32_t>(*state), limit, offset);
read(result);
} else {
auto [_, result] = ExecuteSelect(MODULAR_SELECT + "ORDER BY b.ugc_id DESC LIMIT ? OFFSET ?;", limit, offset);
read(result);
}
return list;
}
std::vector<std::pair<IUgc::eProcessState, uint64_t>> SQLiteDatabase::GetModularBuildProcessCounts() {
auto [_, result] = ExecuteSelect("SELECT is_optimized, COUNT(*) AS count FROM ugc_modular_build GROUP BY is_optimized;");
std::vector<std::pair<IUgc::eProcessState, uint64_t>> counts;
while (!result.eof()) {
counts.emplace_back(static_cast<IUgc::eProcessState>(result.getIntField("is_optimized")), static_cast<uint64_t>(result.getInt64Field("count")));
result.nextRow();
}
return counts;
}

View File

@@ -372,6 +372,18 @@ class TestSQLDatabase : public GameDatabase {
std::optional<IPropertyContents::Model> GetModel(const LWOOBJID modelID) override { return {}; }
std::optional<IProperty::Info> GetPropertyInfo(const LWOOBJID id) override { return {}; }
std::optional<IUgc::Model> GetUgcModel(const LWOOBJID ugcId) override { return {}; }
std::vector<IUgc::PendingModel> GetUgcModelsToProcess(const uint32_t limit) override { return {}; }
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override {}
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override { return {}; }
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override { return 0; }
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const uint32_t offset, const uint32_t limit) override { return {}; }
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetUgcProcessCounts() override { return {}; }
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsToProcess(const uint32_t limit) override { return {}; }
void SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) override {}
std::optional<IUgc::ProcessInfo> GetModularBuildProcessInfo(const LWOOBJID id) override { return {}; }
uint64_t ResetModularBuildProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override { return 0; }
std::vector<IUgc::ProcessInfo> GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const uint32_t offset, const uint32_t limit) override { return {}; }
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetModularBuildProcessCounts() override { return {}; }
void DeleteAccount(const uint32_t accountId) override {};
void InsertAuditLog(uint32_t accountId, const std::string_view accountName, const std::string_view action, const std::string_view description, uint32_t targetAccountId, LWOOBJID targetCharacterId) override {};
std::string GetAuditLogTable(uint32_t start, uint32_t length, const std::string_view search = "", uint32_t orderColumn = 0, bool orderAsc = true) override { return "{}"; };

View File

@@ -82,6 +82,7 @@ std::map<uint32_t, std::string> activeSessions;
SystemAddress authServerMasterPeerSysAddr;
SystemAddress chatServerMasterPeerSysAddr;
SystemAddress dashboardServerMasterPeerSysAddr;
SystemAddress ugcServerMasterPeerSysAddr;
namespace {
// Dashboard player actions waiting for world servers to answer
@@ -444,6 +445,11 @@ int main(int argc, char** argv) {
StartDashboardServer();
}
// The UGC server makes and serves player models' meshes and icons (docs/UgcServer.md)
if (Game::config->GetValue("enable_ugc_server") == "1") {
StartUgcServer();
}
auto t = std::chrono::high_resolution_clock::now();
Packet* packet = nullptr;
constexpr uint32_t logFlushTime = 15 * masterFramerate;
@@ -624,6 +630,9 @@ namespace {
case ServiceType::DASHBOARD:
dashboardServerMasterPeerSysAddr = sysAddr;
break;
case ServiceType::UGC:
ugcServerMasterPeerSysAddr = sysAddr;
break;
default:
break;
}
@@ -1025,6 +1034,19 @@ void HandlePacket(Packet* packet) {
dashboardServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
StartDashboardServer();
}
if (packet->systemAddress == ugcServerMasterPeerSysAddr) {
ugcServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
MasterPackets::ServerInfo offline;
offline.serverType = ServiceType::UGC;
offline.ip = LUString("offline");
MasterPackets::SendTo(dashboardServerMasterPeerSysAddr, offline);
}
StartUgcServer();
}
}
if (packet->length < 4) return;

View File

@@ -143,6 +143,42 @@ uint32_t StartDashboardServer() {
return web_pid;
}
uint32_t StartUgcServer() {
if (Game::ShouldShutdown()) {
LOG("Currently shutting down. UgcServer will not be restarted.");
return 0;
}
auto ugc_path = BinaryPathFinder::GetBinaryDir() / "UgcServer";
#ifdef _WIN32
ugc_path.replace_extension(".exe");
auto ugc_startup = startup;
auto ugc_info = PROCESS_INFORMATION{};
if (!CreateProcessW(ugc_path.wstring().data(), ugc_path.wstring().data(),
nullptr, nullptr, false, 0, nullptr, nullptr,
&ugc_startup, &ugc_info))
{
LOG("Failed to launch UgcServer");
return 0;
}
// get pid and close unused handles
auto ugc_pid = ugc_info.dwProcessId;
CloseHandle(ugc_info.hProcess);
CloseHandle(ugc_info.hThread);
#else // *nix systems
const auto ugc_pid = fork();
if (ugc_pid < 0) {
LOG("Failed to launch UgcServer");
return 0;
} else if (ugc_pid == 0) {
// We are the child process
execl(ugc_path.string().c_str(), ugc_path.string().c_str(), nullptr);
}
#endif
LOG("UgcServer PID is %d", ugc_pid);
return ugc_pid;
}
uint32_t StartWorldServer(LWOMAPID mapID, uint16_t port, LWOINSTANCEID lastInstanceID, int maxPlayers, LWOCLONEID cloneID) {
auto world_path = BinaryPathFinder::GetBinaryDir() / "WorldServer";
#ifdef _WIN32

View File

@@ -4,4 +4,5 @@
uint32_t StartAuthServer();
uint32_t StartChatServer();
uint32_t StartDashboardServer();
uint32_t StartUgcServer();
uint32_t StartWorldServer(LWOMAPID mapID, uint16_t port, LWOINSTANCEID lastInstanceID, int maxPlayers, LWOCLONEID cloneID);

40
dUgcServer/CMakeLists.txt Normal file
View File

@@ -0,0 +1,40 @@
# The pure part of the UGC server (brick data, meshes, rendering, file formats, storage), shared with the tests
set(DUGC_SOURCES
"UgcBricks.cpp"
"UgcFormats.cpp"
"UgcJobs.cpp"
"UgcModel.cpp"
"UgcModular.cpp"
"UgcRender.cpp"
"UgcStorage.cpp"
)
add_library(dUgc STATIC ${DUGC_SOURCES})
target_include_directories(dUgc PUBLIC "." "${PROJECT_SOURCE_DIR}/thirdparty/MD5")
target_link_libraries(dUgc PUBLIC dCommon glm::glm tinyxml2 MD5)
add_executable(UgcServer "UgcServer.cpp" "UgcProcessor.cpp" "UgcCdClient.cpp")
target_include_directories(UgcServer PRIVATE
"${PROJECT_SOURCE_DIR}/dCommon"
"${PROJECT_SOURCE_DIR}/dCommon/dClient"
"${PROJECT_SOURCE_DIR}/dCommon/dEnums"
"${PROJECT_SOURCE_DIR}/dDatabase"
"${PROJECT_SOURCE_DIR}/dDatabase/CDClientDatabase"
"${PROJECT_SOURCE_DIR}/dDatabase/CDClientDatabase/CDClientTables"
"${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase"
"${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase/ITables"
"${PROJECT_SOURCE_DIR}/dNet"
"${PROJECT_SOURCE_DIR}/dWeb"
"${PROJECT_SOURCE_DIR}/dServer"
"${PROJECT_SOURCE_DIR}/thirdparty"
"${PROJECT_SOURCE_DIR}/thirdparty/nlohmann"
)
target_compile_definitions(UgcServer PRIVATE PROJECT_VERSION="\"${PROJECT_VERSION}\"")
target_link_libraries(UgcServer ${COMMON_LIBRARIES} dWeb dServer dUgc)
# Rendering is far too slow unoptimized (minutes a model instead of seconds), so it is optimized in every build type
if(NOT MSVC)
set_source_files_properties("UgcRender.cpp" "UgcModel.cpp" PROPERTIES COMPILE_OPTIONS "-O2")
endif()

208
dUgcServer/UgcBricks.cpp Normal file
View File

@@ -0,0 +1,208 @@
#include "UgcBricks.h"
#include <algorithm>
#include <cctype>
#include <cstring>
#include <fstream>
#include <sstream>
#include "tinyxml2.h"
#include "ZCompression.h"
namespace {
constexpr int32_t GEOMETRY_MAGIC = 0x42473031; // "10GB"
constexpr uint32_t MAX_GEOMETRY_PARTS = 64;
template<typename T>
bool ReadAt(std::string_view data, size_t offset, T& value) {
if (offset > data.size() || sizeof(T) > data.size() - offset) return false;
std::memcpy(&value, data.data() + offset, sizeof(T));
return true;
}
std::string Lower(std::string_view text) {
std::string out(text);
std::transform(out.begin(), out.end(), out.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
return out;
}
}
namespace UgcBricks {
std::optional<Geometry> ParseGeometry(std::string_view data) {
int32_t magic{}, vertexCount{}, indexCount{}, options{};
if (!ReadAt(data, 0, magic) || magic != GEOMETRY_MAGIC || !ReadAt(data, 4, vertexCount) || !ReadAt(data, 8, indexCount) ||
!ReadAt(data, 12, options) || vertexCount < 0 || indexCount < 0 || indexCount % 3 != 0) {
return std::nullopt;
}
const auto vertices = static_cast<size_t>(vertexCount);
const auto indices = static_cast<size_t>(indexCount);
size_t offset = 16;
const bool hasUvs = (options & 3) == 3;
const size_t needed = offset + vertices * 24 + (hasUvs ? vertices * 8 : 0) + indices * 4;
if (needed > data.size()) return std::nullopt;
Geometry geometry;
geometry.positions.resize(vertices * 3);
std::memcpy(geometry.positions.data(), data.data() + offset, vertices * 12);
offset += vertices * 12;
geometry.normals.resize(vertices * 3);
std::memcpy(geometry.normals.data(), data.data() + offset, vertices * 12);
offset += vertices * 12;
if (hasUvs) offset += vertices * 8;
geometry.indices.resize(indices);
std::memcpy(geometry.indices.data(), data.data() + offset, indices * 4);
for (const auto index : geometry.indices) {
if (index >= vertices) return std::nullopt;
}
return geometry;
}
std::map<uint32_t, Material> ParseMaterials(std::string_view xml) {
std::map<uint32_t, Material> materials;
tinyxml2::XMLDocument doc;
if (doc.Parse(xml.data(), xml.size()) != tinyxml2::XML_SUCCESS) return materials;
const auto* root = doc.FirstChildElement("Materials");
if (!root) return materials;
for (const auto* element = root->FirstChildElement("Material"); element; element = element->NextSiblingElement("Material")) {
const auto id = element->UnsignedAttribute("MatID", 0);
if (id == 0) continue;
const auto channel = [element](const char* name, uint32_t fallback) {
return static_cast<uint8_t>(std::min<uint32_t>(element->UnsignedAttribute(name, fallback), 255));
};
materials[id] = Material{ channel("Red", 160), channel("Green", 160), channel("Blue", 160), channel("Alpha", 255) };
}
return materials;
}
std::optional<std::string> ReadZipEntry(std::string_view zip, std::string_view name) {
// The end of central directory record is in the last 64 KiB + 22 bytes
constexpr uint32_t END_SIGNATURE = 0x06054b50;
constexpr uint32_t CENTRAL_SIGNATURE = 0x02014b50;
constexpr uint32_t LOCAL_SIGNATURE = 0x04034b50;
if (zip.size() < 22) return std::nullopt;
size_t end = std::string_view::npos;
const size_t lowest = zip.size() > 22 + 65535 ? zip.size() - 22 - 65535 : 0;
for (size_t i = zip.size() - 22 + 1; i-- > lowest;) {
uint32_t signature{};
if (ReadAt(zip, i, signature) && signature == END_SIGNATURE) {
end = i;
break;
}
}
if (end == std::string_view::npos) return std::nullopt;
uint16_t entries{};
uint32_t directoryOffset{};
if (!ReadAt(zip, end + 10, entries) || !ReadAt(zip, end + 16, directoryOffset)) return std::nullopt;
const auto wanted = Lower(name);
size_t offset = directoryOffset;
for (uint16_t i = 0; i < entries; i++) {
uint32_t signature{}, compressedSize{}, size{}, localOffset{};
uint16_t method{}, nameLength{}, extraLength{}, commentLength{};
if (!ReadAt(zip, offset, signature) || signature != CENTRAL_SIGNATURE || !ReadAt(zip, offset + 10, method) ||
!ReadAt(zip, offset + 20, compressedSize) || !ReadAt(zip, offset + 24, size) || !ReadAt(zip, offset + 28, nameLength) ||
!ReadAt(zip, offset + 30, extraLength) || !ReadAt(zip, offset + 32, commentLength) || !ReadAt(zip, offset + 42, localOffset) ||
offset + 46 + nameLength > zip.size()) {
return std::nullopt;
}
const auto entryName = zip.substr(offset + 46, nameLength);
offset += 46 + nameLength + extraLength + commentLength;
if (Lower(entryName) != wanted) continue;
uint16_t localNameLength{}, localExtraLength{};
if (!ReadAt(zip, localOffset, signature) || signature != LOCAL_SIGNATURE || !ReadAt(zip, localOffset + 26, localNameLength) ||
!ReadAt(zip, localOffset + 28, localExtraLength)) {
return std::nullopt;
}
const size_t dataStart = static_cast<size_t>(localOffset) + 30 + localNameLength + localExtraLength;
if (dataStart > zip.size() || compressedSize > zip.size() - dataStart) return std::nullopt;
const auto data = zip.substr(dataStart, compressedSize);
if (method == 0) return std::string(data);
if (method == 8) return ZCompression::InflateRaw(data, size);
return std::nullopt;
}
return std::nullopt;
}
std::optional<std::filesystem::path> ResolvePath(const std::filesystem::path& root, std::string_view relative) {
std::string normalized(relative);
std::replace(normalized.begin(), normalized.end(), '\\', '/');
std::filesystem::path current = root;
std::stringstream parts(normalized);
std::string part;
std::error_code error;
while (std::getline(parts, part, '/')) {
if (part.empty() || part == ".") continue;
if (part == "..") return std::nullopt;
if (std::filesystem::exists(current / part, error)) {
current /= part;
continue;
}
const auto wanted = Lower(part);
bool found = false;
for (std::filesystem::directory_iterator it(current, error), endIt; !error && it != endIt; it.increment(error)) {
if (Lower(it->path().filename().string()) == wanted) {
current = it->path();
found = true;
break;
}
}
if (!found) return std::nullopt;
}
if (!std::filesystem::is_regular_file(current, error)) return std::nullopt;
return current;
}
std::optional<std::string> ReadFile(const std::filesystem::path& path) {
std::ifstream file(path, std::ios::binary);
if (!file) return std::nullopt;
std::ostringstream contents;
contents << file.rdbuf();
return contents.str();
}
BrickLibrary::BrickLibrary(std::filesystem::path res, uint32_t lod) : m_Res(std::move(res)), m_Lod(std::min<uint32_t>(lod, 2)) {}
bool BrickLibrary::LoadMaterials() {
const auto zipPath = ResolvePath(m_Res, "brickdb.zip");
const auto zip = zipPath ? ReadFile(*zipPath) : std::nullopt;
const auto xml = zip ? ReadZipEntry(*zip, "Materials.xml") : std::nullopt;
if (!xml) return false;
auto materials = ParseMaterials(*xml);
if (materials.empty()) return false;
SetMaterials(std::move(materials));
return true;
}
void BrickLibrary::SetMaterials(std::map<uint32_t, Material> materials) {
std::lock_guard lock(m_Mutex);
m_Materials = std::move(materials);
}
Material BrickLibrary::GetMaterial(uint32_t id) const {
// Written once before the workers start, only read after
const auto it = m_Materials.find(id);
return it != m_Materials.end() ? it->second : Material{};
}
std::shared_ptr<const std::vector<Geometry>> BrickLibrary::GetDesign(uint32_t design) {
{
std::lock_guard lock(m_Mutex);
if (const auto it = m_Designs.find(design); it != m_Designs.end()) return it->second;
}
// Loaded outside the lock; two threads loading the same design at once is harmless
auto parts = std::make_shared<std::vector<Geometry>>();
const auto folder = "brickprimitives/lod" + std::to_string(m_Lod) + "/";
for (uint32_t index = 0; index < MAX_GEOMETRY_PARTS; index++) {
const auto name = std::to_string(design) + ".g" + (index == 0 ? "" : std::to_string(index));
const auto path = ResolvePath(m_Res, folder + name);
const auto data = path ? ReadFile(*path) : std::nullopt;
if (!data) break;
auto geometry = ParseGeometry(*data);
if (!geometry) break;
parts->push_back(std::move(*geometry));
}
std::lock_guard lock(m_Mutex);
return m_Designs.emplace(design, std::move(parts)).first->second;
}
}

78
dUgcServer/UgcBricks.h Normal file
View File

@@ -0,0 +1,78 @@
#pragma once
#include <array>
#include <cstdint>
#include <filesystem>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
/**
* The client's brick data the UGC server builds models from: LDD geometry (res/brickprimitives/lod<n>/<design>.g, .g1,
* ...) and the material colors (Materials.xml in res/brickdb.zip). The parsers are pure; BrickLibrary loads and caches
* files and may be used from any thread.
*/
namespace UgcBricks {
// One LDD .g file: a triangle mesh with a normal per vertex
struct Geometry {
std::vector<float> positions; // x, y, z per vertex
std::vector<float> normals; // x, y, z per vertex
std::vector<uint32_t> indices;
};
// An LDD material: sRGB color and opacity, 0-255
struct Material {
uint8_t r{ 160 };
uint8_t g{ 160 };
uint8_t b{ 160 };
uint8_t a{ 255 };
bool Transparent() const { return a < 255; }
};
// A .g file ("10GB" magic, counts, positions, normals, texture coordinates for decorated parts, indices)
std::optional<Geometry> ParseGeometry(std::string_view data);
// Materials.xml: MatID -> color
std::map<uint32_t, Material> ParseMaterials(std::string_view xml);
// A file from a zip archive (stored or deflated), matched without regard to case; nullopt when it isn't there
std::optional<std::string> ReadZipEntry(std::string_view zip, std::string_view name);
// `relative` (either slash, any case) under `root`, matching each part without regard to case as the client's
// files are named on Windows; nullopt when there is no such file
std::optional<std::filesystem::path> ResolvePath(const std::filesystem::path& root, std::string_view relative);
// Reads a whole file; nullopt when it can't
std::optional<std::string> ReadFile(const std::filesystem::path& path);
class BrickLibrary {
public:
// `res` is the client's res folder; `lod` the brickprimitives level used (0 is the most detailed)
BrickLibrary(std::filesystem::path res, uint32_t lod);
// Loads the material colors; false when brickdb.zip or its Materials.xml can't be read
bool LoadMaterials();
// For tests and callers that have the colors already
void SetMaterials(std::map<uint32_t, Material> materials);
// A material's color (a grey when the id is unknown)
Material GetMaterial(uint32_t id) const;
// Every geometry file of a design, in order (.g, .g1, ...); empty when the design has none. Loaded once.
std::shared_ptr<const std::vector<Geometry>> GetDesign(uint32_t design);
const std::filesystem::path& GetResPath() const { return m_Res; }
private:
std::filesystem::path m_Res;
uint32_t m_Lod;
std::map<uint32_t, Material> m_Materials;
std::mutex m_Mutex;
std::map<uint32_t, std::shared_ptr<const std::vector<Geometry>>> m_Designs;
};
}

View File

@@ -0,0 +1,58 @@
#include "UgcCdClient.h"
#include <optional>
#include "CDClientDatabase.h"
#include "UgcModular.h"
namespace {
constexpr int32_t MODULE_COMPONENT = 28;
constexpr int32_t RENDER_COMPONENT = 2;
}
namespace UgcCdClient {
bool GatherModular(const std::string& modules, UgcJobs::ModularInput& input, std::string& error) {
const auto lots = UgcModular::ParseModuleLots(modules);
if (lots.empty()) {
error = "no modules in \"" + modules + "\"";
return false;
}
std::optional<int32_t> buildType;
for (const auto lot : lots) {
auto moduleQuery = CDClientDatabase::CreatePreppedStmt(
"SELECT m.partCode, m.buildType, m.xml FROM ComponentsRegistry cr JOIN ModuleComponent m ON m.id = cr.component_id "
"WHERE cr.id = ? AND cr.component_type = ? LIMIT 1;");
moduleQuery.bind(1, static_cast<int>(lot));
moduleQuery.bind(2, MODULE_COMPONENT);
auto moduleRow = moduleQuery.execQuery();
if (moduleRow.eof()) {
error = "LOT " + std::to_string(lot) + " is not a module";
return false;
}
UgcJobs::ModuleInput module;
module.lot = lot;
module.partCode = static_cast<uint32_t>(moduleRow.getIntField("partCode", 0));
module.moduleXml = moduleRow.getStringField("xml", "");
if (!buildType) buildType = moduleRow.getIntField("buildType", 0);
auto renderQuery = CDClientDatabase::CreatePreppedStmt(
"SELECT rc.render_asset FROM ComponentsRegistry cr JOIN RenderComponent rc ON rc.id = cr.component_id "
"WHERE cr.id = ? AND cr.component_type = ? LIMIT 1;");
renderQuery.bind(1, static_cast<int>(lot));
renderQuery.bind(2, RENDER_COMPONENT);
auto renderRow = renderQuery.execQuery();
if (!renderRow.eof()) module.renderAsset = renderRow.getStringField("render_asset", "");
input.modules.push_back(std::move(module));
}
auto buildQuery = CDClientDatabase::CreatePreppedStmt("SELECT xml FROM ModularBuildComponent WHERE buildType = ? ORDER BY id LIMIT 1;");
buildQuery.bind(1, *buildType);
auto buildRow = buildQuery.execQuery();
if (buildRow.eof()) {
error = "no ModularBuildComponent for build type " + std::to_string(*buildType);
return false;
}
input.buildXml = buildRow.getStringField("xml", "");
return true;
}
}

12
dUgcServer/UgcCdClient.h Normal file
View File

@@ -0,0 +1,12 @@
#pragma once
#include <string>
#include "UgcJobs.h"
// What the UGC server reads from the CDClient (main thread only: the CDClient connection isn't shared with workers)
namespace UgcCdClient {
// A modular build's modules (ugc_modular_build.ldf_config) with their render assets and module data, and the
// ModularBuildComponent of their build type; false and `error` when it can't be put together
bool GatherModular(const std::string& modules, UgcJobs::ModularInput& input, std::string& error);
}

311
dUgcServer/UgcFormats.cpp Normal file
View File

@@ -0,0 +1,311 @@
#include "UgcFormats.h"
#include <algorithm>
#include <array>
#include <cstring>
#include <limits>
#include "MD5.h"
#include "ZCompression.h"
namespace {
class Writer {
public:
template<typename T>
void Put(T value) {
char bytes[sizeof(T)];
std::memcpy(bytes, &value, sizeof(T));
m_Data.append(bytes, sizeof(T));
}
void U8(uint8_t value) { Put(value); }
void U16(uint16_t value) { Put(value); }
void U32(uint32_t value) { Put(value); }
void I32(int32_t value) { Put(value); }
void Float(float value) { Put(value); }
void SizedString(const std::string& value) {
U32(static_cast<uint32_t>(value.size()));
m_Data += value;
}
void Raw(std::string_view bytes) { m_Data += bytes; }
std::string& Data() { return m_Data; }
private:
std::string m_Data;
};
void PutBigEndian(std::string& out, uint32_t value) {
for (int shift = 24; shift >= 0; shift -= 8) out += static_cast<char>((value >> shift) & 0xFF);
}
uint32_t Crc32(std::string_view data) {
static const auto table = [] {
std::array<uint32_t, 256> values{};
for (uint32_t i = 0; i < 256; i++) {
uint32_t c = i;
for (int k = 0; k < 8; k++) c = (c & 1) ? 0xEDB88320u ^ (c >> 1) : c >> 1;
values[i] = c;
}
return values;
}();
uint32_t crc = 0xFFFFFFFFu;
for (const auto byte : data) crc = table[(crc ^ static_cast<uint8_t>(byte)) & 0xFF] ^ (crc >> 8);
return crc ^ 0xFFFFFFFFu;
}
void PngChunk(std::string& out, const char* type, std::string_view data) {
PutBigEndian(out, static_cast<uint32_t>(data.size()));
std::string typed(type, 4);
typed += data;
out += typed;
PutBigEndian(out, Crc32(typed));
}
// Gamebryo's block writing: a block per call, types and strings collected into the header's tables
class NifBuilder {
public:
int32_t String(const std::string& value) {
if (value.empty()) return -1;
const auto it = std::find(m_Strings.begin(), m_Strings.end(), value);
if (it != m_Strings.end()) return static_cast<int32_t>(it - m_Strings.begin());
m_Strings.push_back(value);
return static_cast<int32_t>(m_Strings.size() - 1);
}
int32_t Add(const std::string& type, std::string data) {
auto it = std::find(m_Types.begin(), m_Types.end(), type);
if (it == m_Types.end()) it = m_Types.insert(m_Types.end(), type);
m_BlockTypes.push_back(static_cast<uint16_t>(it - m_Types.begin()));
m_Blocks.push_back(std::move(data));
return static_cast<int32_t>(m_Blocks.size() - 1);
}
// Reserves a block to fill in later (a parent that lists children made after it)
int32_t Reserve(const std::string& type) { return Add(type, {}); }
void Fill(int32_t block, std::string data) { m_Blocks[block] = std::move(data); }
std::string Finish(int32_t root) {
Writer out;
out.Raw("Gamebryo File Format, Version 20.3.0.9\n");
out.U32(0x14030009);
out.U8(1); // little endian
out.U32(0); // user version
out.U32(static_cast<uint32_t>(m_Blocks.size()));
out.U16(static_cast<uint16_t>(m_Types.size()));
for (const auto& type : m_Types) out.SizedString(type);
for (const auto type : m_BlockTypes) out.U16(type);
for (const auto& block : m_Blocks) out.U32(static_cast<uint32_t>(block.size()));
out.U32(static_cast<uint32_t>(m_Strings.size()));
size_t longest = 0;
for (const auto& value : m_Strings) longest = std::max(longest, value.size());
out.U32(static_cast<uint32_t>(longest));
for (const auto& value : m_Strings) out.SizedString(value);
out.U32(0); // groups
for (const auto& block : m_Blocks) out.Raw(block);
out.U32(1); // roots
out.I32(root);
return std::move(out.Data());
}
private:
std::vector<std::string> m_Types;
std::vector<uint16_t> m_BlockTypes;
std::vector<std::string> m_Blocks;
std::vector<std::string> m_Strings;
};
constexpr uint16_t AV_FLAGS = 14; // the usual NiAVObject flags (selective update), not hidden
void WriteNet(Writer& out, int32_t name) {
out.I32(name);
out.U32(0); // extra data
out.I32(-1); // controller
}
void WriteAv(Writer& out, int32_t name, const std::vector<int32_t>& properties) {
WriteNet(out, name);
out.U16(AV_FLAGS);
for (int i = 0; i < 3; i++) out.Float(0.0f); // translation
for (int row = 0; row < 3; row++) {
for (int col = 0; col < 3; col++) out.Float(row == col ? 1.0f : 0.0f);
}
out.Float(1.0f); // scale
out.U32(static_cast<uint32_t>(properties.size()));
for (const auto property : properties) out.I32(property);
out.I32(-1); // collision object
}
std::string TriShapeData(const UgcModel::Mesh& mesh) {
Writer out;
const auto count = static_cast<uint16_t>(mesh.positions.size());
out.I32(0); // group ID
out.U16(count);
out.U8(0); // keep flags
out.U8(0); // compress flags
out.U8(1); // has vertices
glm::vec3 min(std::numeric_limits<float>::max()), max(-std::numeric_limits<float>::max());
for (const auto& p : mesh.positions) {
out.Float(p.x);
out.Float(p.y);
out.Float(p.z);
min = glm::min(min, p);
max = glm::max(max, p);
}
out.U16(0); // data flags: no texture coordinates or tangents
const bool normals = mesh.normals.size() == mesh.positions.size();
out.U8(normals ? 1 : 0);
if (normals) {
for (const auto& n : mesh.normals) {
out.Float(n.x);
out.Float(n.y);
out.Float(n.z);
}
}
const glm::vec3 center = mesh.positions.empty() ? glm::vec3(0.0f) : (min + max) * 0.5f;
float radius = 0.0f;
for (const auto& p : mesh.positions) radius = std::max(radius, glm::length(p - center));
out.Float(center.x);
out.Float(center.y);
out.Float(center.z);
out.Float(radius);
const bool colors = mesh.colors.size() == mesh.positions.size();
out.U8(colors ? 1 : 0);
if (colors) {
for (const auto& c : mesh.colors) {
out.Float(std::clamp(c.r, 0.0f, 1.0f));
out.Float(std::clamp(c.g, 0.0f, 1.0f));
out.Float(std::clamp(c.b, 0.0f, 1.0f));
out.Float(std::clamp(c.a, 0.0f, 1.0f));
}
}
out.U16(0x4000); // consistency: static
out.I32(-1); // additional data
const auto triangles = static_cast<uint16_t>(mesh.indices.size() / 3);
out.U16(triangles);
out.U32(static_cast<uint32_t>(triangles) * 3);
out.U8(1); // has triangles
for (size_t i = 0; i < static_cast<size_t>(triangles) * 3; i++) out.U16(static_cast<uint16_t>(mesh.indices[i]));
out.U16(0); // match groups
return std::move(out.Data());
}
}
namespace UgcFormats {
std::string WriteNif(const std::string& rootName, const std::vector<NifShape>& shapes) {
NifBuilder nif;
const int32_t root = nif.Reserve("NiNode");
// Shared properties
Writer material;
WriteNet(material, -1);
for (int i = 0; i < 3; i++) material.Float(1.0f); // ambient
for (int i = 0; i < 3; i++) material.Float(1.0f); // diffuse
for (int i = 0; i < 3; i++) material.Float(0.0f); // specular
for (int i = 0; i < 3; i++) material.Float(0.0f); // emissive
material.Float(10.0f); // glossiness
material.Float(1.0f); // alpha
const auto materialBlock = nif.Add("NiMaterialProperty", std::move(material.Data()));
Writer vertexColor;
WriteNet(vertexColor, -1);
vertexColor.U16((2 << 4) | (1 << 3)); // vertex colors are ambient and diffuse; lit
const auto vertexColorBlock = nif.Add("NiVertexColorProperty", std::move(vertexColor.Data()));
int32_t alphaBlock = -1;
std::vector<int32_t> children;
for (const auto& shape : shapes) {
if (!shape.mesh || shape.mesh->Empty() || shape.mesh->positions.size() > 65535 || shape.mesh->TriangleCount() > 65535) continue;
std::vector<int32_t> properties{ materialBlock, vertexColorBlock };
if (shape.transparent) {
if (alphaBlock < 0) {
Writer alpha;
WriteNet(alpha, -1);
alpha.U16(1 | (6 << 1) | (7 << 5)); // blend source alpha over one minus source alpha
alpha.U8(0);
alphaBlock = nif.Add("NiAlphaProperty", std::move(alpha.Data()));
}
properties.push_back(alphaBlock);
}
const auto shapeBlock = nif.Reserve("NiTriShape");
const auto dataBlock = nif.Add("NiTriShapeData", TriShapeData(*shape.mesh));
Writer tri;
WriteAv(tri, nif.String(shape.name), properties);
tri.I32(dataBlock);
tri.I32(-1); // skin instance
tri.U32(0); // materials
tri.I32(-1); // active material
tri.U8(0); // material needs update
nif.Fill(shapeBlock, std::move(tri.Data()));
children.push_back(shapeBlock);
}
Writer node;
WriteAv(node, nif.String(rootName), {});
node.U32(static_cast<uint32_t>(children.size()));
for (const auto child : children) node.I32(child);
node.U32(0); // effects
nif.Fill(root, std::move(node.Data()));
return nif.Finish(root);
}
std::string EncodePng(const UgcRender::Image& image) {
std::string raw;
raw.reserve(static_cast<size_t>(image.height) * (image.width * 4 + 1));
for (int y = 0; y < image.height; y++) {
raw += '\0'; // no filter
raw.append(reinterpret_cast<const char*>(image.rgba.data()) + static_cast<size_t>(y) * image.width * 4, static_cast<size_t>(image.width) * 4);
}
std::string compressed(ZCompression::GetMaxCompressedLength(static_cast<uint32_t>(raw.size())) + 64, '\0');
const auto size = ZCompression::Compress(reinterpret_cast<const uint8_t*>(raw.data()), static_cast<uint32_t>(raw.size()),
reinterpret_cast<uint8_t*>(compressed.data()), static_cast<uint32_t>(compressed.size()));
if (size <= 0) return {};
compressed.resize(static_cast<size_t>(size));
std::string out("\x89PNG\r\n\x1a\n", 8);
std::string header;
PutBigEndian(header, static_cast<uint32_t>(image.width));
PutBigEndian(header, static_cast<uint32_t>(image.height));
header += std::string("\x08\x06\x00\x00\x00", 5); // 8 bits, RGBA, deflate, no filter method, no interlace
PngChunk(out, "IHDR", header);
PngChunk(out, "IDAT", compressed);
PngChunk(out, "IEND", {});
return out;
}
std::string EncodeDds(const UgcRender::Image& image) {
std::array<uint32_t, 31> header{};
header[0] = 124;
header[1] = 0x1 | 0x2 | 0x4 | 0x8 | 0x1000; // caps, height, width, pitch, pixel format
header[2] = static_cast<uint32_t>(image.height);
header[3] = static_cast<uint32_t>(image.width);
header[4] = static_cast<uint32_t>(image.width) * 4; // pitch
header[18] = 32; // pixel format size
header[19] = 0x41; // RGB with alpha
header[21] = 32;
header[22] = 0x00FF0000;
header[23] = 0x0000FF00;
header[24] = 0x000000FF;
header[25] = 0xFF000000;
header[26] = 0x1000; // texture
std::string out = "DDS ";
out.append(reinterpret_cast<const char*>(header.data()), header.size() * 4);
out.reserve(out.size() + image.rgba.size());
for (size_t i = 0; i + 3 < image.rgba.size(); i += 4) {
out += static_cast<char>(image.rgba[i + 2]);
out += static_cast<char>(image.rgba[i + 1]);
out += static_cast<char>(image.rgba[i]);
out += static_cast<char>(image.rgba[i + 3]);
}
return out;
}
std::string Md5Hex(std::string_view data) {
MD5 md5;
md5.update(reinterpret_cast<const unsigned char*>(data.data()), static_cast<MD5::size_type>(data.size()));
md5.finalize();
return md5.hexdigest();
}
std::string ChecksumXml(std::string_view data) {
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Checksum><MD5>" + Md5Hex(data) + "</MD5><Filesize>" + std::to_string(data.size()) + "</Filesize></Checksum>\n";
}
}

40
dUgcServer/UgcFormats.h Normal file
View File

@@ -0,0 +1,40 @@
#pragma once
#include <string>
#include <string_view>
#include <vector>
#include "UgcModel.h"
#include "UgcRender.h"
/**
* The files the UGC server writes: the mesh as a Gamebryo .nif the client loads, icons as .png and .dds, and the
* client's .checksum files for its downloads. Pure (data in, bytes out).
*/
namespace UgcFormats {
struct NifShape {
std::string name;
const UgcModel::Mesh* mesh{};
bool transparent{};
};
/**
* A Gamebryo 20.3.0.9 file (user version 0, as the client's own meshes): a root NiNode named `rootName` with one
* NiTriShape per shape (at most 65535 vertices and triangles each: UgcModel::Split), with vertex colors that are
* the diffuse and ambient color (NiVertexColorProperty), a white NiMaterialProperty, and for transparent shapes an
* NiAlphaProperty blending with the vertex alpha.
*/
std::string WriteNif(const std::string& rootName, const std::vector<NifShape>& shapes);
// PNG (8-bit RGBA)
std::string EncodePng(const UgcRender::Image& image);
// DDS, uncompressed 32-bit BGRA without mipmaps
std::string EncodeDds(const UgcRender::Image& image);
// Lowercase hex MD5 of `data`
std::string Md5Hex(std::string_view data);
// What the client reads from a UGC file's .checksum: the MD5 and size of the file as it is after inflating it
std::string ChecksumXml(std::string_view data);
}

130
dUgcServer/UgcJobs.cpp Normal file
View File

@@ -0,0 +1,130 @@
#include "UgcJobs.h"
#include <sstream>
#include "Sd0.h"
#include "UgcFormats.h"
#include "UgcModel.h"
#include "UgcModular.h"
#include "ZCompression.h"
namespace UgcJobs {
void AddDownload(UgcStorage::Files& files, const std::string& name, const std::string& data) {
files[name + ".gz"] = ZCompression::Gzip(data);
files[name + ".checksum"] = UgcFormats::ChecksumXml(data);
}
std::string LxfmlFromBlob(const std::string& blob) {
if (blob.starts_with("<?xml") || blob.starts_with("<LXFML")) return blob;
std::stringstream stream(blob);
try {
Sd0 sd0(stream);
return sd0.GetAsStringUncompressed();
} catch (...) {
return {};
}
}
namespace {
// The icon files of a model, and false when nothing was drawn
bool AddIcon(UgcStorage::Files& files, const UgcModel::Model& model, const UgcRender::IconOptions& options) {
const auto icon = UgcRender::RenderIcon(model, options);
bool drawn = false;
for (size_t i = 3; i < icon.rgba.size(); i += 4) drawn = drawn || icon.rgba[i] != 0;
files["icon.png"] = UgcFormats::EncodePng(icon);
AddDownload(files, "icon.dds", UgcFormats::EncodeDds(icon));
return drawn;
}
}
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings) {
Outcome outcome;
const auto lxfml = LxfmlFromBlob(blob);
if (lxfml.empty()) {
outcome.error = "the stored LXFML can't be read";
return outcome;
}
std::string error;
const auto parts = UgcModel::ParseLxfml(lxfml, error);
if (parts.empty()) {
outcome.error = error;
return outcome;
}
auto model = UgcModel::Build(parts, library);
if (!model.missingDesigns.empty()) {
outcome.note = "no geometry for design(s)";
for (const auto design : model.missingDesigns) outcome.note += " " + std::to_string(design);
}
if (model.Empty()) {
outcome.error = "none of the model's bricks have geometry";
if (!outcome.note.empty()) outcome.error += " (" + outcome.note + ")";
return outcome;
}
const auto optimized = UgcRender::Optimize(model, settings.optimize);
outcome.aoBaked = settings.optimize.bakeAo;
if (optimized.trianglesRemoved > 0) {
if (!outcome.note.empty()) outcome.note += "; ";
outcome.note += "removed " + std::to_string(optimized.trianglesRemoved) + " of " + std::to_string(optimized.trianglesBefore) + " triangles";
}
// Named like LU Toolbox names its shapes: S<shader>_<Opaque|Alpha>_<name>
std::vector<std::pair<UgcModel::Mesh, bool>> pieces;
for (const bool transparent : { false, true }) {
for (auto& piece : UgcModel::Split(transparent ? model.transparent : model.opaque)) pieces.emplace_back(std::move(piece), transparent);
}
std::vector<UgcFormats::NifShape> shapes;
size_t opaqueIndex = 0, transparentIndex = 0;
for (const auto& [piece, transparent] : pieces) {
const auto index = transparent ? transparentIndex++ : opaqueIndex++;
shapes.push_back({ std::string("S01_") + (transparent ? "Alpha" : "Opaque") + "_Model" + (index > 0 ? std::to_string(index) : ""), &piece, transparent });
}
const auto nif = UgcFormats::WriteNif("SceneNode_Model", shapes);
outcome.files["model.nif"] = nif;
AddDownload(outcome.files, "model.nif", nif);
AddDownload(outcome.files, "model.lxfml", lxfml);
AddIcon(outcome.files, model, settings.icon);
outcome.ok = true;
return outcome;
}
Outcome ProcessModular(const ModularInput& input, const std::filesystem::path& res, const Settings& settings) {
Outcome outcome;
const auto build = UgcModular::ParseBuild(input.buildXml);
if (!build) {
outcome.error = "the build type has no topology in ModularBuildComponent";
return outcome;
}
std::vector<UgcModular::Module> modules;
for (const auto& moduleInput : input.modules) {
const auto path = UgcBricks::ResolvePath(res, moduleInput.renderAsset);
const auto data = path ? UgcBricks::ReadFile(*path) : std::nullopt;
if (!data) {
outcome.note += "module " + std::to_string(moduleInput.lot) + " has no mesh (" + moduleInput.renderAsset + "); ";
continue;
}
std::string error;
auto nif = NifFile::Parse(*data, 0, error);
if (!nif) {
outcome.note += "module " + std::to_string(moduleInput.lot) + ": " + error + "; ";
continue;
}
modules.push_back({ moduleInput.partCode, std::move(*nif), UgcModular::ParseModuleConnections(moduleInput.moduleXml) });
}
if (modules.empty()) {
outcome.error = "none of the modules have a mesh";
if (!outcome.note.empty()) outcome.error += " (" + outcome.note + ")";
return outcome;
}
const auto model = UgcModular::Assemble(*build, modules, outcome.note);
if (model.Empty()) {
outcome.error = "the modules have no triangles";
return outcome;
}
auto options = settings.modularIcon;
options.modelRotation = build->additionalRotation * options.modelRotation;
AddIcon(outcome.files, model, options);
outcome.ok = true;
return outcome;
}
}

55
dUgcServer/UgcJobs.h Normal file
View File

@@ -0,0 +1,55 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <string>
#include <string_view>
#include <vector>
#include "UgcBricks.h"
#include "UgcRender.h"
#include "UgcStorage.h"
/**
* The work a UGC worker thread does for one item: everything from the stored LXFML (or a modular build's modules) to
* the finished files. No database, network or CDClient: the main thread gathers the input and stores the result.
*/
namespace UgcJobs {
struct Settings {
UgcRender::OptimizeOptions optimize;
UgcRender::IconOptions icon;
UgcRender::IconOptions modularIcon;
};
struct Outcome {
bool ok{};
std::string error; // why it failed
std::string note; // what was odd but not fatal (missing bricks, ...)
UgcStorage::Files files; // name -> bytes, when ok
bool aoBaked{};
};
// The client's download of `data` (`name` + ".gz" and ".checksum") added to `files`
void AddDownload(UgcStorage::Files& files, const std::string& name, const std::string& data);
// The LXFML of a ugc row's lxfml column (an sd0 stream, or plain LXFML); empty when it can't be read
std::string LxfmlFromBlob(const std::string& blob);
// A player model: the optimized .nif, its icon and its LXFML for download
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings);
struct ModuleInput {
uint32_t lot{};
uint32_t partCode{};
std::string renderAsset; // RenderComponent.render_asset, relative to the client's res folder
std::string moduleXml; // ModuleComponent.xml
};
struct ModularInput {
std::vector<ModuleInput> modules;
std::string buildXml; // ModularBuildComponent.xml of the modules' build type
};
// A modular build's icon, from its modules' meshes put together
Outcome ProcessModular(const ModularInput& input, const std::filesystem::path& res, const Settings& settings);
}

275
dUgcServer/UgcModel.cpp Normal file
View File

@@ -0,0 +1,275 @@
#include "UgcModel.h"
#include <algorithm>
#include <cmath>
#include <functional>
#include <set>
#include <sstream>
#include <unordered_map>
#include <glm/gtc/matrix_transform.hpp>
#include "NifFile.h"
#include "tinyxml2.h"
namespace {
std::vector<float> ParseFloats(const char* text) {
std::vector<float> values;
if (!text) return values;
std::stringstream stream(text);
std::string item;
while (std::getline(stream, item, ',')) {
try {
values.push_back(std::stof(item));
} catch (...) {
values.push_back(0.0f);
}
}
return values;
}
std::vector<uint32_t> ParseMaterials(const char* text) {
std::vector<uint32_t> values;
if (!text) return values;
std::stringstream stream(text);
std::string item;
while (std::getline(stream, item, ',')) {
try {
values.push_back(static_cast<uint32_t>(std::stoul(item)));
} catch (...) {
values.push_back(0);
}
}
// Material 0 means "the same as the part's first"
for (auto& value : values) {
if (value == 0 && !values.empty()) value = values[0];
}
return values;
}
// LXFML 5 bone: a row-major 3x3 rotation followed by a translation
glm::mat4 BoneMatrix(const std::vector<float>& t) {
glm::mat4 matrix(1.0f);
if (t.size() < 12) return matrix;
matrix[0] = glm::vec4(t[0], t[1], t[2], 0.0f);
matrix[1] = glm::vec4(t[3], t[4], t[5], 0.0f);
matrix[2] = glm::vec4(t[6], t[7], t[8], 0.0f);
matrix[3] = glm::vec4(t[9], t[10], t[11], 1.0f);
return matrix;
}
// LXFML 4: rotate `angle` degrees around (ax, ay, az), then translate
glm::mat4 AxisAngleMatrix(const tinyxml2::XMLElement* element) {
const glm::vec3 axis(element->FloatAttribute("ax"), element->FloatAttribute("ay"), element->FloatAttribute("az"));
const glm::vec3 translation(element->FloatAttribute("tx"), element->FloatAttribute("ty"), element->FloatAttribute("tz"));
glm::mat4 matrix = glm::translate(glm::mat4(1.0f), translation);
if (glm::dot(axis, axis) > 0.0f) matrix = glm::rotate(matrix, glm::radians(element->FloatAttribute("angle")), glm::normalize(axis));
return matrix;
}
bool ParseDesign(const tinyxml2::XMLElement* element, uint32_t& design) {
const char* text = element->Attribute("designID");
if (!text || !*text) return false;
for (const char* c = text; *c; c++) {
if (*c < '0' || *c > '9') return false;
}
try {
design = static_cast<uint32_t>(std::stoul(text));
} catch (...) {
return false;
}
return true;
}
glm::vec4 ToColor(const UgcBricks::Material& material) {
return glm::vec4(material.r, material.g, material.b, material.a) / 255.0f;
}
}
namespace UgcModel {
std::vector<Part> ParseLxfml(std::string_view lxfml, std::string& error) {
std::vector<Part> parts;
tinyxml2::XMLDocument doc;
if (doc.Parse(lxfml.data(), lxfml.size()) != tinyxml2::XML_SUCCESS) {
error = "the LXFML is not valid XML";
return parts;
}
const auto* root = doc.FirstChildElement("LXFML");
if (!root) {
error = "no LXFML element";
return parts;
}
if (const auto* bricks = root->FirstChildElement("Bricks")) {
for (const auto* brick = bricks->FirstChildElement("Brick"); brick; brick = brick->NextSiblingElement("Brick")) {
for (const auto* partElement = brick->FirstChildElement("Part"); partElement; partElement = partElement->NextSiblingElement("Part")) {
const auto* bone = partElement->FirstChildElement("Bone");
Part part;
if (!bone || !ParseDesign(partElement, part.designId)) continue;
const char* materials = partElement->Attribute("materials");
if (!materials) materials = partElement->Attribute("materialID");
part.materials = ParseMaterials(materials ? materials : "0");
part.transform = BoneMatrix(ParseFloats(bone->Attribute("transformation")));
parts.push_back(std::move(part));
}
}
if (!parts.empty()) return parts;
}
// LXFML 4: groups (with their own transforms) nest parts
std::function<void(const tinyxml2::XMLElement*, const glm::mat4&)> walk = [&](const tinyxml2::XMLElement* element, const glm::mat4& parent) {
for (const auto* child = element->FirstChildElement(); child; child = child->NextSiblingElement()) {
const std::string_view name = child->Name();
if (name != "Group" && name != "Part") continue;
const auto matrix = parent * AxisAngleMatrix(child);
if (name == "Group") {
walk(child, matrix);
continue;
}
Part part;
if (!ParseDesign(child, part.designId)) continue;
const char* material = child->Attribute("materialID");
part.materials = ParseMaterials(material ? material : "0");
part.transform = matrix;
parts.push_back(std::move(part));
}
};
if (const auto* scene = root->FirstChildElement("Scene")) {
for (const auto* model = scene->FirstChildElement("Model"); model; model = model->NextSiblingElement("Model")) walk(model, glm::mat4(1.0f));
}
if (parts.empty()) error = "the LXFML has no bricks";
return parts;
}
void Mesh::Append(const Mesh& other) {
const auto base = static_cast<uint32_t>(positions.size());
positions.insert(positions.end(), other.positions.begin(), other.positions.end());
normals.insert(normals.end(), other.normals.begin(), other.normals.end());
colors.insert(colors.end(), other.colors.begin(), other.colors.end());
indices.reserve(indices.size() + other.indices.size());
for (const auto index : other.indices) indices.push_back(base + index);
}
void Mesh::Transform(const glm::mat4& transform) {
const glm::mat3 normalMatrix = glm::transpose(glm::inverse(glm::mat3(transform)));
for (auto& position : positions) position = glm::vec3(transform * glm::vec4(position, 1.0f));
for (auto& normal : normals) {
const auto n = normalMatrix * normal;
const auto length = glm::length(n);
normal = length > 0.0f ? n / length : n;
}
}
bool Model::Bounds(glm::vec3& min, glm::vec3& max) const {
bool any = false;
for (const auto* mesh : { &opaque, &transparent }) {
for (const auto& position : mesh->positions) {
min = any ? glm::min(min, position) : position;
max = any ? glm::max(max, position) : position;
any = true;
}
}
return any;
}
Model Build(const std::vector<Part>& parts, UgcBricks::BrickLibrary& library) {
Model model;
std::set<uint32_t> missing;
for (const auto& part : parts) {
const auto design = library.GetDesign(part.designId);
if (!design || design->empty()) {
missing.insert(part.designId);
continue;
}
model.bricks++;
const glm::mat3 normalMatrix = glm::transpose(glm::inverse(glm::mat3(part.transform)));
for (size_t index = 0; index < design->size(); index++) {
const auto& geometry = (*design)[index];
const auto materialId = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]);
const auto material = library.GetMaterial(materialId);
auto& mesh = material.Transparent() ? model.transparent : model.opaque;
const auto base = static_cast<uint32_t>(mesh.positions.size());
const auto color = ToColor(material);
const size_t vertexCount = geometry.positions.size() / 3;
for (size_t v = 0; v < vertexCount; v++) {
const glm::vec3 position(geometry.positions[v * 3], geometry.positions[v * 3 + 1], geometry.positions[v * 3 + 2]);
glm::vec3 normal(geometry.normals[v * 3], geometry.normals[v * 3 + 1], geometry.normals[v * 3 + 2]);
normal = normalMatrix * normal;
const auto length = glm::length(normal);
if (length > 0.0f) normal /= length;
mesh.positions.push_back(glm::vec3(part.transform * glm::vec4(position, 1.0f)));
mesh.normals.push_back(normal);
mesh.colors.push_back(color);
}
for (const auto i : geometry.indices) mesh.indices.push_back(base + i);
}
}
model.missingDesigns.assign(missing.begin(), missing.end());
return model;
}
Model FromNif(const NifFile::Model& nif) {
Model model;
for (const auto& source : nif.meshes) {
Mesh mesh;
const size_t count = source.positions.size() / 3;
const bool vertexColors = source.material.vertexColorMode == 2 && source.colors.size() == count * 4;
const glm::vec4 materialColor(source.material.diffuse[0], source.material.diffuse[1], source.material.diffuse[2], source.material.alpha);
for (size_t v = 0; v < count; v++) {
mesh.positions.emplace_back(source.positions[v * 3], source.positions[v * 3 + 1], source.positions[v * 3 + 2]);
if (source.normals.size() == count * 3) mesh.normals.emplace_back(source.normals[v * 3], source.normals[v * 3 + 1], source.normals[v * 3 + 2]);
glm::vec4 color = materialColor;
if (vertexColors) color *= glm::vec4(source.colors[v * 4], source.colors[v * 4 + 1], source.colors[v * 4 + 2], source.colors[v * 4 + 3]) / 255.0f;
mesh.colors.push_back(color);
}
mesh.indices.assign(source.indices.begin(), source.indices.end());
// Normals from the faces when the file has none
if (mesh.normals.size() != count) {
mesh.normals.assign(count, glm::vec3(0.0f));
for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) {
const auto& a = mesh.positions[mesh.indices[i]];
const auto face = glm::cross(mesh.positions[mesh.indices[i + 1]] - a, mesh.positions[mesh.indices[i + 2]] - a);
for (int k = 0; k < 3; k++) mesh.normals[mesh.indices[i + k]] += face;
}
for (auto& normal : mesh.normals) {
const auto length = glm::length(normal);
normal = length > 0.0f ? normal / length : glm::vec3(0.0f, 1.0f, 0.0f);
}
}
(source.material.alphaBlend ? model.transparent : model.opaque).Append(mesh);
}
return model;
}
std::vector<Mesh> Split(const Mesh& mesh, size_t maxVertices, size_t maxTriangles) {
std::vector<Mesh> pieces;
if (mesh.positions.size() <= maxVertices && mesh.TriangleCount() <= maxTriangles) {
if (!mesh.Empty()) pieces.push_back(mesh);
return pieces;
}
Mesh current;
std::unordered_map<uint32_t, uint32_t> remap;
const auto flush = [&]() {
if (!current.Empty()) pieces.push_back(std::move(current));
current = Mesh{};
remap.clear();
};
for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) {
size_t newVertices = 0;
for (int k = 0; k < 3; k++) newVertices += remap.contains(mesh.indices[i + k]) ? 0 : 1;
if (current.positions.size() + newVertices > maxVertices || current.TriangleCount() + 1 > maxTriangles) flush();
for (int k = 0; k < 3; k++) {
const auto source = mesh.indices[i + k];
auto [it, added] = remap.try_emplace(source, static_cast<uint32_t>(current.positions.size()));
if (added) {
current.positions.push_back(mesh.positions[source]);
if (source < mesh.normals.size()) current.normals.push_back(mesh.normals[source]);
if (source < mesh.colors.size()) current.colors.push_back(mesh.colors[source]);
}
current.indices.push_back(it->second);
}
}
flush();
return pieces;
}
}

63
dUgcServer/UgcModel.h Normal file
View File

@@ -0,0 +1,63 @@
#pragma once
#include <cstdint>
#include <string>
#include <string_view>
#include <vector>
#include <glm/glm.hpp>
#include "UgcBricks.h"
namespace NifFile {
struct Model;
}
/**
* Brick models as triangle meshes: LXFML parts, the mesh built from them (opaque and transparent bricks apart, with
* the material colors as vertex colors) and the mesh of a client .nif. Pure apart from BrickLibrary's file reads.
*/
namespace UgcModel {
// A brick in a model: its primitive, materials (one per geometry part) and where it is
struct Part {
uint32_t designId{};
std::vector<uint32_t> materials;
glm::mat4 transform{ 1.0f };
};
// The parts of an LXFML 5 (Bricks/Brick/Part with bones) or 4 (Scene/Model/Group/Part with axis angles) model.
// Empty with `error` set when it can't be read.
std::vector<Part> ParseLxfml(std::string_view lxfml, std::string& error);
struct Mesh {
std::vector<glm::vec3> positions;
std::vector<glm::vec3> normals;
std::vector<glm::vec4> colors; // sRGB, 0..1, alpha is opacity
std::vector<uint32_t> indices;
size_t TriangleCount() const { return indices.size() / 3; }
bool Empty() const { return indices.empty(); }
void Append(const Mesh& other);
void Transform(const glm::mat4& transform);
};
struct Model {
Mesh opaque;
Mesh transparent;
std::vector<uint32_t> missingDesigns; // designs without geometry in the client, skipped
size_t bricks{};
bool Empty() const { return opaque.Empty() && transparent.Empty(); }
// The bounds of every vertex; false when there are none
bool Bounds(glm::vec3& min, glm::vec3& max) const;
};
// The mesh of a model's parts
Model Build(const std::vector<Part>& parts, UgcBricks::BrickLibrary& library);
// A client .nif's meshes as one model (vertex colors times material color; transparent when blended)
Model FromNif(const NifFile::Model& nif);
// Splits a mesh into pieces the .nif format can hold (at most `maxVertices` vertices and `maxTriangles` triangles)
std::vector<Mesh> Split(const Mesh& mesh, size_t maxVertices = 65535, size_t maxTriangles = 65535);
}

132
dUgcServer/UgcModular.cpp Normal file
View File

@@ -0,0 +1,132 @@
#include "UgcModular.h"
#include <deque>
#include <set>
#include <glm/gtc/matrix_transform.hpp>
#include <glm/gtc/quaternion.hpp>
#include "tinyxml2.h"
namespace {
const tinyxml2::XMLElement* Child(const tinyxml2::XMLElement* element, const char* name) {
return element ? element->FirstChildElement(name) : nullptr;
}
}
namespace UgcModular {
std::optional<BuildInfo> ParseBuild(std::string_view xml) {
tinyxml2::XMLDocument doc;
if (doc.Parse(xml.data(), xml.size()) != tinyxml2::XML_SUCCESS) return std::nullopt;
const auto* root = doc.FirstChildElement("ModularBuild");
const auto* topology = Child(root, "topology");
if (!topology) return std::nullopt;
BuildInfo build;
if (const auto* rootPart = Child(topology, "rootPart")) build.rootPart = rootPart->UnsignedAttribute("value");
if (const auto* count = Child(topology, "numberOfParts")) build.parts = count->UnsignedAttribute("value");
for (const auto* connection = topology->FirstChildElement("connection"); connection; connection = connection->NextSiblingElement("connection")) {
const char* location = connection->Attribute("myLocation");
build.connections.push_back({ connection->UnsignedAttribute("myPartid"), location ? location : "", connection->UnsignedAttribute("connectingPart") });
}
if (const auto* rotation = Child(Child(Child(root, "Placement"), "AdditionalModelRotation"), "Rotation")) {
glm::quat q(rotation->FloatAttribute("w", 1.0f), rotation->FloatAttribute("x"), rotation->FloatAttribute("y"), rotation->FloatAttribute("z"));
if (glm::length(q) > 0.0f) build.additionalRotation = glm::mat4_cast(glm::normalize(q));
}
return build;
}
std::map<std::string, glm::vec3> ParseModuleConnections(std::string_view xml) {
std::map<std::string, glm::vec3> connections;
tinyxml2::XMLDocument doc;
if (doc.Parse(xml.data(), xml.size()) != tinyxml2::XML_SUCCESS) return connections;
const auto* root = doc.FirstChildElement("ModuleInfo");
if (!root) return connections;
for (const auto* connection = root->FirstChildElement("connection"); connection; connection = connection->NextSiblingElement("connection")) {
const char* name = connection->Attribute("name");
const auto* translation = connection->FirstChildElement("translation");
if (!name || !translation) continue;
connections[name] = glm::vec3(translation->FloatAttribute("x"), translation->FloatAttribute("y"), translation->FloatAttribute("z"));
}
return connections;
}
std::vector<uint32_t> ParseModuleLots(std::string_view ldf) {
std::vector<uint32_t> lots;
size_t start = 0;
while (start <= ldf.size()) {
size_t end = ldf.find_first_of("+;,", start);
if (end == std::string_view::npos) end = ldf.size();
auto item = ldf.substr(start, end - start);
if (const auto colon = item.find(':'); colon != std::string_view::npos) item = item.substr(colon + 1);
uint32_t value = 0;
bool digits = !item.empty();
for (const char c : item) {
if (c < '0' || c > '9') {
digits = false;
break;
}
value = value * 10 + static_cast<uint32_t>(c - '0');
}
if (digits && value != 0) lots.push_back(value);
start = end + 1;
}
return lots;
}
UgcModel::Model Assemble(const BuildInfo& build, const std::vector<Module>& modules, std::string& warnings) {
UgcModel::Model model;
std::map<uint32_t, const Module*> byPart;
for (const auto& module : modules) byPart.emplace(module.partCode, &module);
std::map<uint32_t, glm::mat4> placed;
std::deque<uint32_t> pending;
if (byPart.contains(build.rootPart)) {
placed[build.rootPart] = glm::mat4(1.0f);
pending.push_back(build.rootPart);
} else if (!modules.empty()) {
warnings += "no module for the root part " + std::to_string(build.rootPart) + "; ";
placed[modules.front().partCode] = glm::mat4(1.0f);
pending.push_back(modules.front().partCode);
}
while (!pending.empty()) {
const auto part = pending.front();
pending.pop_front();
const auto* parent = byPart.at(part);
for (const auto& connection : build.connections) {
if (connection.part != part || placed.contains(connection.connectingPart)) continue;
const auto childIt = byPart.find(connection.connectingPart);
if (childIt == byPart.end()) continue; // optional parts
glm::vec3 anchor(0.0f);
if (const auto node = parent->nif.nodes.find(connection.location); node != parent->nif.nodes.end()) {
anchor = glm::vec3(node->second.translation[0], node->second.translation[1], node->second.translation[2]);
} else if (const auto offset = parent->connections.find(connection.location); offset != parent->connections.end()) {
anchor = offset->second;
} else {
warnings += "part " + std::to_string(part) + " has no " + connection.location + "; ";
}
const auto& child = *childIt->second;
if (const auto node = child.nif.nodes.find(connection.location); node != child.nif.nodes.end()) {
anchor -= glm::vec3(node->second.translation[0], node->second.translation[1], node->second.translation[2]);
}
placed[connection.connectingPart] = placed[part] * glm::translate(glm::mat4(1.0f), anchor);
pending.push_back(connection.connectingPart);
}
}
for (const auto& module : modules) {
const auto it = placed.find(module.partCode);
if (it == placed.end()) {
warnings += "part " + std::to_string(module.partCode) + " isn't connected; ";
continue;
}
auto mesh = UgcModel::FromNif(module.nif);
mesh.opaque.Transform(it->second);
mesh.transparent.Transform(it->second);
model.opaque.Append(mesh.opaque);
model.transparent.Append(mesh.transparent);
model.bricks++;
}
return model;
}
}

57
dUgcServer/UgcModular.h Normal file
View File

@@ -0,0 +1,57 @@
#pragma once
#include <cstdint>
#include <map>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
#include <glm/glm.hpp>
#include "NifFile.h"
#include "UgcModel.h"
/**
* Modular builds (cars, rockets) put together from their modules the way the CDClient describes it:
* ModularBuildComponent.xml has the topology (a root part and which part connects to which named location of which
* other part), ModuleComponent.xml each module's connection offsets, and each module's render asset has the named
* attach-point nodes. Pure.
*/
namespace UgcModular {
struct Connection {
uint32_t part{}; // the part the location is on
std::string location; // node name on that part
uint32_t connectingPart{}; // the part placed there
};
struct BuildInfo {
uint32_t rootPart{};
uint32_t parts{};
std::vector<Connection> connections;
glm::mat4 additionalRotation{ 1.0f }; // Placement/AdditionalModelRotation
};
// ModularBuildComponent.xml; nullopt when it has no topology
std::optional<BuildInfo> ParseBuild(std::string_view xml);
// ModuleComponent.xml's connection offsets, by location name
std::map<std::string, glm::vec3> ParseModuleConnections(std::string_view xml);
// The LOTs in a ugc_modular_build.ldf_config ("1:4713+1:4714+1:4715", also with ';' or ',' between)
std::vector<uint32_t> ParseModuleLots(std::string_view ldf);
struct Module {
uint32_t partCode{};
NifFile::Model nif;
std::map<std::string, glm::vec3> connections; // from ParseModuleConnections
};
/**
* The modules placed per the build's topology, starting at the root part: a connection puts the connecting part
* where the parent's node of the location's name is (or the parent module's connection offset of that name when
* its mesh has no such node), less where the connecting part's own node of that name is, if it has one.
* Parts without a module are skipped; `warnings` says what was.
*/
UgcModel::Model Assemble(const BuildInfo& build, const std::vector<Module>& modules, std::string& warnings);
}

235
dUgcServer/UgcProcessor.cpp Normal file
View File

@@ -0,0 +1,235 @@
#include "UgcProcessor.h"
#include "CDClientDatabase.h"
#include "Database.h"
#include "Logger.h"
#include "UgcCdClient.h"
#include "json.hpp"
namespace {
constexpr size_t MAX_ERROR_LENGTH = 1000; // process_error holds 1024
constexpr size_t LOG_LENGTH = 50;
constexpr auto EVICTION_INTERVAL = std::chrono::minutes(5);
constexpr auto RECENT_ANSWER_TIME = std::chrono::seconds(10);
const char* KindName(UgcStorage::Kind kind) {
return kind == UgcStorage::Kind::MODEL ? "model" : "modular";
}
int64_t UnixNow() {
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
}
}
UgcProcessor::UgcProcessor(Config config, UgcStorage& storage, UgcBricks::BrickLibrary& library, UgcJobs::Settings settings)
: m_Config(config), m_Storage(storage), m_Library(library), m_Settings(std::move(settings)) {}
UgcProcessor::~UgcProcessor() {
Stop();
}
void UgcProcessor::Start() {
for (const auto& entry : m_Storage.List()) m_StoredBytes += entry.bytes;
m_NextEviction = std::chrono::steady_clock::now();
m_Stopping = false;
for (size_t i = 0; i < std::max<size_t>(m_Config.threads, 1); i++) m_Threads.emplace_back(&UgcProcessor::Worker, this);
LOG("UGC processing started with %zu worker(s), %llu MB stored", m_Threads.size(), static_cast<unsigned long long>(m_StoredBytes / (1024 * 1024)));
}
void UgcProcessor::Stop() {
{
std::lock_guard lock(m_Mutex);
m_Stopping = true;
m_Jobs.clear();
}
m_Wake.notify_all();
for (auto& thread : m_Threads) {
if (thread.joinable()) thread.join();
}
m_Threads.clear();
}
void UgcProcessor::Worker() {
while (true) {
Job job;
{
std::unique_lock lock(m_Mutex);
m_Wake.wait(lock, [this] { return m_Stopping || !m_Jobs.empty(); });
if (m_Stopping) return;
job = std::move(m_Jobs.front());
m_Jobs.pop_front();
m_Active++;
}
const auto start = std::chrono::steady_clock::now();
Done done{ job.kind, job.id, job.attempts };
try {
done.outcome = job.kind == Kind::MODEL
? UgcJobs::ProcessModel(job.blob, m_Library, m_Settings)
: UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), m_Settings);
} catch (const std::exception& ex) {
done.outcome.ok = false;
done.outcome.error = std::string("crashed: ") + ex.what();
}
if (done.outcome.ok) {
std::string error;
const auto bytes = m_Storage.Write(job.kind, job.id, done.outcome.files, error);
if (bytes) {
done.bytes = *bytes;
} else {
done.outcome.ok = false;
done.outcome.error = error;
}
}
done.outcome.files.clear();
done.milliseconds = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
std::lock_guard lock(m_Mutex);
m_Active--;
m_Done.push_back(std::move(done));
}
}
void UgcProcessor::Poll() {
size_t queued = 0, active = 0;
{
std::lock_guard lock(m_Mutex);
queued = m_Jobs.size();
active = m_Active;
}
// Enough to keep every worker busy until the next poll
const size_t wanted = std::max<size_t>(m_Threads.size() * 2, m_Config.pollBatch);
if (queued + active >= wanted) return;
const auto limit = static_cast<uint32_t>(wanted - queued - active + m_InFlight.size());
std::vector<Job> jobs;
for (auto& model : Database::Get()->GetUgcModelsToProcess(limit)) {
if (m_InFlight.contains({ Kind::MODEL, model.id })) continue;
jobs.push_back(Job{ Kind::MODEL, model.id, model.attempts, std::move(model.lxfml) });
}
for (auto& build : Database::Get()->GetModularBuildsToProcess(limit)) {
if (m_InFlight.contains({ Kind::MODULAR, build.id })) continue;
Job job{ Kind::MODULAR, build.id, build.attempts };
std::string error;
if (!UgcCdClient::GatherModular(build.modules, job.modular, error)) {
// Nothing a worker could do: record it right away
Record(Done{ Kind::MODULAR, build.id, build.attempts, UgcJobs::Outcome{ false, error } });
continue;
}
jobs.push_back(std::move(job));
}
if (jobs.empty()) return;
{
std::lock_guard lock(m_Mutex);
for (auto& job : jobs) {
m_InFlight.insert({ job.kind, job.id });
m_Jobs.push_back(std::move(job));
}
}
m_Wake.notify_all();
}
void UgcProcessor::Record(const Done& done) {
auto error = done.outcome.error.substr(0, MAX_ERROR_LENGTH);
const auto attempts = done.attempts + 1;
const auto state = done.outcome.ok ? IUgc::eProcessState::DONE
: attempts >= m_Config.maxAttempts ? IUgc::eProcessState::FAILED : IUgc::eProcessState::PENDING;
if (done.kind == Kind::MODEL) {
Database::Get()->SetUgcModelProcessed(done.id, state, attempts, error, done.outcome.ok && done.outcome.aoBaked);
} else {
Database::Get()->SetModularBuildProcessed(done.id, state, attempts, error);
}
m_Recent.erase({ done.kind, done.id });
if (done.outcome.ok) {
m_Made++;
m_StoredBytes += done.bytes;
LOG_DEBUG("Made %s %llu in %.0f ms%s%s", KindName(done.kind), static_cast<unsigned long long>(done.id), done.milliseconds,
done.outcome.note.empty() ? "" : ": ", done.outcome.note.c_str());
} else {
m_Failed++;
LOG("Couldn't make %s %llu (attempt %u): %s", KindName(done.kind), static_cast<unsigned long long>(done.id), attempts, error.c_str());
}
m_Log.push_back({ done.kind, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? done.outcome.note : error, UnixNow() });
while (m_Log.size() > LOG_LENGTH) m_Log.pop_front();
}
void UgcProcessor::Collect() {
std::deque<Done> finished;
{
std::lock_guard lock(m_Mutex);
finished.swap(m_Done);
}
for (const auto& done : finished) {
m_InFlight.erase({ done.kind, done.id });
Record(done);
}
// Something finished: look for more now rather than at the next interval
if (!finished.empty()) m_NextPoll = std::min(m_NextPoll, std::chrono::steady_clock::now() + std::chrono::milliseconds(100));
}
void UgcProcessor::Update() {
Collect();
const auto now = std::chrono::steady_clock::now();
if (now >= m_NextPoll) {
m_NextPoll = now + std::chrono::milliseconds(m_Config.pollIntervalMs);
Poll();
}
if (m_Config.maxStorageBytes > 0 && (m_StoredBytes > m_Config.maxStorageBytes || now >= m_NextEviction)) {
m_NextEviction = now + EVICTION_INTERVAL;
// Deleted files stay marked made: they're made again when someone asks for them (Request)
const auto removed = m_Storage.Evict(m_Config.maxStorageBytes);
m_StoredBytes = 0;
for (const auto& entry : m_Storage.List()) m_StoredBytes += entry.bytes;
if (!removed.empty()) {
m_Evicted += removed.size();
LOG("Deleted the files of %zu item(s) used longest ago to stay under %llu MB", removed.size(), static_cast<unsigned long long>(m_Config.maxStorageBytes / (1024 * 1024)));
}
}
// Forget old answers
std::erase_if(m_Recent, [now](const auto& item) { return now - item.second.first > RECENT_ANSWER_TIME; });
}
UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) {
if (m_InFlight.contains({ kind, id })) return Availability::QUEUED;
if (m_Storage.File(kind, id, "icon.png")) {
m_Storage.Touch(kind, id);
return Availability::READY;
}
if (const auto it = m_Recent.find({ kind, id }); it != m_Recent.end()) return it->second.second;
const auto info = kind == Kind::MODEL ? Database::Get()->GetUgcProcessInfo(id) : Database::Get()->GetModularBuildProcessInfo(id);
Availability answer = Availability::UNKNOWN;
if (info && info->state != IUgc::eProcessState::FAILED) {
// Made before but the files are gone (deleted to save space): make them again, first
if (info->state == IUgc::eProcessState::DONE) {
if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false);
else Database::Get()->ResetModularBuildProcessing(id, false);
}
m_NextPoll = std::chrono::steady_clock::now();
answer = Availability::QUEUED;
}
m_Recent[{ kind, id }] = { std::chrono::steady_clock::now(), answer };
return answer;
}
nlohmann::json UgcProcessor::Status() const {
nlohmann::json status;
{
std::lock_guard lock(m_Mutex);
status["queued"] = m_Jobs.size();
status["active"] = m_Active;
}
status["workers"] = m_Threads.size();
status["made"] = m_Made;
status["failed"] = m_Failed;
status["evicted"] = m_Evicted;
status["storedBytes"] = m_StoredBytes;
status["maxStorageBytes"] = m_Config.maxStorageBytes;
auto& recent = status["recent"] = nlohmann::json::array();
for (auto it = m_Log.rbegin(); it != m_Log.rend(); ++it) {
recent.push_back({ { "kind", KindName(it->kind) }, { "id", std::to_string(it->id) }, { "ok", it->ok },
{ "ms", static_cast<int64_t>(it->milliseconds) }, { "message", it->message }, { "time", it->time } });
}
return status;
}

115
dUgcServer/UgcProcessor.h Normal file
View File

@@ -0,0 +1,115 @@
#pragma once
#include <chrono>
#include <condition_variable>
#include <cstdint>
#include <deque>
#include <filesystem>
#include <map>
#include <mutex>
#include <set>
#include <string>
#include <thread>
#include <vector>
#include "dCommonVars.h"
#include "json_fwd.hpp"
#include "UgcBricks.h"
#include "UgcJobs.h"
#include "UgcStorage.h"
/**
* The UGC server's queue. The main thread (Update) takes pending rows from the database, gathers each one's input
* (the stored LXFML, or a modular build's modules from the CDClient) and hands it to the worker threads; they make
* and write the files; the main thread records the outcome in the database. Workers never touch the database, the
* CDClient or the network.
*/
class UgcProcessor {
public:
using Kind = UgcStorage::Kind;
struct Config {
uint32_t pollIntervalMs{ 2000 };
uint32_t pollBatch{ 32 };
uint32_t maxAttempts{ 3 };
uint64_t maxStorageBytes{}; // 0: no limit
size_t threads{ 2 };
};
UgcProcessor(Config config, UgcStorage& storage, UgcBricks::BrickLibrary& library, UgcJobs::Settings settings);
~UgcProcessor();
void Start();
// Waits for the jobs that are running; queued ones are dropped (they stay pending in the database)
void Stop();
// Main thread: poll, dispatch, record results, keep the storage under its cap
void Update();
enum class Availability {
READY, // the files are there
QUEUED, // it exists and is being made (again)
UNKNOWN, // no such item, or it failed
};
// Main thread: whether an item's files can be served, queuing it to be made when it exists but they're missing
Availability Request(Kind kind, LWOOBJID id);
// Main thread: what the server is doing, for /status
nlohmann::json Status() const;
private:
struct Job {
Kind kind{};
LWOOBJID id{};
uint32_t attempts{};
std::string blob; // models: the stored LXFML
UgcJobs::ModularInput modular; // modular builds
};
struct Done {
Kind kind{};
LWOOBJID id{};
uint32_t attempts{};
UgcJobs::Outcome outcome;
uint64_t bytes{};
double milliseconds{};
};
void Poll();
void Collect();
void Record(const Done& done);
void Worker();
Config m_Config;
UgcStorage& m_Storage;
UgcBricks::BrickLibrary& m_Library;
UgcJobs::Settings m_Settings;
mutable std::mutex m_Mutex;
std::condition_variable m_Wake;
std::deque<Job> m_Jobs;
std::deque<Done> m_Done;
size_t m_Active{};
bool m_Stopping{};
std::vector<std::thread> m_Threads;
// Main thread only
std::set<std::pair<Kind, LWOOBJID>> m_InFlight;
std::chrono::steady_clock::time_point m_NextPoll{};
std::chrono::steady_clock::time_point m_NextEviction{};
std::map<std::pair<Kind, LWOOBJID>, std::pair<std::chrono::steady_clock::time_point, Availability>> m_Recent; // answers for missing files
uint64_t m_StoredBytes{};
uint64_t m_Made{};
uint64_t m_Failed{};
uint64_t m_Evicted{};
struct LogEntry {
Kind kind{};
LWOOBJID id{};
bool ok{};
double milliseconds{};
std::string message;
int64_t time{};
};
std::deque<LogEntry> m_Log; // the last results
};

360
dUgcServer/UgcRender.cpp Normal file
View File

@@ -0,0 +1,360 @@
#include "UgcRender.h"
#include <algorithm>
#include <cmath>
#include <limits>
#include <numeric>
#include <glm/gtc/matrix_transform.hpp>
namespace {
constexpr float INF = std::numeric_limits<float>::infinity();
float Edge(const glm::vec3& a, const glm::vec3& b, float px, float py) {
return (b.x - a.x) * (py - a.y) - (b.y - a.y) * (px - a.x);
}
// Calls fragment(x, y, z, w0, w1, w2) for every pixel centre inside the screen-space triangle (x, y in pixels)
template<typename Fragment>
void Rasterize(int width, int height, const glm::vec3& a, const glm::vec3& b, const glm::vec3& c, Fragment&& fragment) {
const float area = Edge(a, b, c.x, c.y);
if (!(std::abs(area) > 1e-9f)) return;
const float inverse = 1.0f / area;
const int minX = std::max(0, static_cast<int>(std::floor(std::min({ a.x, b.x, c.x }))));
const int maxX = std::min(width - 1, static_cast<int>(std::ceil(std::max({ a.x, b.x, c.x }))));
const int minY = std::max(0, static_cast<int>(std::floor(std::min({ a.y, b.y, c.y }))));
const int maxY = std::min(height - 1, static_cast<int>(std::ceil(std::max({ a.y, b.y, c.y }))));
for (int y = minY; y <= maxY; y++) {
const float py = y + 0.5f;
for (int x = minX; x <= maxX; x++) {
const float px = x + 0.5f;
const float w0 = Edge(b, c, px, py) * inverse;
const float w1 = Edge(c, a, px, py) * inverse;
const float w2 = 1.0f - w0 - w1;
if (w0 < 0.0f || w1 < 0.0f || w2 < 0.0f) continue;
fragment(x, y, w0 * a.z + w1 * b.z + w2 * c.z, w0, w1, w2);
}
}
}
float ToLinear(float c) { return std::pow(std::clamp(c, 0.0f, 1.0f), 2.2f); }
float ToSrgb(float c) { return std::pow(std::clamp(c, 0.0f, 1.0f), 1.0f / 2.2f); }
// Looking at a sphere (center, radius) from direction `dir` (towards the viewer), square orthographic view
struct OrthoView {
glm::vec3 center{};
glm::vec3 dir{};
glm::vec3 right{};
glm::vec3 up{};
float radius{};
int resolution{};
OrthoView(const glm::vec3& center_, float radius_, const glm::vec3& dir_, int resolution_)
: center(center_), dir(dir_), radius(radius_), resolution(resolution_) {
const glm::vec3 worldUp = std::abs(dir.y) < 0.99f ? glm::vec3(0.0f, 1.0f, 0.0f) : glm::vec3(1.0f, 0.0f, 0.0f);
right = glm::normalize(glm::cross(worldUp, dir));
up = glm::cross(dir, right);
}
// x, y in pixels; z the distance from the camera plane (smaller is nearer)
glm::vec3 Project(const glm::vec3& p) const {
const auto offset = p - center;
const float scale = 0.5f * resolution / radius;
return { resolution * 0.5f + glm::dot(offset, right) * scale, resolution * 0.5f - glm::dot(offset, up) * scale, radius - glm::dot(offset, dir) };
}
float PixelSize() const { return 2.0f * radius / resolution; }
};
void Bounds(const UgcModel::Model& model, glm::vec3& center, float& radius) {
glm::vec3 min{}, max{};
if (!model.Bounds(min, max)) {
center = glm::vec3(0.0f);
radius = 1.0f;
return;
}
center = (min + max) * 0.5f;
radius = std::max(glm::length(max - min) * 0.5f, 0.01f);
}
}
namespace UgcRender {
std::vector<glm::vec3> SphereDirections() {
const float t = (1.0f + std::sqrt(5.0f)) / 2.0f;
const std::vector<glm::vec3> corners = {
{ -1, t, 0 }, { 1, t, 0 }, { -1, -t, 0 }, { 1, -t, 0 },
{ 0, -1, t }, { 0, 1, t }, { 0, -1, -t }, { 0, 1, -t },
{ t, 0, -1 }, { t, 0, 1 }, { -t, 0, -1 }, { -t, 0, 1 },
};
std::vector<glm::vec3> directions;
for (const auto& corner : corners) directions.push_back(glm::normalize(corner));
// Edge centres: the pairs of corners that are neighbours (the shortest distance apart)
const float edge = glm::length(corners[0] - corners[1]);
for (size_t i = 0; i < corners.size(); i++) {
for (size_t j = i + 1; j < corners.size(); j++) {
if (std::abs(glm::length(corners[i] - corners[j]) - edge) < 1e-3f) directions.push_back(glm::normalize(corners[i] + corners[j]));
}
}
return directions;
}
OptimizeResult Optimize(UgcModel::Model& model, const OptimizeOptions& options) {
OptimizeResult result;
auto& opaque = model.opaque;
result.trianglesBefore = opaque.TriangleCount() + model.transparent.TriangleCount();
if (model.Empty() || (!options.removeHidden && !options.bakeAo)) return result;
glm::vec3 center{};
float radius{};
Bounds(model, center, radius);
radius *= 1.02f;
const int resolution = std::clamp(options.resolution, 64, 4096);
const size_t pixels = static_cast<size_t>(resolution) * resolution;
std::vector<float> depth(pixels);
std::vector<uint32_t> ids(pixels);
std::vector<glm::vec3> screen(opaque.positions.size());
const size_t triangles = opaque.TriangleCount();
std::vector<bool> visible(triangles, false);
std::vector<bool> facing(triangles, true);
std::vector<glm::vec3> faceNormals(triangles, glm::vec3(0.0f));
for (size_t t = 0; t < triangles; t++) {
if (opaque.normals.size() != opaque.positions.size()) break;
faceNormals[t] = glm::normalize(opaque.normals[opaque.indices[t * 3]] + opaque.normals[opaque.indices[t * 3 + 1]] + opaque.normals[opaque.indices[t * 3 + 2]] + glm::vec3(1e-6f));
}
// Without normals nothing is culled
if (opaque.normals.size() != opaque.positions.size()) std::fill(faceNormals.begin(), faceNormals.end(), glm::vec3(0.0f));
std::vector<float> aoOpaque(opaque.positions.size()), weightOpaque(opaque.positions.size());
std::vector<float> aoTransparent(model.transparent.positions.size()), weightTransparent(model.transparent.positions.size());
for (const auto& direction : SphereDirections()) {
const OrthoView view(center, radius, direction, resolution);
const float bias = view.PixelSize();
std::fill(depth.begin(), depth.end(), INF);
std::fill(ids.begin(), ids.end(), 0);
for (size_t v = 0; v < opaque.positions.size(); v++) screen[v] = view.Project(opaque.positions[v]);
// Faces turned away can't be seen from here (they're seen from the directions they face); their vertex
// normals say which way they face, which doesn't depend on the files' winding
for (size_t t = 0; t < triangles; t++) facing[t] = glm::dot(faceNormals[t], direction) > -0.1f;
for (size_t t = 0; t < triangles; t++) {
if (!facing[t]) continue;
const auto id = static_cast<uint32_t>(t + 1);
Rasterize(resolution, resolution, screen[opaque.indices[t * 3]], screen[opaque.indices[t * 3 + 1]], screen[opaque.indices[t * 3 + 2]],
[&](int x, int y, float z, float, float, float) {
auto& stored = depth[static_cast<size_t>(y) * resolution + x];
if (z < stored) {
stored = z;
ids[static_cast<size_t>(y) * resolution + x] = id;
}
});
}
for (const auto id : ids) {
if (id != 0) visible[id - 1] = true;
}
// Whether a point is in front of what was drawn around its pixel
const auto exposed = [&](const glm::vec3& point, float allowance, bool neighbours) {
const auto p = view.Project(point);
const int px = static_cast<int>(std::floor(p.x)), py = static_cast<int>(std::floor(p.y));
const int reach = neighbours ? 1 : 0;
for (int dy = -reach; dy <= reach; dy++) {
for (int dx = -reach; dx <= reach; dx++) {
const int x = px + dx, y = py + dy;
if (x < 0 || y < 0 || x >= resolution || y >= resolution) return true;
if (p.z <= depth[static_cast<size_t>(y) * resolution + x] + allowance) return true;
}
}
return false;
};
// Triangles too small or thin to cover a pixel centre: kept when their centre isn't behind what was drawn.
// Bigger ones that show would have covered one.
if (options.removeHidden) {
const float smallArea = 2.0f; // pixels
for (size_t t = 0; t < triangles; t++) {
if (visible[t] || !facing[t]) continue;
const auto& a = screen[opaque.indices[t * 3]];
const auto& b = screen[opaque.indices[t * 3 + 1]];
const auto& c = screen[opaque.indices[t * 3 + 2]];
if (std::abs(Edge(a, b, c.x, c.y)) * 0.5f > smallArea) continue;
const auto centre = (a + b + c) / 3.0f;
const int x = static_cast<int>(centre.x), y = static_cast<int>(centre.y);
if (x < 0 || y < 0 || x >= resolution || y >= resolution || centre.z <= depth[static_cast<size_t>(y) * resolution + x] + bias) visible[t] = true;
}
}
if (options.bakeAo) {
const auto accumulate = [&](const UgcModel::Mesh& mesh, std::vector<float>& ao, std::vector<float>& weight) {
for (size_t v = 0; v < mesh.positions.size(); v++) {
const float w = v < mesh.normals.size() ? glm::dot(mesh.normals[v], direction) : 1.0f;
if (w <= 0.0f) continue;
weight[v] += w;
const auto normal = v < mesh.normals.size() ? mesh.normals[v] : glm::vec3(0.0f);
if (exposed(mesh.positions[v] + normal * bias, bias, false)) ao[v] += w;
}
};
accumulate(opaque, aoOpaque, weightOpaque);
accumulate(model.transparent, aoTransparent, weightTransparent);
}
}
if (options.bakeAo) {
const float strength = std::clamp(options.aoStrength, 0.0f, 1.0f);
const auto apply = [strength](UgcModel::Mesh& mesh, const std::vector<float>& ao, const std::vector<float>& weight) {
for (size_t v = 0; v < mesh.colors.size() && v < ao.size(); v++) {
if (weight[v] <= 0.0f) continue;
const float factor = 1.0f - strength * (1.0f - ao[v] / weight[v]);
auto& color = mesh.colors[v];
color.r = ToSrgb(ToLinear(color.r) * factor);
color.g = ToSrgb(ToLinear(color.g) * factor);
color.b = ToSrgb(ToLinear(color.b) * factor);
}
};
apply(opaque, aoOpaque, weightOpaque);
apply(model.transparent, aoTransparent, weightTransparent);
}
if (options.removeHidden) {
UgcModel::Mesh kept;
std::vector<uint32_t> remap(opaque.positions.size(), UINT32_MAX);
for (size_t t = 0; t < triangles; t++) {
if (!visible[t]) {
result.trianglesRemoved++;
continue;
}
for (int k = 0; k < 3; k++) {
const auto source = opaque.indices[t * 3 + k];
if (remap[source] == UINT32_MAX) {
remap[source] = static_cast<uint32_t>(kept.positions.size());
kept.positions.push_back(opaque.positions[source]);
if (source < opaque.normals.size()) kept.normals.push_back(opaque.normals[source]);
if (source < opaque.colors.size()) kept.colors.push_back(opaque.colors[source]);
}
kept.indices.push_back(remap[source]);
}
}
opaque = std::move(kept);
}
return result;
}
Image RenderIcon(const UgcModel::Model& source, const IconOptions& options) {
const int size = std::clamp(options.size, 8, 1024);
const int supersample = std::clamp(options.supersample, 1, 8);
const int n = size * supersample;
Image image{ size, size, std::vector<uint8_t>(static_cast<size_t>(size) * size * 4, 0) };
if (source.Empty()) return image;
UgcModel::Model model = source;
model.opaque.Transform(options.modelRotation);
model.transparent.Transform(options.modelRotation);
glm::vec3 center{};
float radius{};
Bounds(model, center, radius);
const float yaw = glm::radians(options.yawDegrees), pitch = glm::radians(options.pitchDegrees);
const glm::vec3 dir(std::sin(yaw) * std::cos(pitch), std::sin(pitch), std::cos(yaw) * std::cos(pitch));
const float fov = glm::radians(std::clamp(options.fovDegrees, 1.0f, 120.0f));
const float distance = radius / std::sin(fov * 0.5f);
const glm::vec3 eye = center + dir * distance;
const glm::mat4 viewProjection = glm::perspective(fov, 1.0f, std::max(distance - radius * 1.5f, distance * 0.01f), distance + radius * 1.5f) *
glm::lookAt(eye, center, glm::vec3(0.0f, 1.0f, 0.0f));
// Frame the model: its projected bounds, scaled to fill the icon less the margin
float minX = INF, minY = INF, maxX = -INF, maxY = -INF;
for (const auto* mesh : { &model.opaque, &model.transparent }) {
for (const auto& position : mesh->positions) {
const auto clip = viewProjection * glm::vec4(position, 1.0f);
if (clip.w <= 0.0f) continue;
minX = std::min(minX, clip.x / clip.w);
maxX = std::max(maxX, clip.x / clip.w);
minY = std::min(minY, clip.y / clip.w);
maxY = std::max(maxY, clip.y / clip.w);
}
}
if (minX > maxX) return image;
const float centerX = (minX + maxX) * 0.5f, centerY = (minY + maxY) * 0.5f;
const float scale = 2.0f / (std::max({ maxX - minX, maxY - minY, 1e-6f }) * std::max(options.margin, 0.1f));
const auto project = [&](const glm::vec3& position) {
const auto clip = viewProjection * glm::vec4(position, 1.0f);
const float w = clip.w > 1e-6f ? clip.w : 1e-6f;
return glm::vec3((0.5f + (clip.x / w - centerX) * scale * 0.5f) * n, (0.5f - (clip.y / w - centerY) * scale * 0.5f) * n, clip.z / w);
};
// Linear, premultiplied
std::vector<glm::vec4> color(static_cast<size_t>(n) * n, glm::vec4(0.0f));
std::vector<float> depth(static_cast<size_t>(n) * n, INF);
const glm::vec3 light = glm::normalize(dir + glm::vec3(-0.35f, 1.1f, 0.25f));
const auto shade = [&](const UgcModel::Mesh& mesh, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) {
glm::vec3 normal(0.0f, 1.0f, 0.0f);
if (mesh.normals.size() == mesh.positions.size()) {
normal = mesh.normals[i0] * w0 + mesh.normals[i1] * w1 + mesh.normals[i2] * w2;
const auto length = glm::length(normal);
normal = length > 0.0f ? normal / length : glm::vec3(0.0f, 1.0f, 0.0f);
if (glm::dot(normal, dir) < 0.0f) normal = -normal; // the back of a face
}
glm::vec4 base(0.63f, 0.63f, 0.63f, 1.0f);
if (mesh.colors.size() == mesh.positions.size()) base = mesh.colors[i0] * w0 + mesh.colors[i1] * w1 + mesh.colors[i2] * w2;
const float lighting = 0.55f + 0.6f * std::max(0.0f, glm::dot(normal, light));
return glm::vec4(ToLinear(base.r) * lighting, ToLinear(base.g) * lighting, ToLinear(base.b) * lighting, std::clamp(base.a, 0.0f, 1.0f));
};
// Opaque first, with the depth buffer
{
const auto& mesh = model.opaque;
std::vector<glm::vec3> screen(mesh.positions.size());
for (size_t v = 0; v < mesh.positions.size(); v++) screen[v] = project(mesh.positions[v]);
for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) {
const auto i0 = mesh.indices[i], i1 = mesh.indices[i + 1], i2 = mesh.indices[i + 2];
Rasterize(n, n, screen[i0], screen[i1], screen[i2], [&](int x, int y, float z, float w0, float w1, float w2) {
const size_t index = static_cast<size_t>(y) * n + x;
if (z >= depth[index]) return;
depth[index] = z;
const auto shaded = shade(mesh, i0, i1, i2, w0, w1, w2);
color[index] = glm::vec4(glm::vec3(shaded), 1.0f);
});
}
}
// Then transparent triangles, farthest first, blended over it
{
const auto& mesh = model.transparent;
std::vector<glm::vec3> screen(mesh.positions.size());
for (size_t v = 0; v < mesh.positions.size(); v++) screen[v] = project(mesh.positions[v]);
std::vector<size_t> order(mesh.indices.size() / 3);
std::iota(order.begin(), order.end(), 0);
const auto depthOf = [&](size_t t) { return screen[mesh.indices[t * 3]].z + screen[mesh.indices[t * 3 + 1]].z + screen[mesh.indices[t * 3 + 2]].z; };
std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { return depthOf(a) > depthOf(b); });
for (const auto t : order) {
const auto i0 = mesh.indices[t * 3], i1 = mesh.indices[t * 3 + 1], i2 = mesh.indices[t * 3 + 2];
Rasterize(n, n, screen[i0], screen[i1], screen[i2], [&](int x, int y, float z, float w0, float w1, float w2) {
const size_t index = static_cast<size_t>(y) * n + x;
if (z >= depth[index]) return;
const auto shaded = shade(mesh, i0, i1, i2, w0, w1, w2);
const float alpha = shaded.a;
auto& target = color[index];
target = glm::vec4(glm::vec3(shaded) * alpha + glm::vec3(target) * (1.0f - alpha), alpha + target.a * (1.0f - alpha));
});
}
}
// Box filter down to the icon's size
const float samples = static_cast<float>(supersample * supersample);
for (int y = 0; y < size; y++) {
for (int x = 0; x < size; x++) {
glm::vec4 sum(0.0f);
for (int sy = 0; sy < supersample; sy++) {
for (int sx = 0; sx < supersample; sx++) sum += color[static_cast<size_t>(y * supersample + sy) * n + (x * supersample + sx)];
}
sum /= samples;
uint8_t* out = &image.rgba[(static_cast<size_t>(y) * size + x) * 4];
if (sum.a <= 0.0f) continue;
out[0] = static_cast<uint8_t>(std::lround(ToSrgb(sum.r / sum.a) * 255.0f));
out[1] = static_cast<uint8_t>(std::lround(ToSrgb(sum.g / sum.a) * 255.0f));
out[2] = static_cast<uint8_t>(std::lround(ToSrgb(sum.b / sum.a) * 255.0f));
out[3] = static_cast<uint8_t>(std::lround(std::clamp(sum.a, 0.0f, 1.0f) * 255.0f));
}
}
return image;
}
}

56
dUgcServer/UgcRender.h Normal file
View File

@@ -0,0 +1,56 @@
#pragma once
#include <cstdint>
#include <vector>
#include <glm/glm.hpp>
#include "UgcModel.h"
/**
* A small software rasterizer (no GPU or display needed) for what the UGC server draws: the icons, and the renders
* from many directions that find the faces nobody can see and bake ambient occlusion into the vertex colors.
*/
namespace UgcRender {
// 8-bit RGBA, rows top to bottom, not premultiplied
struct Image {
int width{};
int height{};
std::vector<uint8_t> rgba;
};
struct IconOptions {
int size{ 128 };
int supersample{ 4 };
float yawDegrees{ 35.0f }; // camera around the model, from +Z towards +X
float pitchDegrees{ 25.0f }; // camera above the model
float fovDegrees{ 30.0f };
float margin{ 1.03f }; // 1 fills the icon, more leaves a border
glm::mat4 modelRotation{ 1.0f }; // applied to the model before the camera looks at it
};
// The model drawn from the icon's camera, framed to fit, on a transparent background
Image RenderIcon(const UgcModel::Model& model, const IconOptions& options);
struct OptimizeOptions {
int resolution{ 1024 }; // of each direction's render
bool removeHidden{ true };
bool bakeAo{ true };
float aoStrength{ 0.6f }; // 0 leaves the colors, 1 makes fully hidden corners black
};
struct OptimizeResult {
size_t trianglesBefore{};
size_t trianglesRemoved{};
};
/**
* Renders the opaque mesh from 42 directions around it: triangles that show in none of them are removed (with a
* conservative test, so small visible ones stay), and each vertex's share of the directions it can be seen from
* darkens its color (ambient occlusion). Transparent bricks hide nothing but are darkened too.
*/
OptimizeResult Optimize(UgcModel::Model& model, const OptimizeOptions& options);
// The 42 directions Optimize renders from (an icosahedron's corners and edge centres), unit length
std::vector<glm::vec3> SphereDirections();
}

360
dUgcServer/UgcServer.cpp Normal file
View File

@@ -0,0 +1,360 @@
// The UGC server: makes the meshes and icons of what players build and serves them, and the models' LXFML, to the
// game client over HTTP. See docs/UgcServer.md.
#include <algorithm>
#include <chrono>
#include <csignal>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <random>
#include <thread>
#include "BinaryPathFinder.h"
#include "CDClientDatabase.h"
#include "ConfigSync.h"
#include "Database.h"
#include "Diagnostics.h"
#include "Game.h"
#include "GeneralUtils.h"
#include "Logger.h"
#include "Server.h"
#include "ServiceType.h"
#include "Web.h"
#include "dConfig.h"
#include "dServer.h"
#include "eHTTPMethod.h"
#include "json.hpp"
#include "UgcBricks.h"
#include "UgcCdClient.h"
#include "UgcJobs.h"
#include "UgcModel.h"
#include "UgcProcessor.h"
#include "UgcStorage.h"
namespace Game {
Logger* logger = nullptr;
dServer* server = nullptr;
dConfig* config = nullptr;
Game::signal_t lastSignal = 0;
std::mt19937 randomEngine;
}
namespace {
dServer* g_Server = nullptr;
UgcProcessor* g_Processor = nullptr;
UgcStorage* g_Storage = nullptr;
template<typename T>
T Setting(const std::string& key, T fallback) {
return GeneralUtils::TryParse<T>(Game::config->GetValue(key)).value_or(fallback);
}
UgcJobs::Settings ReadSettings() {
UgcJobs::Settings settings;
settings.optimize.removeHidden = Setting<int32_t>("remove_hidden_faces", 1) != 0;
settings.optimize.bakeAo = Setting<int32_t>("bake_ao", 1) != 0;
settings.optimize.aoStrength = Setting<float>("ao_strength", 0.6f);
settings.optimize.resolution = Setting<int32_t>("optimize_resolution", 1024);
settings.icon.size = Setting<int32_t>("icon_size", 128);
settings.icon.yawDegrees = Setting<float>("icon_yaw", 35.0f);
settings.icon.pitchDegrees = Setting<float>("icon_pitch", 25.0f);
settings.icon.margin = Setting<float>("icon_margin", 1.03f);
settings.modularIcon = settings.icon;
settings.modularIcon.yawDegrees = Setting<float>("modular_icon_yaw", 35.0f);
settings.modularIcon.pitchDegrees = Setting<float>("modular_icon_pitch", 20.0f);
return settings;
}
std::vector<std::string> Segments(const std::string& path) {
std::vector<std::string> segments;
size_t start = 0;
while (start < path.size()) {
auto end = path.find('/', start);
if (end == std::string::npos) end = path.size();
if (end > start) segments.push_back(path.substr(start, end - start));
start = end + 1;
}
return segments;
}
std::string Lower(std::string text) {
std::transform(text.begin(), text.end(), text.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
return text;
}
void NotFound(HTTPReply& reply) {
reply.status = eHTTPStatusCode::NOT_FOUND;
reply.message = "Not Found";
reply.contentType = eContentType::TEXT_PLAIN;
}
void ServeFile(HTTPReply& reply, UgcStorage::Kind kind, LWOOBJID id, const std::string& name, eContentType type, bool crossOrigin) {
switch (g_Processor->Request(kind, id)) {
case UgcProcessor::Availability::READY:
break;
case UgcProcessor::Availability::QUEUED:
// The client asks again later on 408
reply.status = eHTTPStatusCode::REQUEST_TIMEOUT;
reply.message = "Being made";
reply.contentType = eContentType::TEXT_PLAIN;
reply.headers.push_back("Retry-After: 5");
return;
case UgcProcessor::Availability::UNKNOWN:
NotFound(reply);
return;
}
const auto path = g_Storage->File(kind, id, name);
if (!path) {
NotFound(reply);
return;
}
reply.status = eHTTPStatusCode::OK;
reply.file = path->string();
reply.contentType = type;
reply.headers.push_back("Cache-Control: public, max-age=3600");
if (crossOrigin) reply.headers.push_back("Access-Control-Allow-Origin: *");
}
/**
* A game client download (3D services): <client_path>/UGCC<dc>/[3DOPTIMIZED/|IMAGE128DDS/]<dc><id><.ext>(.gz|.checksum).
* `segments` are the path's parts after client_path.
*/
void ServeClientDownload(HTTPReply& reply, const std::vector<std::string>& segments) {
NotFound(reply);
if (segments.size() < 2 || segments.size() > 3) return;
const auto folder = Lower(segments[0]);
if (!folder.starts_with("ugcc")) return;
const auto datacenter = folder.substr(4);
const std::string typeFolder = segments.size() == 3 ? Lower(segments[1]) : "";
auto file = Lower(segments.back());
std::string suffix;
if (file.ends_with(".gz")) suffix = ".gz";
else if (file.ends_with(".checksum")) suffix = ".checksum";
else return;
file.resize(file.size() - suffix.size());
const auto dot = file.rfind('.');
if (dot == std::string::npos) return;
const auto extension = file.substr(dot);
auto number = file.substr(0, dot);
// The file name starts with the datacenter id
if (!number.starts_with(datacenter)) return;
number = number.substr(datacenter.size());
const auto id = GeneralUtils::TryParse<uint64_t>(number);
if (!id || number.empty()) return;
const auto blueprint = static_cast<LWOOBJID>(*id);
const auto type = suffix == ".gz" ? eContentType::APPLICATION_OCTET_STREAM : eContentType::TEXT_PLAIN;
if (extension == ".lxfml" && typeFolder.empty()) {
ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.lxfml" + suffix, type, false);
} else if (extension == ".nif" && typeFolder == "3doptimized") {
ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.nif" + suffix, type, false);
} else if (extension == ".dds" && typeFolder == "image128dds") {
// A player model's icon, else a modular build's
ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "icon.dds" + suffix, type, false);
if (reply.status == eHTTPStatusCode::NOT_FOUND) ServeFile(reply, UgcStorage::Kind::MODULAR, blueprint, "icon.dds" + suffix, type, false);
}
// .hkx: no physics is made
}
void RegisterRoutes() {
const auto prefix = Segments(Lower(Game::config->GetValue("client_path").empty() ? "/ugc" : Game::config->GetValue("client_path")));
std::string base;
for (const auto& segment : prefix) base += "/" + segment;
const auto clientRoute = [prefixSize = prefix.size()](HTTPReply& reply, const HTTPContext& context) {
auto segments = Segments(context.originalPath);
if (segments.size() <= prefixSize) {
NotFound(reply);
return;
}
segments.erase(segments.begin(), segments.begin() + static_cast<std::ptrdiff_t>(prefixSize));
ServeClientDownload(reply, segments);
};
Game::web.RegisterHTTPRoute({ .path = base + "/:folder/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = clientRoute });
Game::web.RegisterHTTPRoute({ .path = base + "/:folder/:type/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = clientRoute });
// Previews for the dashboard
Game::web.RegisterHTTPRoute({ .path = "/files/:kind/:id/:file", .method = eHTTPMethod::GET, .middleware = {},
.handle = [](HTTPReply& reply, const HTTPContext& context) {
const auto segments = Segments(context.originalPath);
NotFound(reply);
reply.headers.push_back("Access-Control-Allow-Origin: *");
if (segments.size() != 4) return;
const auto id = GeneralUtils::TryParse<uint64_t>(segments[2]);
const auto kind = segments[1] == "model" ? std::optional(UgcStorage::Kind::MODEL) : segments[1] == "modular" ? std::optional(UgcStorage::Kind::MODULAR) : std::nullopt;
const auto& name = segments[3];
if (!id || !kind || (name != "icon.png" && name != "model.nif")) return;
reply.headers.clear();
ServeFile(reply, *kind, static_cast<LWOOBJID>(*id), name, name == "icon.png" ? eContentType::IMAGE_PNG : eContentType::APPLICATION_OCTET_STREAM, true);
} });
Game::web.RegisterHTTPRoute({ .path = "/status", .method = eHTTPMethod::GET, .middleware = {},
.handle = [](HTTPReply& reply, const HTTPContext&) {
reply.status = eHTTPStatusCode::OK;
reply.contentType = eContentType::APPLICATION_JSON;
reply.message = g_Processor->Status().dump();
reply.headers.push_back("Access-Control-Allow-Origin: *");
reply.headers.push_back("Cache-Control: no-store");
} });
LOG("Serving the client's downloads under %s/UGCC<datacenter>/", base.c_str());
}
std::filesystem::path ResPath() {
const auto client = Game::config->GetValue("client_location");
return client.empty() ? std::filesystem::path{} : std::filesystem::path(client) / "res";
}
// Command line tools: make one model's or modular build's files into a folder, without a database
int MakeFromCommandLine(const std::string& mode, const std::string& input, const std::filesystem::path& output) {
const auto res = ResPath();
const auto settings = ReadSettings();
UgcBricks::BrickLibrary library(res, Setting<uint32_t>("brick_lod", 0));
if (!library.LoadMaterials()) std::cerr << "Couldn't read Materials.xml from " << (res / "brickdb.zip") << "; bricks will be grey\n";
UgcJobs::Outcome outcome;
const auto start = std::chrono::steady_clock::now();
if (mode == "--make-model") {
const auto data = UgcBricks::ReadFile(input);
if (!data) {
std::cerr << "Can't read " << input << "\n";
return EXIT_FAILURE;
}
outcome = UgcJobs::ProcessModel(*data, library, settings);
} else {
CDClientDatabase::Connect((BinaryPathFinder::GetBinaryDir() / "resServer/CDServer.sqlite").string());
UgcJobs::ModularInput modular;
std::string error;
if (!UgcCdClient::GatherModular(input, modular, error)) {
std::cerr << error << "\n";
return EXIT_FAILURE;
}
outcome = UgcJobs::ProcessModular(modular, res, settings);
}
const auto ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
if (!outcome.ok) {
std::cerr << "Failed: " << outcome.error << "\n";
return EXIT_FAILURE;
}
std::filesystem::create_directories(output);
for (const auto& [name, data] : outcome.files) {
std::ofstream(output / name, std::ios::binary).write(data.data(), static_cast<std::streamsize>(data.size()));
}
std::cout << "Made " << outcome.files.size() << " files in " << ms << " ms" << (outcome.note.empty() ? "" : ": " + outcome.note) << "\n";
return EXIT_SUCCESS;
}
}
int main(int argc, char** argv) {
Diagnostics::SetProduceMemoryDump(true);
std::signal(SIGINT, Game::OnSignal);
std::signal(SIGTERM, Game::OnSignal);
Game::config = new dConfig("ugcconfig.ini");
// UgcServer --make-model <file.lxfml> <folder> or --make-modular "1:4713+1:4714+1:4715" <folder>
if (argc == 4 && (std::string(argv[1]) == "--make-model" || std::string(argv[1]) == "--make-modular")) {
return MakeFromCommandLine(argv[1], argv[2], argv[3]);
}
Server::SetupLogger("UgcServer");
if (!Game::logger) return EXIT_FAILURE;
Game::config->LogSettings();
LOG("Starting UGC Server");
const auto res = ResPath();
if (res.empty() || !std::filesystem::exists(res)) {
LOG("client_location is not set or has no res folder; the UGC server needs the client's brick data");
return EXIT_FAILURE;
}
try {
CDClientDatabase::Connect((BinaryPathFinder::GetBinaryDir() / "resServer/CDServer.sqlite").string());
} catch (std::exception& ex) {
LOG("Failed to connect to CDClient database: %s", ex.what());
return EXIT_FAILURE;
}
try {
Database::Connect();
} catch (std::exception& ex) {
LOG("Failed to connect to the database: %s", ex.what());
return EXIT_FAILURE;
}
// Settings edited on the dashboard (server_config table) are layered over the files from here on
Game::config->SetDatabaseSync(ConfigSync::Sync);
std::string masterIP = "localhost";
uint32_t masterPort = 1000;
std::string masterPassword;
if (const auto masterInfo = Database::Get()->GetMasterInfo()) {
masterIP = masterInfo->ip;
masterPort = masterInfo->port;
masterPassword = masterInfo->password;
}
// The master starts it again when this link drops
g_Server = new dServer(masterIP, Setting<uint32_t>("net_port", 2012), 0, 16, false, false, Game::logger, masterIP, masterPort,
ServiceType::UGC, Game::config, &Game::lastSignal, masterPassword);
Game::server = g_Server;
UgcBricks::BrickLibrary library(res, Setting<uint32_t>("brick_lod", 0));
if (!library.LoadMaterials()) LOG("Couldn't read Materials.xml from %s; bricks will be grey", (res / "brickdb.zip").string().c_str());
auto outputDir = std::filesystem::path(Game::config->GetValue("ugc_output_dir").empty() ? "ugc" : Game::config->GetValue("ugc_output_dir"));
if (outputDir.is_relative()) outputDir = BinaryPathFinder::GetBinaryDir() / outputDir;
UgcStorage storage(outputDir);
g_Storage = &storage;
UgcProcessor::Config processorConfig;
processorConfig.pollIntervalMs = std::max<uint32_t>(Setting<uint32_t>("poll_interval_ms", 2000), 100);
processorConfig.pollBatch = std::max<uint32_t>(Setting<uint32_t>("poll_batch", 32), 1);
processorConfig.maxAttempts = std::max<uint32_t>(Setting<uint32_t>("max_attempts", 3), 1);
processorConfig.maxStorageBytes = Setting<uint64_t>("ugc_max_storage_mb", 2048) * 1024 * 1024;
const auto threads = Setting<uint32_t>("worker_threads", 0);
processorConfig.threads = threads > 0 ? threads : std::max<size_t>(std::thread::hardware_concurrency() / 2, 1);
UgcProcessor processor(processorConfig, storage, library, ReadSettings());
g_Processor = &processor;
const auto listenIp = Game::config->GetValue("listen_ip").empty() ? std::string("0.0.0.0") : Game::config->GetValue("listen_ip");
const auto port = Setting<uint32_t>("port", 2008);
if (!Game::web.Startup(listenIp, port)) {
LOG("Failed to start the web server on %s:%u", listenIp.c_str(), port);
return EXIT_FAILURE;
}
RegisterRoutes();
processor.Start();
LOG("UGC Server started on %s:%u, files in %s", listenIp.c_str(), port, outputDir.string().c_str());
auto lastTick = std::chrono::steady_clock::now();
while (!Game::ShouldShutdown()) {
// The poll's wait for network traffic is also this loop's sleep
Game::web.ReceiveRequests(10);
const auto now = std::chrono::steady_clock::now();
if (now - lastTick < std::chrono::milliseconds(16)) continue;
lastTick = now;
Packet* packet = g_Server->ReceiveFromMaster();
while (packet) {
g_Server->DeallocateMasterPacket(packet);
packet = g_Server->ReceiveFromMaster();
}
processor.Update();
}
LOG("Stopping the UGC server");
processor.Stop();
Game::web.Shutdown();
g_Processor = nullptr;
Database::Destroy("UgcServer");
delete g_Server;
g_Server = nullptr;
Game::server = nullptr;
delete Game::logger;
Game::logger = nullptr;
delete Game::config;
Game::config = nullptr;
return EXIT_SUCCESS;
}

132
dUgcServer/UgcStorage.cpp Normal file
View File

@@ -0,0 +1,132 @@
#include "UgcStorage.h"
#include <algorithm>
#include <array>
#include <fstream>
#include <random>
namespace {
constexpr std::array KNOWN_FILES = {
"model.nif", "model.nif.gz", "model.nif.checksum",
"model.lxfml.gz", "model.lxfml.checksum",
"icon.dds.gz", "icon.dds.checksum", "icon.png",
};
const char* KindFolder(UgcStorage::Kind kind) {
return kind == UgcStorage::Kind::MODEL ? "models" : "modular";
}
std::string RandomSuffix() {
static thread_local std::mt19937_64 random{ std::random_device{}() };
return std::to_string(random());
}
}
UgcStorage::UgcStorage(std::filesystem::path root) : m_Root(std::move(root)) {}
bool UgcStorage::IsKnownFile(const std::string& name) {
return std::find(KNOWN_FILES.begin(), KNOWN_FILES.end(), name) != KNOWN_FILES.end();
}
std::filesystem::path UgcStorage::Folder(Kind kind, LWOOBJID id) const {
const auto id64 = static_cast<uint64_t>(id);
return m_Root / KindFolder(kind) / std::to_string(id64 % 1000) / std::to_string(id64);
}
std::optional<std::filesystem::path> UgcStorage::File(Kind kind, LWOOBJID id, const std::string& name) const {
if (!IsKnownFile(name)) return std::nullopt;
auto path = Folder(kind, id) / name;
std::error_code error;
if (!std::filesystem::is_regular_file(path, error)) return std::nullopt;
return path;
}
std::optional<uint64_t> UgcStorage::Write(Kind kind, LWOOBJID id, const Files& files, std::string& error) const {
const auto folder = Folder(kind, id);
const auto temporary = folder.parent_path() / (".tmp-" + folder.filename().string() + "-" + RandomSuffix());
std::error_code code;
std::filesystem::create_directories(temporary, code);
if (code) {
error = "could not create " + temporary.string() + ": " + code.message();
return std::nullopt;
}
uint64_t bytes = 0;
for (const auto& [name, data] : files) {
std::ofstream out(temporary / name, std::ios::binary | std::ios::trunc);
out.write(data.data(), static_cast<std::streamsize>(data.size()));
if (!out) {
error = "could not write " + (temporary / name).string();
std::filesystem::remove_all(temporary, code);
return std::nullopt;
}
bytes += data.size();
}
// Swap the old folder out and the new one in; the old one is deleted after
const auto old = folder.parent_path() / (".old-" + folder.filename().string() + "-" + RandomSuffix());
const bool hadOld = std::filesystem::exists(folder, code);
if (hadOld) std::filesystem::rename(folder, old, code);
std::filesystem::rename(temporary, folder, code);
if (code) {
error = "could not move the files into " + folder.string() + ": " + code.message();
std::filesystem::remove_all(temporary, code);
return std::nullopt;
}
if (hadOld) std::filesystem::remove_all(old, code);
return bytes;
}
void UgcStorage::Remove(Kind kind, LWOOBJID id) const {
std::error_code error;
std::filesystem::remove_all(Folder(kind, id), error);
}
void UgcStorage::Touch(Kind kind, LWOOBJID id) const {
std::error_code error;
std::filesystem::last_write_time(Folder(kind, id), std::filesystem::file_time_type::clock::now(), error);
}
std::vector<UgcStorage::Entry> UgcStorage::List() const {
std::vector<Entry> entries;
std::error_code error;
for (const auto kind : { Kind::MODEL, Kind::MODULAR }) {
const auto base = m_Root / KindFolder(kind);
for (std::filesystem::directory_iterator bucket(base, error), end; !error && bucket != end; bucket.increment(error)) {
if (!bucket->is_directory(error)) continue;
for (std::filesystem::directory_iterator item(bucket->path(), error), itemEnd; !error && item != itemEnd; item.increment(error)) {
const auto name = item->path().filename().string();
if (name.empty() || name[0] == '.' || !item->is_directory(error)) continue;
Entry entry;
entry.kind = kind;
try {
entry.id = static_cast<LWOOBJID>(std::stoull(name));
} catch (...) {
continue;
}
entry.used = std::filesystem::last_write_time(item->path(), error);
for (std::filesystem::directory_iterator file(item->path(), error), fileEnd; !error && file != fileEnd; file.increment(error)) {
if (file->is_regular_file(error)) entry.bytes += file->file_size(error);
}
entries.push_back(entry);
}
error.clear();
}
error.clear();
}
return entries;
}
std::vector<UgcStorage::Entry> UgcStorage::Evict(uint64_t maxBytes) const {
auto entries = List();
uint64_t total = 0;
for (const auto& entry : entries) total += entry.bytes;
std::vector<Entry> removed;
if (total <= maxBytes) return removed;
std::sort(entries.begin(), entries.end(), [](const Entry& a, const Entry& b) { return a.used < b.used; });
for (const auto& entry : entries) {
if (total <= maxBytes) break;
Remove(entry.kind, entry.id);
total -= std::min(total, entry.bytes);
removed.push_back(entry);
}
return removed;
}

60
dUgcServer/UgcStorage.h Normal file
View File

@@ -0,0 +1,60 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <map>
#include <optional>
#include <string>
#include <vector>
#include "dCommonVars.h"
/**
* Where the UGC server keeps what it makes: <root>/models/<id % 1000>/<id>/ for player models and
* <root>/modular/<id % 1000>/<id>/ for modular builds. An item's files are written to a temporary folder and renamed
* into place, so a request never sees half of them. Plain file operations, safe from any thread for different items.
*/
class UgcStorage {
public:
enum class Kind { MODEL, MODULAR };
// The files an item has, by name (see FileName)
using Files = std::map<std::string, std::string>;
explicit UgcStorage(std::filesystem::path root);
const std::filesystem::path& GetRoot() const { return m_Root; }
std::filesystem::path Folder(Kind kind, LWOOBJID id) const;
// A file of an item, if it is there; `name` must be one of the names the server writes
std::optional<std::filesystem::path> File(Kind kind, LWOOBJID id, const std::string& name) const;
// Replaces an item's files with `files`; the bytes written, nullopt (and `error`) on failure
std::optional<uint64_t> Write(Kind kind, LWOOBJID id, const Files& files, std::string& error) const;
// Deletes an item's files
void Remove(Kind kind, LWOOBJID id) const;
// Marks an item used now, for eviction
void Touch(Kind kind, LWOOBJID id) const;
struct Entry {
Kind kind{};
LWOOBJID id{};
uint64_t bytes{};
std::filesystem::file_time_type used{};
};
// Everything stored, with sizes and when it was last used
std::vector<Entry> List() const;
// Removes the least recently used items until the total is at most `maxBytes`; what was removed
std::vector<Entry> Evict(uint64_t maxBytes) const;
// Names the server writes (and serves); anything else is refused
static bool IsKnownFile(const std::string& name);
private:
std::filesystem::path m_Root;
};

View File

@@ -77,13 +77,16 @@ Modular builds (`ugc_modular_build` rows, `ldf_config` like `1:4713+1:4714+1:471
Files live under `ugc_output_dir` (default `ugc` next to the server binaries):
```
ugc/models/<id % 1000>/<id>/model.nif(.gz, .checksum), model.lxfml(.gz, .checksum), icon.dds(.gz, .checksum), icon.png
ugc/modular/<id % 1000>/<id>/icon.dds(.gz, .checksum), icon.png
ugc/models/<id % 1000>/<id>/model.nif, model.nif.gz, model.nif.checksum, model.lxfml.gz, model.lxfml.checksum,
icon.dds.gz, icon.dds.checksum, icon.png
ugc/modular/<id % 1000>/<id>/icon.dds.gz, icon.dds.checksum, icon.png
```
A model's files are written to a temporary folder and renamed into place, so a half written model is never served.
`ugc_max_storage_mb` (default 2048, 0 for no limit) caps the folder: when it is over, the models whose files were used
longest ago are deleted and marked unprocessed, and are made again the next time they are asked for.
longest ago are deleted. Their rows stay `is_optimized = 1`; when something asks for their files the server sets
them back to 0 and makes them again (the request answers 408 meanwhile), so deleted files are only made again when
wanted.
## Database state
@@ -102,6 +105,15 @@ The database is the queue: the UGC server looks for rows with `is_optimized = 0`
`max_attempts` (default 3) times. Reprocessing (dashboard) sets rows back to `is_optimized = 0, process_attempts = 0`.
No master messages are needed for any of it.
### Marking models for processing (for code that writes `ugc` rows)
* A new row needs nothing: `is_optimized` defaults to 0 (`InsertNewUgcModel` writes 0).
* When a model's LXFML changes, `UpdateUgcModelData` sets `is_optimized = 0, process_attempts = 0, process_error = ''`
in the same statement. Anything that writes `ugc.lxfml` another way must do the same, or call
`ResetUgcModelProcessing(id, false)`.
* Deleting the row is enough when a model is deleted; its files are left until the storage cap removes them.
* The same holds for `ugc_modular_build` (`InsertUgcBuild` rows start at 0; `ResetModularBuildProcessing`).
## Threads
The main thread owns RakNet (the master link), mongoose (HTTP) and the database. `worker_threads` (default: half the
@@ -119,11 +131,14 @@ files are what every player on a property sees anyway):
`<client_path>/UGCC<dc>/IMAGE128DDS/<dc><id>.dds.gz|.checksum` for the client (`client_path`, default `/ugc`, is
the client's `UGCSERVERDIR`). HKX answers 404. A model that exists but isn't made yet (or was evicted) is moved to
the front of the queue and answers 408 so the client asks again.
* `/files/model/<id>/icon.png|model.nif|model.lxfml` and `/files/modular/<id>/icon.png` for the dashboard's previews
* `/files/model/<id>/icon.png|model.nif` and `/files/modular/<id>/icon.png` for the dashboard's previews
(with `Access-Control-Allow-Origin: *`).
* `/status`: JSON with the queue length, what the workers are doing and totals since start.
Files are sent from disk (mongoose streams them) with `Cache-Control: public, max-age=86400` and an ETag of the MD5.
Files are sent from disk (mongoose streams them, with its own ETag) with `Cache-Control: public, max-age=3600`.
`UgcServer --make-model <file.lxfml or sd0> <folder>` and `UgcServer --make-modular "1:4713+1:4714+1:4715" <folder>`
make one item's files into a folder without a database, for trying settings.
## Dashboard

View File

@@ -0,0 +1,57 @@
/* UGC server state (see docs/UgcServer.md). ugc.is_optimized: 0 waiting to be made, 1 made, 2 failed; processed_at is
the last attempt (Unix seconds), process_attempts how many there have been and process_error the last failure. The
same columns on ugc_modular_build for the builds' icons. Columns are only added when they are not there yet. */
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN processed_at BIGINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'processed_at');
PREPARE dlu_column_stmt FROM @dlu_column;
EXECUTE dlu_column_stmt;
DEALLOCATE PREPARE dlu_column_stmt;
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN process_attempts INT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'process_attempts');
PREPARE dlu_column_stmt FROM @dlu_column;
EXECUTE dlu_column_stmt;
DEALLOCATE PREPARE dlu_column_stmt;
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN process_error VARCHAR(1024) NOT NULL DEFAULT ''''', 'DO 0') FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'process_error');
PREPARE dlu_column_stmt FROM @dlu_column;
EXECUTE dlu_column_stmt;
DEALLOCATE PREPARE dlu_column_stmt;
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN is_optimized INT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'is_optimized');
PREPARE dlu_column_stmt FROM @dlu_column;
EXECUTE dlu_column_stmt;
DEALLOCATE PREPARE dlu_column_stmt;
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN processed_at BIGINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'processed_at');
PREPARE dlu_column_stmt FROM @dlu_column;
EXECUTE dlu_column_stmt;
DEALLOCATE PREPARE dlu_column_stmt;
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN process_attempts INT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'process_attempts');
PREPARE dlu_column_stmt FROM @dlu_column;
EXECUTE dlu_column_stmt;
DEALLOCATE PREPARE dlu_column_stmt;
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN process_error VARCHAR(1024) NOT NULL DEFAULT ''''', 'DO 0') FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'process_error');
PREPARE dlu_column_stmt FROM @dlu_column;
EXECUTE dlu_column_stmt;
DEALLOCATE PREPARE dlu_column_stmt;
/* What the UGC server looks for */
SET @dlu_index = (SELECT IF(COUNT(*) = 0, 'CREATE INDEX ugc_is_optimized ON ugc (is_optimized)', 'DO 0') FROM information_schema.statistics
WHERE table_schema = DATABASE() AND table_name = 'ugc' AND index_name = 'ugc_is_optimized');
PREPARE dlu_index_stmt FROM @dlu_index;
EXECUTE dlu_index_stmt;
DEALLOCATE PREPARE dlu_index_stmt;
SET @dlu_index = (SELECT IF(COUNT(*) = 0, 'CREATE INDEX ugc_modular_build_is_optimized ON ugc_modular_build (is_optimized)', 'DO 0') FROM information_schema.statistics
WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND index_name = 'ugc_modular_build_is_optimized');
PREPARE dlu_index_stmt FROM @dlu_index;
EXECUTE dlu_index_stmt;
DEALLOCATE PREPARE dlu_index_stmt;

View File

@@ -0,0 +1,10 @@
/* UGC server state: see the MySQL migration. */
ALTER TABLE ugc ADD COLUMN processed_at BIGINT NOT NULL DEFAULT 0;
ALTER TABLE ugc ADD COLUMN process_attempts INTEGER NOT NULL DEFAULT 0;
ALTER TABLE ugc ADD COLUMN process_error TEXT NOT NULL DEFAULT '';
ALTER TABLE ugc_modular_build ADD COLUMN is_optimized INTEGER NOT NULL DEFAULT 0;
ALTER TABLE ugc_modular_build ADD COLUMN processed_at BIGINT NOT NULL DEFAULT 0;
ALTER TABLE ugc_modular_build ADD COLUMN process_attempts INTEGER NOT NULL DEFAULT 0;
ALTER TABLE ugc_modular_build ADD COLUMN process_error TEXT NOT NULL DEFAULT '';
CREATE INDEX IF NOT EXISTS ugc_is_optimized ON ugc (is_optimized);
CREATE INDEX IF NOT EXISTS ugc_modular_build_is_optimized ON ugc_modular_build (is_optimized);

View File

@@ -15,3 +15,7 @@ master_password=3.25DARKFLAME1
# Enable the web dashboard (0 = disabled, 1 = enabled)
# Dashboard settings are configured in dashboardconfig.ini
enable_dashboard=0
# Start the UGC server, which makes and serves the meshes and icons of what players build (0 = disabled, 1 = enabled)
# Its settings are in ugcconfig.ini; see docs/UgcServer.md for the client's boot.cfg settings
enable_ugc_server=0

48
resources/ugcconfig.ini Normal file
View File

@@ -0,0 +1,48 @@
# UGC server: makes the meshes and icons of what players build and serves them to the game client.
# The master starts it when enable_ugc_server=1 (masterconfig.ini). See docs/UgcServer.md.
# HTTP port the game client downloads from (the client's UGCSERVERPORT)
port=2008
# The IP address to bind to. The game client has to reach it, so the default is every interface.
listen_ip=0.0.0.0
# UDP port for the connection to the master server (the next port is used too). It must not clash with another
# server's ports: auth uses 1500-1501, chat 2005-2006, the dashboard 2010-2011, worlds 3000 and up.
net_port=2012
# The client's UGCSERVERDIR: the path the downloads are under (the client adds /UGCC<DATACENTERID>/...)
client_path=/ugc
# Where the made files are kept (relative to the server binaries), and at most how much space they may take in MB.
# When there's more, the files used longest ago are deleted; they're made again when asked for. 0: no limit.
ugc_output_dir=ugc
ugc_max_storage_mb=2048
# Worker threads making files (0: half the CPU cores)
worker_threads=0
# How often to look for models to make (milliseconds), how many to take at once, and how often a model may fail
# before it's given up on (the dashboard can try again)
poll_interval_ms=2000
poll_batch=32
max_attempts=3
# Brick detail used for meshes and icons: 0 (most detailed) to 2
brick_lod=0
# Remove faces that can't be seen from anywhere, and darken hidden corners (ambient occlusion baked into the vertex
# colors): 0 or 1. ao_strength: 0 (none) to 1. optimize_resolution: detail of the renders that decide both.
remove_hidden_faces=1
bake_ao=1
ao_strength=0.6
optimize_resolution=1024
# Icons: size in pixels, the camera's angle around (yaw, from the model's front) and above (pitch) the model in degrees,
# and the empty border (1 fills the icon). Modular builds (cars, rockets) have their own angles.
icon_size=128
icon_yaw=35
icon_pitch=25
icon_margin=1.03
modular_icon_yaw=35
modular_icon_pitch=20

View File

@@ -9,3 +9,4 @@ add_subdirectory(dCommonTests)
add_subdirectory(dGameTests)
add_subdirectory(dWebTests)
add_subdirectory(dDatabaseTests)
add_subdirectory(dUgcTests)

View File

@@ -1833,6 +1833,7 @@ static_assert(static_cast<int64_t>(ServiceType::WORLD) == 4);
static_assert(static_cast<int64_t>(ServiceType::CLIENT) == 5);
static_assert(static_cast<int64_t>(ServiceType::MASTER) == 6);
static_assert(static_cast<int64_t>(ServiceType::UNKNOWN) == 7);
static_assert(static_cast<int64_t>(ServiceType::UGC) == 8);
// The asserts above are compile time; this test only makes the file show up in test runs.
TEST(MessageIdPinTests, MessageTypeValuesArePinned) { SUCCEED(); }

View File

@@ -704,6 +704,50 @@ TEST_F(ParitySeeded, UgcModel) {
auto model = db.GetUgcModel(1152921510000500001LL);
return model ? json{ model->id, model->modelID, model->lxfmlData.str() } : json();
});
// The UGC server's processing state (processed_at is the time of the call, so only whether it's set is compared)
const auto infoJson = [](const std::optional<IUgc::ProcessInfo>& info) {
return info ? json{ info->id, info->characterId, info->characterName, static_cast<int32_t>(info->state), info->attempts,
info->processedAt > 0, info->error, info->bakeAo, info->details } : json();
};
const auto countsJson = [](const std::vector<std::pair<IUgc::eProcessState, uint64_t>>& counts) {
json out = json::array();
for (const auto& [state, count] : counts) out.push_back({ static_cast<int32_t>(state), count });
return out;
};
Both("GetUgcModelsToProcess", [](GameDatabase& db) {
json out = json::array();
for (const auto& model : db.GetUgcModelsToProcess(10)) out.push_back({ model.id, model.lxfml, model.attempts });
return out;
});
Both("SetUgcModelProcessed failed", [&](GameDatabase& db) {
db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::FAILED, 3, "no bricks", false);
return json{ infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), countsJson(db.GetUgcProcessCounts()), db.GetUgcModelsToProcess(10).size() };
});
Both("GetUgcProcessList", [&](GameDatabase& db) {
json out = json::array();
for (const auto& info : db.GetUgcProcessList(IUgc::eProcessState::FAILED, 0, 10)) out.push_back(infoJson(info));
out.push_back(db.GetUgcProcessList(std::nullopt, 0, 10).size());
out.push_back(db.GetUgcProcessList(IUgc::eProcessState::DONE, 0, 10).size());
return out;
});
Both("ResetUgcModelProcessing", [&](GameDatabase& db) {
const auto changed = db.ResetUgcModelProcessing(std::nullopt, true);
db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::DONE, 1, "", true);
return json{ changed, infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), infoJson(db.GetUgcProcessInfo(1)) };
});
Both("Modular build processing", [&](GameDatabase& db) {
db.InsertUgcBuild("1:4713+1:4714+1:4715", 1152921510000500002LL, CHAR_ALICE);
json out = json::array();
for (const auto& build : db.GetModularBuildsToProcess(10)) out.push_back({ build.id, build.modules, build.attempts });
db.SetModularBuildProcessed(1152921510000500002LL, IUgc::eProcessState::PENDING, 1, "no mesh");
out.push_back(infoJson(db.GetModularBuildProcessInfo(1152921510000500002LL)));
out.push_back(countsJson(db.GetModularBuildProcessCounts()));
out.push_back(db.GetModularBuildProcessList(std::nullopt, 0, 10).size());
out.push_back(db.ResetModularBuildProcessing(1152921510000500002LL, false));
out.push_back(infoJson(db.GetModularBuildProcessInfo(1152921510000500002LL)));
return out;
});
}
TEST_F(ParitySeeded, LogsAndAudit) {

View File

@@ -0,0 +1,3 @@
add_executable(dUgcTests "UgcTests.cpp")
target_link_libraries(dUgcTests ${COMMON_LIBRARIES} dUgc GTest::gtest_main)
gtest_discover_tests(dUgcTests)

View File

@@ -0,0 +1,291 @@
#include <gtest/gtest.h>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <glm/gtc/matrix_transform.hpp>
#include "Game.h"
#include "NifFile.h"
#include "UgcBricks.h"
#include "UgcFormats.h"
#include "UgcModel.h"
#include "UgcModular.h"
#include "UgcRender.h"
#include "UgcStorage.h"
#include "ZCompression.h"
class Logger;
class dConfig;
namespace Game {
Logger* logger = nullptr;
dConfig* config = nullptr;
}
namespace {
// A closed box as an LDD .g file: 8 corners, 12 triangles
std::string BoxGeometry(glm::vec3 min, glm::vec3 max) {
std::vector<float> positions, normals;
for (int i = 0; i < 8; i++) {
const glm::vec3 p((i & 1) ? max.x : min.x, (i & 2) ? max.y : min.y, (i & 4) ? max.z : min.z);
const auto n = glm::normalize(p - (min + max) * 0.5f);
positions.insert(positions.end(), { p.x, p.y, p.z });
normals.insert(normals.end(), { n.x, n.y, n.z });
}
const std::vector<uint32_t> indices = { 0, 2, 1, 1, 2, 3, 4, 5, 6, 5, 7, 6, 0, 1, 4, 1, 5, 4, 2, 6, 3, 3, 6, 7, 0, 4, 2, 2, 4, 6, 1, 3, 5, 3, 7, 5 };
std::string out;
const int32_t header[4] = { 0x42473031, 8, static_cast<int32_t>(indices.size()), 0 };
out.append(reinterpret_cast<const char*>(header), sizeof(header));
out.append(reinterpret_cast<const char*>(positions.data()), positions.size() * 4);
out.append(reinterpret_cast<const char*>(normals.data()), normals.size() * 4);
out.append(reinterpret_cast<const char*>(indices.data()), indices.size() * 4);
return out;
}
std::filesystem::path TempFolder(const std::string& name) {
auto path = std::filesystem::temp_directory_path() / ("dlu_ugc_test_" + name + "_" + std::to_string(::testing::UnitTest::GetInstance()->random_seed()));
std::filesystem::remove_all(path);
std::filesystem::create_directories(path);
return path;
}
// A res folder with brick 3001 (a 1x1x1 box) and brick 3002 (a big box)
std::filesystem::path MakeRes() {
const auto res = TempFolder("res");
std::filesystem::create_directories(res / "brickprimitives" / "lod0");
std::ofstream(res / "brickprimitives" / "lod0" / "3001.g", std::ios::binary) << BoxGeometry(glm::vec3(0.0f), glm::vec3(1.0f));
std::ofstream(res / "brickprimitives" / "lod0" / "3002.g", std::ios::binary) << BoxGeometry(glm::vec3(-4.0f), glm::vec3(4.0f));
return res;
}
const char* LXFML5 = R"(<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
<LXFML versionMajor="5" versionMinor="0"><Bricks>
<Brick refID="0" designID="3001"><Part refID="0" designID="3001" materials="21,0"><Bone refID="0" transformation="1,0,0,0,1,0,0,0,1,10,0,0"/></Part></Brick>
<Brick refID="1" designID="3001"><Part refID="1" designID="3001" materials="40"><Bone refID="1" transformation="1,0,0,0,1,0,0,0,1,0,5,0"/></Part></Brick>
<Brick refID="2" designID="9999"><Part refID="2" designID="9999" materials="1"><Bone refID="2" transformation="1,0,0,0,1,0,0,0,1,0,0,0"/></Part></Brick>
</Bricks></LXFML>)";
}
TEST(UgcCompression, GzipRoundTrip) {
const std::string data(10000, 'x');
const auto gz = ZCompression::Gzip(data);
ASSERT_GE(gz.size(), 2u);
EXPECT_EQ(static_cast<uint8_t>(gz[0]), 0x1f);
EXPECT_EQ(static_cast<uint8_t>(gz[1]), 0x8b);
EXPECT_EQ(ZCompression::Gunzip(gz), data);
EXPECT_FALSE(ZCompression::Gunzip("not gzip"));
}
TEST(UgcBricks, ParsesGeometryAndRejectsBadData) {
const auto geometry = UgcBricks::ParseGeometry(BoxGeometry(glm::vec3(0.0f), glm::vec3(1.0f)));
ASSERT_TRUE(geometry);
EXPECT_EQ(geometry->positions.size(), 24u);
EXPECT_EQ(geometry->indices.size(), 36u);
EXPECT_FALSE(UgcBricks::ParseGeometry("10GB"));
auto broken = BoxGeometry(glm::vec3(0.0f), glm::vec3(1.0f));
broken.resize(broken.size() - 4);
EXPECT_FALSE(UgcBricks::ParseGeometry(broken));
}
TEST(UgcBricks, ParsesMaterials) {
const auto materials = UgcBricks::ParseMaterials(R"(<Materials><Material MatID="21" Red="222" Green="0" Blue="13" Alpha="255"/><Material MatID="40" Red="238" Green="238" Blue="238" Alpha="150"/></Materials>)");
ASSERT_EQ(materials.size(), 2u);
EXPECT_EQ(materials.at(21).r, 222);
EXPECT_FALSE(materials.at(21).Transparent());
EXPECT_TRUE(materials.at(40).Transparent());
}
TEST(UgcBricks, ReadsStoredZipEntries) {
// A zip with one stored file, "Materials.xml"
const std::string name = "Materials.xml", content = "<Materials/>";
std::string zip;
const auto u16 = [&zip](uint16_t v) { zip.append(reinterpret_cast<const char*>(&v), 2); };
const auto u32 = [&zip](uint32_t v) { zip.append(reinterpret_cast<const char*>(&v), 4); };
u32(0x04034b50); u16(20); u16(0); u16(0); u16(0); u16(0); u32(0); u32(content.size()); u32(content.size()); u16(name.size()); u16(0);
zip += name + content;
const auto central = static_cast<uint32_t>(zip.size());
u32(0x02014b50); u16(20); u16(20); u16(0); u16(0); u16(0); u16(0); u32(0); u32(content.size()); u32(content.size()); u16(name.size());
u16(0); u16(0); u16(0); u16(0); u32(0); u32(0);
zip += name;
const auto centralSize = static_cast<uint32_t>(zip.size()) - central;
u32(0x06054b50); u16(0); u16(0); u16(1); u16(1); u32(centralSize); u32(central); u16(0);
EXPECT_EQ(UgcBricks::ReadZipEntry(zip, "materials.XML"), content);
EXPECT_FALSE(UgcBricks::ReadZipEntry(zip, "Other.xml"));
}
TEST(UgcModel, ParsesLxfml5And4) {
std::string error;
const auto parts = UgcModel::ParseLxfml(LXFML5, error);
ASSERT_EQ(parts.size(), 3u);
EXPECT_EQ(parts[0].designId, 3001u);
EXPECT_EQ(parts[0].materials, (std::vector<uint32_t>{ 21, 21 })); // 0: the part's first material
EXPECT_FLOAT_EQ(parts[0].transform[3].x, 10.0f);
const auto v4 = UgcModel::ParseLxfml(R"(<LXFML versionMajor="4"><Scene><Model><Group ax="0" ay="1" az="0" angle="90" tx="1" ty="0" tz="0">
<Part designID="3001" materialID="21" ax="0" ay="1" az="0" angle="0" tx="0" ty="2" tz="0"/></Group></Model></Scene></LXFML>)", error);
ASSERT_EQ(v4.size(), 1u);
const auto origin = v4[0].transform * glm::vec4(0, 0, 0, 1);
EXPECT_NEAR(origin.x, 1.0f, 1e-5f);
EXPECT_NEAR(origin.y, 2.0f, 1e-5f);
EXPECT_TRUE(UgcModel::ParseLxfml("<nope", error).empty());
EXPECT_FALSE(error.empty());
}
TEST(UgcModel, BuildsOpaqueAndTransparentMeshes) {
UgcBricks::BrickLibrary library(MakeRes(), 0);
library.SetMaterials({ { 21, { 222, 0, 13, 255 } }, { 40, { 238, 238, 238, 150 } } });
std::string error;
const auto model = UgcModel::Build(UgcModel::ParseLxfml(LXFML5, error), library);
EXPECT_EQ(model.bricks, 2u);
EXPECT_EQ(model.missingDesigns, std::vector<uint32_t>{ 9999 });
EXPECT_EQ(model.opaque.TriangleCount(), 12u);
EXPECT_EQ(model.transparent.TriangleCount(), 12u);
EXPECT_NEAR(model.opaque.colors[0].r, 222.0f / 255.0f, 1e-5f);
EXPECT_NEAR(model.transparent.colors[0].a, 150.0f / 255.0f, 1e-5f);
EXPECT_NEAR(model.opaque.positions[0].x, 10.0f, 1e-5f);
}
TEST(UgcModel, SplitsBigMeshes) {
UgcModel::Mesh mesh;
for (uint32_t i = 0; i < 30; i++) {
mesh.positions.push_back(glm::vec3(static_cast<float>(i)));
mesh.normals.push_back(glm::vec3(0, 1, 0));
mesh.colors.push_back(glm::vec4(1.0f));
}
for (uint32_t i = 0; i + 2 < 30; i += 3) mesh.indices.insert(mesh.indices.end(), { i, i + 1, i + 2 });
const auto pieces = UgcModel::Split(mesh, 9, 100);
ASSERT_EQ(pieces.size(), 4u); // 10 triangles, 3 fit per piece
size_t triangles = 0;
for (const auto& piece : pieces) {
EXPECT_LE(piece.positions.size(), 9u);
triangles += piece.TriangleCount();
}
EXPECT_EQ(triangles, 10u);
}
TEST(UgcRender, RemovesWhatIsInsideAndDrawsIcons) {
UgcBricks::BrickLibrary library(MakeRes(), 0);
library.SetMaterials({ { 21, { 222, 0, 13, 255 } } });
std::string error;
// A small box inside the big one: its faces can't be seen
const auto parts = UgcModel::ParseLxfml(R"(<LXFML versionMajor="5"><Bricks>
<Brick><Part designID="3002" materials="21"><Bone transformation="1,0,0,0,1,0,0,0,1,0,0,0"/></Part></Brick>
<Brick><Part designID="3001" materials="21"><Bone transformation="1,0,0,0,1,0,0,0,1,0,0,0"/></Part></Brick>
</Bricks></LXFML>)", error);
auto model = UgcModel::Build(parts, library);
ASSERT_EQ(model.opaque.TriangleCount(), 24u);
const auto result = UgcRender::Optimize(model, UgcRender::OptimizeOptions{ 256, true, true, 0.6f });
EXPECT_EQ(result.trianglesRemoved, 12u);
EXPECT_EQ(model.opaque.TriangleCount(), 12u);
EXPECT_EQ(model.opaque.positions.size(), 8u);
const auto icon = UgcRender::RenderIcon(model, UgcRender::IconOptions{ 32, 2 });
ASSERT_EQ(icon.rgba.size(), 32u * 32u * 4u);
EXPECT_EQ(icon.rgba[3], 0); // a corner is background
EXPECT_EQ(icon.rgba[(16 * 32 + 16) * 4 + 3], 255); // the middle is the box
EXPECT_GT(icon.rgba[(16 * 32 + 16) * 4], icon.rgba[(16 * 32 + 16) * 4 + 1]); // red
EXPECT_EQ(UgcRender::SphereDirections().size(), 42u);
}
TEST(UgcFormats, NifReadsBack) {
UgcModel::Mesh opaque, transparent;
opaque.positions = { { 0, 0, 0 }, { 1, 0, 0 }, { 0, 1, 0 } };
opaque.normals = { { 0, 0, 1 }, { 0, 0, 1 }, { 0, 0, 1 } };
opaque.colors = { { 1, 0, 0, 1 }, { 1, 0, 0, 1 }, { 1, 0, 0, 1 } };
opaque.indices = { 0, 1, 2 };
transparent = opaque;
for (auto& color : transparent.colors) color.a = 0.5f;
const auto nif = UgcFormats::WriteNif("SceneNode_Model", { { "S01_Opaque_Model", &opaque, false }, { "S01_Alpha_Model", &transparent, true } });
ASSERT_TRUE(nif.starts_with("Gamebryo File Format, Version 20.3.0.9\n"));
std::string error;
const auto model = NifFile::Parse(nif, 0, error);
ASSERT_TRUE(model) << error;
EXPECT_TRUE(model->skipped.empty());
ASSERT_EQ(model->meshes.size(), 2u);
EXPECT_EQ(model->meshes[0].indices.size(), 3u);
EXPECT_EQ(model->meshes[0].colors[0], 255);
EXPECT_FALSE(model->meshes[0].material.alphaBlend);
EXPECT_TRUE(model->meshes[1].material.alphaBlend);
EXPECT_EQ(model->meshes[1].colors[3], 128);
EXPECT_EQ(model->meshes[0].material.vertexColorMode, 2);
EXPECT_TRUE(model->nodes.contains("SceneNode_Model"));
}
TEST(UgcFormats, ImagesAndChecksums) {
UgcRender::Image image{ 2, 2, std::vector<uint8_t>(16, 0) };
image.rgba[0] = 10; // red of the first pixel
image.rgba[3] = 255;
const auto png = UgcFormats::EncodePng(image);
EXPECT_TRUE(png.starts_with("\x89PNG\r\n\x1a\n"));
const auto dds = UgcFormats::EncodeDds(image);
ASSERT_EQ(dds.size(), 128u + 16u);
EXPECT_TRUE(dds.starts_with("DDS "));
EXPECT_EQ(dds[128 + 2], 10); // stored BGRA
EXPECT_EQ(UgcFormats::Md5Hex("abc"), "900150983cd24fb0d6963f7d28e17f72");
EXPECT_NE(UgcFormats::ChecksumXml("abc").find("<Checksum><MD5>900150983cd24fb0d6963f7d28e17f72</MD5><Filesize>3</Filesize></Checksum>"), std::string::npos);
}
TEST(UgcModular, ParsesTheCdClientData) {
EXPECT_EQ(UgcModular::ParseModuleLots("1:4713+1:4714+1:4715"), (std::vector<uint32_t>{ 4713, 4714, 4715 }));
EXPECT_EQ(UgcModular::ParseModuleLots("1:8129;1:x;1:8130"), (std::vector<uint32_t>{ 8129, 8130 }));
const auto build = UgcModular::ParseBuild(R"(<ModularBuild><topology><numberOfParts value="3" /><rootPart value="2" />
<connection myPartid="2" myLocation="CP_A1" connectingPart="1" /><connection myPartid="1" myLocation="CP_B2" connectingPart="0"/></topology>
<Placement><AdditionalModelRotation><Rotation w="0.707" x="0" y="-0.707" z="0" /></AdditionalModelRotation></Placement></ModularBuild>)");
ASSERT_TRUE(build);
EXPECT_EQ(build->rootPart, 2u);
ASSERT_EQ(build->connections.size(), 2u);
EXPECT_EQ(build->connections[1].location, "CP_B2");
const auto connections = UgcModular::ParseModuleConnections(R"(<ModuleInfo moduleLOT="4714"><connection name="CP_B2"><translation x="0" y="0" z="5.2" /></connection></ModuleInfo>)");
EXPECT_FLOAT_EQ(connections.at("CP_B2").z, 5.2f);
EXPECT_FALSE(UgcModular::ParseBuild("<ModularBuild/>"));
}
TEST(UgcModular, PutsPartsOnTheirAttachPoints) {
const auto triangle = [] {
NifFile::Model nif;
NifFile::Mesh mesh;
mesh.positions = { 0, 0, 0, 1, 0, 0, 0, 1, 0 };
mesh.indices = { 0, 1, 2 };
nif.meshes.push_back(mesh);
return nif;
};
UgcModular::BuildInfo build;
build.rootPart = 2;
build.connections = { { 2, "CP_A1", 1 }, { 1, "CP_B2", 0 } };
std::vector<UgcModular::Module> modules(3);
modules[0].partCode = 2; // bottom: node CP_A1 at y 5
modules[0].nif = triangle();
modules[0].nif.nodes["CP_A1"].translation = { 0, 5, 0 };
modules[1].partCode = 1; // middle: no node, a connection offset of y 3 in the CDClient
modules[1].nif = triangle();
modules[1].connections["CP_B2"] = glm::vec3(0, 3, 0);
modules[2].partCode = 0; // top: its own CP_B2 node at y 1 lines up with the middle's
modules[2].nif = triangle();
modules[2].nif.nodes["CP_B2"].translation = { 0, 1, 0 };
std::string warnings;
const auto model = UgcModular::Assemble(build, modules, warnings);
EXPECT_TRUE(warnings.empty()) << warnings;
ASSERT_EQ(model.opaque.positions.size(), 9u);
EXPECT_FLOAT_EQ(model.opaque.positions[0].y, 0.0f);
EXPECT_FLOAT_EQ(model.opaque.positions[3].y, 5.0f);
EXPECT_FLOAT_EQ(model.opaque.positions[6].y, 7.0f); // 5 + 3 - 1
}
TEST(UgcStorage, WritesListsAndEvicts) {
UgcStorage storage(TempFolder("storage"));
std::string error;
ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODEL, 1001, { { "icon.png", std::string(100, 'a') } }, error)) << error;
ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODULAR, 2002, { { "icon.png", std::string(100, 'b') } }, error)) << error;
ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODEL, 1001, { { "icon.png", std::string(50, 'c') } }, error)) << error; // replaced
EXPECT_TRUE(storage.File(UgcStorage::Kind::MODEL, 1001, "icon.png"));
EXPECT_FALSE(storage.File(UgcStorage::Kind::MODEL, 1001, "../../etc/passwd"));
EXPECT_EQ(storage.List().size(), 2u);
std::filesystem::last_write_time(storage.Folder(UgcStorage::Kind::MODULAR, 2002), std::filesystem::file_time_type::clock::now() - std::chrono::hours(1));
const auto removed = storage.Evict(60);
ASSERT_EQ(removed.size(), 1u);
EXPECT_EQ(removed[0].id, 2002);
EXPECT_TRUE(storage.File(UgcStorage::Kind::MODEL, 1001, "icon.png"));
std::filesystem::remove_all(storage.GetRoot());
}

View File

@@ -38,7 +38,6 @@ set(DWEBTESTS_SOURCES
"WorkerPoolTests.cpp"
"OnceCacheTests.cpp"
"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/WorkerPool.cpp"
"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/NifFile.cpp"
"BackupFilesTests.cpp"
"SecurityFixesTests.cpp"
"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/BackupFiles.cpp"