mirror of
https://github.com/DarkflameUniverse/DarkflameServer.git
synced 2026-10-12 15:53:44 +00:00
feat: UGC server that makes and serves player models' meshes and icons
A new server (dUgcServer, started by master with enable_ugc_server=1) that takes unprocessed ugc and ugc_modular_build rows from the database and makes what the client downloads with UGCUSE3DSERVICES: an optimized NIF (hidden faces removed, ambient occlusion baked into vertex colors) and a 128px DDS icon for player models, rendered by a software rasterizer from the client's LDD brick primitives, and icons for cars and rockets assembled from their modules per ModularBuildComponent/ModuleComponent. It serves them, with the models' LXFML, over HTTP in the client's UGCC<dc>/3DOPTIMIZED and IMAGE128DDS layout with .gz and .checksum files, and keeps its folder under a size cap. Processing state lives in ugc.is_optimized plus new processed_at, process_attempts and process_error columns (and the same on ugc_modular_build). ServiceType::UGC is appended. NifFile moves to dCommon and records named node transforms for the modules' attach points. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -133,7 +133,7 @@ endif()
|
||||
message(STATUS "Variable: DLU_CONFIG_DIR = ${DLU_CONFIG_DIR}")
|
||||
|
||||
# Copy resource files on first build
|
||||
set(RESOURCE_FILES "sharedconfig.ini" "authconfig.ini" "chatconfig.ini" "worldconfig.ini" "masterconfig.ini" "dashboardconfig.ini" "blocklist.dcf")
|
||||
set(RESOURCE_FILES "sharedconfig.ini" "authconfig.ini" "chatconfig.ini" "worldconfig.ini" "masterconfig.ini" "dashboardconfig.ini" "ugcconfig.ini" "blocklist.dcf")
|
||||
message(STATUS "Checking resource file integrity")
|
||||
|
||||
include(Utils)
|
||||
@@ -330,6 +330,7 @@ add_subdirectory(dWorldServer)
|
||||
add_subdirectory(dAuthServer)
|
||||
add_subdirectory(dChatServer)
|
||||
add_subdirectory(dDashboardServer)
|
||||
add_subdirectory(dUgcServer)
|
||||
add_subdirectory(dMasterServer) # Add MasterServer last so it can rely on the other binaries
|
||||
|
||||
target_precompile_headers(
|
||||
|
||||
@@ -31,6 +31,7 @@ set(DCOMMON_SOURCES
|
||||
"Raw.cpp"
|
||||
"Process.cpp"
|
||||
"Permissions.cpp"
|
||||
"NifFile.cpp"
|
||||
)
|
||||
|
||||
# Workaround for compiler bug where the optimized code could result in a memcpy of 0 bytes, even though that isnt possible.
|
||||
|
||||
@@ -383,9 +383,17 @@ namespace {
|
||||
const auto children = reader.Array<int32_t>(childCount);
|
||||
const auto effectCount = reader.U32();
|
||||
reader.Skip(static_cast<uint64_t>(effectCount) * 4);
|
||||
if (!reader.Ok() || (av.flags & APP_CULLED)) return;
|
||||
if (!reader.Ok()) return;
|
||||
|
||||
const auto world = parent.Then(av.transform);
|
||||
// Recorded even when hidden: attach points often are
|
||||
if (!av.name.empty() && !m_Model.nodes.contains(av.name)) {
|
||||
NifFile::NodeTransform node;
|
||||
for (int i = 0; i < 9; i++) node.rotation[i] = world.r[i] * world.s;
|
||||
node.translation = world.t;
|
||||
m_Model.nodes.emplace(av.name, node);
|
||||
}
|
||||
if (av.flags & APP_CULLED) return;
|
||||
properties = Inherit(properties, av.properties);
|
||||
if (const auto tag = NifFile::ShaderTag(av.name); tag >= 0) properties.shaderTag = tag;
|
||||
|
||||
@@ -72,9 +72,16 @@ namespace NifFile {
|
||||
// What a texture's alpha does under a shader (mapShaders.gameValue); -1 is fixed function (opacity)
|
||||
eTextureAlpha TextureAlphaFor(int32_t shader);
|
||||
|
||||
// Where a node is in the model's space: p' = rotation * p + translation (row-major, scale folded in)
|
||||
struct NodeTransform {
|
||||
std::array<float, 9> rotation{ 1, 0, 0, 0, 1, 0, 0, 0, 1 };
|
||||
std::array<float, 3> translation{};
|
||||
};
|
||||
|
||||
struct Model {
|
||||
uint32_t version{};
|
||||
std::vector<Mesh> meshes;
|
||||
std::map<std::string, NodeTransform> nodes; // named nodes that are drawn (the first of each name), e.g. attach points
|
||||
std::array<float, 3> min{};
|
||||
std::array<float, 3> max{};
|
||||
std::map<std::string, uint32_t> skipped; // block types in the file that aren't drawn, with how many
|
||||
@@ -46,5 +46,55 @@ namespace ZCompression {
|
||||
inflateEnd(&zInfo); // zlib function
|
||||
return(nRet);
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
// Inflate everything in `data` with zlib's `windowBits` (31: gzip, -15: raw); nullopt on any error
|
||||
std::optional<std::string> Inflate(std::string_view data, int windowBits, size_t sizeHint) {
|
||||
z_stream stream{};
|
||||
if (inflateInit2(&stream, windowBits) != Z_OK) return std::nullopt;
|
||||
std::string out;
|
||||
out.reserve(sizeHint);
|
||||
stream.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(data.data()));
|
||||
stream.avail_in = static_cast<uInt>(data.size());
|
||||
char buffer[65536];
|
||||
int result = Z_OK;
|
||||
while (result == Z_OK) {
|
||||
stream.next_out = reinterpret_cast<Bytef*>(buffer);
|
||||
stream.avail_out = sizeof(buffer);
|
||||
result = inflate(&stream, Z_NO_FLUSH);
|
||||
if (result != Z_OK && result != Z_STREAM_END) break;
|
||||
out.append(buffer, sizeof(buffer) - stream.avail_out);
|
||||
if (result == Z_OK && stream.avail_in == 0 && stream.avail_out != 0) break; // truncated
|
||||
}
|
||||
inflateEnd(&stream);
|
||||
if (result != Z_STREAM_END) return std::nullopt;
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
std::string Gzip(std::string_view data) {
|
||||
z_stream stream{};
|
||||
// 31: a gzip header and trailer instead of zlib's
|
||||
if (deflateInit2(&stream, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY) != Z_OK) return {};
|
||||
std::string out;
|
||||
out.resize(deflateBound(&stream, static_cast<uLong>(data.size())) + 32);
|
||||
stream.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(data.data()));
|
||||
stream.avail_in = static_cast<uInt>(data.size());
|
||||
stream.next_out = reinterpret_cast<Bytef*>(out.data());
|
||||
stream.avail_out = static_cast<uInt>(out.size());
|
||||
const auto result = deflate(&stream, Z_FINISH);
|
||||
out.resize(stream.total_out);
|
||||
deflateEnd(&stream);
|
||||
return result == Z_STREAM_END ? out : std::string{};
|
||||
}
|
||||
|
||||
std::optional<std::string> Gunzip(std::string_view data) {
|
||||
return Inflate(data, 31, data.size() * 4);
|
||||
}
|
||||
|
||||
std::optional<std::string> InflateRaw(std::string_view data, size_t size) {
|
||||
auto out = Inflate(data, -15, size);
|
||||
if (!out || out->size() != size) return std::nullopt;
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace ZCompression {
|
||||
uint32_t GetMaxCompressedLength(uint32_t nLenSrc);
|
||||
@@ -8,5 +11,14 @@ namespace ZCompression {
|
||||
int32_t Compress(const uint8_t* abSrc, uint32_t nLenSrc, uint8_t* abDst, uint32_t nLenDst);
|
||||
|
||||
int32_t Decompress(const uint8_t* abSrc, uint32_t nLenSrc, uint8_t* abDst, uint32_t nLenDst, int32_t& nErr);
|
||||
|
||||
// `data` as a gzip file (RFC 1952), what HTTP clients and the game client's UGC downloads inflate
|
||||
std::string Gzip(std::string_view data);
|
||||
|
||||
// A gzip file's contents; nullopt when it isn't one or is damaged
|
||||
std::optional<std::string> Gunzip(std::string_view data);
|
||||
|
||||
// Raw deflate data (no zlib or gzip header, as in zip files) that inflates to `size` bytes; nullopt when it doesn't
|
||||
std::optional<std::string> InflateRaw(std::string_view data, size_t size);
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,8 @@ enum class ServiceType : uint16_t {
|
||||
WORLD,
|
||||
CLIENT,
|
||||
MASTER,
|
||||
UNKNOWN
|
||||
UNKNOWN,
|
||||
UGC // the UGC server (dUgcServer): appended, the values above are on the wire
|
||||
};
|
||||
|
||||
#endif //!__SERVICETYPE__H__
|
||||
|
||||
@@ -40,7 +40,6 @@ set(DASHBOARDROUTES_SOURCES
|
||||
"ChallengeRoutes.cpp"
|
||||
"InstanceLoad.cpp"
|
||||
"WorldView.cpp"
|
||||
"NifFile.cpp"
|
||||
"Scenery.cpp"
|
||||
"WorkerPool.cpp"
|
||||
"Workers.cpp"
|
||||
|
||||
@@ -16,6 +16,7 @@ namespace {
|
||||
const std::string CHAT = "chatconfig.ini";
|
||||
const std::string WORLD = "worldconfig.ini";
|
||||
const std::string DASHBOARD = "dashboardconfig.ini";
|
||||
const std::string UGC = "ugcconfig.ini";
|
||||
|
||||
constexpr double PORT_MIN = 1;
|
||||
constexpr double PORT_MAX = 65535;
|
||||
@@ -127,6 +128,7 @@ namespace {
|
||||
c.AddSection("Startup");
|
||||
c.Add(Bool(MASTER, "prestart_servers", "Start auth, chat and char servers", "Master starts the other servers itself.", true, true));
|
||||
c.Add(Bool(MASTER, "enable_dashboard", "Start the web dashboard", "", false, true));
|
||||
c.Add(Bool(MASTER, "enable_ugc_server", "Start the UGC server", "Makes and serves the meshes and icons of what players build (docs/UgcServer.md).", false, true));
|
||||
c.Add(Bool(SHARED, "skip_account_creation", "Skip the first-account prompt", "For non-interactive setups: master doesn't ask for an account when there are none.", false, true));
|
||||
|
||||
c.AddSection("Game client");
|
||||
@@ -400,6 +402,30 @@ namespace {
|
||||
c.AddSection("Character history");
|
||||
c.Add(Days("snapshot_days", "Snapshots", "Older snapshots are deleted, but each character keeps its newest few (below).", "90", 1));
|
||||
c.Add(Unit(Int(DASHBOARD, "snapshot_keep", "Always keep per character", "", "10", 1, 1000), "snapshots"));
|
||||
c.AddSection("UGC server", "Makes the meshes and icons of what players build and serves them to the game client. See docs/UgcServer.md.",
|
||||
eLayout::ROWS, Condition{ MASTER, "enable_ugc_server", { "1" } });
|
||||
c.Add(Port(UGC, "port", "Download port", "The game client's UGCSERVERPORT.", "2008"));
|
||||
c.Add(Format(Text(UGC, "listen_ip", "Listen address", "The game client has to reach it.", "0.0.0.0", true), eFormat::HOST));
|
||||
c.Add(Port(UGC, "net_port", "Master connection port", "UDP; the next port is used too.", "2012"));
|
||||
c.Add(Text(UGC, "client_path", "Download path", "The game client's UGCSERVERDIR.", "/ugc", true));
|
||||
c.Add(Format(Text(UGC, "ugc_output_dir", "Files folder", "Relative to the server binaries.", "ugc", true), eFormat::PATH));
|
||||
c.Add(Unit(Int(UGC, "ugc_max_storage_mb", "Most space for files", "The files used longest ago are deleted past this and made again when asked for. 0: no limit.", "2048", 0, std::nullopt, true), "MB"));
|
||||
c.Add(Int(UGC, "worker_threads", "Worker threads", "0: half the CPU cores.", "0", 0, 64, true));
|
||||
c.Add(Unit(Int(UGC, "poll_interval_ms", "Look for new models every", "", "2000", 100, 600000, true), "ms"));
|
||||
c.Add(Int(UGC, "poll_batch", "Models taken at once", "", "32", 1, 1000, true));
|
||||
c.Add(Int(UGC, "max_attempts", "Attempts before giving up", "", "3", 1, 100, true));
|
||||
c.Add(Int(UGC, "brick_lod", "Brick detail", "0 is the most detailed, 2 the least.", "0", 0, 2, true));
|
||||
c.Add(Bool(UGC, "remove_hidden_faces", "Remove faces nobody can see", "", true, true));
|
||||
c.Add(Bool(UGC, "bake_ao", "Darken hidden corners", "Ambient occlusion baked into the vertex colors.", true, true));
|
||||
c.Add(Float(UGC, "ao_strength", "Darkening strength", "0 to 1.", "0.6", 0, 1));
|
||||
c.Add(Unit(Int(UGC, "optimize_resolution", "Detail of the visibility renders", "", "1024", 64, 4096, true), "pixels"));
|
||||
c.Add(Unit(Int(UGC, "icon_size", "Icon size", "", "128", 16, 1024, true), "pixels"));
|
||||
c.Add(Unit(Float(UGC, "icon_yaw", "Icon angle around", "", "35", -360, 360), "degrees"));
|
||||
c.Add(Unit(Float(UGC, "icon_pitch", "Icon angle above", "", "25", -90, 90), "degrees"));
|
||||
c.Add(Float(UGC, "icon_margin", "Icon border", "1 fills the icon.", "1.03", 0.1, 10));
|
||||
c.Add(Unit(Float(UGC, "modular_icon_yaw", "Car and rocket icon angle around", "", "35", -360, 360), "degrees"));
|
||||
c.Add(Unit(Float(UGC, "modular_icon_pitch", "Car and rocket icon angle above", "", "20", -90, 90), "degrees"));
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
@@ -456,7 +482,7 @@ namespace SettingsCatalog {
|
||||
|
||||
const std::vector<std::pair<std::string, std::string>>& Files() {
|
||||
static const std::vector<std::pair<std::string, std::string>> files{
|
||||
{ SHARED, "Shared" }, { WORLD, "World" }, { AUTH, "Auth" }, { CHAT, "Chat" }, { MASTER, "Master" }, { DASHBOARD, "Dashboard" }
|
||||
{ SHARED, "Shared" }, { WORLD, "World" }, { AUTH, "Auth" }, { CHAT, "Chat" }, { MASTER, "Master" }, { DASHBOARD, "Dashboard" }, { UGC, "UGC" }
|
||||
};
|
||||
return files;
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
class IUgc {
|
||||
public:
|
||||
@@ -31,5 +32,53 @@ public:
|
||||
virtual void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) = 0;
|
||||
|
||||
virtual std::optional<IUgc::Model> GetUgcModel(const LWOOBJID ugcId) = 0;
|
||||
|
||||
// ---- Processing by the UGC server (is_optimized, processed_at, process_attempts, process_error) ----
|
||||
|
||||
// is_optimized's values
|
||||
enum class eProcessState : int32_t {
|
||||
PENDING = 0, // not made yet (or to be made again)
|
||||
DONE = 1, // the UGC server's files are made
|
||||
FAILED = 2, // gave up after the allowed attempts
|
||||
};
|
||||
|
||||
// A row's processing state, for the UGC server and the dashboard
|
||||
struct ProcessInfo {
|
||||
LWOOBJID id{};
|
||||
LWOOBJID characterId{};
|
||||
std::string characterName; // empty when the character is gone
|
||||
eProcessState state{};
|
||||
uint32_t attempts{};
|
||||
int64_t processedAt{}; // Unix seconds of the last attempt, 0 for none
|
||||
std::string error;
|
||||
bool bakeAo{};
|
||||
std::string details; // modular builds: the modules (ldf_config)
|
||||
};
|
||||
|
||||
// A model waiting to be made: its id, stored LXFML (sd0) and the attempts so far
|
||||
struct PendingModel {
|
||||
LWOOBJID id{};
|
||||
std::string lxfml;
|
||||
uint32_t attempts{};
|
||||
};
|
||||
|
||||
// Up to `limit` pending models, the least tried and then the newest first
|
||||
virtual std::vector<PendingModel> GetUgcModelsToProcess(const uint32_t limit) = 0;
|
||||
|
||||
// Records an attempt: the new state, how many attempts there have been, why it failed (empty when it didn't) and
|
||||
// whether lighting was baked in; processed_at becomes now
|
||||
virtual void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) = 0;
|
||||
|
||||
virtual std::optional<ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) = 0;
|
||||
|
||||
// Sets models back to pending with no attempts: one (`id`), or all of them (`id` nullopt; only the failed ones
|
||||
// with `failedOnly`). Returns how many rows changed.
|
||||
virtual uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) = 0;
|
||||
|
||||
// A page of models (all, or those in `state`), the newest first
|
||||
virtual std::vector<ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const uint32_t offset, const uint32_t limit) = 0;
|
||||
|
||||
// How many models are in each state
|
||||
virtual std::vector<std::pair<eProcessState, uint64_t>> GetUgcProcessCounts() = 0;
|
||||
};
|
||||
#endif //!__IUGC__H__
|
||||
|
||||
@@ -4,11 +4,30 @@
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "IUgc.h"
|
||||
|
||||
class IUgcModularBuild {
|
||||
public:
|
||||
virtual void InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional<LWOOBJID> characterId) = 0;
|
||||
virtual void DeleteUgcBuild(const LWOOBJID bigId) = 0;
|
||||
|
||||
// ---- Icons made by the UGC server; the same columns and meanings as IUgc's processing ----
|
||||
|
||||
// A modular build waiting for its icon: its id, modules (ldf_config) and the attempts so far
|
||||
struct PendingBuild {
|
||||
LWOOBJID id{};
|
||||
std::string modules;
|
||||
uint32_t attempts{};
|
||||
};
|
||||
|
||||
virtual std::vector<PendingBuild> GetModularBuildsToProcess(const uint32_t limit) = 0;
|
||||
virtual void SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) = 0;
|
||||
virtual std::optional<IUgc::ProcessInfo> GetModularBuildProcessInfo(const LWOOBJID id) = 0;
|
||||
virtual uint64_t ResetModularBuildProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) = 0;
|
||||
virtual std::vector<IUgc::ProcessInfo> GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const uint32_t offset, const uint32_t limit) = 0;
|
||||
virtual std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetModularBuildProcessCounts() = 0;
|
||||
};
|
||||
|
||||
#endif //!IUGCMODULARBUILD_H
|
||||
|
||||
@@ -393,6 +393,18 @@ public:
|
||||
nlohmann::json GetCharacterById(const LWOOBJID charId) override;
|
||||
std::optional<IPropertyContents::Model> GetModel(const LWOOBJID modelID) override;
|
||||
std::optional<IUgc::Model> GetUgcModel(const LWOOBJID ugcId) override;
|
||||
std::vector<IUgc::PendingModel> GetUgcModelsToProcess(const uint32_t limit) override;
|
||||
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override;
|
||||
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override;
|
||||
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
|
||||
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const uint32_t offset, const uint32_t limit) override;
|
||||
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetUgcProcessCounts() override;
|
||||
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsToProcess(const uint32_t limit) override;
|
||||
void SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) override;
|
||||
std::optional<IUgc::ProcessInfo> GetModularBuildProcessInfo(const LWOOBJID id) override;
|
||||
uint64_t ResetModularBuildProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
|
||||
std::vector<IUgc::ProcessInfo> GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const uint32_t offset, const uint32_t limit) override;
|
||||
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetModularBuildProcessCounts() override;
|
||||
std::optional<IProperty::Info> GetPropertyInfo(const LWOOBJID id) override;
|
||||
std::string GetPropertiesTable(uint32_t start, uint32_t length, const std::string_view search = "", uint32_t orderColumn = 0, bool orderAsc = true, bool pendingOnly = false) override;
|
||||
uint32_t GetPropertyCount() override;
|
||||
|
||||
@@ -1,5 +1,27 @@
|
||||
#include "MySQLDatabase.h"
|
||||
|
||||
#include <chrono>
|
||||
|
||||
namespace {
|
||||
IUgc::ProcessInfo ReadUgcProcessInfo(PreparedStmtResultSet& result, bool modular) {
|
||||
IUgc::ProcessInfo info;
|
||||
info.id = result->getInt64("id");
|
||||
info.characterId = result->getInt64("character_id");
|
||||
info.characterName = std::string(result->getString("character_name").c_str());
|
||||
info.state = static_cast<IUgc::eProcessState>(result->getInt("is_optimized"));
|
||||
info.attempts = static_cast<uint32_t>(result->getInt("process_attempts"));
|
||||
info.processedAt = result->getInt64("processed_at");
|
||||
info.error = std::string(result->getString("process_error").c_str());
|
||||
if (modular) info.details = std::string(result->getString("ldf_config").c_str());
|
||||
else info.bakeAo = result->getInt("bake_ao") != 0;
|
||||
return info;
|
||||
}
|
||||
|
||||
int64_t UnixNow() {
|
||||
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
}
|
||||
|
||||
IUgc::Model ReadModel(PreparedStmtResultSet& result) {
|
||||
IUgc::Model model;
|
||||
|
||||
@@ -67,7 +89,8 @@ void MySQLDatabase::DeleteUgcModelData(const LWOOBJID& modelId) {
|
||||
|
||||
void MySQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) {
|
||||
const std::istream stream(lxfml.rdbuf());
|
||||
ExecuteUpdate("UPDATE ugc SET lxfml = ? WHERE id = ?;", &stream, modelId);
|
||||
// The UGC server makes the model's files again
|
||||
ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", &stream, modelId);
|
||||
}
|
||||
|
||||
std::optional<IUgc::Model> MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) {
|
||||
@@ -81,3 +104,66 @@ std::optional<IUgc::Model> MySQLDatabase::GetUgcModel(const LWOOBJID ugcId) {
|
||||
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
std::vector<IUgc::PendingModel> MySQLDatabase::GetUgcModelsToProcess(const uint32_t limit) {
|
||||
auto result = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 ORDER BY process_attempts ASC, id DESC LIMIT ?;", limit);
|
||||
std::vector<IUgc::PendingModel> models;
|
||||
while (result->next()) {
|
||||
auto& model = models.emplace_back();
|
||||
model.id = result->getInt64("id");
|
||||
std::unique_ptr<std::istream> blob(result->getBlob("lxfml"));
|
||||
std::stringstream contents;
|
||||
contents << blob->rdbuf();
|
||||
model.lxfml = contents.str();
|
||||
model.attempts = static_cast<uint32_t>(result->getInt("process_attempts"));
|
||||
}
|
||||
return models;
|
||||
}
|
||||
|
||||
void MySQLDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) {
|
||||
ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ? WHERE id = ?;",
|
||||
static_cast<int32_t>(state), attempts, error, bakeAo, UnixNow(), id);
|
||||
}
|
||||
|
||||
std::optional<IUgc::ProcessInfo> MySQLDatabase::GetUgcProcessInfo(const LWOOBJID id) {
|
||||
auto result = ExecuteSelect(
|
||||
"SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao "
|
||||
"FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id WHERE u.id = ? LIMIT 1;", id);
|
||||
if (!result->next()) return std::nullopt;
|
||||
return ReadUgcProcessInfo(result, false);
|
||||
}
|
||||
|
||||
uint64_t MySQLDatabase::ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) {
|
||||
if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", *id);
|
||||
if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;");
|
||||
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '';");
|
||||
}
|
||||
|
||||
std::vector<IUgc::ProcessInfo> MySQLDatabase::GetUgcProcessList(const std::optional<eProcessState> state, const uint32_t offset, const uint32_t limit) {
|
||||
const std::string select =
|
||||
"SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao "
|
||||
"FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id ";
|
||||
std::vector<IUgc::ProcessInfo> list;
|
||||
auto read = [&list](PreparedStmtResultSet& result) {
|
||||
while (result->next()) {
|
||||
list.push_back(ReadUgcProcessInfo(result, false));
|
||||
}
|
||||
};
|
||||
if (state) {
|
||||
auto result = ExecuteSelect(select + "WHERE u.is_optimized = ? ORDER BY u.id DESC LIMIT ? OFFSET ?;", static_cast<int32_t>(*state), limit, offset);
|
||||
read(result);
|
||||
} else {
|
||||
auto result = ExecuteSelect(select + "ORDER BY u.id DESC LIMIT ? OFFSET ?;", limit, offset);
|
||||
read(result);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
std::vector<std::pair<IUgc::eProcessState, uint64_t>> MySQLDatabase::GetUgcProcessCounts() {
|
||||
auto result = ExecuteSelect("SELECT is_optimized, COUNT(*) AS count FROM ugc GROUP BY is_optimized;");
|
||||
std::vector<std::pair<eProcessState, uint64_t>> counts;
|
||||
while (result->next()) {
|
||||
counts.emplace_back(static_cast<eProcessState>(result->getInt("is_optimized")), static_cast<uint64_t>(result->getInt64("count")));
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,27 @@
|
||||
#include "MySQLDatabase.h"
|
||||
|
||||
#include <chrono>
|
||||
|
||||
namespace {
|
||||
IUgc::ProcessInfo ReadModularProcessInfo(PreparedStmtResultSet& result, bool modular) {
|
||||
IUgc::ProcessInfo info;
|
||||
info.id = result->getInt64("id");
|
||||
info.characterId = result->getInt64("character_id");
|
||||
info.characterName = std::string(result->getString("character_name").c_str());
|
||||
info.state = static_cast<IUgc::eProcessState>(result->getInt("is_optimized"));
|
||||
info.attempts = static_cast<uint32_t>(result->getInt("process_attempts"));
|
||||
info.processedAt = result->getInt64("processed_at");
|
||||
info.error = std::string(result->getString("process_error").c_str());
|
||||
if (modular) info.details = std::string(result->getString("ldf_config").c_str());
|
||||
else info.bakeAo = result->getInt("bake_ao") != 0;
|
||||
return info;
|
||||
}
|
||||
|
||||
int64_t UnixNow() {
|
||||
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
}
|
||||
|
||||
void MySQLDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional<LWOOBJID> characterId) {
|
||||
ExecuteInsert("INSERT INTO ugc_modular_build (ugc_id, ldf_config, character_id) VALUES (?,?,?)", bigId, modules, characterId);
|
||||
}
|
||||
@@ -7,3 +29,61 @@ void MySQLDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID bi
|
||||
void MySQLDatabase::DeleteUgcBuild(const LWOOBJID bigId) {
|
||||
ExecuteDelete("DELETE FROM ugc_modular_build WHERE ugc_id = ?;", bigId);
|
||||
}
|
||||
|
||||
std::vector<IUgcModularBuild::PendingBuild> MySQLDatabase::GetModularBuildsToProcess(const uint32_t limit) {
|
||||
auto result = ExecuteSelect("SELECT ugc_id, ldf_config, process_attempts FROM ugc_modular_build WHERE is_optimized = 0 ORDER BY process_attempts ASC, ugc_id DESC LIMIT ?;", limit);
|
||||
std::vector<PendingBuild> builds;
|
||||
while (result->next()) {
|
||||
builds.push_back({ result->getInt64("ugc_id"), std::string(result->getString("ldf_config").c_str()), static_cast<uint32_t>(result->getInt("process_attempts")) });
|
||||
}
|
||||
return builds;
|
||||
}
|
||||
|
||||
void MySQLDatabase::SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) {
|
||||
ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = ?, process_attempts = ?, process_error = ?, processed_at = ? WHERE ugc_id = ?;",
|
||||
static_cast<int32_t>(state), attempts, error, UnixNow(), id);
|
||||
}
|
||||
|
||||
namespace {
|
||||
const std::string MODULAR_SELECT =
|
||||
"SELECT b.ugc_id AS id, b.character_id, c.name AS character_name, b.is_optimized, b.process_attempts, b.processed_at, b.process_error, b.ldf_config "
|
||||
"FROM ugc_modular_build AS b LEFT JOIN charinfo AS c ON c.id = b.character_id ";
|
||||
}
|
||||
|
||||
std::optional<IUgc::ProcessInfo> MySQLDatabase::GetModularBuildProcessInfo(const LWOOBJID id) {
|
||||
auto result = ExecuteSelect(MODULAR_SELECT + "WHERE b.ugc_id = ? LIMIT 1;", id);
|
||||
if (!result->next()) return std::nullopt;
|
||||
return ReadModularProcessInfo(result, true);
|
||||
}
|
||||
|
||||
uint64_t MySQLDatabase::ResetModularBuildProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) {
|
||||
if (id) return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE ugc_id = ?;", *id);
|
||||
if (failedOnly) return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;");
|
||||
return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '';");
|
||||
}
|
||||
|
||||
std::vector<IUgc::ProcessInfo> MySQLDatabase::GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const uint32_t offset, const uint32_t limit) {
|
||||
std::vector<IUgc::ProcessInfo> list;
|
||||
auto read = [&list](PreparedStmtResultSet& result) {
|
||||
while (result->next()) {
|
||||
list.push_back(ReadModularProcessInfo(result, true));
|
||||
}
|
||||
};
|
||||
if (state) {
|
||||
auto result = ExecuteSelect(MODULAR_SELECT + "WHERE b.is_optimized = ? ORDER BY b.ugc_id DESC LIMIT ? OFFSET ?;", static_cast<int32_t>(*state), limit, offset);
|
||||
read(result);
|
||||
} else {
|
||||
auto result = ExecuteSelect(MODULAR_SELECT + "ORDER BY b.ugc_id DESC LIMIT ? OFFSET ?;", limit, offset);
|
||||
read(result);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
std::vector<std::pair<IUgc::eProcessState, uint64_t>> MySQLDatabase::GetModularBuildProcessCounts() {
|
||||
auto result = ExecuteSelect("SELECT is_optimized, COUNT(*) AS count FROM ugc_modular_build GROUP BY is_optimized;");
|
||||
std::vector<std::pair<IUgc::eProcessState, uint64_t>> counts;
|
||||
while (result->next()) {
|
||||
counts.emplace_back(static_cast<IUgc::eProcessState>(result->getInt("is_optimized")), static_cast<uint64_t>(result->getInt64("count")));
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
@@ -377,6 +377,18 @@ public:
|
||||
nlohmann::json GetCharacterById(const LWOOBJID charId) override;
|
||||
std::optional<IPropertyContents::Model> GetModel(const LWOOBJID modelID) override;
|
||||
std::optional<IUgc::Model> GetUgcModel(const LWOOBJID ugcId) override;
|
||||
std::vector<IUgc::PendingModel> GetUgcModelsToProcess(const uint32_t limit) override;
|
||||
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override;
|
||||
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override;
|
||||
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
|
||||
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const uint32_t offset, const uint32_t limit) override;
|
||||
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetUgcProcessCounts() override;
|
||||
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsToProcess(const uint32_t limit) override;
|
||||
void SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) override;
|
||||
std::optional<IUgc::ProcessInfo> GetModularBuildProcessInfo(const LWOOBJID id) override;
|
||||
uint64_t ResetModularBuildProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override;
|
||||
std::vector<IUgc::ProcessInfo> GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const uint32_t offset, const uint32_t limit) override;
|
||||
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetModularBuildProcessCounts() override;
|
||||
std::optional<IProperty::Info> GetPropertyInfo(const LWOOBJID id) override;
|
||||
std::string GetPropertiesTable(uint32_t start, uint32_t length, const std::string_view search = "", uint32_t orderColumn = 0, bool orderAsc = true, bool pendingOnly = false) override;
|
||||
uint32_t GetPropertyCount() override;
|
||||
|
||||
@@ -1,5 +1,27 @@
|
||||
#include "SQLiteDatabase.h"
|
||||
|
||||
#include <chrono>
|
||||
|
||||
namespace {
|
||||
IUgc::ProcessInfo ReadUgcProcessInfo(CppSQLite3Query& result, bool modular) {
|
||||
IUgc::ProcessInfo info;
|
||||
info.id = result.getInt64Field("id");
|
||||
info.characterId = result.getInt64Field("character_id");
|
||||
info.characterName = result.getStringField("character_name", "");
|
||||
info.state = static_cast<IUgc::eProcessState>(result.getIntField("is_optimized"));
|
||||
info.attempts = static_cast<uint32_t>(result.getIntField("process_attempts"));
|
||||
info.processedAt = result.getInt64Field("processed_at");
|
||||
info.error = result.getStringField("process_error", "");
|
||||
if (modular) info.details = result.getStringField("ldf_config", "");
|
||||
else info.bakeAo = result.getIntField("bake_ao") != 0;
|
||||
return info;
|
||||
}
|
||||
|
||||
int64_t UnixNow() {
|
||||
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
}
|
||||
|
||||
IUgc::Model ReadModel(CppSQLite3Query& result) {
|
||||
IUgc::Model model;
|
||||
|
||||
@@ -68,7 +90,8 @@ void SQLiteDatabase::DeleteUgcModelData(const LWOOBJID& modelId) {
|
||||
|
||||
void SQLiteDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) {
|
||||
const std::istream stream(lxfml.rdbuf());
|
||||
ExecuteUpdate("UPDATE ugc SET lxfml = ? WHERE id = ?;", &stream, modelId);
|
||||
// The UGC server makes the model's files again
|
||||
ExecuteUpdate("UPDATE ugc SET lxfml = ?, is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", &stream, modelId);
|
||||
}
|
||||
|
||||
std::optional<IUgc::Model> SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) {
|
||||
@@ -82,3 +105,68 @@ std::optional<IUgc::Model> SQLiteDatabase::GetUgcModel(const LWOOBJID ugcId) {
|
||||
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
std::vector<IUgc::PendingModel> SQLiteDatabase::GetUgcModelsToProcess(const uint32_t limit) {
|
||||
auto [_, result] = ExecuteSelect("SELECT id, lxfml, process_attempts FROM ugc WHERE is_optimized = 0 ORDER BY process_attempts ASC, id DESC LIMIT ?;", limit);
|
||||
std::vector<IUgc::PendingModel> models;
|
||||
while (!result.eof()) {
|
||||
auto& model = models.emplace_back();
|
||||
model.id = result.getInt64Field("id");
|
||||
int blobSize{};
|
||||
const auto* blob = result.getBlobField("lxfml", blobSize);
|
||||
model.lxfml.assign(reinterpret_cast<const char*>(blob), blobSize);
|
||||
model.attempts = static_cast<uint32_t>(result.getIntField("process_attempts"));
|
||||
result.nextRow();
|
||||
}
|
||||
return models;
|
||||
}
|
||||
|
||||
void SQLiteDatabase::SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) {
|
||||
ExecuteUpdate("UPDATE ugc SET is_optimized = ?, process_attempts = ?, process_error = ?, bake_ao = ?, processed_at = ? WHERE id = ?;",
|
||||
static_cast<int32_t>(state), attempts, error, bakeAo, UnixNow(), id);
|
||||
}
|
||||
|
||||
std::optional<IUgc::ProcessInfo> SQLiteDatabase::GetUgcProcessInfo(const LWOOBJID id) {
|
||||
auto [_, result] = ExecuteSelect(
|
||||
"SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao "
|
||||
"FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id WHERE u.id = ? LIMIT 1;", id);
|
||||
if (result.eof()) return std::nullopt;
|
||||
return ReadUgcProcessInfo(result, false);
|
||||
}
|
||||
|
||||
uint64_t SQLiteDatabase::ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) {
|
||||
if (id) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE id = ?;", *id);
|
||||
if (failedOnly) return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;");
|
||||
return ExecuteUpdate("UPDATE ugc SET is_optimized = 0, process_attempts = 0, process_error = '';");
|
||||
}
|
||||
|
||||
std::vector<IUgc::ProcessInfo> SQLiteDatabase::GetUgcProcessList(const std::optional<eProcessState> state, const uint32_t offset, const uint32_t limit) {
|
||||
const std::string select =
|
||||
"SELECT u.id, u.character_id, c.name AS character_name, u.is_optimized, u.process_attempts, u.processed_at, u.process_error, u.bake_ao "
|
||||
"FROM ugc AS u LEFT JOIN charinfo AS c ON c.id = u.character_id ";
|
||||
std::vector<IUgc::ProcessInfo> list;
|
||||
auto read = [&list](CppSQLite3Query& result) {
|
||||
while (!result.eof()) {
|
||||
list.push_back(ReadUgcProcessInfo(result, false));
|
||||
result.nextRow();
|
||||
}
|
||||
};
|
||||
if (state) {
|
||||
auto [_, result] = ExecuteSelect(select + "WHERE u.is_optimized = ? ORDER BY u.id DESC LIMIT ? OFFSET ?;", static_cast<int32_t>(*state), limit, offset);
|
||||
read(result);
|
||||
} else {
|
||||
auto [_, result] = ExecuteSelect(select + "ORDER BY u.id DESC LIMIT ? OFFSET ?;", limit, offset);
|
||||
read(result);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
std::vector<std::pair<IUgc::eProcessState, uint64_t>> SQLiteDatabase::GetUgcProcessCounts() {
|
||||
auto [_, result] = ExecuteSelect("SELECT is_optimized, COUNT(*) AS count FROM ugc GROUP BY is_optimized;");
|
||||
std::vector<std::pair<eProcessState, uint64_t>> counts;
|
||||
while (!result.eof()) {
|
||||
counts.emplace_back(static_cast<eProcessState>(result.getIntField("is_optimized")), static_cast<uint64_t>(result.getInt64Field("count")));
|
||||
result.nextRow();
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,27 @@
|
||||
#include "SQLiteDatabase.h"
|
||||
|
||||
#include <chrono>
|
||||
|
||||
namespace {
|
||||
IUgc::ProcessInfo ReadModularProcessInfo(CppSQLite3Query& result, bool modular) {
|
||||
IUgc::ProcessInfo info;
|
||||
info.id = result.getInt64Field("id");
|
||||
info.characterId = result.getInt64Field("character_id");
|
||||
info.characterName = result.getStringField("character_name", "");
|
||||
info.state = static_cast<IUgc::eProcessState>(result.getIntField("is_optimized"));
|
||||
info.attempts = static_cast<uint32_t>(result.getIntField("process_attempts"));
|
||||
info.processedAt = result.getInt64Field("processed_at");
|
||||
info.error = result.getStringField("process_error", "");
|
||||
if (modular) info.details = result.getStringField("ldf_config", "");
|
||||
else info.bakeAo = result.getIntField("bake_ao") != 0;
|
||||
return info;
|
||||
}
|
||||
|
||||
int64_t UnixNow() {
|
||||
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
}
|
||||
|
||||
void SQLiteDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional<LWOOBJID> characterId) {
|
||||
ExecuteInsert("INSERT INTO ugc_modular_build (ugc_id, ldf_config, character_id) VALUES (?,?,?)", bigId, modules, characterId);
|
||||
}
|
||||
@@ -7,3 +29,64 @@ void SQLiteDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID b
|
||||
void SQLiteDatabase::DeleteUgcBuild(const LWOOBJID bigId) {
|
||||
ExecuteDelete("DELETE FROM ugc_modular_build WHERE ugc_id = ?;", bigId);
|
||||
}
|
||||
|
||||
std::vector<IUgcModularBuild::PendingBuild> SQLiteDatabase::GetModularBuildsToProcess(const uint32_t limit) {
|
||||
auto [_, result] = ExecuteSelect("SELECT ugc_id, ldf_config, process_attempts FROM ugc_modular_build WHERE is_optimized = 0 ORDER BY process_attempts ASC, ugc_id DESC LIMIT ?;", limit);
|
||||
std::vector<PendingBuild> builds;
|
||||
while (!result.eof()) {
|
||||
builds.push_back({ result.getInt64Field("ugc_id"), result.getStringField("ldf_config", ""), static_cast<uint32_t>(result.getIntField("process_attempts")) });
|
||||
result.nextRow();
|
||||
}
|
||||
return builds;
|
||||
}
|
||||
|
||||
void SQLiteDatabase::SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) {
|
||||
ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = ?, process_attempts = ?, process_error = ?, processed_at = ? WHERE ugc_id = ?;",
|
||||
static_cast<int32_t>(state), attempts, error, UnixNow(), id);
|
||||
}
|
||||
|
||||
namespace {
|
||||
const std::string MODULAR_SELECT =
|
||||
"SELECT b.ugc_id AS id, b.character_id, c.name AS character_name, b.is_optimized, b.process_attempts, b.processed_at, b.process_error, b.ldf_config "
|
||||
"FROM ugc_modular_build AS b LEFT JOIN charinfo AS c ON c.id = b.character_id ";
|
||||
}
|
||||
|
||||
std::optional<IUgc::ProcessInfo> SQLiteDatabase::GetModularBuildProcessInfo(const LWOOBJID id) {
|
||||
auto [_, result] = ExecuteSelect(MODULAR_SELECT + "WHERE b.ugc_id = ? LIMIT 1;", id);
|
||||
if (result.eof()) return std::nullopt;
|
||||
return ReadModularProcessInfo(result, true);
|
||||
}
|
||||
|
||||
uint64_t SQLiteDatabase::ResetModularBuildProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) {
|
||||
if (id) return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE ugc_id = ?;", *id);
|
||||
if (failedOnly) return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '' WHERE is_optimized = 2;");
|
||||
return ExecuteUpdate("UPDATE ugc_modular_build SET is_optimized = 0, process_attempts = 0, process_error = '';");
|
||||
}
|
||||
|
||||
std::vector<IUgc::ProcessInfo> SQLiteDatabase::GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const uint32_t offset, const uint32_t limit) {
|
||||
std::vector<IUgc::ProcessInfo> list;
|
||||
auto read = [&list](CppSQLite3Query& result) {
|
||||
while (!result.eof()) {
|
||||
list.push_back(ReadModularProcessInfo(result, true));
|
||||
result.nextRow();
|
||||
}
|
||||
};
|
||||
if (state) {
|
||||
auto [_, result] = ExecuteSelect(MODULAR_SELECT + "WHERE b.is_optimized = ? ORDER BY b.ugc_id DESC LIMIT ? OFFSET ?;", static_cast<int32_t>(*state), limit, offset);
|
||||
read(result);
|
||||
} else {
|
||||
auto [_, result] = ExecuteSelect(MODULAR_SELECT + "ORDER BY b.ugc_id DESC LIMIT ? OFFSET ?;", limit, offset);
|
||||
read(result);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
std::vector<std::pair<IUgc::eProcessState, uint64_t>> SQLiteDatabase::GetModularBuildProcessCounts() {
|
||||
auto [_, result] = ExecuteSelect("SELECT is_optimized, COUNT(*) AS count FROM ugc_modular_build GROUP BY is_optimized;");
|
||||
std::vector<std::pair<IUgc::eProcessState, uint64_t>> counts;
|
||||
while (!result.eof()) {
|
||||
counts.emplace_back(static_cast<IUgc::eProcessState>(result.getIntField("is_optimized")), static_cast<uint64_t>(result.getInt64Field("count")));
|
||||
result.nextRow();
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
@@ -372,6 +372,18 @@ class TestSQLDatabase : public GameDatabase {
|
||||
std::optional<IPropertyContents::Model> GetModel(const LWOOBJID modelID) override { return {}; }
|
||||
std::optional<IProperty::Info> GetPropertyInfo(const LWOOBJID id) override { return {}; }
|
||||
std::optional<IUgc::Model> GetUgcModel(const LWOOBJID ugcId) override { return {}; }
|
||||
std::vector<IUgc::PendingModel> GetUgcModelsToProcess(const uint32_t limit) override { return {}; }
|
||||
void SetUgcModelProcessed(const LWOOBJID id, const eProcessState state, const uint32_t attempts, const std::string_view error, const bool bakeAo) override {}
|
||||
std::optional<IUgc::ProcessInfo> GetUgcProcessInfo(const LWOOBJID id) override { return {}; }
|
||||
uint64_t ResetUgcModelProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override { return 0; }
|
||||
std::vector<IUgc::ProcessInfo> GetUgcProcessList(const std::optional<eProcessState> state, const uint32_t offset, const uint32_t limit) override { return {}; }
|
||||
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetUgcProcessCounts() override { return {}; }
|
||||
std::vector<IUgcModularBuild::PendingBuild> GetModularBuildsToProcess(const uint32_t limit) override { return {}; }
|
||||
void SetModularBuildProcessed(const LWOOBJID id, const IUgc::eProcessState state, const uint32_t attempts, const std::string_view error) override {}
|
||||
std::optional<IUgc::ProcessInfo> GetModularBuildProcessInfo(const LWOOBJID id) override { return {}; }
|
||||
uint64_t ResetModularBuildProcessing(const std::optional<LWOOBJID> id, const bool failedOnly) override { return 0; }
|
||||
std::vector<IUgc::ProcessInfo> GetModularBuildProcessList(const std::optional<IUgc::eProcessState> state, const uint32_t offset, const uint32_t limit) override { return {}; }
|
||||
std::vector<std::pair<IUgc::eProcessState, uint64_t>> GetModularBuildProcessCounts() override { return {}; }
|
||||
void DeleteAccount(const uint32_t accountId) override {};
|
||||
void InsertAuditLog(uint32_t accountId, const std::string_view accountName, const std::string_view action, const std::string_view description, uint32_t targetAccountId, LWOOBJID targetCharacterId) override {};
|
||||
std::string GetAuditLogTable(uint32_t start, uint32_t length, const std::string_view search = "", uint32_t orderColumn = 0, bool orderAsc = true) override { return "{}"; };
|
||||
|
||||
@@ -82,6 +82,7 @@ std::map<uint32_t, std::string> activeSessions;
|
||||
SystemAddress authServerMasterPeerSysAddr;
|
||||
SystemAddress chatServerMasterPeerSysAddr;
|
||||
SystemAddress dashboardServerMasterPeerSysAddr;
|
||||
SystemAddress ugcServerMasterPeerSysAddr;
|
||||
|
||||
namespace {
|
||||
// Dashboard player actions waiting for world servers to answer
|
||||
@@ -444,6 +445,11 @@ int main(int argc, char** argv) {
|
||||
StartDashboardServer();
|
||||
}
|
||||
|
||||
// The UGC server makes and serves player models' meshes and icons (docs/UgcServer.md)
|
||||
if (Game::config->GetValue("enable_ugc_server") == "1") {
|
||||
StartUgcServer();
|
||||
}
|
||||
|
||||
auto t = std::chrono::high_resolution_clock::now();
|
||||
Packet* packet = nullptr;
|
||||
constexpr uint32_t logFlushTime = 15 * masterFramerate;
|
||||
@@ -624,6 +630,9 @@ namespace {
|
||||
case ServiceType::DASHBOARD:
|
||||
dashboardServerMasterPeerSysAddr = sysAddr;
|
||||
break;
|
||||
case ServiceType::UGC:
|
||||
ugcServerMasterPeerSysAddr = sysAddr;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
@@ -1025,6 +1034,19 @@ void HandlePacket(Packet* packet) {
|
||||
dashboardServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
|
||||
StartDashboardServer();
|
||||
}
|
||||
|
||||
if (packet->systemAddress == ugcServerMasterPeerSysAddr) {
|
||||
ugcServerMasterPeerSysAddr = UNASSIGNED_SYSTEM_ADDRESS;
|
||||
|
||||
if (dashboardServerMasterPeerSysAddr != UNASSIGNED_SYSTEM_ADDRESS) {
|
||||
MasterPackets::ServerInfo offline;
|
||||
offline.serverType = ServiceType::UGC;
|
||||
offline.ip = LUString("offline");
|
||||
MasterPackets::SendTo(dashboardServerMasterPeerSysAddr, offline);
|
||||
}
|
||||
|
||||
StartUgcServer();
|
||||
}
|
||||
}
|
||||
|
||||
if (packet->length < 4) return;
|
||||
|
||||
@@ -143,6 +143,42 @@ uint32_t StartDashboardServer() {
|
||||
return web_pid;
|
||||
}
|
||||
|
||||
uint32_t StartUgcServer() {
|
||||
if (Game::ShouldShutdown()) {
|
||||
LOG("Currently shutting down. UgcServer will not be restarted.");
|
||||
return 0;
|
||||
}
|
||||
auto ugc_path = BinaryPathFinder::GetBinaryDir() / "UgcServer";
|
||||
#ifdef _WIN32
|
||||
ugc_path.replace_extension(".exe");
|
||||
auto ugc_startup = startup;
|
||||
auto ugc_info = PROCESS_INFORMATION{};
|
||||
if (!CreateProcessW(ugc_path.wstring().data(), ugc_path.wstring().data(),
|
||||
nullptr, nullptr, false, 0, nullptr, nullptr,
|
||||
&ugc_startup, &ugc_info))
|
||||
{
|
||||
LOG("Failed to launch UgcServer");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// get pid and close unused handles
|
||||
auto ugc_pid = ugc_info.dwProcessId;
|
||||
CloseHandle(ugc_info.hProcess);
|
||||
CloseHandle(ugc_info.hThread);
|
||||
#else // *nix systems
|
||||
const auto ugc_pid = fork();
|
||||
if (ugc_pid < 0) {
|
||||
LOG("Failed to launch UgcServer");
|
||||
return 0;
|
||||
} else if (ugc_pid == 0) {
|
||||
// We are the child process
|
||||
execl(ugc_path.string().c_str(), ugc_path.string().c_str(), nullptr);
|
||||
}
|
||||
#endif
|
||||
LOG("UgcServer PID is %d", ugc_pid);
|
||||
return ugc_pid;
|
||||
}
|
||||
|
||||
uint32_t StartWorldServer(LWOMAPID mapID, uint16_t port, LWOINSTANCEID lastInstanceID, int maxPlayers, LWOCLONEID cloneID) {
|
||||
auto world_path = BinaryPathFinder::GetBinaryDir() / "WorldServer";
|
||||
#ifdef _WIN32
|
||||
|
||||
@@ -4,4 +4,5 @@
|
||||
uint32_t StartAuthServer();
|
||||
uint32_t StartChatServer();
|
||||
uint32_t StartDashboardServer();
|
||||
uint32_t StartUgcServer();
|
||||
uint32_t StartWorldServer(LWOMAPID mapID, uint16_t port, LWOINSTANCEID lastInstanceID, int maxPlayers, LWOCLONEID cloneID);
|
||||
|
||||
40
dUgcServer/CMakeLists.txt
Normal file
40
dUgcServer/CMakeLists.txt
Normal file
@@ -0,0 +1,40 @@
|
||||
# The pure part of the UGC server (brick data, meshes, rendering, file formats, storage), shared with the tests
|
||||
set(DUGC_SOURCES
|
||||
"UgcBricks.cpp"
|
||||
"UgcFormats.cpp"
|
||||
"UgcJobs.cpp"
|
||||
"UgcModel.cpp"
|
||||
"UgcModular.cpp"
|
||||
"UgcRender.cpp"
|
||||
"UgcStorage.cpp"
|
||||
)
|
||||
|
||||
add_library(dUgc STATIC ${DUGC_SOURCES})
|
||||
target_include_directories(dUgc PUBLIC "." "${PROJECT_SOURCE_DIR}/thirdparty/MD5")
|
||||
target_link_libraries(dUgc PUBLIC dCommon glm::glm tinyxml2 MD5)
|
||||
|
||||
add_executable(UgcServer "UgcServer.cpp" "UgcProcessor.cpp" "UgcCdClient.cpp")
|
||||
|
||||
target_include_directories(UgcServer PRIVATE
|
||||
"${PROJECT_SOURCE_DIR}/dCommon"
|
||||
"${PROJECT_SOURCE_DIR}/dCommon/dClient"
|
||||
"${PROJECT_SOURCE_DIR}/dCommon/dEnums"
|
||||
"${PROJECT_SOURCE_DIR}/dDatabase"
|
||||
"${PROJECT_SOURCE_DIR}/dDatabase/CDClientDatabase"
|
||||
"${PROJECT_SOURCE_DIR}/dDatabase/CDClientDatabase/CDClientTables"
|
||||
"${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase"
|
||||
"${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase/ITables"
|
||||
"${PROJECT_SOURCE_DIR}/dNet"
|
||||
"${PROJECT_SOURCE_DIR}/dWeb"
|
||||
"${PROJECT_SOURCE_DIR}/dServer"
|
||||
"${PROJECT_SOURCE_DIR}/thirdparty"
|
||||
"${PROJECT_SOURCE_DIR}/thirdparty/nlohmann"
|
||||
)
|
||||
|
||||
target_compile_definitions(UgcServer PRIVATE PROJECT_VERSION="\"${PROJECT_VERSION}\"")
|
||||
target_link_libraries(UgcServer ${COMMON_LIBRARIES} dWeb dServer dUgc)
|
||||
|
||||
# Rendering is far too slow unoptimized (minutes a model instead of seconds), so it is optimized in every build type
|
||||
if(NOT MSVC)
|
||||
set_source_files_properties("UgcRender.cpp" "UgcModel.cpp" PROPERTIES COMPILE_OPTIONS "-O2")
|
||||
endif()
|
||||
208
dUgcServer/UgcBricks.cpp
Normal file
208
dUgcServer/UgcBricks.cpp
Normal file
@@ -0,0 +1,208 @@
|
||||
#include "UgcBricks.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
|
||||
#include "tinyxml2.h"
|
||||
#include "ZCompression.h"
|
||||
|
||||
namespace {
|
||||
constexpr int32_t GEOMETRY_MAGIC = 0x42473031; // "10GB"
|
||||
constexpr uint32_t MAX_GEOMETRY_PARTS = 64;
|
||||
|
||||
template<typename T>
|
||||
bool ReadAt(std::string_view data, size_t offset, T& value) {
|
||||
if (offset > data.size() || sizeof(T) > data.size() - offset) return false;
|
||||
std::memcpy(&value, data.data() + offset, sizeof(T));
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string Lower(std::string_view text) {
|
||||
std::string out(text);
|
||||
std::transform(out.begin(), out.end(), out.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
namespace UgcBricks {
|
||||
std::optional<Geometry> ParseGeometry(std::string_view data) {
|
||||
int32_t magic{}, vertexCount{}, indexCount{}, options{};
|
||||
if (!ReadAt(data, 0, magic) || magic != GEOMETRY_MAGIC || !ReadAt(data, 4, vertexCount) || !ReadAt(data, 8, indexCount) ||
|
||||
!ReadAt(data, 12, options) || vertexCount < 0 || indexCount < 0 || indexCount % 3 != 0) {
|
||||
return std::nullopt;
|
||||
}
|
||||
const auto vertices = static_cast<size_t>(vertexCount);
|
||||
const auto indices = static_cast<size_t>(indexCount);
|
||||
size_t offset = 16;
|
||||
const bool hasUvs = (options & 3) == 3;
|
||||
const size_t needed = offset + vertices * 24 + (hasUvs ? vertices * 8 : 0) + indices * 4;
|
||||
if (needed > data.size()) return std::nullopt;
|
||||
|
||||
Geometry geometry;
|
||||
geometry.positions.resize(vertices * 3);
|
||||
std::memcpy(geometry.positions.data(), data.data() + offset, vertices * 12);
|
||||
offset += vertices * 12;
|
||||
geometry.normals.resize(vertices * 3);
|
||||
std::memcpy(geometry.normals.data(), data.data() + offset, vertices * 12);
|
||||
offset += vertices * 12;
|
||||
if (hasUvs) offset += vertices * 8;
|
||||
geometry.indices.resize(indices);
|
||||
std::memcpy(geometry.indices.data(), data.data() + offset, indices * 4);
|
||||
for (const auto index : geometry.indices) {
|
||||
if (index >= vertices) return std::nullopt;
|
||||
}
|
||||
return geometry;
|
||||
}
|
||||
|
||||
std::map<uint32_t, Material> ParseMaterials(std::string_view xml) {
|
||||
std::map<uint32_t, Material> materials;
|
||||
tinyxml2::XMLDocument doc;
|
||||
if (doc.Parse(xml.data(), xml.size()) != tinyxml2::XML_SUCCESS) return materials;
|
||||
const auto* root = doc.FirstChildElement("Materials");
|
||||
if (!root) return materials;
|
||||
for (const auto* element = root->FirstChildElement("Material"); element; element = element->NextSiblingElement("Material")) {
|
||||
const auto id = element->UnsignedAttribute("MatID", 0);
|
||||
if (id == 0) continue;
|
||||
const auto channel = [element](const char* name, uint32_t fallback) {
|
||||
return static_cast<uint8_t>(std::min<uint32_t>(element->UnsignedAttribute(name, fallback), 255));
|
||||
};
|
||||
materials[id] = Material{ channel("Red", 160), channel("Green", 160), channel("Blue", 160), channel("Alpha", 255) };
|
||||
}
|
||||
return materials;
|
||||
}
|
||||
|
||||
std::optional<std::string> ReadZipEntry(std::string_view zip, std::string_view name) {
|
||||
// The end of central directory record is in the last 64 KiB + 22 bytes
|
||||
constexpr uint32_t END_SIGNATURE = 0x06054b50;
|
||||
constexpr uint32_t CENTRAL_SIGNATURE = 0x02014b50;
|
||||
constexpr uint32_t LOCAL_SIGNATURE = 0x04034b50;
|
||||
if (zip.size() < 22) return std::nullopt;
|
||||
size_t end = std::string_view::npos;
|
||||
const size_t lowest = zip.size() > 22 + 65535 ? zip.size() - 22 - 65535 : 0;
|
||||
for (size_t i = zip.size() - 22 + 1; i-- > lowest;) {
|
||||
uint32_t signature{};
|
||||
if (ReadAt(zip, i, signature) && signature == END_SIGNATURE) {
|
||||
end = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (end == std::string_view::npos) return std::nullopt;
|
||||
uint16_t entries{};
|
||||
uint32_t directoryOffset{};
|
||||
if (!ReadAt(zip, end + 10, entries) || !ReadAt(zip, end + 16, directoryOffset)) return std::nullopt;
|
||||
|
||||
const auto wanted = Lower(name);
|
||||
size_t offset = directoryOffset;
|
||||
for (uint16_t i = 0; i < entries; i++) {
|
||||
uint32_t signature{}, compressedSize{}, size{}, localOffset{};
|
||||
uint16_t method{}, nameLength{}, extraLength{}, commentLength{};
|
||||
if (!ReadAt(zip, offset, signature) || signature != CENTRAL_SIGNATURE || !ReadAt(zip, offset + 10, method) ||
|
||||
!ReadAt(zip, offset + 20, compressedSize) || !ReadAt(zip, offset + 24, size) || !ReadAt(zip, offset + 28, nameLength) ||
|
||||
!ReadAt(zip, offset + 30, extraLength) || !ReadAt(zip, offset + 32, commentLength) || !ReadAt(zip, offset + 42, localOffset) ||
|
||||
offset + 46 + nameLength > zip.size()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
const auto entryName = zip.substr(offset + 46, nameLength);
|
||||
offset += 46 + nameLength + extraLength + commentLength;
|
||||
if (Lower(entryName) != wanted) continue;
|
||||
|
||||
uint16_t localNameLength{}, localExtraLength{};
|
||||
if (!ReadAt(zip, localOffset, signature) || signature != LOCAL_SIGNATURE || !ReadAt(zip, localOffset + 26, localNameLength) ||
|
||||
!ReadAt(zip, localOffset + 28, localExtraLength)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
const size_t dataStart = static_cast<size_t>(localOffset) + 30 + localNameLength + localExtraLength;
|
||||
if (dataStart > zip.size() || compressedSize > zip.size() - dataStart) return std::nullopt;
|
||||
const auto data = zip.substr(dataStart, compressedSize);
|
||||
if (method == 0) return std::string(data);
|
||||
if (method == 8) return ZCompression::InflateRaw(data, size);
|
||||
return std::nullopt;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> ResolvePath(const std::filesystem::path& root, std::string_view relative) {
|
||||
std::string normalized(relative);
|
||||
std::replace(normalized.begin(), normalized.end(), '\\', '/');
|
||||
std::filesystem::path current = root;
|
||||
std::stringstream parts(normalized);
|
||||
std::string part;
|
||||
std::error_code error;
|
||||
while (std::getline(parts, part, '/')) {
|
||||
if (part.empty() || part == ".") continue;
|
||||
if (part == "..") return std::nullopt;
|
||||
if (std::filesystem::exists(current / part, error)) {
|
||||
current /= part;
|
||||
continue;
|
||||
}
|
||||
const auto wanted = Lower(part);
|
||||
bool found = false;
|
||||
for (std::filesystem::directory_iterator it(current, error), endIt; !error && it != endIt; it.increment(error)) {
|
||||
if (Lower(it->path().filename().string()) == wanted) {
|
||||
current = it->path();
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found) return std::nullopt;
|
||||
}
|
||||
if (!std::filesystem::is_regular_file(current, error)) return std::nullopt;
|
||||
return current;
|
||||
}
|
||||
|
||||
std::optional<std::string> ReadFile(const std::filesystem::path& path) {
|
||||
std::ifstream file(path, std::ios::binary);
|
||||
if (!file) return std::nullopt;
|
||||
std::ostringstream contents;
|
||||
contents << file.rdbuf();
|
||||
return contents.str();
|
||||
}
|
||||
|
||||
BrickLibrary::BrickLibrary(std::filesystem::path res, uint32_t lod) : m_Res(std::move(res)), m_Lod(std::min<uint32_t>(lod, 2)) {}
|
||||
|
||||
bool BrickLibrary::LoadMaterials() {
|
||||
const auto zipPath = ResolvePath(m_Res, "brickdb.zip");
|
||||
const auto zip = zipPath ? ReadFile(*zipPath) : std::nullopt;
|
||||
const auto xml = zip ? ReadZipEntry(*zip, "Materials.xml") : std::nullopt;
|
||||
if (!xml) return false;
|
||||
auto materials = ParseMaterials(*xml);
|
||||
if (materials.empty()) return false;
|
||||
SetMaterials(std::move(materials));
|
||||
return true;
|
||||
}
|
||||
|
||||
void BrickLibrary::SetMaterials(std::map<uint32_t, Material> materials) {
|
||||
std::lock_guard lock(m_Mutex);
|
||||
m_Materials = std::move(materials);
|
||||
}
|
||||
|
||||
Material BrickLibrary::GetMaterial(uint32_t id) const {
|
||||
// Written once before the workers start, only read after
|
||||
const auto it = m_Materials.find(id);
|
||||
return it != m_Materials.end() ? it->second : Material{};
|
||||
}
|
||||
|
||||
std::shared_ptr<const std::vector<Geometry>> BrickLibrary::GetDesign(uint32_t design) {
|
||||
{
|
||||
std::lock_guard lock(m_Mutex);
|
||||
if (const auto it = m_Designs.find(design); it != m_Designs.end()) return it->second;
|
||||
}
|
||||
// Loaded outside the lock; two threads loading the same design at once is harmless
|
||||
auto parts = std::make_shared<std::vector<Geometry>>();
|
||||
const auto folder = "brickprimitives/lod" + std::to_string(m_Lod) + "/";
|
||||
for (uint32_t index = 0; index < MAX_GEOMETRY_PARTS; index++) {
|
||||
const auto name = std::to_string(design) + ".g" + (index == 0 ? "" : std::to_string(index));
|
||||
const auto path = ResolvePath(m_Res, folder + name);
|
||||
const auto data = path ? ReadFile(*path) : std::nullopt;
|
||||
if (!data) break;
|
||||
auto geometry = ParseGeometry(*data);
|
||||
if (!geometry) break;
|
||||
parts->push_back(std::move(*geometry));
|
||||
}
|
||||
std::lock_guard lock(m_Mutex);
|
||||
return m_Designs.emplace(design, std::move(parts)).first->second;
|
||||
}
|
||||
}
|
||||
78
dUgcServer/UgcBricks.h
Normal file
78
dUgcServer/UgcBricks.h
Normal file
@@ -0,0 +1,78 @@
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
/**
|
||||
* The client's brick data the UGC server builds models from: LDD geometry (res/brickprimitives/lod<n>/<design>.g, .g1,
|
||||
* ...) and the material colors (Materials.xml in res/brickdb.zip). The parsers are pure; BrickLibrary loads and caches
|
||||
* files and may be used from any thread.
|
||||
*/
|
||||
namespace UgcBricks {
|
||||
// One LDD .g file: a triangle mesh with a normal per vertex
|
||||
struct Geometry {
|
||||
std::vector<float> positions; // x, y, z per vertex
|
||||
std::vector<float> normals; // x, y, z per vertex
|
||||
std::vector<uint32_t> indices;
|
||||
};
|
||||
|
||||
// An LDD material: sRGB color and opacity, 0-255
|
||||
struct Material {
|
||||
uint8_t r{ 160 };
|
||||
uint8_t g{ 160 };
|
||||
uint8_t b{ 160 };
|
||||
uint8_t a{ 255 };
|
||||
bool Transparent() const { return a < 255; }
|
||||
};
|
||||
|
||||
// A .g file ("10GB" magic, counts, positions, normals, texture coordinates for decorated parts, indices)
|
||||
std::optional<Geometry> ParseGeometry(std::string_view data);
|
||||
|
||||
// Materials.xml: MatID -> color
|
||||
std::map<uint32_t, Material> ParseMaterials(std::string_view xml);
|
||||
|
||||
// A file from a zip archive (stored or deflated), matched without regard to case; nullopt when it isn't there
|
||||
std::optional<std::string> ReadZipEntry(std::string_view zip, std::string_view name);
|
||||
|
||||
// `relative` (either slash, any case) under `root`, matching each part without regard to case as the client's
|
||||
// files are named on Windows; nullopt when there is no such file
|
||||
std::optional<std::filesystem::path> ResolvePath(const std::filesystem::path& root, std::string_view relative);
|
||||
|
||||
// Reads a whole file; nullopt when it can't
|
||||
std::optional<std::string> ReadFile(const std::filesystem::path& path);
|
||||
|
||||
class BrickLibrary {
|
||||
public:
|
||||
// `res` is the client's res folder; `lod` the brickprimitives level used (0 is the most detailed)
|
||||
BrickLibrary(std::filesystem::path res, uint32_t lod);
|
||||
|
||||
// Loads the material colors; false when brickdb.zip or its Materials.xml can't be read
|
||||
bool LoadMaterials();
|
||||
|
||||
// For tests and callers that have the colors already
|
||||
void SetMaterials(std::map<uint32_t, Material> materials);
|
||||
|
||||
// A material's color (a grey when the id is unknown)
|
||||
Material GetMaterial(uint32_t id) const;
|
||||
|
||||
// Every geometry file of a design, in order (.g, .g1, ...); empty when the design has none. Loaded once.
|
||||
std::shared_ptr<const std::vector<Geometry>> GetDesign(uint32_t design);
|
||||
|
||||
const std::filesystem::path& GetResPath() const { return m_Res; }
|
||||
|
||||
private:
|
||||
std::filesystem::path m_Res;
|
||||
uint32_t m_Lod;
|
||||
std::map<uint32_t, Material> m_Materials;
|
||||
std::mutex m_Mutex;
|
||||
std::map<uint32_t, std::shared_ptr<const std::vector<Geometry>>> m_Designs;
|
||||
};
|
||||
}
|
||||
58
dUgcServer/UgcCdClient.cpp
Normal file
58
dUgcServer/UgcCdClient.cpp
Normal file
@@ -0,0 +1,58 @@
|
||||
#include "UgcCdClient.h"
|
||||
|
||||
#include <optional>
|
||||
|
||||
#include "CDClientDatabase.h"
|
||||
#include "UgcModular.h"
|
||||
|
||||
namespace {
|
||||
constexpr int32_t MODULE_COMPONENT = 28;
|
||||
constexpr int32_t RENDER_COMPONENT = 2;
|
||||
}
|
||||
|
||||
namespace UgcCdClient {
|
||||
bool GatherModular(const std::string& modules, UgcJobs::ModularInput& input, std::string& error) {
|
||||
const auto lots = UgcModular::ParseModuleLots(modules);
|
||||
if (lots.empty()) {
|
||||
error = "no modules in \"" + modules + "\"";
|
||||
return false;
|
||||
}
|
||||
std::optional<int32_t> buildType;
|
||||
for (const auto lot : lots) {
|
||||
auto moduleQuery = CDClientDatabase::CreatePreppedStmt(
|
||||
"SELECT m.partCode, m.buildType, m.xml FROM ComponentsRegistry cr JOIN ModuleComponent m ON m.id = cr.component_id "
|
||||
"WHERE cr.id = ? AND cr.component_type = ? LIMIT 1;");
|
||||
moduleQuery.bind(1, static_cast<int>(lot));
|
||||
moduleQuery.bind(2, MODULE_COMPONENT);
|
||||
auto moduleRow = moduleQuery.execQuery();
|
||||
if (moduleRow.eof()) {
|
||||
error = "LOT " + std::to_string(lot) + " is not a module";
|
||||
return false;
|
||||
}
|
||||
UgcJobs::ModuleInput module;
|
||||
module.lot = lot;
|
||||
module.partCode = static_cast<uint32_t>(moduleRow.getIntField("partCode", 0));
|
||||
module.moduleXml = moduleRow.getStringField("xml", "");
|
||||
if (!buildType) buildType = moduleRow.getIntField("buildType", 0);
|
||||
|
||||
auto renderQuery = CDClientDatabase::CreatePreppedStmt(
|
||||
"SELECT rc.render_asset FROM ComponentsRegistry cr JOIN RenderComponent rc ON rc.id = cr.component_id "
|
||||
"WHERE cr.id = ? AND cr.component_type = ? LIMIT 1;");
|
||||
renderQuery.bind(1, static_cast<int>(lot));
|
||||
renderQuery.bind(2, RENDER_COMPONENT);
|
||||
auto renderRow = renderQuery.execQuery();
|
||||
if (!renderRow.eof()) module.renderAsset = renderRow.getStringField("render_asset", "");
|
||||
input.modules.push_back(std::move(module));
|
||||
}
|
||||
|
||||
auto buildQuery = CDClientDatabase::CreatePreppedStmt("SELECT xml FROM ModularBuildComponent WHERE buildType = ? ORDER BY id LIMIT 1;");
|
||||
buildQuery.bind(1, *buildType);
|
||||
auto buildRow = buildQuery.execQuery();
|
||||
if (buildRow.eof()) {
|
||||
error = "no ModularBuildComponent for build type " + std::to_string(*buildType);
|
||||
return false;
|
||||
}
|
||||
input.buildXml = buildRow.getStringField("xml", "");
|
||||
return true;
|
||||
}
|
||||
}
|
||||
12
dUgcServer/UgcCdClient.h
Normal file
12
dUgcServer/UgcCdClient.h
Normal file
@@ -0,0 +1,12 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "UgcJobs.h"
|
||||
|
||||
// What the UGC server reads from the CDClient (main thread only: the CDClient connection isn't shared with workers)
|
||||
namespace UgcCdClient {
|
||||
// A modular build's modules (ugc_modular_build.ldf_config) with their render assets and module data, and the
|
||||
// ModularBuildComponent of their build type; false and `error` when it can't be put together
|
||||
bool GatherModular(const std::string& modules, UgcJobs::ModularInput& input, std::string& error);
|
||||
}
|
||||
311
dUgcServer/UgcFormats.cpp
Normal file
311
dUgcServer/UgcFormats.cpp
Normal file
@@ -0,0 +1,311 @@
|
||||
#include "UgcFormats.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
|
||||
#include "MD5.h"
|
||||
#include "ZCompression.h"
|
||||
|
||||
namespace {
|
||||
class Writer {
|
||||
public:
|
||||
template<typename T>
|
||||
void Put(T value) {
|
||||
char bytes[sizeof(T)];
|
||||
std::memcpy(bytes, &value, sizeof(T));
|
||||
m_Data.append(bytes, sizeof(T));
|
||||
}
|
||||
void U8(uint8_t value) { Put(value); }
|
||||
void U16(uint16_t value) { Put(value); }
|
||||
void U32(uint32_t value) { Put(value); }
|
||||
void I32(int32_t value) { Put(value); }
|
||||
void Float(float value) { Put(value); }
|
||||
void SizedString(const std::string& value) {
|
||||
U32(static_cast<uint32_t>(value.size()));
|
||||
m_Data += value;
|
||||
}
|
||||
void Raw(std::string_view bytes) { m_Data += bytes; }
|
||||
std::string& Data() { return m_Data; }
|
||||
|
||||
private:
|
||||
std::string m_Data;
|
||||
};
|
||||
|
||||
void PutBigEndian(std::string& out, uint32_t value) {
|
||||
for (int shift = 24; shift >= 0; shift -= 8) out += static_cast<char>((value >> shift) & 0xFF);
|
||||
}
|
||||
|
||||
uint32_t Crc32(std::string_view data) {
|
||||
static const auto table = [] {
|
||||
std::array<uint32_t, 256> values{};
|
||||
for (uint32_t i = 0; i < 256; i++) {
|
||||
uint32_t c = i;
|
||||
for (int k = 0; k < 8; k++) c = (c & 1) ? 0xEDB88320u ^ (c >> 1) : c >> 1;
|
||||
values[i] = c;
|
||||
}
|
||||
return values;
|
||||
}();
|
||||
uint32_t crc = 0xFFFFFFFFu;
|
||||
for (const auto byte : data) crc = table[(crc ^ static_cast<uint8_t>(byte)) & 0xFF] ^ (crc >> 8);
|
||||
return crc ^ 0xFFFFFFFFu;
|
||||
}
|
||||
|
||||
void PngChunk(std::string& out, const char* type, std::string_view data) {
|
||||
PutBigEndian(out, static_cast<uint32_t>(data.size()));
|
||||
std::string typed(type, 4);
|
||||
typed += data;
|
||||
out += typed;
|
||||
PutBigEndian(out, Crc32(typed));
|
||||
}
|
||||
|
||||
// Gamebryo's block writing: a block per call, types and strings collected into the header's tables
|
||||
class NifBuilder {
|
||||
public:
|
||||
int32_t String(const std::string& value) {
|
||||
if (value.empty()) return -1;
|
||||
const auto it = std::find(m_Strings.begin(), m_Strings.end(), value);
|
||||
if (it != m_Strings.end()) return static_cast<int32_t>(it - m_Strings.begin());
|
||||
m_Strings.push_back(value);
|
||||
return static_cast<int32_t>(m_Strings.size() - 1);
|
||||
}
|
||||
|
||||
int32_t Add(const std::string& type, std::string data) {
|
||||
auto it = std::find(m_Types.begin(), m_Types.end(), type);
|
||||
if (it == m_Types.end()) it = m_Types.insert(m_Types.end(), type);
|
||||
m_BlockTypes.push_back(static_cast<uint16_t>(it - m_Types.begin()));
|
||||
m_Blocks.push_back(std::move(data));
|
||||
return static_cast<int32_t>(m_Blocks.size() - 1);
|
||||
}
|
||||
|
||||
// Reserves a block to fill in later (a parent that lists children made after it)
|
||||
int32_t Reserve(const std::string& type) { return Add(type, {}); }
|
||||
void Fill(int32_t block, std::string data) { m_Blocks[block] = std::move(data); }
|
||||
|
||||
std::string Finish(int32_t root) {
|
||||
Writer out;
|
||||
out.Raw("Gamebryo File Format, Version 20.3.0.9\n");
|
||||
out.U32(0x14030009);
|
||||
out.U8(1); // little endian
|
||||
out.U32(0); // user version
|
||||
out.U32(static_cast<uint32_t>(m_Blocks.size()));
|
||||
out.U16(static_cast<uint16_t>(m_Types.size()));
|
||||
for (const auto& type : m_Types) out.SizedString(type);
|
||||
for (const auto type : m_BlockTypes) out.U16(type);
|
||||
for (const auto& block : m_Blocks) out.U32(static_cast<uint32_t>(block.size()));
|
||||
out.U32(static_cast<uint32_t>(m_Strings.size()));
|
||||
size_t longest = 0;
|
||||
for (const auto& value : m_Strings) longest = std::max(longest, value.size());
|
||||
out.U32(static_cast<uint32_t>(longest));
|
||||
for (const auto& value : m_Strings) out.SizedString(value);
|
||||
out.U32(0); // groups
|
||||
for (const auto& block : m_Blocks) out.Raw(block);
|
||||
out.U32(1); // roots
|
||||
out.I32(root);
|
||||
return std::move(out.Data());
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<std::string> m_Types;
|
||||
std::vector<uint16_t> m_BlockTypes;
|
||||
std::vector<std::string> m_Blocks;
|
||||
std::vector<std::string> m_Strings;
|
||||
};
|
||||
|
||||
constexpr uint16_t AV_FLAGS = 14; // the usual NiAVObject flags (selective update), not hidden
|
||||
|
||||
void WriteNet(Writer& out, int32_t name) {
|
||||
out.I32(name);
|
||||
out.U32(0); // extra data
|
||||
out.I32(-1); // controller
|
||||
}
|
||||
|
||||
void WriteAv(Writer& out, int32_t name, const std::vector<int32_t>& properties) {
|
||||
WriteNet(out, name);
|
||||
out.U16(AV_FLAGS);
|
||||
for (int i = 0; i < 3; i++) out.Float(0.0f); // translation
|
||||
for (int row = 0; row < 3; row++) {
|
||||
for (int col = 0; col < 3; col++) out.Float(row == col ? 1.0f : 0.0f);
|
||||
}
|
||||
out.Float(1.0f); // scale
|
||||
out.U32(static_cast<uint32_t>(properties.size()));
|
||||
for (const auto property : properties) out.I32(property);
|
||||
out.I32(-1); // collision object
|
||||
}
|
||||
|
||||
std::string TriShapeData(const UgcModel::Mesh& mesh) {
|
||||
Writer out;
|
||||
const auto count = static_cast<uint16_t>(mesh.positions.size());
|
||||
out.I32(0); // group ID
|
||||
out.U16(count);
|
||||
out.U8(0); // keep flags
|
||||
out.U8(0); // compress flags
|
||||
out.U8(1); // has vertices
|
||||
glm::vec3 min(std::numeric_limits<float>::max()), max(-std::numeric_limits<float>::max());
|
||||
for (const auto& p : mesh.positions) {
|
||||
out.Float(p.x);
|
||||
out.Float(p.y);
|
||||
out.Float(p.z);
|
||||
min = glm::min(min, p);
|
||||
max = glm::max(max, p);
|
||||
}
|
||||
out.U16(0); // data flags: no texture coordinates or tangents
|
||||
const bool normals = mesh.normals.size() == mesh.positions.size();
|
||||
out.U8(normals ? 1 : 0);
|
||||
if (normals) {
|
||||
for (const auto& n : mesh.normals) {
|
||||
out.Float(n.x);
|
||||
out.Float(n.y);
|
||||
out.Float(n.z);
|
||||
}
|
||||
}
|
||||
const glm::vec3 center = mesh.positions.empty() ? glm::vec3(0.0f) : (min + max) * 0.5f;
|
||||
float radius = 0.0f;
|
||||
for (const auto& p : mesh.positions) radius = std::max(radius, glm::length(p - center));
|
||||
out.Float(center.x);
|
||||
out.Float(center.y);
|
||||
out.Float(center.z);
|
||||
out.Float(radius);
|
||||
const bool colors = mesh.colors.size() == mesh.positions.size();
|
||||
out.U8(colors ? 1 : 0);
|
||||
if (colors) {
|
||||
for (const auto& c : mesh.colors) {
|
||||
out.Float(std::clamp(c.r, 0.0f, 1.0f));
|
||||
out.Float(std::clamp(c.g, 0.0f, 1.0f));
|
||||
out.Float(std::clamp(c.b, 0.0f, 1.0f));
|
||||
out.Float(std::clamp(c.a, 0.0f, 1.0f));
|
||||
}
|
||||
}
|
||||
out.U16(0x4000); // consistency: static
|
||||
out.I32(-1); // additional data
|
||||
const auto triangles = static_cast<uint16_t>(mesh.indices.size() / 3);
|
||||
out.U16(triangles);
|
||||
out.U32(static_cast<uint32_t>(triangles) * 3);
|
||||
out.U8(1); // has triangles
|
||||
for (size_t i = 0; i < static_cast<size_t>(triangles) * 3; i++) out.U16(static_cast<uint16_t>(mesh.indices[i]));
|
||||
out.U16(0); // match groups
|
||||
return std::move(out.Data());
|
||||
}
|
||||
}
|
||||
|
||||
namespace UgcFormats {
|
||||
std::string WriteNif(const std::string& rootName, const std::vector<NifShape>& shapes) {
|
||||
NifBuilder nif;
|
||||
const int32_t root = nif.Reserve("NiNode");
|
||||
|
||||
// Shared properties
|
||||
Writer material;
|
||||
WriteNet(material, -1);
|
||||
for (int i = 0; i < 3; i++) material.Float(1.0f); // ambient
|
||||
for (int i = 0; i < 3; i++) material.Float(1.0f); // diffuse
|
||||
for (int i = 0; i < 3; i++) material.Float(0.0f); // specular
|
||||
for (int i = 0; i < 3; i++) material.Float(0.0f); // emissive
|
||||
material.Float(10.0f); // glossiness
|
||||
material.Float(1.0f); // alpha
|
||||
const auto materialBlock = nif.Add("NiMaterialProperty", std::move(material.Data()));
|
||||
|
||||
Writer vertexColor;
|
||||
WriteNet(vertexColor, -1);
|
||||
vertexColor.U16((2 << 4) | (1 << 3)); // vertex colors are ambient and diffuse; lit
|
||||
const auto vertexColorBlock = nif.Add("NiVertexColorProperty", std::move(vertexColor.Data()));
|
||||
|
||||
int32_t alphaBlock = -1;
|
||||
std::vector<int32_t> children;
|
||||
for (const auto& shape : shapes) {
|
||||
if (!shape.mesh || shape.mesh->Empty() || shape.mesh->positions.size() > 65535 || shape.mesh->TriangleCount() > 65535) continue;
|
||||
std::vector<int32_t> properties{ materialBlock, vertexColorBlock };
|
||||
if (shape.transparent) {
|
||||
if (alphaBlock < 0) {
|
||||
Writer alpha;
|
||||
WriteNet(alpha, -1);
|
||||
alpha.U16(1 | (6 << 1) | (7 << 5)); // blend source alpha over one minus source alpha
|
||||
alpha.U8(0);
|
||||
alphaBlock = nif.Add("NiAlphaProperty", std::move(alpha.Data()));
|
||||
}
|
||||
properties.push_back(alphaBlock);
|
||||
}
|
||||
const auto shapeBlock = nif.Reserve("NiTriShape");
|
||||
const auto dataBlock = nif.Add("NiTriShapeData", TriShapeData(*shape.mesh));
|
||||
Writer tri;
|
||||
WriteAv(tri, nif.String(shape.name), properties);
|
||||
tri.I32(dataBlock);
|
||||
tri.I32(-1); // skin instance
|
||||
tri.U32(0); // materials
|
||||
tri.I32(-1); // active material
|
||||
tri.U8(0); // material needs update
|
||||
nif.Fill(shapeBlock, std::move(tri.Data()));
|
||||
children.push_back(shapeBlock);
|
||||
}
|
||||
|
||||
Writer node;
|
||||
WriteAv(node, nif.String(rootName), {});
|
||||
node.U32(static_cast<uint32_t>(children.size()));
|
||||
for (const auto child : children) node.I32(child);
|
||||
node.U32(0); // effects
|
||||
nif.Fill(root, std::move(node.Data()));
|
||||
return nif.Finish(root);
|
||||
}
|
||||
|
||||
std::string EncodePng(const UgcRender::Image& image) {
|
||||
std::string raw;
|
||||
raw.reserve(static_cast<size_t>(image.height) * (image.width * 4 + 1));
|
||||
for (int y = 0; y < image.height; y++) {
|
||||
raw += '\0'; // no filter
|
||||
raw.append(reinterpret_cast<const char*>(image.rgba.data()) + static_cast<size_t>(y) * image.width * 4, static_cast<size_t>(image.width) * 4);
|
||||
}
|
||||
std::string compressed(ZCompression::GetMaxCompressedLength(static_cast<uint32_t>(raw.size())) + 64, '\0');
|
||||
const auto size = ZCompression::Compress(reinterpret_cast<const uint8_t*>(raw.data()), static_cast<uint32_t>(raw.size()),
|
||||
reinterpret_cast<uint8_t*>(compressed.data()), static_cast<uint32_t>(compressed.size()));
|
||||
if (size <= 0) return {};
|
||||
compressed.resize(static_cast<size_t>(size));
|
||||
|
||||
std::string out("\x89PNG\r\n\x1a\n", 8);
|
||||
std::string header;
|
||||
PutBigEndian(header, static_cast<uint32_t>(image.width));
|
||||
PutBigEndian(header, static_cast<uint32_t>(image.height));
|
||||
header += std::string("\x08\x06\x00\x00\x00", 5); // 8 bits, RGBA, deflate, no filter method, no interlace
|
||||
PngChunk(out, "IHDR", header);
|
||||
PngChunk(out, "IDAT", compressed);
|
||||
PngChunk(out, "IEND", {});
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string EncodeDds(const UgcRender::Image& image) {
|
||||
std::array<uint32_t, 31> header{};
|
||||
header[0] = 124;
|
||||
header[1] = 0x1 | 0x2 | 0x4 | 0x8 | 0x1000; // caps, height, width, pitch, pixel format
|
||||
header[2] = static_cast<uint32_t>(image.height);
|
||||
header[3] = static_cast<uint32_t>(image.width);
|
||||
header[4] = static_cast<uint32_t>(image.width) * 4; // pitch
|
||||
header[18] = 32; // pixel format size
|
||||
header[19] = 0x41; // RGB with alpha
|
||||
header[21] = 32;
|
||||
header[22] = 0x00FF0000;
|
||||
header[23] = 0x0000FF00;
|
||||
header[24] = 0x000000FF;
|
||||
header[25] = 0xFF000000;
|
||||
header[26] = 0x1000; // texture
|
||||
std::string out = "DDS ";
|
||||
out.append(reinterpret_cast<const char*>(header.data()), header.size() * 4);
|
||||
out.reserve(out.size() + image.rgba.size());
|
||||
for (size_t i = 0; i + 3 < image.rgba.size(); i += 4) {
|
||||
out += static_cast<char>(image.rgba[i + 2]);
|
||||
out += static_cast<char>(image.rgba[i + 1]);
|
||||
out += static_cast<char>(image.rgba[i]);
|
||||
out += static_cast<char>(image.rgba[i + 3]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string Md5Hex(std::string_view data) {
|
||||
MD5 md5;
|
||||
md5.update(reinterpret_cast<const unsigned char*>(data.data()), static_cast<MD5::size_type>(data.size()));
|
||||
md5.finalize();
|
||||
return md5.hexdigest();
|
||||
}
|
||||
|
||||
std::string ChecksumXml(std::string_view data) {
|
||||
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Checksum><MD5>" + Md5Hex(data) + "</MD5><Filesize>" + std::to_string(data.size()) + "</Filesize></Checksum>\n";
|
||||
}
|
||||
}
|
||||
40
dUgcServer/UgcFormats.h
Normal file
40
dUgcServer/UgcFormats.h
Normal file
@@ -0,0 +1,40 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "UgcModel.h"
|
||||
#include "UgcRender.h"
|
||||
|
||||
/**
|
||||
* The files the UGC server writes: the mesh as a Gamebryo .nif the client loads, icons as .png and .dds, and the
|
||||
* client's .checksum files for its downloads. Pure (data in, bytes out).
|
||||
*/
|
||||
namespace UgcFormats {
|
||||
struct NifShape {
|
||||
std::string name;
|
||||
const UgcModel::Mesh* mesh{};
|
||||
bool transparent{};
|
||||
};
|
||||
|
||||
/**
|
||||
* A Gamebryo 20.3.0.9 file (user version 0, as the client's own meshes): a root NiNode named `rootName` with one
|
||||
* NiTriShape per shape (at most 65535 vertices and triangles each: UgcModel::Split), with vertex colors that are
|
||||
* the diffuse and ambient color (NiVertexColorProperty), a white NiMaterialProperty, and for transparent shapes an
|
||||
* NiAlphaProperty blending with the vertex alpha.
|
||||
*/
|
||||
std::string WriteNif(const std::string& rootName, const std::vector<NifShape>& shapes);
|
||||
|
||||
// PNG (8-bit RGBA)
|
||||
std::string EncodePng(const UgcRender::Image& image);
|
||||
|
||||
// DDS, uncompressed 32-bit BGRA without mipmaps
|
||||
std::string EncodeDds(const UgcRender::Image& image);
|
||||
|
||||
// Lowercase hex MD5 of `data`
|
||||
std::string Md5Hex(std::string_view data);
|
||||
|
||||
// What the client reads from a UGC file's .checksum: the MD5 and size of the file as it is after inflating it
|
||||
std::string ChecksumXml(std::string_view data);
|
||||
}
|
||||
130
dUgcServer/UgcJobs.cpp
Normal file
130
dUgcServer/UgcJobs.cpp
Normal file
@@ -0,0 +1,130 @@
|
||||
#include "UgcJobs.h"
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "Sd0.h"
|
||||
#include "UgcFormats.h"
|
||||
#include "UgcModel.h"
|
||||
#include "UgcModular.h"
|
||||
#include "ZCompression.h"
|
||||
|
||||
namespace UgcJobs {
|
||||
void AddDownload(UgcStorage::Files& files, const std::string& name, const std::string& data) {
|
||||
files[name + ".gz"] = ZCompression::Gzip(data);
|
||||
files[name + ".checksum"] = UgcFormats::ChecksumXml(data);
|
||||
}
|
||||
|
||||
std::string LxfmlFromBlob(const std::string& blob) {
|
||||
if (blob.starts_with("<?xml") || blob.starts_with("<LXFML")) return blob;
|
||||
std::stringstream stream(blob);
|
||||
try {
|
||||
Sd0 sd0(stream);
|
||||
return sd0.GetAsStringUncompressed();
|
||||
} catch (...) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
// The icon files of a model, and false when nothing was drawn
|
||||
bool AddIcon(UgcStorage::Files& files, const UgcModel::Model& model, const UgcRender::IconOptions& options) {
|
||||
const auto icon = UgcRender::RenderIcon(model, options);
|
||||
bool drawn = false;
|
||||
for (size_t i = 3; i < icon.rgba.size(); i += 4) drawn = drawn || icon.rgba[i] != 0;
|
||||
files["icon.png"] = UgcFormats::EncodePng(icon);
|
||||
AddDownload(files, "icon.dds", UgcFormats::EncodeDds(icon));
|
||||
return drawn;
|
||||
}
|
||||
}
|
||||
|
||||
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings) {
|
||||
Outcome outcome;
|
||||
const auto lxfml = LxfmlFromBlob(blob);
|
||||
if (lxfml.empty()) {
|
||||
outcome.error = "the stored LXFML can't be read";
|
||||
return outcome;
|
||||
}
|
||||
std::string error;
|
||||
const auto parts = UgcModel::ParseLxfml(lxfml, error);
|
||||
if (parts.empty()) {
|
||||
outcome.error = error;
|
||||
return outcome;
|
||||
}
|
||||
auto model = UgcModel::Build(parts, library);
|
||||
if (!model.missingDesigns.empty()) {
|
||||
outcome.note = "no geometry for design(s)";
|
||||
for (const auto design : model.missingDesigns) outcome.note += " " + std::to_string(design);
|
||||
}
|
||||
if (model.Empty()) {
|
||||
outcome.error = "none of the model's bricks have geometry";
|
||||
if (!outcome.note.empty()) outcome.error += " (" + outcome.note + ")";
|
||||
return outcome;
|
||||
}
|
||||
|
||||
const auto optimized = UgcRender::Optimize(model, settings.optimize);
|
||||
outcome.aoBaked = settings.optimize.bakeAo;
|
||||
if (optimized.trianglesRemoved > 0) {
|
||||
if (!outcome.note.empty()) outcome.note += "; ";
|
||||
outcome.note += "removed " + std::to_string(optimized.trianglesRemoved) + " of " + std::to_string(optimized.trianglesBefore) + " triangles";
|
||||
}
|
||||
|
||||
// Named like LU Toolbox names its shapes: S<shader>_<Opaque|Alpha>_<name>
|
||||
std::vector<std::pair<UgcModel::Mesh, bool>> pieces;
|
||||
for (const bool transparent : { false, true }) {
|
||||
for (auto& piece : UgcModel::Split(transparent ? model.transparent : model.opaque)) pieces.emplace_back(std::move(piece), transparent);
|
||||
}
|
||||
std::vector<UgcFormats::NifShape> shapes;
|
||||
size_t opaqueIndex = 0, transparentIndex = 0;
|
||||
for (const auto& [piece, transparent] : pieces) {
|
||||
const auto index = transparent ? transparentIndex++ : opaqueIndex++;
|
||||
shapes.push_back({ std::string("S01_") + (transparent ? "Alpha" : "Opaque") + "_Model" + (index > 0 ? std::to_string(index) : ""), &piece, transparent });
|
||||
}
|
||||
const auto nif = UgcFormats::WriteNif("SceneNode_Model", shapes);
|
||||
outcome.files["model.nif"] = nif;
|
||||
AddDownload(outcome.files, "model.nif", nif);
|
||||
AddDownload(outcome.files, "model.lxfml", lxfml);
|
||||
AddIcon(outcome.files, model, settings.icon);
|
||||
outcome.ok = true;
|
||||
return outcome;
|
||||
}
|
||||
|
||||
Outcome ProcessModular(const ModularInput& input, const std::filesystem::path& res, const Settings& settings) {
|
||||
Outcome outcome;
|
||||
const auto build = UgcModular::ParseBuild(input.buildXml);
|
||||
if (!build) {
|
||||
outcome.error = "the build type has no topology in ModularBuildComponent";
|
||||
return outcome;
|
||||
}
|
||||
std::vector<UgcModular::Module> modules;
|
||||
for (const auto& moduleInput : input.modules) {
|
||||
const auto path = UgcBricks::ResolvePath(res, moduleInput.renderAsset);
|
||||
const auto data = path ? UgcBricks::ReadFile(*path) : std::nullopt;
|
||||
if (!data) {
|
||||
outcome.note += "module " + std::to_string(moduleInput.lot) + " has no mesh (" + moduleInput.renderAsset + "); ";
|
||||
continue;
|
||||
}
|
||||
std::string error;
|
||||
auto nif = NifFile::Parse(*data, 0, error);
|
||||
if (!nif) {
|
||||
outcome.note += "module " + std::to_string(moduleInput.lot) + ": " + error + "; ";
|
||||
continue;
|
||||
}
|
||||
modules.push_back({ moduleInput.partCode, std::move(*nif), UgcModular::ParseModuleConnections(moduleInput.moduleXml) });
|
||||
}
|
||||
if (modules.empty()) {
|
||||
outcome.error = "none of the modules have a mesh";
|
||||
if (!outcome.note.empty()) outcome.error += " (" + outcome.note + ")";
|
||||
return outcome;
|
||||
}
|
||||
const auto model = UgcModular::Assemble(*build, modules, outcome.note);
|
||||
if (model.Empty()) {
|
||||
outcome.error = "the modules have no triangles";
|
||||
return outcome;
|
||||
}
|
||||
auto options = settings.modularIcon;
|
||||
options.modelRotation = build->additionalRotation * options.modelRotation;
|
||||
AddIcon(outcome.files, model, options);
|
||||
outcome.ok = true;
|
||||
return outcome;
|
||||
}
|
||||
}
|
||||
55
dUgcServer/UgcJobs.h
Normal file
55
dUgcServer/UgcJobs.h
Normal file
@@ -0,0 +1,55 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "UgcBricks.h"
|
||||
#include "UgcRender.h"
|
||||
#include "UgcStorage.h"
|
||||
|
||||
/**
|
||||
* The work a UGC worker thread does for one item: everything from the stored LXFML (or a modular build's modules) to
|
||||
* the finished files. No database, network or CDClient: the main thread gathers the input and stores the result.
|
||||
*/
|
||||
namespace UgcJobs {
|
||||
struct Settings {
|
||||
UgcRender::OptimizeOptions optimize;
|
||||
UgcRender::IconOptions icon;
|
||||
UgcRender::IconOptions modularIcon;
|
||||
};
|
||||
|
||||
struct Outcome {
|
||||
bool ok{};
|
||||
std::string error; // why it failed
|
||||
std::string note; // what was odd but not fatal (missing bricks, ...)
|
||||
UgcStorage::Files files; // name -> bytes, when ok
|
||||
bool aoBaked{};
|
||||
};
|
||||
|
||||
// The client's download of `data` (`name` + ".gz" and ".checksum") added to `files`
|
||||
void AddDownload(UgcStorage::Files& files, const std::string& name, const std::string& data);
|
||||
|
||||
// The LXFML of a ugc row's lxfml column (an sd0 stream, or plain LXFML); empty when it can't be read
|
||||
std::string LxfmlFromBlob(const std::string& blob);
|
||||
|
||||
// A player model: the optimized .nif, its icon and its LXFML for download
|
||||
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings);
|
||||
|
||||
struct ModuleInput {
|
||||
uint32_t lot{};
|
||||
uint32_t partCode{};
|
||||
std::string renderAsset; // RenderComponent.render_asset, relative to the client's res folder
|
||||
std::string moduleXml; // ModuleComponent.xml
|
||||
};
|
||||
|
||||
struct ModularInput {
|
||||
std::vector<ModuleInput> modules;
|
||||
std::string buildXml; // ModularBuildComponent.xml of the modules' build type
|
||||
};
|
||||
|
||||
// A modular build's icon, from its modules' meshes put together
|
||||
Outcome ProcessModular(const ModularInput& input, const std::filesystem::path& res, const Settings& settings);
|
||||
}
|
||||
275
dUgcServer/UgcModel.cpp
Normal file
275
dUgcServer/UgcModel.cpp
Normal file
@@ -0,0 +1,275 @@
|
||||
#include "UgcModel.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <functional>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <glm/gtc/matrix_transform.hpp>
|
||||
|
||||
#include "NifFile.h"
|
||||
#include "tinyxml2.h"
|
||||
|
||||
namespace {
|
||||
std::vector<float> ParseFloats(const char* text) {
|
||||
std::vector<float> values;
|
||||
if (!text) return values;
|
||||
std::stringstream stream(text);
|
||||
std::string item;
|
||||
while (std::getline(stream, item, ',')) {
|
||||
try {
|
||||
values.push_back(std::stof(item));
|
||||
} catch (...) {
|
||||
values.push_back(0.0f);
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
std::vector<uint32_t> ParseMaterials(const char* text) {
|
||||
std::vector<uint32_t> values;
|
||||
if (!text) return values;
|
||||
std::stringstream stream(text);
|
||||
std::string item;
|
||||
while (std::getline(stream, item, ',')) {
|
||||
try {
|
||||
values.push_back(static_cast<uint32_t>(std::stoul(item)));
|
||||
} catch (...) {
|
||||
values.push_back(0);
|
||||
}
|
||||
}
|
||||
// Material 0 means "the same as the part's first"
|
||||
for (auto& value : values) {
|
||||
if (value == 0 && !values.empty()) value = values[0];
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
// LXFML 5 bone: a row-major 3x3 rotation followed by a translation
|
||||
glm::mat4 BoneMatrix(const std::vector<float>& t) {
|
||||
glm::mat4 matrix(1.0f);
|
||||
if (t.size() < 12) return matrix;
|
||||
matrix[0] = glm::vec4(t[0], t[1], t[2], 0.0f);
|
||||
matrix[1] = glm::vec4(t[3], t[4], t[5], 0.0f);
|
||||
matrix[2] = glm::vec4(t[6], t[7], t[8], 0.0f);
|
||||
matrix[3] = glm::vec4(t[9], t[10], t[11], 1.0f);
|
||||
return matrix;
|
||||
}
|
||||
|
||||
// LXFML 4: rotate `angle` degrees around (ax, ay, az), then translate
|
||||
glm::mat4 AxisAngleMatrix(const tinyxml2::XMLElement* element) {
|
||||
const glm::vec3 axis(element->FloatAttribute("ax"), element->FloatAttribute("ay"), element->FloatAttribute("az"));
|
||||
const glm::vec3 translation(element->FloatAttribute("tx"), element->FloatAttribute("ty"), element->FloatAttribute("tz"));
|
||||
glm::mat4 matrix = glm::translate(glm::mat4(1.0f), translation);
|
||||
if (glm::dot(axis, axis) > 0.0f) matrix = glm::rotate(matrix, glm::radians(element->FloatAttribute("angle")), glm::normalize(axis));
|
||||
return matrix;
|
||||
}
|
||||
|
||||
bool ParseDesign(const tinyxml2::XMLElement* element, uint32_t& design) {
|
||||
const char* text = element->Attribute("designID");
|
||||
if (!text || !*text) return false;
|
||||
for (const char* c = text; *c; c++) {
|
||||
if (*c < '0' || *c > '9') return false;
|
||||
}
|
||||
try {
|
||||
design = static_cast<uint32_t>(std::stoul(text));
|
||||
} catch (...) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
glm::vec4 ToColor(const UgcBricks::Material& material) {
|
||||
return glm::vec4(material.r, material.g, material.b, material.a) / 255.0f;
|
||||
}
|
||||
}
|
||||
|
||||
namespace UgcModel {
|
||||
std::vector<Part> ParseLxfml(std::string_view lxfml, std::string& error) {
|
||||
std::vector<Part> parts;
|
||||
tinyxml2::XMLDocument doc;
|
||||
if (doc.Parse(lxfml.data(), lxfml.size()) != tinyxml2::XML_SUCCESS) {
|
||||
error = "the LXFML is not valid XML";
|
||||
return parts;
|
||||
}
|
||||
const auto* root = doc.FirstChildElement("LXFML");
|
||||
if (!root) {
|
||||
error = "no LXFML element";
|
||||
return parts;
|
||||
}
|
||||
|
||||
if (const auto* bricks = root->FirstChildElement("Bricks")) {
|
||||
for (const auto* brick = bricks->FirstChildElement("Brick"); brick; brick = brick->NextSiblingElement("Brick")) {
|
||||
for (const auto* partElement = brick->FirstChildElement("Part"); partElement; partElement = partElement->NextSiblingElement("Part")) {
|
||||
const auto* bone = partElement->FirstChildElement("Bone");
|
||||
Part part;
|
||||
if (!bone || !ParseDesign(partElement, part.designId)) continue;
|
||||
const char* materials = partElement->Attribute("materials");
|
||||
if (!materials) materials = partElement->Attribute("materialID");
|
||||
part.materials = ParseMaterials(materials ? materials : "0");
|
||||
part.transform = BoneMatrix(ParseFloats(bone->Attribute("transformation")));
|
||||
parts.push_back(std::move(part));
|
||||
}
|
||||
}
|
||||
if (!parts.empty()) return parts;
|
||||
}
|
||||
|
||||
// LXFML 4: groups (with their own transforms) nest parts
|
||||
std::function<void(const tinyxml2::XMLElement*, const glm::mat4&)> walk = [&](const tinyxml2::XMLElement* element, const glm::mat4& parent) {
|
||||
for (const auto* child = element->FirstChildElement(); child; child = child->NextSiblingElement()) {
|
||||
const std::string_view name = child->Name();
|
||||
if (name != "Group" && name != "Part") continue;
|
||||
const auto matrix = parent * AxisAngleMatrix(child);
|
||||
if (name == "Group") {
|
||||
walk(child, matrix);
|
||||
continue;
|
||||
}
|
||||
Part part;
|
||||
if (!ParseDesign(child, part.designId)) continue;
|
||||
const char* material = child->Attribute("materialID");
|
||||
part.materials = ParseMaterials(material ? material : "0");
|
||||
part.transform = matrix;
|
||||
parts.push_back(std::move(part));
|
||||
}
|
||||
};
|
||||
if (const auto* scene = root->FirstChildElement("Scene")) {
|
||||
for (const auto* model = scene->FirstChildElement("Model"); model; model = model->NextSiblingElement("Model")) walk(model, glm::mat4(1.0f));
|
||||
}
|
||||
if (parts.empty()) error = "the LXFML has no bricks";
|
||||
return parts;
|
||||
}
|
||||
|
||||
void Mesh::Append(const Mesh& other) {
|
||||
const auto base = static_cast<uint32_t>(positions.size());
|
||||
positions.insert(positions.end(), other.positions.begin(), other.positions.end());
|
||||
normals.insert(normals.end(), other.normals.begin(), other.normals.end());
|
||||
colors.insert(colors.end(), other.colors.begin(), other.colors.end());
|
||||
indices.reserve(indices.size() + other.indices.size());
|
||||
for (const auto index : other.indices) indices.push_back(base + index);
|
||||
}
|
||||
|
||||
void Mesh::Transform(const glm::mat4& transform) {
|
||||
const glm::mat3 normalMatrix = glm::transpose(glm::inverse(glm::mat3(transform)));
|
||||
for (auto& position : positions) position = glm::vec3(transform * glm::vec4(position, 1.0f));
|
||||
for (auto& normal : normals) {
|
||||
const auto n = normalMatrix * normal;
|
||||
const auto length = glm::length(n);
|
||||
normal = length > 0.0f ? n / length : n;
|
||||
}
|
||||
}
|
||||
|
||||
bool Model::Bounds(glm::vec3& min, glm::vec3& max) const {
|
||||
bool any = false;
|
||||
for (const auto* mesh : { &opaque, &transparent }) {
|
||||
for (const auto& position : mesh->positions) {
|
||||
min = any ? glm::min(min, position) : position;
|
||||
max = any ? glm::max(max, position) : position;
|
||||
any = true;
|
||||
}
|
||||
}
|
||||
return any;
|
||||
}
|
||||
|
||||
Model Build(const std::vector<Part>& parts, UgcBricks::BrickLibrary& library) {
|
||||
Model model;
|
||||
std::set<uint32_t> missing;
|
||||
for (const auto& part : parts) {
|
||||
const auto design = library.GetDesign(part.designId);
|
||||
if (!design || design->empty()) {
|
||||
missing.insert(part.designId);
|
||||
continue;
|
||||
}
|
||||
model.bricks++;
|
||||
const glm::mat3 normalMatrix = glm::transpose(glm::inverse(glm::mat3(part.transform)));
|
||||
for (size_t index = 0; index < design->size(); index++) {
|
||||
const auto& geometry = (*design)[index];
|
||||
const auto materialId = index < part.materials.size() ? part.materials[index] : (part.materials.empty() ? 0 : part.materials[0]);
|
||||
const auto material = library.GetMaterial(materialId);
|
||||
auto& mesh = material.Transparent() ? model.transparent : model.opaque;
|
||||
const auto base = static_cast<uint32_t>(mesh.positions.size());
|
||||
const auto color = ToColor(material);
|
||||
const size_t vertexCount = geometry.positions.size() / 3;
|
||||
for (size_t v = 0; v < vertexCount; v++) {
|
||||
const glm::vec3 position(geometry.positions[v * 3], geometry.positions[v * 3 + 1], geometry.positions[v * 3 + 2]);
|
||||
glm::vec3 normal(geometry.normals[v * 3], geometry.normals[v * 3 + 1], geometry.normals[v * 3 + 2]);
|
||||
normal = normalMatrix * normal;
|
||||
const auto length = glm::length(normal);
|
||||
if (length > 0.0f) normal /= length;
|
||||
mesh.positions.push_back(glm::vec3(part.transform * glm::vec4(position, 1.0f)));
|
||||
mesh.normals.push_back(normal);
|
||||
mesh.colors.push_back(color);
|
||||
}
|
||||
for (const auto i : geometry.indices) mesh.indices.push_back(base + i);
|
||||
}
|
||||
}
|
||||
model.missingDesigns.assign(missing.begin(), missing.end());
|
||||
return model;
|
||||
}
|
||||
|
||||
Model FromNif(const NifFile::Model& nif) {
|
||||
Model model;
|
||||
for (const auto& source : nif.meshes) {
|
||||
Mesh mesh;
|
||||
const size_t count = source.positions.size() / 3;
|
||||
const bool vertexColors = source.material.vertexColorMode == 2 && source.colors.size() == count * 4;
|
||||
const glm::vec4 materialColor(source.material.diffuse[0], source.material.diffuse[1], source.material.diffuse[2], source.material.alpha);
|
||||
for (size_t v = 0; v < count; v++) {
|
||||
mesh.positions.emplace_back(source.positions[v * 3], source.positions[v * 3 + 1], source.positions[v * 3 + 2]);
|
||||
if (source.normals.size() == count * 3) mesh.normals.emplace_back(source.normals[v * 3], source.normals[v * 3 + 1], source.normals[v * 3 + 2]);
|
||||
glm::vec4 color = materialColor;
|
||||
if (vertexColors) color *= glm::vec4(source.colors[v * 4], source.colors[v * 4 + 1], source.colors[v * 4 + 2], source.colors[v * 4 + 3]) / 255.0f;
|
||||
mesh.colors.push_back(color);
|
||||
}
|
||||
mesh.indices.assign(source.indices.begin(), source.indices.end());
|
||||
// Normals from the faces when the file has none
|
||||
if (mesh.normals.size() != count) {
|
||||
mesh.normals.assign(count, glm::vec3(0.0f));
|
||||
for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) {
|
||||
const auto& a = mesh.positions[mesh.indices[i]];
|
||||
const auto face = glm::cross(mesh.positions[mesh.indices[i + 1]] - a, mesh.positions[mesh.indices[i + 2]] - a);
|
||||
for (int k = 0; k < 3; k++) mesh.normals[mesh.indices[i + k]] += face;
|
||||
}
|
||||
for (auto& normal : mesh.normals) {
|
||||
const auto length = glm::length(normal);
|
||||
normal = length > 0.0f ? normal / length : glm::vec3(0.0f, 1.0f, 0.0f);
|
||||
}
|
||||
}
|
||||
(source.material.alphaBlend ? model.transparent : model.opaque).Append(mesh);
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
std::vector<Mesh> Split(const Mesh& mesh, size_t maxVertices, size_t maxTriangles) {
|
||||
std::vector<Mesh> pieces;
|
||||
if (mesh.positions.size() <= maxVertices && mesh.TriangleCount() <= maxTriangles) {
|
||||
if (!mesh.Empty()) pieces.push_back(mesh);
|
||||
return pieces;
|
||||
}
|
||||
Mesh current;
|
||||
std::unordered_map<uint32_t, uint32_t> remap;
|
||||
const auto flush = [&]() {
|
||||
if (!current.Empty()) pieces.push_back(std::move(current));
|
||||
current = Mesh{};
|
||||
remap.clear();
|
||||
};
|
||||
for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) {
|
||||
size_t newVertices = 0;
|
||||
for (int k = 0; k < 3; k++) newVertices += remap.contains(mesh.indices[i + k]) ? 0 : 1;
|
||||
if (current.positions.size() + newVertices > maxVertices || current.TriangleCount() + 1 > maxTriangles) flush();
|
||||
for (int k = 0; k < 3; k++) {
|
||||
const auto source = mesh.indices[i + k];
|
||||
auto [it, added] = remap.try_emplace(source, static_cast<uint32_t>(current.positions.size()));
|
||||
if (added) {
|
||||
current.positions.push_back(mesh.positions[source]);
|
||||
if (source < mesh.normals.size()) current.normals.push_back(mesh.normals[source]);
|
||||
if (source < mesh.colors.size()) current.colors.push_back(mesh.colors[source]);
|
||||
}
|
||||
current.indices.push_back(it->second);
|
||||
}
|
||||
}
|
||||
flush();
|
||||
return pieces;
|
||||
}
|
||||
}
|
||||
63
dUgcServer/UgcModel.h
Normal file
63
dUgcServer/UgcModel.h
Normal file
@@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include <glm/glm.hpp>
|
||||
|
||||
#include "UgcBricks.h"
|
||||
|
||||
namespace NifFile {
|
||||
struct Model;
|
||||
}
|
||||
|
||||
/**
|
||||
* Brick models as triangle meshes: LXFML parts, the mesh built from them (opaque and transparent bricks apart, with
|
||||
* the material colors as vertex colors) and the mesh of a client .nif. Pure apart from BrickLibrary's file reads.
|
||||
*/
|
||||
namespace UgcModel {
|
||||
// A brick in a model: its primitive, materials (one per geometry part) and where it is
|
||||
struct Part {
|
||||
uint32_t designId{};
|
||||
std::vector<uint32_t> materials;
|
||||
glm::mat4 transform{ 1.0f };
|
||||
};
|
||||
|
||||
// The parts of an LXFML 5 (Bricks/Brick/Part with bones) or 4 (Scene/Model/Group/Part with axis angles) model.
|
||||
// Empty with `error` set when it can't be read.
|
||||
std::vector<Part> ParseLxfml(std::string_view lxfml, std::string& error);
|
||||
|
||||
struct Mesh {
|
||||
std::vector<glm::vec3> positions;
|
||||
std::vector<glm::vec3> normals;
|
||||
std::vector<glm::vec4> colors; // sRGB, 0..1, alpha is opacity
|
||||
std::vector<uint32_t> indices;
|
||||
|
||||
size_t TriangleCount() const { return indices.size() / 3; }
|
||||
bool Empty() const { return indices.empty(); }
|
||||
void Append(const Mesh& other);
|
||||
void Transform(const glm::mat4& transform);
|
||||
};
|
||||
|
||||
struct Model {
|
||||
Mesh opaque;
|
||||
Mesh transparent;
|
||||
std::vector<uint32_t> missingDesigns; // designs without geometry in the client, skipped
|
||||
size_t bricks{};
|
||||
|
||||
bool Empty() const { return opaque.Empty() && transparent.Empty(); }
|
||||
// The bounds of every vertex; false when there are none
|
||||
bool Bounds(glm::vec3& min, glm::vec3& max) const;
|
||||
};
|
||||
|
||||
// The mesh of a model's parts
|
||||
Model Build(const std::vector<Part>& parts, UgcBricks::BrickLibrary& library);
|
||||
|
||||
// A client .nif's meshes as one model (vertex colors times material color; transparent when blended)
|
||||
Model FromNif(const NifFile::Model& nif);
|
||||
|
||||
// Splits a mesh into pieces the .nif format can hold (at most `maxVertices` vertices and `maxTriangles` triangles)
|
||||
std::vector<Mesh> Split(const Mesh& mesh, size_t maxVertices = 65535, size_t maxTriangles = 65535);
|
||||
}
|
||||
132
dUgcServer/UgcModular.cpp
Normal file
132
dUgcServer/UgcModular.cpp
Normal file
@@ -0,0 +1,132 @@
|
||||
#include "UgcModular.h"
|
||||
|
||||
#include <deque>
|
||||
#include <set>
|
||||
|
||||
#include <glm/gtc/matrix_transform.hpp>
|
||||
#include <glm/gtc/quaternion.hpp>
|
||||
|
||||
#include "tinyxml2.h"
|
||||
|
||||
namespace {
|
||||
const tinyxml2::XMLElement* Child(const tinyxml2::XMLElement* element, const char* name) {
|
||||
return element ? element->FirstChildElement(name) : nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
namespace UgcModular {
|
||||
std::optional<BuildInfo> ParseBuild(std::string_view xml) {
|
||||
tinyxml2::XMLDocument doc;
|
||||
if (doc.Parse(xml.data(), xml.size()) != tinyxml2::XML_SUCCESS) return std::nullopt;
|
||||
const auto* root = doc.FirstChildElement("ModularBuild");
|
||||
const auto* topology = Child(root, "topology");
|
||||
if (!topology) return std::nullopt;
|
||||
BuildInfo build;
|
||||
if (const auto* rootPart = Child(topology, "rootPart")) build.rootPart = rootPart->UnsignedAttribute("value");
|
||||
if (const auto* count = Child(topology, "numberOfParts")) build.parts = count->UnsignedAttribute("value");
|
||||
for (const auto* connection = topology->FirstChildElement("connection"); connection; connection = connection->NextSiblingElement("connection")) {
|
||||
const char* location = connection->Attribute("myLocation");
|
||||
build.connections.push_back({ connection->UnsignedAttribute("myPartid"), location ? location : "", connection->UnsignedAttribute("connectingPart") });
|
||||
}
|
||||
if (const auto* rotation = Child(Child(Child(root, "Placement"), "AdditionalModelRotation"), "Rotation")) {
|
||||
glm::quat q(rotation->FloatAttribute("w", 1.0f), rotation->FloatAttribute("x"), rotation->FloatAttribute("y"), rotation->FloatAttribute("z"));
|
||||
if (glm::length(q) > 0.0f) build.additionalRotation = glm::mat4_cast(glm::normalize(q));
|
||||
}
|
||||
return build;
|
||||
}
|
||||
|
||||
std::map<std::string, glm::vec3> ParseModuleConnections(std::string_view xml) {
|
||||
std::map<std::string, glm::vec3> connections;
|
||||
tinyxml2::XMLDocument doc;
|
||||
if (doc.Parse(xml.data(), xml.size()) != tinyxml2::XML_SUCCESS) return connections;
|
||||
const auto* root = doc.FirstChildElement("ModuleInfo");
|
||||
if (!root) return connections;
|
||||
for (const auto* connection = root->FirstChildElement("connection"); connection; connection = connection->NextSiblingElement("connection")) {
|
||||
const char* name = connection->Attribute("name");
|
||||
const auto* translation = connection->FirstChildElement("translation");
|
||||
if (!name || !translation) continue;
|
||||
connections[name] = glm::vec3(translation->FloatAttribute("x"), translation->FloatAttribute("y"), translation->FloatAttribute("z"));
|
||||
}
|
||||
return connections;
|
||||
}
|
||||
|
||||
std::vector<uint32_t> ParseModuleLots(std::string_view ldf) {
|
||||
std::vector<uint32_t> lots;
|
||||
size_t start = 0;
|
||||
while (start <= ldf.size()) {
|
||||
size_t end = ldf.find_first_of("+;,", start);
|
||||
if (end == std::string_view::npos) end = ldf.size();
|
||||
auto item = ldf.substr(start, end - start);
|
||||
if (const auto colon = item.find(':'); colon != std::string_view::npos) item = item.substr(colon + 1);
|
||||
uint32_t value = 0;
|
||||
bool digits = !item.empty();
|
||||
for (const char c : item) {
|
||||
if (c < '0' || c > '9') {
|
||||
digits = false;
|
||||
break;
|
||||
}
|
||||
value = value * 10 + static_cast<uint32_t>(c - '0');
|
||||
}
|
||||
if (digits && value != 0) lots.push_back(value);
|
||||
start = end + 1;
|
||||
}
|
||||
return lots;
|
||||
}
|
||||
|
||||
UgcModel::Model Assemble(const BuildInfo& build, const std::vector<Module>& modules, std::string& warnings) {
|
||||
UgcModel::Model model;
|
||||
std::map<uint32_t, const Module*> byPart;
|
||||
for (const auto& module : modules) byPart.emplace(module.partCode, &module);
|
||||
|
||||
std::map<uint32_t, glm::mat4> placed;
|
||||
std::deque<uint32_t> pending;
|
||||
if (byPart.contains(build.rootPart)) {
|
||||
placed[build.rootPart] = glm::mat4(1.0f);
|
||||
pending.push_back(build.rootPart);
|
||||
} else if (!modules.empty()) {
|
||||
warnings += "no module for the root part " + std::to_string(build.rootPart) + "; ";
|
||||
placed[modules.front().partCode] = glm::mat4(1.0f);
|
||||
pending.push_back(modules.front().partCode);
|
||||
}
|
||||
|
||||
while (!pending.empty()) {
|
||||
const auto part = pending.front();
|
||||
pending.pop_front();
|
||||
const auto* parent = byPart.at(part);
|
||||
for (const auto& connection : build.connections) {
|
||||
if (connection.part != part || placed.contains(connection.connectingPart)) continue;
|
||||
const auto childIt = byPart.find(connection.connectingPart);
|
||||
if (childIt == byPart.end()) continue; // optional parts
|
||||
glm::vec3 anchor(0.0f);
|
||||
if (const auto node = parent->nif.nodes.find(connection.location); node != parent->nif.nodes.end()) {
|
||||
anchor = glm::vec3(node->second.translation[0], node->second.translation[1], node->second.translation[2]);
|
||||
} else if (const auto offset = parent->connections.find(connection.location); offset != parent->connections.end()) {
|
||||
anchor = offset->second;
|
||||
} else {
|
||||
warnings += "part " + std::to_string(part) + " has no " + connection.location + "; ";
|
||||
}
|
||||
const auto& child = *childIt->second;
|
||||
if (const auto node = child.nif.nodes.find(connection.location); node != child.nif.nodes.end()) {
|
||||
anchor -= glm::vec3(node->second.translation[0], node->second.translation[1], node->second.translation[2]);
|
||||
}
|
||||
placed[connection.connectingPart] = placed[part] * glm::translate(glm::mat4(1.0f), anchor);
|
||||
pending.push_back(connection.connectingPart);
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto& module : modules) {
|
||||
const auto it = placed.find(module.partCode);
|
||||
if (it == placed.end()) {
|
||||
warnings += "part " + std::to_string(module.partCode) + " isn't connected; ";
|
||||
continue;
|
||||
}
|
||||
auto mesh = UgcModel::FromNif(module.nif);
|
||||
mesh.opaque.Transform(it->second);
|
||||
mesh.transparent.Transform(it->second);
|
||||
model.opaque.Append(mesh.opaque);
|
||||
model.transparent.Append(mesh.transparent);
|
||||
model.bricks++;
|
||||
}
|
||||
return model;
|
||||
}
|
||||
}
|
||||
57
dUgcServer/UgcModular.h
Normal file
57
dUgcServer/UgcModular.h
Normal file
@@ -0,0 +1,57 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include <glm/glm.hpp>
|
||||
|
||||
#include "NifFile.h"
|
||||
#include "UgcModel.h"
|
||||
|
||||
/**
|
||||
* Modular builds (cars, rockets) put together from their modules the way the CDClient describes it:
|
||||
* ModularBuildComponent.xml has the topology (a root part and which part connects to which named location of which
|
||||
* other part), ModuleComponent.xml each module's connection offsets, and each module's render asset has the named
|
||||
* attach-point nodes. Pure.
|
||||
*/
|
||||
namespace UgcModular {
|
||||
struct Connection {
|
||||
uint32_t part{}; // the part the location is on
|
||||
std::string location; // node name on that part
|
||||
uint32_t connectingPart{}; // the part placed there
|
||||
};
|
||||
|
||||
struct BuildInfo {
|
||||
uint32_t rootPart{};
|
||||
uint32_t parts{};
|
||||
std::vector<Connection> connections;
|
||||
glm::mat4 additionalRotation{ 1.0f }; // Placement/AdditionalModelRotation
|
||||
};
|
||||
|
||||
// ModularBuildComponent.xml; nullopt when it has no topology
|
||||
std::optional<BuildInfo> ParseBuild(std::string_view xml);
|
||||
|
||||
// ModuleComponent.xml's connection offsets, by location name
|
||||
std::map<std::string, glm::vec3> ParseModuleConnections(std::string_view xml);
|
||||
|
||||
// The LOTs in a ugc_modular_build.ldf_config ("1:4713+1:4714+1:4715", also with ';' or ',' between)
|
||||
std::vector<uint32_t> ParseModuleLots(std::string_view ldf);
|
||||
|
||||
struct Module {
|
||||
uint32_t partCode{};
|
||||
NifFile::Model nif;
|
||||
std::map<std::string, glm::vec3> connections; // from ParseModuleConnections
|
||||
};
|
||||
|
||||
/**
|
||||
* The modules placed per the build's topology, starting at the root part: a connection puts the connecting part
|
||||
* where the parent's node of the location's name is (or the parent module's connection offset of that name when
|
||||
* its mesh has no such node), less where the connecting part's own node of that name is, if it has one.
|
||||
* Parts without a module are skipped; `warnings` says what was.
|
||||
*/
|
||||
UgcModel::Model Assemble(const BuildInfo& build, const std::vector<Module>& modules, std::string& warnings);
|
||||
}
|
||||
235
dUgcServer/UgcProcessor.cpp
Normal file
235
dUgcServer/UgcProcessor.cpp
Normal file
@@ -0,0 +1,235 @@
|
||||
#include "UgcProcessor.h"
|
||||
|
||||
#include "CDClientDatabase.h"
|
||||
#include "Database.h"
|
||||
#include "Logger.h"
|
||||
#include "UgcCdClient.h"
|
||||
#include "json.hpp"
|
||||
|
||||
namespace {
|
||||
constexpr size_t MAX_ERROR_LENGTH = 1000; // process_error holds 1024
|
||||
constexpr size_t LOG_LENGTH = 50;
|
||||
constexpr auto EVICTION_INTERVAL = std::chrono::minutes(5);
|
||||
constexpr auto RECENT_ANSWER_TIME = std::chrono::seconds(10);
|
||||
|
||||
const char* KindName(UgcStorage::Kind kind) {
|
||||
return kind == UgcStorage::Kind::MODEL ? "model" : "modular";
|
||||
}
|
||||
|
||||
int64_t UnixNow() {
|
||||
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
}
|
||||
|
||||
UgcProcessor::UgcProcessor(Config config, UgcStorage& storage, UgcBricks::BrickLibrary& library, UgcJobs::Settings settings)
|
||||
: m_Config(config), m_Storage(storage), m_Library(library), m_Settings(std::move(settings)) {}
|
||||
|
||||
UgcProcessor::~UgcProcessor() {
|
||||
Stop();
|
||||
}
|
||||
|
||||
void UgcProcessor::Start() {
|
||||
for (const auto& entry : m_Storage.List()) m_StoredBytes += entry.bytes;
|
||||
m_NextEviction = std::chrono::steady_clock::now();
|
||||
m_Stopping = false;
|
||||
for (size_t i = 0; i < std::max<size_t>(m_Config.threads, 1); i++) m_Threads.emplace_back(&UgcProcessor::Worker, this);
|
||||
LOG("UGC processing started with %zu worker(s), %llu MB stored", m_Threads.size(), static_cast<unsigned long long>(m_StoredBytes / (1024 * 1024)));
|
||||
}
|
||||
|
||||
void UgcProcessor::Stop() {
|
||||
{
|
||||
std::lock_guard lock(m_Mutex);
|
||||
m_Stopping = true;
|
||||
m_Jobs.clear();
|
||||
}
|
||||
m_Wake.notify_all();
|
||||
for (auto& thread : m_Threads) {
|
||||
if (thread.joinable()) thread.join();
|
||||
}
|
||||
m_Threads.clear();
|
||||
}
|
||||
|
||||
void UgcProcessor::Worker() {
|
||||
while (true) {
|
||||
Job job;
|
||||
{
|
||||
std::unique_lock lock(m_Mutex);
|
||||
m_Wake.wait(lock, [this] { return m_Stopping || !m_Jobs.empty(); });
|
||||
if (m_Stopping) return;
|
||||
job = std::move(m_Jobs.front());
|
||||
m_Jobs.pop_front();
|
||||
m_Active++;
|
||||
}
|
||||
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
Done done{ job.kind, job.id, job.attempts };
|
||||
try {
|
||||
done.outcome = job.kind == Kind::MODEL
|
||||
? UgcJobs::ProcessModel(job.blob, m_Library, m_Settings)
|
||||
: UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), m_Settings);
|
||||
} catch (const std::exception& ex) {
|
||||
done.outcome.ok = false;
|
||||
done.outcome.error = std::string("crashed: ") + ex.what();
|
||||
}
|
||||
if (done.outcome.ok) {
|
||||
std::string error;
|
||||
const auto bytes = m_Storage.Write(job.kind, job.id, done.outcome.files, error);
|
||||
if (bytes) {
|
||||
done.bytes = *bytes;
|
||||
} else {
|
||||
done.outcome.ok = false;
|
||||
done.outcome.error = error;
|
||||
}
|
||||
}
|
||||
done.outcome.files.clear();
|
||||
done.milliseconds = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
|
||||
|
||||
std::lock_guard lock(m_Mutex);
|
||||
m_Active--;
|
||||
m_Done.push_back(std::move(done));
|
||||
}
|
||||
}
|
||||
|
||||
void UgcProcessor::Poll() {
|
||||
size_t queued = 0, active = 0;
|
||||
{
|
||||
std::lock_guard lock(m_Mutex);
|
||||
queued = m_Jobs.size();
|
||||
active = m_Active;
|
||||
}
|
||||
// Enough to keep every worker busy until the next poll
|
||||
const size_t wanted = std::max<size_t>(m_Threads.size() * 2, m_Config.pollBatch);
|
||||
if (queued + active >= wanted) return;
|
||||
const auto limit = static_cast<uint32_t>(wanted - queued - active + m_InFlight.size());
|
||||
|
||||
std::vector<Job> jobs;
|
||||
for (auto& model : Database::Get()->GetUgcModelsToProcess(limit)) {
|
||||
if (m_InFlight.contains({ Kind::MODEL, model.id })) continue;
|
||||
jobs.push_back(Job{ Kind::MODEL, model.id, model.attempts, std::move(model.lxfml) });
|
||||
}
|
||||
for (auto& build : Database::Get()->GetModularBuildsToProcess(limit)) {
|
||||
if (m_InFlight.contains({ Kind::MODULAR, build.id })) continue;
|
||||
Job job{ Kind::MODULAR, build.id, build.attempts };
|
||||
std::string error;
|
||||
if (!UgcCdClient::GatherModular(build.modules, job.modular, error)) {
|
||||
// Nothing a worker could do: record it right away
|
||||
Record(Done{ Kind::MODULAR, build.id, build.attempts, UgcJobs::Outcome{ false, error } });
|
||||
continue;
|
||||
}
|
||||
jobs.push_back(std::move(job));
|
||||
}
|
||||
if (jobs.empty()) return;
|
||||
{
|
||||
std::lock_guard lock(m_Mutex);
|
||||
for (auto& job : jobs) {
|
||||
m_InFlight.insert({ job.kind, job.id });
|
||||
m_Jobs.push_back(std::move(job));
|
||||
}
|
||||
}
|
||||
m_Wake.notify_all();
|
||||
}
|
||||
|
||||
void UgcProcessor::Record(const Done& done) {
|
||||
auto error = done.outcome.error.substr(0, MAX_ERROR_LENGTH);
|
||||
const auto attempts = done.attempts + 1;
|
||||
const auto state = done.outcome.ok ? IUgc::eProcessState::DONE
|
||||
: attempts >= m_Config.maxAttempts ? IUgc::eProcessState::FAILED : IUgc::eProcessState::PENDING;
|
||||
if (done.kind == Kind::MODEL) {
|
||||
Database::Get()->SetUgcModelProcessed(done.id, state, attempts, error, done.outcome.ok && done.outcome.aoBaked);
|
||||
} else {
|
||||
Database::Get()->SetModularBuildProcessed(done.id, state, attempts, error);
|
||||
}
|
||||
m_Recent.erase({ done.kind, done.id });
|
||||
|
||||
if (done.outcome.ok) {
|
||||
m_Made++;
|
||||
m_StoredBytes += done.bytes;
|
||||
LOG_DEBUG("Made %s %llu in %.0f ms%s%s", KindName(done.kind), static_cast<unsigned long long>(done.id), done.milliseconds,
|
||||
done.outcome.note.empty() ? "" : ": ", done.outcome.note.c_str());
|
||||
} else {
|
||||
m_Failed++;
|
||||
LOG("Couldn't make %s %llu (attempt %u): %s", KindName(done.kind), static_cast<unsigned long long>(done.id), attempts, error.c_str());
|
||||
}
|
||||
m_Log.push_back({ done.kind, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? done.outcome.note : error, UnixNow() });
|
||||
while (m_Log.size() > LOG_LENGTH) m_Log.pop_front();
|
||||
}
|
||||
|
||||
void UgcProcessor::Collect() {
|
||||
std::deque<Done> finished;
|
||||
{
|
||||
std::lock_guard lock(m_Mutex);
|
||||
finished.swap(m_Done);
|
||||
}
|
||||
for (const auto& done : finished) {
|
||||
m_InFlight.erase({ done.kind, done.id });
|
||||
Record(done);
|
||||
}
|
||||
// Something finished: look for more now rather than at the next interval
|
||||
if (!finished.empty()) m_NextPoll = std::min(m_NextPoll, std::chrono::steady_clock::now() + std::chrono::milliseconds(100));
|
||||
}
|
||||
|
||||
void UgcProcessor::Update() {
|
||||
Collect();
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
if (now >= m_NextPoll) {
|
||||
m_NextPoll = now + std::chrono::milliseconds(m_Config.pollIntervalMs);
|
||||
Poll();
|
||||
}
|
||||
if (m_Config.maxStorageBytes > 0 && (m_StoredBytes > m_Config.maxStorageBytes || now >= m_NextEviction)) {
|
||||
m_NextEviction = now + EVICTION_INTERVAL;
|
||||
// Deleted files stay marked made: they're made again when someone asks for them (Request)
|
||||
const auto removed = m_Storage.Evict(m_Config.maxStorageBytes);
|
||||
m_StoredBytes = 0;
|
||||
for (const auto& entry : m_Storage.List()) m_StoredBytes += entry.bytes;
|
||||
if (!removed.empty()) {
|
||||
m_Evicted += removed.size();
|
||||
LOG("Deleted the files of %zu item(s) used longest ago to stay under %llu MB", removed.size(), static_cast<unsigned long long>(m_Config.maxStorageBytes / (1024 * 1024)));
|
||||
}
|
||||
}
|
||||
// Forget old answers
|
||||
std::erase_if(m_Recent, [now](const auto& item) { return now - item.second.first > RECENT_ANSWER_TIME; });
|
||||
}
|
||||
|
||||
UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) {
|
||||
if (m_InFlight.contains({ kind, id })) return Availability::QUEUED;
|
||||
if (m_Storage.File(kind, id, "icon.png")) {
|
||||
m_Storage.Touch(kind, id);
|
||||
return Availability::READY;
|
||||
}
|
||||
if (const auto it = m_Recent.find({ kind, id }); it != m_Recent.end()) return it->second.second;
|
||||
|
||||
const auto info = kind == Kind::MODEL ? Database::Get()->GetUgcProcessInfo(id) : Database::Get()->GetModularBuildProcessInfo(id);
|
||||
Availability answer = Availability::UNKNOWN;
|
||||
if (info && info->state != IUgc::eProcessState::FAILED) {
|
||||
// Made before but the files are gone (deleted to save space): make them again, first
|
||||
if (info->state == IUgc::eProcessState::DONE) {
|
||||
if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false);
|
||||
else Database::Get()->ResetModularBuildProcessing(id, false);
|
||||
}
|
||||
m_NextPoll = std::chrono::steady_clock::now();
|
||||
answer = Availability::QUEUED;
|
||||
}
|
||||
m_Recent[{ kind, id }] = { std::chrono::steady_clock::now(), answer };
|
||||
return answer;
|
||||
}
|
||||
|
||||
nlohmann::json UgcProcessor::Status() const {
|
||||
nlohmann::json status;
|
||||
{
|
||||
std::lock_guard lock(m_Mutex);
|
||||
status["queued"] = m_Jobs.size();
|
||||
status["active"] = m_Active;
|
||||
}
|
||||
status["workers"] = m_Threads.size();
|
||||
status["made"] = m_Made;
|
||||
status["failed"] = m_Failed;
|
||||
status["evicted"] = m_Evicted;
|
||||
status["storedBytes"] = m_StoredBytes;
|
||||
status["maxStorageBytes"] = m_Config.maxStorageBytes;
|
||||
auto& recent = status["recent"] = nlohmann::json::array();
|
||||
for (auto it = m_Log.rbegin(); it != m_Log.rend(); ++it) {
|
||||
recent.push_back({ { "kind", KindName(it->kind) }, { "id", std::to_string(it->id) }, { "ok", it->ok },
|
||||
{ "ms", static_cast<int64_t>(it->milliseconds) }, { "message", it->message }, { "time", it->time } });
|
||||
}
|
||||
return status;
|
||||
}
|
||||
115
dUgcServer/UgcProcessor.h
Normal file
115
dUgcServer/UgcProcessor.h
Normal file
@@ -0,0 +1,115 @@
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <cstdint>
|
||||
#include <deque>
|
||||
#include <filesystem>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include "dCommonVars.h"
|
||||
#include "json_fwd.hpp"
|
||||
#include "UgcBricks.h"
|
||||
#include "UgcJobs.h"
|
||||
#include "UgcStorage.h"
|
||||
|
||||
/**
|
||||
* The UGC server's queue. The main thread (Update) takes pending rows from the database, gathers each one's input
|
||||
* (the stored LXFML, or a modular build's modules from the CDClient) and hands it to the worker threads; they make
|
||||
* and write the files; the main thread records the outcome in the database. Workers never touch the database, the
|
||||
* CDClient or the network.
|
||||
*/
|
||||
class UgcProcessor {
|
||||
public:
|
||||
using Kind = UgcStorage::Kind;
|
||||
|
||||
struct Config {
|
||||
uint32_t pollIntervalMs{ 2000 };
|
||||
uint32_t pollBatch{ 32 };
|
||||
uint32_t maxAttempts{ 3 };
|
||||
uint64_t maxStorageBytes{}; // 0: no limit
|
||||
size_t threads{ 2 };
|
||||
};
|
||||
|
||||
UgcProcessor(Config config, UgcStorage& storage, UgcBricks::BrickLibrary& library, UgcJobs::Settings settings);
|
||||
~UgcProcessor();
|
||||
|
||||
void Start();
|
||||
// Waits for the jobs that are running; queued ones are dropped (they stay pending in the database)
|
||||
void Stop();
|
||||
|
||||
// Main thread: poll, dispatch, record results, keep the storage under its cap
|
||||
void Update();
|
||||
|
||||
enum class Availability {
|
||||
READY, // the files are there
|
||||
QUEUED, // it exists and is being made (again)
|
||||
UNKNOWN, // no such item, or it failed
|
||||
};
|
||||
|
||||
// Main thread: whether an item's files can be served, queuing it to be made when it exists but they're missing
|
||||
Availability Request(Kind kind, LWOOBJID id);
|
||||
|
||||
// Main thread: what the server is doing, for /status
|
||||
nlohmann::json Status() const;
|
||||
|
||||
private:
|
||||
struct Job {
|
||||
Kind kind{};
|
||||
LWOOBJID id{};
|
||||
uint32_t attempts{};
|
||||
std::string blob; // models: the stored LXFML
|
||||
UgcJobs::ModularInput modular; // modular builds
|
||||
};
|
||||
|
||||
struct Done {
|
||||
Kind kind{};
|
||||
LWOOBJID id{};
|
||||
uint32_t attempts{};
|
||||
UgcJobs::Outcome outcome;
|
||||
uint64_t bytes{};
|
||||
double milliseconds{};
|
||||
};
|
||||
|
||||
void Poll();
|
||||
void Collect();
|
||||
void Record(const Done& done);
|
||||
void Worker();
|
||||
|
||||
Config m_Config;
|
||||
UgcStorage& m_Storage;
|
||||
UgcBricks::BrickLibrary& m_Library;
|
||||
UgcJobs::Settings m_Settings;
|
||||
|
||||
mutable std::mutex m_Mutex;
|
||||
std::condition_variable m_Wake;
|
||||
std::deque<Job> m_Jobs;
|
||||
std::deque<Done> m_Done;
|
||||
size_t m_Active{};
|
||||
bool m_Stopping{};
|
||||
std::vector<std::thread> m_Threads;
|
||||
|
||||
// Main thread only
|
||||
std::set<std::pair<Kind, LWOOBJID>> m_InFlight;
|
||||
std::chrono::steady_clock::time_point m_NextPoll{};
|
||||
std::chrono::steady_clock::time_point m_NextEviction{};
|
||||
std::map<std::pair<Kind, LWOOBJID>, std::pair<std::chrono::steady_clock::time_point, Availability>> m_Recent; // answers for missing files
|
||||
uint64_t m_StoredBytes{};
|
||||
uint64_t m_Made{};
|
||||
uint64_t m_Failed{};
|
||||
uint64_t m_Evicted{};
|
||||
struct LogEntry {
|
||||
Kind kind{};
|
||||
LWOOBJID id{};
|
||||
bool ok{};
|
||||
double milliseconds{};
|
||||
std::string message;
|
||||
int64_t time{};
|
||||
};
|
||||
std::deque<LogEntry> m_Log; // the last results
|
||||
};
|
||||
360
dUgcServer/UgcRender.cpp
Normal file
360
dUgcServer/UgcRender.cpp
Normal file
@@ -0,0 +1,360 @@
|
||||
#include "UgcRender.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
#include <numeric>
|
||||
|
||||
#include <glm/gtc/matrix_transform.hpp>
|
||||
|
||||
namespace {
|
||||
constexpr float INF = std::numeric_limits<float>::infinity();
|
||||
|
||||
float Edge(const glm::vec3& a, const glm::vec3& b, float px, float py) {
|
||||
return (b.x - a.x) * (py - a.y) - (b.y - a.y) * (px - a.x);
|
||||
}
|
||||
|
||||
// Calls fragment(x, y, z, w0, w1, w2) for every pixel centre inside the screen-space triangle (x, y in pixels)
|
||||
template<typename Fragment>
|
||||
void Rasterize(int width, int height, const glm::vec3& a, const glm::vec3& b, const glm::vec3& c, Fragment&& fragment) {
|
||||
const float area = Edge(a, b, c.x, c.y);
|
||||
if (!(std::abs(area) > 1e-9f)) return;
|
||||
const float inverse = 1.0f / area;
|
||||
const int minX = std::max(0, static_cast<int>(std::floor(std::min({ a.x, b.x, c.x }))));
|
||||
const int maxX = std::min(width - 1, static_cast<int>(std::ceil(std::max({ a.x, b.x, c.x }))));
|
||||
const int minY = std::max(0, static_cast<int>(std::floor(std::min({ a.y, b.y, c.y }))));
|
||||
const int maxY = std::min(height - 1, static_cast<int>(std::ceil(std::max({ a.y, b.y, c.y }))));
|
||||
for (int y = minY; y <= maxY; y++) {
|
||||
const float py = y + 0.5f;
|
||||
for (int x = minX; x <= maxX; x++) {
|
||||
const float px = x + 0.5f;
|
||||
const float w0 = Edge(b, c, px, py) * inverse;
|
||||
const float w1 = Edge(c, a, px, py) * inverse;
|
||||
const float w2 = 1.0f - w0 - w1;
|
||||
if (w0 < 0.0f || w1 < 0.0f || w2 < 0.0f) continue;
|
||||
fragment(x, y, w0 * a.z + w1 * b.z + w2 * c.z, w0, w1, w2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
float ToLinear(float c) { return std::pow(std::clamp(c, 0.0f, 1.0f), 2.2f); }
|
||||
float ToSrgb(float c) { return std::pow(std::clamp(c, 0.0f, 1.0f), 1.0f / 2.2f); }
|
||||
|
||||
// Looking at a sphere (center, radius) from direction `dir` (towards the viewer), square orthographic view
|
||||
struct OrthoView {
|
||||
glm::vec3 center{};
|
||||
glm::vec3 dir{};
|
||||
glm::vec3 right{};
|
||||
glm::vec3 up{};
|
||||
float radius{};
|
||||
int resolution{};
|
||||
|
||||
OrthoView(const glm::vec3& center_, float radius_, const glm::vec3& dir_, int resolution_)
|
||||
: center(center_), dir(dir_), radius(radius_), resolution(resolution_) {
|
||||
const glm::vec3 worldUp = std::abs(dir.y) < 0.99f ? glm::vec3(0.0f, 1.0f, 0.0f) : glm::vec3(1.0f, 0.0f, 0.0f);
|
||||
right = glm::normalize(glm::cross(worldUp, dir));
|
||||
up = glm::cross(dir, right);
|
||||
}
|
||||
|
||||
// x, y in pixels; z the distance from the camera plane (smaller is nearer)
|
||||
glm::vec3 Project(const glm::vec3& p) const {
|
||||
const auto offset = p - center;
|
||||
const float scale = 0.5f * resolution / radius;
|
||||
return { resolution * 0.5f + glm::dot(offset, right) * scale, resolution * 0.5f - glm::dot(offset, up) * scale, radius - glm::dot(offset, dir) };
|
||||
}
|
||||
|
||||
float PixelSize() const { return 2.0f * radius / resolution; }
|
||||
};
|
||||
|
||||
void Bounds(const UgcModel::Model& model, glm::vec3& center, float& radius) {
|
||||
glm::vec3 min{}, max{};
|
||||
if (!model.Bounds(min, max)) {
|
||||
center = glm::vec3(0.0f);
|
||||
radius = 1.0f;
|
||||
return;
|
||||
}
|
||||
center = (min + max) * 0.5f;
|
||||
radius = std::max(glm::length(max - min) * 0.5f, 0.01f);
|
||||
}
|
||||
}
|
||||
|
||||
namespace UgcRender {
|
||||
std::vector<glm::vec3> SphereDirections() {
|
||||
const float t = (1.0f + std::sqrt(5.0f)) / 2.0f;
|
||||
const std::vector<glm::vec3> corners = {
|
||||
{ -1, t, 0 }, { 1, t, 0 }, { -1, -t, 0 }, { 1, -t, 0 },
|
||||
{ 0, -1, t }, { 0, 1, t }, { 0, -1, -t }, { 0, 1, -t },
|
||||
{ t, 0, -1 }, { t, 0, 1 }, { -t, 0, -1 }, { -t, 0, 1 },
|
||||
};
|
||||
std::vector<glm::vec3> directions;
|
||||
for (const auto& corner : corners) directions.push_back(glm::normalize(corner));
|
||||
// Edge centres: the pairs of corners that are neighbours (the shortest distance apart)
|
||||
const float edge = glm::length(corners[0] - corners[1]);
|
||||
for (size_t i = 0; i < corners.size(); i++) {
|
||||
for (size_t j = i + 1; j < corners.size(); j++) {
|
||||
if (std::abs(glm::length(corners[i] - corners[j]) - edge) < 1e-3f) directions.push_back(glm::normalize(corners[i] + corners[j]));
|
||||
}
|
||||
}
|
||||
return directions;
|
||||
}
|
||||
|
||||
OptimizeResult Optimize(UgcModel::Model& model, const OptimizeOptions& options) {
|
||||
OptimizeResult result;
|
||||
auto& opaque = model.opaque;
|
||||
result.trianglesBefore = opaque.TriangleCount() + model.transparent.TriangleCount();
|
||||
if (model.Empty() || (!options.removeHidden && !options.bakeAo)) return result;
|
||||
|
||||
glm::vec3 center{};
|
||||
float radius{};
|
||||
Bounds(model, center, radius);
|
||||
radius *= 1.02f;
|
||||
const int resolution = std::clamp(options.resolution, 64, 4096);
|
||||
const size_t pixels = static_cast<size_t>(resolution) * resolution;
|
||||
std::vector<float> depth(pixels);
|
||||
std::vector<uint32_t> ids(pixels);
|
||||
std::vector<glm::vec3> screen(opaque.positions.size());
|
||||
|
||||
const size_t triangles = opaque.TriangleCount();
|
||||
std::vector<bool> visible(triangles, false);
|
||||
std::vector<bool> facing(triangles, true);
|
||||
std::vector<glm::vec3> faceNormals(triangles, glm::vec3(0.0f));
|
||||
for (size_t t = 0; t < triangles; t++) {
|
||||
if (opaque.normals.size() != opaque.positions.size()) break;
|
||||
faceNormals[t] = glm::normalize(opaque.normals[opaque.indices[t * 3]] + opaque.normals[opaque.indices[t * 3 + 1]] + opaque.normals[opaque.indices[t * 3 + 2]] + glm::vec3(1e-6f));
|
||||
}
|
||||
// Without normals nothing is culled
|
||||
if (opaque.normals.size() != opaque.positions.size()) std::fill(faceNormals.begin(), faceNormals.end(), glm::vec3(0.0f));
|
||||
std::vector<float> aoOpaque(opaque.positions.size()), weightOpaque(opaque.positions.size());
|
||||
std::vector<float> aoTransparent(model.transparent.positions.size()), weightTransparent(model.transparent.positions.size());
|
||||
|
||||
for (const auto& direction : SphereDirections()) {
|
||||
const OrthoView view(center, radius, direction, resolution);
|
||||
const float bias = view.PixelSize();
|
||||
std::fill(depth.begin(), depth.end(), INF);
|
||||
std::fill(ids.begin(), ids.end(), 0);
|
||||
for (size_t v = 0; v < opaque.positions.size(); v++) screen[v] = view.Project(opaque.positions[v]);
|
||||
// Faces turned away can't be seen from here (they're seen from the directions they face); their vertex
|
||||
// normals say which way they face, which doesn't depend on the files' winding
|
||||
for (size_t t = 0; t < triangles; t++) facing[t] = glm::dot(faceNormals[t], direction) > -0.1f;
|
||||
for (size_t t = 0; t < triangles; t++) {
|
||||
if (!facing[t]) continue;
|
||||
const auto id = static_cast<uint32_t>(t + 1);
|
||||
Rasterize(resolution, resolution, screen[opaque.indices[t * 3]], screen[opaque.indices[t * 3 + 1]], screen[opaque.indices[t * 3 + 2]],
|
||||
[&](int x, int y, float z, float, float, float) {
|
||||
auto& stored = depth[static_cast<size_t>(y) * resolution + x];
|
||||
if (z < stored) {
|
||||
stored = z;
|
||||
ids[static_cast<size_t>(y) * resolution + x] = id;
|
||||
}
|
||||
});
|
||||
}
|
||||
for (const auto id : ids) {
|
||||
if (id != 0) visible[id - 1] = true;
|
||||
}
|
||||
|
||||
// Whether a point is in front of what was drawn around its pixel
|
||||
const auto exposed = [&](const glm::vec3& point, float allowance, bool neighbours) {
|
||||
const auto p = view.Project(point);
|
||||
const int px = static_cast<int>(std::floor(p.x)), py = static_cast<int>(std::floor(p.y));
|
||||
const int reach = neighbours ? 1 : 0;
|
||||
for (int dy = -reach; dy <= reach; dy++) {
|
||||
for (int dx = -reach; dx <= reach; dx++) {
|
||||
const int x = px + dx, y = py + dy;
|
||||
if (x < 0 || y < 0 || x >= resolution || y >= resolution) return true;
|
||||
if (p.z <= depth[static_cast<size_t>(y) * resolution + x] + allowance) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// Triangles too small or thin to cover a pixel centre: kept when their centre isn't behind what was drawn.
|
||||
// Bigger ones that show would have covered one.
|
||||
if (options.removeHidden) {
|
||||
const float smallArea = 2.0f; // pixels
|
||||
for (size_t t = 0; t < triangles; t++) {
|
||||
if (visible[t] || !facing[t]) continue;
|
||||
const auto& a = screen[opaque.indices[t * 3]];
|
||||
const auto& b = screen[opaque.indices[t * 3 + 1]];
|
||||
const auto& c = screen[opaque.indices[t * 3 + 2]];
|
||||
if (std::abs(Edge(a, b, c.x, c.y)) * 0.5f > smallArea) continue;
|
||||
const auto centre = (a + b + c) / 3.0f;
|
||||
const int x = static_cast<int>(centre.x), y = static_cast<int>(centre.y);
|
||||
if (x < 0 || y < 0 || x >= resolution || y >= resolution || centre.z <= depth[static_cast<size_t>(y) * resolution + x] + bias) visible[t] = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (options.bakeAo) {
|
||||
const auto accumulate = [&](const UgcModel::Mesh& mesh, std::vector<float>& ao, std::vector<float>& weight) {
|
||||
for (size_t v = 0; v < mesh.positions.size(); v++) {
|
||||
const float w = v < mesh.normals.size() ? glm::dot(mesh.normals[v], direction) : 1.0f;
|
||||
if (w <= 0.0f) continue;
|
||||
weight[v] += w;
|
||||
const auto normal = v < mesh.normals.size() ? mesh.normals[v] : glm::vec3(0.0f);
|
||||
if (exposed(mesh.positions[v] + normal * bias, bias, false)) ao[v] += w;
|
||||
}
|
||||
};
|
||||
accumulate(opaque, aoOpaque, weightOpaque);
|
||||
accumulate(model.transparent, aoTransparent, weightTransparent);
|
||||
}
|
||||
}
|
||||
|
||||
if (options.bakeAo) {
|
||||
const float strength = std::clamp(options.aoStrength, 0.0f, 1.0f);
|
||||
const auto apply = [strength](UgcModel::Mesh& mesh, const std::vector<float>& ao, const std::vector<float>& weight) {
|
||||
for (size_t v = 0; v < mesh.colors.size() && v < ao.size(); v++) {
|
||||
if (weight[v] <= 0.0f) continue;
|
||||
const float factor = 1.0f - strength * (1.0f - ao[v] / weight[v]);
|
||||
auto& color = mesh.colors[v];
|
||||
color.r = ToSrgb(ToLinear(color.r) * factor);
|
||||
color.g = ToSrgb(ToLinear(color.g) * factor);
|
||||
color.b = ToSrgb(ToLinear(color.b) * factor);
|
||||
}
|
||||
};
|
||||
apply(opaque, aoOpaque, weightOpaque);
|
||||
apply(model.transparent, aoTransparent, weightTransparent);
|
||||
}
|
||||
|
||||
if (options.removeHidden) {
|
||||
UgcModel::Mesh kept;
|
||||
std::vector<uint32_t> remap(opaque.positions.size(), UINT32_MAX);
|
||||
for (size_t t = 0; t < triangles; t++) {
|
||||
if (!visible[t]) {
|
||||
result.trianglesRemoved++;
|
||||
continue;
|
||||
}
|
||||
for (int k = 0; k < 3; k++) {
|
||||
const auto source = opaque.indices[t * 3 + k];
|
||||
if (remap[source] == UINT32_MAX) {
|
||||
remap[source] = static_cast<uint32_t>(kept.positions.size());
|
||||
kept.positions.push_back(opaque.positions[source]);
|
||||
if (source < opaque.normals.size()) kept.normals.push_back(opaque.normals[source]);
|
||||
if (source < opaque.colors.size()) kept.colors.push_back(opaque.colors[source]);
|
||||
}
|
||||
kept.indices.push_back(remap[source]);
|
||||
}
|
||||
}
|
||||
opaque = std::move(kept);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Image RenderIcon(const UgcModel::Model& source, const IconOptions& options) {
|
||||
const int size = std::clamp(options.size, 8, 1024);
|
||||
const int supersample = std::clamp(options.supersample, 1, 8);
|
||||
const int n = size * supersample;
|
||||
Image image{ size, size, std::vector<uint8_t>(static_cast<size_t>(size) * size * 4, 0) };
|
||||
if (source.Empty()) return image;
|
||||
|
||||
UgcModel::Model model = source;
|
||||
model.opaque.Transform(options.modelRotation);
|
||||
model.transparent.Transform(options.modelRotation);
|
||||
glm::vec3 center{};
|
||||
float radius{};
|
||||
Bounds(model, center, radius);
|
||||
|
||||
const float yaw = glm::radians(options.yawDegrees), pitch = glm::radians(options.pitchDegrees);
|
||||
const glm::vec3 dir(std::sin(yaw) * std::cos(pitch), std::sin(pitch), std::cos(yaw) * std::cos(pitch));
|
||||
const float fov = glm::radians(std::clamp(options.fovDegrees, 1.0f, 120.0f));
|
||||
const float distance = radius / std::sin(fov * 0.5f);
|
||||
const glm::vec3 eye = center + dir * distance;
|
||||
const glm::mat4 viewProjection = glm::perspective(fov, 1.0f, std::max(distance - radius * 1.5f, distance * 0.01f), distance + radius * 1.5f) *
|
||||
glm::lookAt(eye, center, glm::vec3(0.0f, 1.0f, 0.0f));
|
||||
|
||||
// Frame the model: its projected bounds, scaled to fill the icon less the margin
|
||||
float minX = INF, minY = INF, maxX = -INF, maxY = -INF;
|
||||
for (const auto* mesh : { &model.opaque, &model.transparent }) {
|
||||
for (const auto& position : mesh->positions) {
|
||||
const auto clip = viewProjection * glm::vec4(position, 1.0f);
|
||||
if (clip.w <= 0.0f) continue;
|
||||
minX = std::min(minX, clip.x / clip.w);
|
||||
maxX = std::max(maxX, clip.x / clip.w);
|
||||
minY = std::min(minY, clip.y / clip.w);
|
||||
maxY = std::max(maxY, clip.y / clip.w);
|
||||
}
|
||||
}
|
||||
if (minX > maxX) return image;
|
||||
const float centerX = (minX + maxX) * 0.5f, centerY = (minY + maxY) * 0.5f;
|
||||
const float scale = 2.0f / (std::max({ maxX - minX, maxY - minY, 1e-6f }) * std::max(options.margin, 0.1f));
|
||||
const auto project = [&](const glm::vec3& position) {
|
||||
const auto clip = viewProjection * glm::vec4(position, 1.0f);
|
||||
const float w = clip.w > 1e-6f ? clip.w : 1e-6f;
|
||||
return glm::vec3((0.5f + (clip.x / w - centerX) * scale * 0.5f) * n, (0.5f - (clip.y / w - centerY) * scale * 0.5f) * n, clip.z / w);
|
||||
};
|
||||
|
||||
// Linear, premultiplied
|
||||
std::vector<glm::vec4> color(static_cast<size_t>(n) * n, glm::vec4(0.0f));
|
||||
std::vector<float> depth(static_cast<size_t>(n) * n, INF);
|
||||
const glm::vec3 light = glm::normalize(dir + glm::vec3(-0.35f, 1.1f, 0.25f));
|
||||
const auto shade = [&](const UgcModel::Mesh& mesh, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) {
|
||||
glm::vec3 normal(0.0f, 1.0f, 0.0f);
|
||||
if (mesh.normals.size() == mesh.positions.size()) {
|
||||
normal = mesh.normals[i0] * w0 + mesh.normals[i1] * w1 + mesh.normals[i2] * w2;
|
||||
const auto length = glm::length(normal);
|
||||
normal = length > 0.0f ? normal / length : glm::vec3(0.0f, 1.0f, 0.0f);
|
||||
if (glm::dot(normal, dir) < 0.0f) normal = -normal; // the back of a face
|
||||
}
|
||||
glm::vec4 base(0.63f, 0.63f, 0.63f, 1.0f);
|
||||
if (mesh.colors.size() == mesh.positions.size()) base = mesh.colors[i0] * w0 + mesh.colors[i1] * w1 + mesh.colors[i2] * w2;
|
||||
const float lighting = 0.55f + 0.6f * std::max(0.0f, glm::dot(normal, light));
|
||||
return glm::vec4(ToLinear(base.r) * lighting, ToLinear(base.g) * lighting, ToLinear(base.b) * lighting, std::clamp(base.a, 0.0f, 1.0f));
|
||||
};
|
||||
|
||||
// Opaque first, with the depth buffer
|
||||
{
|
||||
const auto& mesh = model.opaque;
|
||||
std::vector<glm::vec3> screen(mesh.positions.size());
|
||||
for (size_t v = 0; v < mesh.positions.size(); v++) screen[v] = project(mesh.positions[v]);
|
||||
for (size_t i = 0; i + 2 < mesh.indices.size(); i += 3) {
|
||||
const auto i0 = mesh.indices[i], i1 = mesh.indices[i + 1], i2 = mesh.indices[i + 2];
|
||||
Rasterize(n, n, screen[i0], screen[i1], screen[i2], [&](int x, int y, float z, float w0, float w1, float w2) {
|
||||
const size_t index = static_cast<size_t>(y) * n + x;
|
||||
if (z >= depth[index]) return;
|
||||
depth[index] = z;
|
||||
const auto shaded = shade(mesh, i0, i1, i2, w0, w1, w2);
|
||||
color[index] = glm::vec4(glm::vec3(shaded), 1.0f);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Then transparent triangles, farthest first, blended over it
|
||||
{
|
||||
const auto& mesh = model.transparent;
|
||||
std::vector<glm::vec3> screen(mesh.positions.size());
|
||||
for (size_t v = 0; v < mesh.positions.size(); v++) screen[v] = project(mesh.positions[v]);
|
||||
std::vector<size_t> order(mesh.indices.size() / 3);
|
||||
std::iota(order.begin(), order.end(), 0);
|
||||
const auto depthOf = [&](size_t t) { return screen[mesh.indices[t * 3]].z + screen[mesh.indices[t * 3 + 1]].z + screen[mesh.indices[t * 3 + 2]].z; };
|
||||
std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { return depthOf(a) > depthOf(b); });
|
||||
for (const auto t : order) {
|
||||
const auto i0 = mesh.indices[t * 3], i1 = mesh.indices[t * 3 + 1], i2 = mesh.indices[t * 3 + 2];
|
||||
Rasterize(n, n, screen[i0], screen[i1], screen[i2], [&](int x, int y, float z, float w0, float w1, float w2) {
|
||||
const size_t index = static_cast<size_t>(y) * n + x;
|
||||
if (z >= depth[index]) return;
|
||||
const auto shaded = shade(mesh, i0, i1, i2, w0, w1, w2);
|
||||
const float alpha = shaded.a;
|
||||
auto& target = color[index];
|
||||
target = glm::vec4(glm::vec3(shaded) * alpha + glm::vec3(target) * (1.0f - alpha), alpha + target.a * (1.0f - alpha));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Box filter down to the icon's size
|
||||
const float samples = static_cast<float>(supersample * supersample);
|
||||
for (int y = 0; y < size; y++) {
|
||||
for (int x = 0; x < size; x++) {
|
||||
glm::vec4 sum(0.0f);
|
||||
for (int sy = 0; sy < supersample; sy++) {
|
||||
for (int sx = 0; sx < supersample; sx++) sum += color[static_cast<size_t>(y * supersample + sy) * n + (x * supersample + sx)];
|
||||
}
|
||||
sum /= samples;
|
||||
uint8_t* out = &image.rgba[(static_cast<size_t>(y) * size + x) * 4];
|
||||
if (sum.a <= 0.0f) continue;
|
||||
out[0] = static_cast<uint8_t>(std::lround(ToSrgb(sum.r / sum.a) * 255.0f));
|
||||
out[1] = static_cast<uint8_t>(std::lround(ToSrgb(sum.g / sum.a) * 255.0f));
|
||||
out[2] = static_cast<uint8_t>(std::lround(ToSrgb(sum.b / sum.a) * 255.0f));
|
||||
out[3] = static_cast<uint8_t>(std::lround(std::clamp(sum.a, 0.0f, 1.0f) * 255.0f));
|
||||
}
|
||||
}
|
||||
return image;
|
||||
}
|
||||
}
|
||||
56
dUgcServer/UgcRender.h
Normal file
56
dUgcServer/UgcRender.h
Normal file
@@ -0,0 +1,56 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
|
||||
#include <glm/glm.hpp>
|
||||
|
||||
#include "UgcModel.h"
|
||||
|
||||
/**
|
||||
* A small software rasterizer (no GPU or display needed) for what the UGC server draws: the icons, and the renders
|
||||
* from many directions that find the faces nobody can see and bake ambient occlusion into the vertex colors.
|
||||
*/
|
||||
namespace UgcRender {
|
||||
// 8-bit RGBA, rows top to bottom, not premultiplied
|
||||
struct Image {
|
||||
int width{};
|
||||
int height{};
|
||||
std::vector<uint8_t> rgba;
|
||||
};
|
||||
|
||||
struct IconOptions {
|
||||
int size{ 128 };
|
||||
int supersample{ 4 };
|
||||
float yawDegrees{ 35.0f }; // camera around the model, from +Z towards +X
|
||||
float pitchDegrees{ 25.0f }; // camera above the model
|
||||
float fovDegrees{ 30.0f };
|
||||
float margin{ 1.03f }; // 1 fills the icon, more leaves a border
|
||||
glm::mat4 modelRotation{ 1.0f }; // applied to the model before the camera looks at it
|
||||
};
|
||||
|
||||
// The model drawn from the icon's camera, framed to fit, on a transparent background
|
||||
Image RenderIcon(const UgcModel::Model& model, const IconOptions& options);
|
||||
|
||||
struct OptimizeOptions {
|
||||
int resolution{ 1024 }; // of each direction's render
|
||||
bool removeHidden{ true };
|
||||
bool bakeAo{ true };
|
||||
float aoStrength{ 0.6f }; // 0 leaves the colors, 1 makes fully hidden corners black
|
||||
};
|
||||
|
||||
struct OptimizeResult {
|
||||
size_t trianglesBefore{};
|
||||
size_t trianglesRemoved{};
|
||||
};
|
||||
|
||||
/**
|
||||
* Renders the opaque mesh from 42 directions around it: triangles that show in none of them are removed (with a
|
||||
* conservative test, so small visible ones stay), and each vertex's share of the directions it can be seen from
|
||||
* darkens its color (ambient occlusion). Transparent bricks hide nothing but are darkened too.
|
||||
*/
|
||||
OptimizeResult Optimize(UgcModel::Model& model, const OptimizeOptions& options);
|
||||
|
||||
// The 42 directions Optimize renders from (an icosahedron's corners and edge centres), unit length
|
||||
std::vector<glm::vec3> SphereDirections();
|
||||
}
|
||||
360
dUgcServer/UgcServer.cpp
Normal file
360
dUgcServer/UgcServer.cpp
Normal file
@@ -0,0 +1,360 @@
|
||||
// The UGC server: makes the meshes and icons of what players build and serves them, and the models' LXFML, to the
|
||||
// game client over HTTP. See docs/UgcServer.md.
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <csignal>
|
||||
#include <cstdlib>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <random>
|
||||
#include <thread>
|
||||
|
||||
#include "BinaryPathFinder.h"
|
||||
#include "CDClientDatabase.h"
|
||||
#include "ConfigSync.h"
|
||||
#include "Database.h"
|
||||
#include "Diagnostics.h"
|
||||
#include "Game.h"
|
||||
#include "GeneralUtils.h"
|
||||
#include "Logger.h"
|
||||
#include "Server.h"
|
||||
#include "ServiceType.h"
|
||||
#include "Web.h"
|
||||
#include "dConfig.h"
|
||||
#include "dServer.h"
|
||||
#include "eHTTPMethod.h"
|
||||
#include "json.hpp"
|
||||
|
||||
#include "UgcBricks.h"
|
||||
#include "UgcCdClient.h"
|
||||
#include "UgcJobs.h"
|
||||
#include "UgcModel.h"
|
||||
#include "UgcProcessor.h"
|
||||
#include "UgcStorage.h"
|
||||
|
||||
namespace Game {
|
||||
Logger* logger = nullptr;
|
||||
dServer* server = nullptr;
|
||||
dConfig* config = nullptr;
|
||||
Game::signal_t lastSignal = 0;
|
||||
std::mt19937 randomEngine;
|
||||
}
|
||||
|
||||
namespace {
|
||||
dServer* g_Server = nullptr;
|
||||
UgcProcessor* g_Processor = nullptr;
|
||||
UgcStorage* g_Storage = nullptr;
|
||||
|
||||
template<typename T>
|
||||
T Setting(const std::string& key, T fallback) {
|
||||
return GeneralUtils::TryParse<T>(Game::config->GetValue(key)).value_or(fallback);
|
||||
}
|
||||
|
||||
UgcJobs::Settings ReadSettings() {
|
||||
UgcJobs::Settings settings;
|
||||
settings.optimize.removeHidden = Setting<int32_t>("remove_hidden_faces", 1) != 0;
|
||||
settings.optimize.bakeAo = Setting<int32_t>("bake_ao", 1) != 0;
|
||||
settings.optimize.aoStrength = Setting<float>("ao_strength", 0.6f);
|
||||
settings.optimize.resolution = Setting<int32_t>("optimize_resolution", 1024);
|
||||
settings.icon.size = Setting<int32_t>("icon_size", 128);
|
||||
settings.icon.yawDegrees = Setting<float>("icon_yaw", 35.0f);
|
||||
settings.icon.pitchDegrees = Setting<float>("icon_pitch", 25.0f);
|
||||
settings.icon.margin = Setting<float>("icon_margin", 1.03f);
|
||||
settings.modularIcon = settings.icon;
|
||||
settings.modularIcon.yawDegrees = Setting<float>("modular_icon_yaw", 35.0f);
|
||||
settings.modularIcon.pitchDegrees = Setting<float>("modular_icon_pitch", 20.0f);
|
||||
return settings;
|
||||
}
|
||||
|
||||
std::vector<std::string> Segments(const std::string& path) {
|
||||
std::vector<std::string> segments;
|
||||
size_t start = 0;
|
||||
while (start < path.size()) {
|
||||
auto end = path.find('/', start);
|
||||
if (end == std::string::npos) end = path.size();
|
||||
if (end > start) segments.push_back(path.substr(start, end - start));
|
||||
start = end + 1;
|
||||
}
|
||||
return segments;
|
||||
}
|
||||
|
||||
std::string Lower(std::string text) {
|
||||
std::transform(text.begin(), text.end(), text.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
|
||||
return text;
|
||||
}
|
||||
|
||||
void NotFound(HTTPReply& reply) {
|
||||
reply.status = eHTTPStatusCode::NOT_FOUND;
|
||||
reply.message = "Not Found";
|
||||
reply.contentType = eContentType::TEXT_PLAIN;
|
||||
}
|
||||
|
||||
void ServeFile(HTTPReply& reply, UgcStorage::Kind kind, LWOOBJID id, const std::string& name, eContentType type, bool crossOrigin) {
|
||||
switch (g_Processor->Request(kind, id)) {
|
||||
case UgcProcessor::Availability::READY:
|
||||
break;
|
||||
case UgcProcessor::Availability::QUEUED:
|
||||
// The client asks again later on 408
|
||||
reply.status = eHTTPStatusCode::REQUEST_TIMEOUT;
|
||||
reply.message = "Being made";
|
||||
reply.contentType = eContentType::TEXT_PLAIN;
|
||||
reply.headers.push_back("Retry-After: 5");
|
||||
return;
|
||||
case UgcProcessor::Availability::UNKNOWN:
|
||||
NotFound(reply);
|
||||
return;
|
||||
}
|
||||
const auto path = g_Storage->File(kind, id, name);
|
||||
if (!path) {
|
||||
NotFound(reply);
|
||||
return;
|
||||
}
|
||||
reply.status = eHTTPStatusCode::OK;
|
||||
reply.file = path->string();
|
||||
reply.contentType = type;
|
||||
reply.headers.push_back("Cache-Control: public, max-age=3600");
|
||||
if (crossOrigin) reply.headers.push_back("Access-Control-Allow-Origin: *");
|
||||
}
|
||||
|
||||
/**
|
||||
* A game client download (3D services): <client_path>/UGCC<dc>/[3DOPTIMIZED/|IMAGE128DDS/]<dc><id><.ext>(.gz|.checksum).
|
||||
* `segments` are the path's parts after client_path.
|
||||
*/
|
||||
void ServeClientDownload(HTTPReply& reply, const std::vector<std::string>& segments) {
|
||||
NotFound(reply);
|
||||
if (segments.size() < 2 || segments.size() > 3) return;
|
||||
const auto folder = Lower(segments[0]);
|
||||
if (!folder.starts_with("ugcc")) return;
|
||||
const auto datacenter = folder.substr(4);
|
||||
const std::string typeFolder = segments.size() == 3 ? Lower(segments[1]) : "";
|
||||
auto file = Lower(segments.back());
|
||||
|
||||
std::string suffix;
|
||||
if (file.ends_with(".gz")) suffix = ".gz";
|
||||
else if (file.ends_with(".checksum")) suffix = ".checksum";
|
||||
else return;
|
||||
file.resize(file.size() - suffix.size());
|
||||
const auto dot = file.rfind('.');
|
||||
if (dot == std::string::npos) return;
|
||||
const auto extension = file.substr(dot);
|
||||
auto number = file.substr(0, dot);
|
||||
// The file name starts with the datacenter id
|
||||
if (!number.starts_with(datacenter)) return;
|
||||
number = number.substr(datacenter.size());
|
||||
const auto id = GeneralUtils::TryParse<uint64_t>(number);
|
||||
if (!id || number.empty()) return;
|
||||
const auto blueprint = static_cast<LWOOBJID>(*id);
|
||||
|
||||
const auto type = suffix == ".gz" ? eContentType::APPLICATION_OCTET_STREAM : eContentType::TEXT_PLAIN;
|
||||
if (extension == ".lxfml" && typeFolder.empty()) {
|
||||
ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.lxfml" + suffix, type, false);
|
||||
} else if (extension == ".nif" && typeFolder == "3doptimized") {
|
||||
ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "model.nif" + suffix, type, false);
|
||||
} else if (extension == ".dds" && typeFolder == "image128dds") {
|
||||
// A player model's icon, else a modular build's
|
||||
ServeFile(reply, UgcStorage::Kind::MODEL, blueprint, "icon.dds" + suffix, type, false);
|
||||
if (reply.status == eHTTPStatusCode::NOT_FOUND) ServeFile(reply, UgcStorage::Kind::MODULAR, blueprint, "icon.dds" + suffix, type, false);
|
||||
}
|
||||
// .hkx: no physics is made
|
||||
}
|
||||
|
||||
void RegisterRoutes() {
|
||||
const auto prefix = Segments(Lower(Game::config->GetValue("client_path").empty() ? "/ugc" : Game::config->GetValue("client_path")));
|
||||
std::string base;
|
||||
for (const auto& segment : prefix) base += "/" + segment;
|
||||
|
||||
const auto clientRoute = [prefixSize = prefix.size()](HTTPReply& reply, const HTTPContext& context) {
|
||||
auto segments = Segments(context.originalPath);
|
||||
if (segments.size() <= prefixSize) {
|
||||
NotFound(reply);
|
||||
return;
|
||||
}
|
||||
segments.erase(segments.begin(), segments.begin() + static_cast<std::ptrdiff_t>(prefixSize));
|
||||
ServeClientDownload(reply, segments);
|
||||
};
|
||||
Game::web.RegisterHTTPRoute({ .path = base + "/:folder/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = clientRoute });
|
||||
Game::web.RegisterHTTPRoute({ .path = base + "/:folder/:type/:file", .method = eHTTPMethod::GET, .middleware = {}, .handle = clientRoute });
|
||||
|
||||
// Previews for the dashboard
|
||||
Game::web.RegisterHTTPRoute({ .path = "/files/:kind/:id/:file", .method = eHTTPMethod::GET, .middleware = {},
|
||||
.handle = [](HTTPReply& reply, const HTTPContext& context) {
|
||||
const auto segments = Segments(context.originalPath);
|
||||
NotFound(reply);
|
||||
reply.headers.push_back("Access-Control-Allow-Origin: *");
|
||||
if (segments.size() != 4) return;
|
||||
const auto id = GeneralUtils::TryParse<uint64_t>(segments[2]);
|
||||
const auto kind = segments[1] == "model" ? std::optional(UgcStorage::Kind::MODEL) : segments[1] == "modular" ? std::optional(UgcStorage::Kind::MODULAR) : std::nullopt;
|
||||
const auto& name = segments[3];
|
||||
if (!id || !kind || (name != "icon.png" && name != "model.nif")) return;
|
||||
reply.headers.clear();
|
||||
ServeFile(reply, *kind, static_cast<LWOOBJID>(*id), name, name == "icon.png" ? eContentType::IMAGE_PNG : eContentType::APPLICATION_OCTET_STREAM, true);
|
||||
} });
|
||||
|
||||
Game::web.RegisterHTTPRoute({ .path = "/status", .method = eHTTPMethod::GET, .middleware = {},
|
||||
.handle = [](HTTPReply& reply, const HTTPContext&) {
|
||||
reply.status = eHTTPStatusCode::OK;
|
||||
reply.contentType = eContentType::APPLICATION_JSON;
|
||||
reply.message = g_Processor->Status().dump();
|
||||
reply.headers.push_back("Access-Control-Allow-Origin: *");
|
||||
reply.headers.push_back("Cache-Control: no-store");
|
||||
} });
|
||||
LOG("Serving the client's downloads under %s/UGCC<datacenter>/", base.c_str());
|
||||
}
|
||||
|
||||
std::filesystem::path ResPath() {
|
||||
const auto client = Game::config->GetValue("client_location");
|
||||
return client.empty() ? std::filesystem::path{} : std::filesystem::path(client) / "res";
|
||||
}
|
||||
|
||||
// Command line tools: make one model's or modular build's files into a folder, without a database
|
||||
int MakeFromCommandLine(const std::string& mode, const std::string& input, const std::filesystem::path& output) {
|
||||
const auto res = ResPath();
|
||||
const auto settings = ReadSettings();
|
||||
UgcBricks::BrickLibrary library(res, Setting<uint32_t>("brick_lod", 0));
|
||||
if (!library.LoadMaterials()) std::cerr << "Couldn't read Materials.xml from " << (res / "brickdb.zip") << "; bricks will be grey\n";
|
||||
UgcJobs::Outcome outcome;
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
if (mode == "--make-model") {
|
||||
const auto data = UgcBricks::ReadFile(input);
|
||||
if (!data) {
|
||||
std::cerr << "Can't read " << input << "\n";
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
outcome = UgcJobs::ProcessModel(*data, library, settings);
|
||||
} else {
|
||||
CDClientDatabase::Connect((BinaryPathFinder::GetBinaryDir() / "resServer/CDServer.sqlite").string());
|
||||
UgcJobs::ModularInput modular;
|
||||
std::string error;
|
||||
if (!UgcCdClient::GatherModular(input, modular, error)) {
|
||||
std::cerr << error << "\n";
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
outcome = UgcJobs::ProcessModular(modular, res, settings);
|
||||
}
|
||||
const auto ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
|
||||
if (!outcome.ok) {
|
||||
std::cerr << "Failed: " << outcome.error << "\n";
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
std::filesystem::create_directories(output);
|
||||
for (const auto& [name, data] : outcome.files) {
|
||||
std::ofstream(output / name, std::ios::binary).write(data.data(), static_cast<std::streamsize>(data.size()));
|
||||
}
|
||||
std::cout << "Made " << outcome.files.size() << " files in " << ms << " ms" << (outcome.note.empty() ? "" : ": " + outcome.note) << "\n";
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
Diagnostics::SetProduceMemoryDump(true);
|
||||
std::signal(SIGINT, Game::OnSignal);
|
||||
std::signal(SIGTERM, Game::OnSignal);
|
||||
|
||||
Game::config = new dConfig("ugcconfig.ini");
|
||||
|
||||
// UgcServer --make-model <file.lxfml> <folder> or --make-modular "1:4713+1:4714+1:4715" <folder>
|
||||
if (argc == 4 && (std::string(argv[1]) == "--make-model" || std::string(argv[1]) == "--make-modular")) {
|
||||
return MakeFromCommandLine(argv[1], argv[2], argv[3]);
|
||||
}
|
||||
|
||||
Server::SetupLogger("UgcServer");
|
||||
if (!Game::logger) return EXIT_FAILURE;
|
||||
Game::config->LogSettings();
|
||||
LOG("Starting UGC Server");
|
||||
|
||||
const auto res = ResPath();
|
||||
if (res.empty() || !std::filesystem::exists(res)) {
|
||||
LOG("client_location is not set or has no res folder; the UGC server needs the client's brick data");
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
try {
|
||||
CDClientDatabase::Connect((BinaryPathFinder::GetBinaryDir() / "resServer/CDServer.sqlite").string());
|
||||
} catch (std::exception& ex) {
|
||||
LOG("Failed to connect to CDClient database: %s", ex.what());
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
try {
|
||||
Database::Connect();
|
||||
} catch (std::exception& ex) {
|
||||
LOG("Failed to connect to the database: %s", ex.what());
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
// Settings edited on the dashboard (server_config table) are layered over the files from here on
|
||||
Game::config->SetDatabaseSync(ConfigSync::Sync);
|
||||
|
||||
std::string masterIP = "localhost";
|
||||
uint32_t masterPort = 1000;
|
||||
std::string masterPassword;
|
||||
if (const auto masterInfo = Database::Get()->GetMasterInfo()) {
|
||||
masterIP = masterInfo->ip;
|
||||
masterPort = masterInfo->port;
|
||||
masterPassword = masterInfo->password;
|
||||
}
|
||||
|
||||
// The master starts it again when this link drops
|
||||
g_Server = new dServer(masterIP, Setting<uint32_t>("net_port", 2012), 0, 16, false, false, Game::logger, masterIP, masterPort,
|
||||
ServiceType::UGC, Game::config, &Game::lastSignal, masterPassword);
|
||||
Game::server = g_Server;
|
||||
|
||||
UgcBricks::BrickLibrary library(res, Setting<uint32_t>("brick_lod", 0));
|
||||
if (!library.LoadMaterials()) LOG("Couldn't read Materials.xml from %s; bricks will be grey", (res / "brickdb.zip").string().c_str());
|
||||
|
||||
auto outputDir = std::filesystem::path(Game::config->GetValue("ugc_output_dir").empty() ? "ugc" : Game::config->GetValue("ugc_output_dir"));
|
||||
if (outputDir.is_relative()) outputDir = BinaryPathFinder::GetBinaryDir() / outputDir;
|
||||
UgcStorage storage(outputDir);
|
||||
g_Storage = &storage;
|
||||
|
||||
UgcProcessor::Config processorConfig;
|
||||
processorConfig.pollIntervalMs = std::max<uint32_t>(Setting<uint32_t>("poll_interval_ms", 2000), 100);
|
||||
processorConfig.pollBatch = std::max<uint32_t>(Setting<uint32_t>("poll_batch", 32), 1);
|
||||
processorConfig.maxAttempts = std::max<uint32_t>(Setting<uint32_t>("max_attempts", 3), 1);
|
||||
processorConfig.maxStorageBytes = Setting<uint64_t>("ugc_max_storage_mb", 2048) * 1024 * 1024;
|
||||
const auto threads = Setting<uint32_t>("worker_threads", 0);
|
||||
processorConfig.threads = threads > 0 ? threads : std::max<size_t>(std::thread::hardware_concurrency() / 2, 1);
|
||||
UgcProcessor processor(processorConfig, storage, library, ReadSettings());
|
||||
g_Processor = &processor;
|
||||
|
||||
const auto listenIp = Game::config->GetValue("listen_ip").empty() ? std::string("0.0.0.0") : Game::config->GetValue("listen_ip");
|
||||
const auto port = Setting<uint32_t>("port", 2008);
|
||||
if (!Game::web.Startup(listenIp, port)) {
|
||||
LOG("Failed to start the web server on %s:%u", listenIp.c_str(), port);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
RegisterRoutes();
|
||||
processor.Start();
|
||||
LOG("UGC Server started on %s:%u, files in %s", listenIp.c_str(), port, outputDir.string().c_str());
|
||||
|
||||
auto lastTick = std::chrono::steady_clock::now();
|
||||
while (!Game::ShouldShutdown()) {
|
||||
// The poll's wait for network traffic is also this loop's sleep
|
||||
Game::web.ReceiveRequests(10);
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
if (now - lastTick < std::chrono::milliseconds(16)) continue;
|
||||
lastTick = now;
|
||||
|
||||
Packet* packet = g_Server->ReceiveFromMaster();
|
||||
while (packet) {
|
||||
g_Server->DeallocateMasterPacket(packet);
|
||||
packet = g_Server->ReceiveFromMaster();
|
||||
}
|
||||
processor.Update();
|
||||
}
|
||||
|
||||
LOG("Stopping the UGC server");
|
||||
processor.Stop();
|
||||
Game::web.Shutdown();
|
||||
g_Processor = nullptr;
|
||||
Database::Destroy("UgcServer");
|
||||
delete g_Server;
|
||||
g_Server = nullptr;
|
||||
Game::server = nullptr;
|
||||
delete Game::logger;
|
||||
Game::logger = nullptr;
|
||||
delete Game::config;
|
||||
Game::config = nullptr;
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
132
dUgcServer/UgcStorage.cpp
Normal file
132
dUgcServer/UgcStorage.cpp
Normal file
@@ -0,0 +1,132 @@
|
||||
#include "UgcStorage.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <fstream>
|
||||
#include <random>
|
||||
|
||||
namespace {
|
||||
constexpr std::array KNOWN_FILES = {
|
||||
"model.nif", "model.nif.gz", "model.nif.checksum",
|
||||
"model.lxfml.gz", "model.lxfml.checksum",
|
||||
"icon.dds.gz", "icon.dds.checksum", "icon.png",
|
||||
};
|
||||
|
||||
const char* KindFolder(UgcStorage::Kind kind) {
|
||||
return kind == UgcStorage::Kind::MODEL ? "models" : "modular";
|
||||
}
|
||||
|
||||
std::string RandomSuffix() {
|
||||
static thread_local std::mt19937_64 random{ std::random_device{}() };
|
||||
return std::to_string(random());
|
||||
}
|
||||
}
|
||||
|
||||
UgcStorage::UgcStorage(std::filesystem::path root) : m_Root(std::move(root)) {}
|
||||
|
||||
bool UgcStorage::IsKnownFile(const std::string& name) {
|
||||
return std::find(KNOWN_FILES.begin(), KNOWN_FILES.end(), name) != KNOWN_FILES.end();
|
||||
}
|
||||
|
||||
std::filesystem::path UgcStorage::Folder(Kind kind, LWOOBJID id) const {
|
||||
const auto id64 = static_cast<uint64_t>(id);
|
||||
return m_Root / KindFolder(kind) / std::to_string(id64 % 1000) / std::to_string(id64);
|
||||
}
|
||||
|
||||
std::optional<std::filesystem::path> UgcStorage::File(Kind kind, LWOOBJID id, const std::string& name) const {
|
||||
if (!IsKnownFile(name)) return std::nullopt;
|
||||
auto path = Folder(kind, id) / name;
|
||||
std::error_code error;
|
||||
if (!std::filesystem::is_regular_file(path, error)) return std::nullopt;
|
||||
return path;
|
||||
}
|
||||
|
||||
std::optional<uint64_t> UgcStorage::Write(Kind kind, LWOOBJID id, const Files& files, std::string& error) const {
|
||||
const auto folder = Folder(kind, id);
|
||||
const auto temporary = folder.parent_path() / (".tmp-" + folder.filename().string() + "-" + RandomSuffix());
|
||||
std::error_code code;
|
||||
std::filesystem::create_directories(temporary, code);
|
||||
if (code) {
|
||||
error = "could not create " + temporary.string() + ": " + code.message();
|
||||
return std::nullopt;
|
||||
}
|
||||
uint64_t bytes = 0;
|
||||
for (const auto& [name, data] : files) {
|
||||
std::ofstream out(temporary / name, std::ios::binary | std::ios::trunc);
|
||||
out.write(data.data(), static_cast<std::streamsize>(data.size()));
|
||||
if (!out) {
|
||||
error = "could not write " + (temporary / name).string();
|
||||
std::filesystem::remove_all(temporary, code);
|
||||
return std::nullopt;
|
||||
}
|
||||
bytes += data.size();
|
||||
}
|
||||
// Swap the old folder out and the new one in; the old one is deleted after
|
||||
const auto old = folder.parent_path() / (".old-" + folder.filename().string() + "-" + RandomSuffix());
|
||||
const bool hadOld = std::filesystem::exists(folder, code);
|
||||
if (hadOld) std::filesystem::rename(folder, old, code);
|
||||
std::filesystem::rename(temporary, folder, code);
|
||||
if (code) {
|
||||
error = "could not move the files into " + folder.string() + ": " + code.message();
|
||||
std::filesystem::remove_all(temporary, code);
|
||||
return std::nullopt;
|
||||
}
|
||||
if (hadOld) std::filesystem::remove_all(old, code);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
void UgcStorage::Remove(Kind kind, LWOOBJID id) const {
|
||||
std::error_code error;
|
||||
std::filesystem::remove_all(Folder(kind, id), error);
|
||||
}
|
||||
|
||||
void UgcStorage::Touch(Kind kind, LWOOBJID id) const {
|
||||
std::error_code error;
|
||||
std::filesystem::last_write_time(Folder(kind, id), std::filesystem::file_time_type::clock::now(), error);
|
||||
}
|
||||
|
||||
std::vector<UgcStorage::Entry> UgcStorage::List() const {
|
||||
std::vector<Entry> entries;
|
||||
std::error_code error;
|
||||
for (const auto kind : { Kind::MODEL, Kind::MODULAR }) {
|
||||
const auto base = m_Root / KindFolder(kind);
|
||||
for (std::filesystem::directory_iterator bucket(base, error), end; !error && bucket != end; bucket.increment(error)) {
|
||||
if (!bucket->is_directory(error)) continue;
|
||||
for (std::filesystem::directory_iterator item(bucket->path(), error), itemEnd; !error && item != itemEnd; item.increment(error)) {
|
||||
const auto name = item->path().filename().string();
|
||||
if (name.empty() || name[0] == '.' || !item->is_directory(error)) continue;
|
||||
Entry entry;
|
||||
entry.kind = kind;
|
||||
try {
|
||||
entry.id = static_cast<LWOOBJID>(std::stoull(name));
|
||||
} catch (...) {
|
||||
continue;
|
||||
}
|
||||
entry.used = std::filesystem::last_write_time(item->path(), error);
|
||||
for (std::filesystem::directory_iterator file(item->path(), error), fileEnd; !error && file != fileEnd; file.increment(error)) {
|
||||
if (file->is_regular_file(error)) entry.bytes += file->file_size(error);
|
||||
}
|
||||
entries.push_back(entry);
|
||||
}
|
||||
error.clear();
|
||||
}
|
||||
error.clear();
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
std::vector<UgcStorage::Entry> UgcStorage::Evict(uint64_t maxBytes) const {
|
||||
auto entries = List();
|
||||
uint64_t total = 0;
|
||||
for (const auto& entry : entries) total += entry.bytes;
|
||||
std::vector<Entry> removed;
|
||||
if (total <= maxBytes) return removed;
|
||||
std::sort(entries.begin(), entries.end(), [](const Entry& a, const Entry& b) { return a.used < b.used; });
|
||||
for (const auto& entry : entries) {
|
||||
if (total <= maxBytes) break;
|
||||
Remove(entry.kind, entry.id);
|
||||
total -= std::min(total, entry.bytes);
|
||||
removed.push_back(entry);
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
60
dUgcServer/UgcStorage.h
Normal file
60
dUgcServer/UgcStorage.h
Normal file
@@ -0,0 +1,60 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "dCommonVars.h"
|
||||
|
||||
/**
|
||||
* Where the UGC server keeps what it makes: <root>/models/<id % 1000>/<id>/ for player models and
|
||||
* <root>/modular/<id % 1000>/<id>/ for modular builds. An item's files are written to a temporary folder and renamed
|
||||
* into place, so a request never sees half of them. Plain file operations, safe from any thread for different items.
|
||||
*/
|
||||
class UgcStorage {
|
||||
public:
|
||||
enum class Kind { MODEL, MODULAR };
|
||||
|
||||
// The files an item has, by name (see FileName)
|
||||
using Files = std::map<std::string, std::string>;
|
||||
|
||||
explicit UgcStorage(std::filesystem::path root);
|
||||
|
||||
const std::filesystem::path& GetRoot() const { return m_Root; }
|
||||
|
||||
std::filesystem::path Folder(Kind kind, LWOOBJID id) const;
|
||||
|
||||
// A file of an item, if it is there; `name` must be one of the names the server writes
|
||||
std::optional<std::filesystem::path> File(Kind kind, LWOOBJID id, const std::string& name) const;
|
||||
|
||||
// Replaces an item's files with `files`; the bytes written, nullopt (and `error`) on failure
|
||||
std::optional<uint64_t> Write(Kind kind, LWOOBJID id, const Files& files, std::string& error) const;
|
||||
|
||||
// Deletes an item's files
|
||||
void Remove(Kind kind, LWOOBJID id) const;
|
||||
|
||||
// Marks an item used now, for eviction
|
||||
void Touch(Kind kind, LWOOBJID id) const;
|
||||
|
||||
struct Entry {
|
||||
Kind kind{};
|
||||
LWOOBJID id{};
|
||||
uint64_t bytes{};
|
||||
std::filesystem::file_time_type used{};
|
||||
};
|
||||
|
||||
// Everything stored, with sizes and when it was last used
|
||||
std::vector<Entry> List() const;
|
||||
|
||||
// Removes the least recently used items until the total is at most `maxBytes`; what was removed
|
||||
std::vector<Entry> Evict(uint64_t maxBytes) const;
|
||||
|
||||
// Names the server writes (and serves); anything else is refused
|
||||
static bool IsKnownFile(const std::string& name);
|
||||
|
||||
private:
|
||||
std::filesystem::path m_Root;
|
||||
};
|
||||
@@ -77,13 +77,16 @@ Modular builds (`ugc_modular_build` rows, `ldf_config` like `1:4713+1:4714+1:471
|
||||
Files live under `ugc_output_dir` (default `ugc` next to the server binaries):
|
||||
|
||||
```
|
||||
ugc/models/<id % 1000>/<id>/model.nif(.gz, .checksum), model.lxfml(.gz, .checksum), icon.dds(.gz, .checksum), icon.png
|
||||
ugc/modular/<id % 1000>/<id>/icon.dds(.gz, .checksum), icon.png
|
||||
ugc/models/<id % 1000>/<id>/model.nif, model.nif.gz, model.nif.checksum, model.lxfml.gz, model.lxfml.checksum,
|
||||
icon.dds.gz, icon.dds.checksum, icon.png
|
||||
ugc/modular/<id % 1000>/<id>/icon.dds.gz, icon.dds.checksum, icon.png
|
||||
```
|
||||
|
||||
A model's files are written to a temporary folder and renamed into place, so a half written model is never served.
|
||||
`ugc_max_storage_mb` (default 2048, 0 for no limit) caps the folder: when it is over, the models whose files were used
|
||||
longest ago are deleted and marked unprocessed, and are made again the next time they are asked for.
|
||||
longest ago are deleted. Their rows stay `is_optimized = 1`; when something asks for their files the server sets
|
||||
them back to 0 and makes them again (the request answers 408 meanwhile), so deleted files are only made again when
|
||||
wanted.
|
||||
|
||||
## Database state
|
||||
|
||||
@@ -102,6 +105,15 @@ The database is the queue: the UGC server looks for rows with `is_optimized = 0`
|
||||
`max_attempts` (default 3) times. Reprocessing (dashboard) sets rows back to `is_optimized = 0, process_attempts = 0`.
|
||||
No master messages are needed for any of it.
|
||||
|
||||
### Marking models for processing (for code that writes `ugc` rows)
|
||||
|
||||
* A new row needs nothing: `is_optimized` defaults to 0 (`InsertNewUgcModel` writes 0).
|
||||
* When a model's LXFML changes, `UpdateUgcModelData` sets `is_optimized = 0, process_attempts = 0, process_error = ''`
|
||||
in the same statement. Anything that writes `ugc.lxfml` another way must do the same, or call
|
||||
`ResetUgcModelProcessing(id, false)`.
|
||||
* Deleting the row is enough when a model is deleted; its files are left until the storage cap removes them.
|
||||
* The same holds for `ugc_modular_build` (`InsertUgcBuild` rows start at 0; `ResetModularBuildProcessing`).
|
||||
|
||||
## Threads
|
||||
|
||||
The main thread owns RakNet (the master link), mongoose (HTTP) and the database. `worker_threads` (default: half the
|
||||
@@ -119,11 +131,14 @@ files are what every player on a property sees anyway):
|
||||
`<client_path>/UGCC<dc>/IMAGE128DDS/<dc><id>.dds.gz|.checksum` for the client (`client_path`, default `/ugc`, is
|
||||
the client's `UGCSERVERDIR`). HKX answers 404. A model that exists but isn't made yet (or was evicted) is moved to
|
||||
the front of the queue and answers 408 so the client asks again.
|
||||
* `/files/model/<id>/icon.png|model.nif|model.lxfml` and `/files/modular/<id>/icon.png` for the dashboard's previews
|
||||
* `/files/model/<id>/icon.png|model.nif` and `/files/modular/<id>/icon.png` for the dashboard's previews
|
||||
(with `Access-Control-Allow-Origin: *`).
|
||||
* `/status`: JSON with the queue length, what the workers are doing and totals since start.
|
||||
|
||||
Files are sent from disk (mongoose streams them) with `Cache-Control: public, max-age=86400` and an ETag of the MD5.
|
||||
Files are sent from disk (mongoose streams them, with its own ETag) with `Cache-Control: public, max-age=3600`.
|
||||
|
||||
`UgcServer --make-model <file.lxfml or sd0> <folder>` and `UgcServer --make-modular "1:4713+1:4714+1:4715" <folder>`
|
||||
make one item's files into a folder without a database, for trying settings.
|
||||
|
||||
## Dashboard
|
||||
|
||||
|
||||
57
migrations/dlu/mysql/81_ugc_processing.sql
Normal file
57
migrations/dlu/mysql/81_ugc_processing.sql
Normal file
@@ -0,0 +1,57 @@
|
||||
/* UGC server state (see docs/UgcServer.md). ugc.is_optimized: 0 waiting to be made, 1 made, 2 failed; processed_at is
|
||||
the last attempt (Unix seconds), process_attempts how many there have been and process_error the last failure. The
|
||||
same columns on ugc_modular_build for the builds' icons. Columns are only added when they are not there yet. */
|
||||
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN processed_at BIGINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'processed_at');
|
||||
PREPARE dlu_column_stmt FROM @dlu_column;
|
||||
EXECUTE dlu_column_stmt;
|
||||
DEALLOCATE PREPARE dlu_column_stmt;
|
||||
|
||||
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN process_attempts INT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'process_attempts');
|
||||
PREPARE dlu_column_stmt FROM @dlu_column;
|
||||
EXECUTE dlu_column_stmt;
|
||||
DEALLOCATE PREPARE dlu_column_stmt;
|
||||
|
||||
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc ADD COLUMN process_error VARCHAR(1024) NOT NULL DEFAULT ''''', 'DO 0') FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE() AND table_name = 'ugc' AND column_name = 'process_error');
|
||||
PREPARE dlu_column_stmt FROM @dlu_column;
|
||||
EXECUTE dlu_column_stmt;
|
||||
DEALLOCATE PREPARE dlu_column_stmt;
|
||||
|
||||
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN is_optimized INT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'is_optimized');
|
||||
PREPARE dlu_column_stmt FROM @dlu_column;
|
||||
EXECUTE dlu_column_stmt;
|
||||
DEALLOCATE PREPARE dlu_column_stmt;
|
||||
|
||||
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN processed_at BIGINT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'processed_at');
|
||||
PREPARE dlu_column_stmt FROM @dlu_column;
|
||||
EXECUTE dlu_column_stmt;
|
||||
DEALLOCATE PREPARE dlu_column_stmt;
|
||||
|
||||
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN process_attempts INT NOT NULL DEFAULT 0', 'DO 0') FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'process_attempts');
|
||||
PREPARE dlu_column_stmt FROM @dlu_column;
|
||||
EXECUTE dlu_column_stmt;
|
||||
DEALLOCATE PREPARE dlu_column_stmt;
|
||||
|
||||
SET @dlu_column = (SELECT IF(COUNT(*) = 0, 'ALTER TABLE ugc_modular_build ADD COLUMN process_error VARCHAR(1024) NOT NULL DEFAULT ''''', 'DO 0') FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND column_name = 'process_error');
|
||||
PREPARE dlu_column_stmt FROM @dlu_column;
|
||||
EXECUTE dlu_column_stmt;
|
||||
DEALLOCATE PREPARE dlu_column_stmt;
|
||||
|
||||
/* What the UGC server looks for */
|
||||
SET @dlu_index = (SELECT IF(COUNT(*) = 0, 'CREATE INDEX ugc_is_optimized ON ugc (is_optimized)', 'DO 0') FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE() AND table_name = 'ugc' AND index_name = 'ugc_is_optimized');
|
||||
PREPARE dlu_index_stmt FROM @dlu_index;
|
||||
EXECUTE dlu_index_stmt;
|
||||
DEALLOCATE PREPARE dlu_index_stmt;
|
||||
|
||||
SET @dlu_index = (SELECT IF(COUNT(*) = 0, 'CREATE INDEX ugc_modular_build_is_optimized ON ugc_modular_build (is_optimized)', 'DO 0') FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE() AND table_name = 'ugc_modular_build' AND index_name = 'ugc_modular_build_is_optimized');
|
||||
PREPARE dlu_index_stmt FROM @dlu_index;
|
||||
EXECUTE dlu_index_stmt;
|
||||
DEALLOCATE PREPARE dlu_index_stmt;
|
||||
10
migrations/dlu/sqlite/64_ugc_processing.sql
Normal file
10
migrations/dlu/sqlite/64_ugc_processing.sql
Normal file
@@ -0,0 +1,10 @@
|
||||
/* UGC server state: see the MySQL migration. */
|
||||
ALTER TABLE ugc ADD COLUMN processed_at BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE ugc ADD COLUMN process_attempts INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE ugc ADD COLUMN process_error TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE ugc_modular_build ADD COLUMN is_optimized INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE ugc_modular_build ADD COLUMN processed_at BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE ugc_modular_build ADD COLUMN process_attempts INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE ugc_modular_build ADD COLUMN process_error TEXT NOT NULL DEFAULT '';
|
||||
CREATE INDEX IF NOT EXISTS ugc_is_optimized ON ugc (is_optimized);
|
||||
CREATE INDEX IF NOT EXISTS ugc_modular_build_is_optimized ON ugc_modular_build (is_optimized);
|
||||
@@ -15,3 +15,7 @@ master_password=3.25DARKFLAME1
|
||||
# Enable the web dashboard (0 = disabled, 1 = enabled)
|
||||
# Dashboard settings are configured in dashboardconfig.ini
|
||||
enable_dashboard=0
|
||||
|
||||
# Start the UGC server, which makes and serves the meshes and icons of what players build (0 = disabled, 1 = enabled)
|
||||
# Its settings are in ugcconfig.ini; see docs/UgcServer.md for the client's boot.cfg settings
|
||||
enable_ugc_server=0
|
||||
|
||||
48
resources/ugcconfig.ini
Normal file
48
resources/ugcconfig.ini
Normal file
@@ -0,0 +1,48 @@
|
||||
# UGC server: makes the meshes and icons of what players build and serves them to the game client.
|
||||
# The master starts it when enable_ugc_server=1 (masterconfig.ini). See docs/UgcServer.md.
|
||||
|
||||
# HTTP port the game client downloads from (the client's UGCSERVERPORT)
|
||||
port=2008
|
||||
|
||||
# The IP address to bind to. The game client has to reach it, so the default is every interface.
|
||||
listen_ip=0.0.0.0
|
||||
|
||||
# UDP port for the connection to the master server (the next port is used too). It must not clash with another
|
||||
# server's ports: auth uses 1500-1501, chat 2005-2006, the dashboard 2010-2011, worlds 3000 and up.
|
||||
net_port=2012
|
||||
|
||||
# The client's UGCSERVERDIR: the path the downloads are under (the client adds /UGCC<DATACENTERID>/...)
|
||||
client_path=/ugc
|
||||
|
||||
# Where the made files are kept (relative to the server binaries), and at most how much space they may take in MB.
|
||||
# When there's more, the files used longest ago are deleted; they're made again when asked for. 0: no limit.
|
||||
ugc_output_dir=ugc
|
||||
ugc_max_storage_mb=2048
|
||||
|
||||
# Worker threads making files (0: half the CPU cores)
|
||||
worker_threads=0
|
||||
|
||||
# How often to look for models to make (milliseconds), how many to take at once, and how often a model may fail
|
||||
# before it's given up on (the dashboard can try again)
|
||||
poll_interval_ms=2000
|
||||
poll_batch=32
|
||||
max_attempts=3
|
||||
|
||||
# Brick detail used for meshes and icons: 0 (most detailed) to 2
|
||||
brick_lod=0
|
||||
|
||||
# Remove faces that can't be seen from anywhere, and darken hidden corners (ambient occlusion baked into the vertex
|
||||
# colors): 0 or 1. ao_strength: 0 (none) to 1. optimize_resolution: detail of the renders that decide both.
|
||||
remove_hidden_faces=1
|
||||
bake_ao=1
|
||||
ao_strength=0.6
|
||||
optimize_resolution=1024
|
||||
|
||||
# Icons: size in pixels, the camera's angle around (yaw, from the model's front) and above (pitch) the model in degrees,
|
||||
# and the empty border (1 fills the icon). Modular builds (cars, rockets) have their own angles.
|
||||
icon_size=128
|
||||
icon_yaw=35
|
||||
icon_pitch=25
|
||||
icon_margin=1.03
|
||||
modular_icon_yaw=35
|
||||
modular_icon_pitch=20
|
||||
@@ -9,3 +9,4 @@ add_subdirectory(dCommonTests)
|
||||
add_subdirectory(dGameTests)
|
||||
add_subdirectory(dWebTests)
|
||||
add_subdirectory(dDatabaseTests)
|
||||
add_subdirectory(dUgcTests)
|
||||
|
||||
@@ -1833,6 +1833,7 @@ static_assert(static_cast<int64_t>(ServiceType::WORLD) == 4);
|
||||
static_assert(static_cast<int64_t>(ServiceType::CLIENT) == 5);
|
||||
static_assert(static_cast<int64_t>(ServiceType::MASTER) == 6);
|
||||
static_assert(static_cast<int64_t>(ServiceType::UNKNOWN) == 7);
|
||||
static_assert(static_cast<int64_t>(ServiceType::UGC) == 8);
|
||||
|
||||
// The asserts above are compile time; this test only makes the file show up in test runs.
|
||||
TEST(MessageIdPinTests, MessageTypeValuesArePinned) { SUCCEED(); }
|
||||
|
||||
@@ -704,6 +704,50 @@ TEST_F(ParitySeeded, UgcModel) {
|
||||
auto model = db.GetUgcModel(1152921510000500001LL);
|
||||
return model ? json{ model->id, model->modelID, model->lxfmlData.str() } : json();
|
||||
});
|
||||
|
||||
// The UGC server's processing state (processed_at is the time of the call, so only whether it's set is compared)
|
||||
const auto infoJson = [](const std::optional<IUgc::ProcessInfo>& info) {
|
||||
return info ? json{ info->id, info->characterId, info->characterName, static_cast<int32_t>(info->state), info->attempts,
|
||||
info->processedAt > 0, info->error, info->bakeAo, info->details } : json();
|
||||
};
|
||||
const auto countsJson = [](const std::vector<std::pair<IUgc::eProcessState, uint64_t>>& counts) {
|
||||
json out = json::array();
|
||||
for (const auto& [state, count] : counts) out.push_back({ static_cast<int32_t>(state), count });
|
||||
return out;
|
||||
};
|
||||
Both("GetUgcModelsToProcess", [](GameDatabase& db) {
|
||||
json out = json::array();
|
||||
for (const auto& model : db.GetUgcModelsToProcess(10)) out.push_back({ model.id, model.lxfml, model.attempts });
|
||||
return out;
|
||||
});
|
||||
Both("SetUgcModelProcessed failed", [&](GameDatabase& db) {
|
||||
db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::FAILED, 3, "no bricks", false);
|
||||
return json{ infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), countsJson(db.GetUgcProcessCounts()), db.GetUgcModelsToProcess(10).size() };
|
||||
});
|
||||
Both("GetUgcProcessList", [&](GameDatabase& db) {
|
||||
json out = json::array();
|
||||
for (const auto& info : db.GetUgcProcessList(IUgc::eProcessState::FAILED, 0, 10)) out.push_back(infoJson(info));
|
||||
out.push_back(db.GetUgcProcessList(std::nullopt, 0, 10).size());
|
||||
out.push_back(db.GetUgcProcessList(IUgc::eProcessState::DONE, 0, 10).size());
|
||||
return out;
|
||||
});
|
||||
Both("ResetUgcModelProcessing", [&](GameDatabase& db) {
|
||||
const auto changed = db.ResetUgcModelProcessing(std::nullopt, true);
|
||||
db.SetUgcModelProcessed(1152921510000500001LL, IUgc::eProcessState::DONE, 1, "", true);
|
||||
return json{ changed, infoJson(db.GetUgcProcessInfo(1152921510000500001LL)), infoJson(db.GetUgcProcessInfo(1)) };
|
||||
});
|
||||
Both("Modular build processing", [&](GameDatabase& db) {
|
||||
db.InsertUgcBuild("1:4713+1:4714+1:4715", 1152921510000500002LL, CHAR_ALICE);
|
||||
json out = json::array();
|
||||
for (const auto& build : db.GetModularBuildsToProcess(10)) out.push_back({ build.id, build.modules, build.attempts });
|
||||
db.SetModularBuildProcessed(1152921510000500002LL, IUgc::eProcessState::PENDING, 1, "no mesh");
|
||||
out.push_back(infoJson(db.GetModularBuildProcessInfo(1152921510000500002LL)));
|
||||
out.push_back(countsJson(db.GetModularBuildProcessCounts()));
|
||||
out.push_back(db.GetModularBuildProcessList(std::nullopt, 0, 10).size());
|
||||
out.push_back(db.ResetModularBuildProcessing(1152921510000500002LL, false));
|
||||
out.push_back(infoJson(db.GetModularBuildProcessInfo(1152921510000500002LL)));
|
||||
return out;
|
||||
});
|
||||
}
|
||||
|
||||
TEST_F(ParitySeeded, LogsAndAudit) {
|
||||
|
||||
3
tests/dUgcTests/CMakeLists.txt
Normal file
3
tests/dUgcTests/CMakeLists.txt
Normal file
@@ -0,0 +1,3 @@
|
||||
add_executable(dUgcTests "UgcTests.cpp")
|
||||
target_link_libraries(dUgcTests ${COMMON_LIBRARIES} dUgc GTest::gtest_main)
|
||||
gtest_discover_tests(dUgcTests)
|
||||
291
tests/dUgcTests/UgcTests.cpp
Normal file
291
tests/dUgcTests/UgcTests.cpp
Normal file
@@ -0,0 +1,291 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
|
||||
#include <glm/gtc/matrix_transform.hpp>
|
||||
|
||||
#include "Game.h"
|
||||
#include "NifFile.h"
|
||||
#include "UgcBricks.h"
|
||||
#include "UgcFormats.h"
|
||||
#include "UgcModel.h"
|
||||
#include "UgcModular.h"
|
||||
#include "UgcRender.h"
|
||||
#include "UgcStorage.h"
|
||||
#include "ZCompression.h"
|
||||
|
||||
class Logger;
|
||||
class dConfig;
|
||||
namespace Game {
|
||||
Logger* logger = nullptr;
|
||||
dConfig* config = nullptr;
|
||||
}
|
||||
|
||||
namespace {
|
||||
// A closed box as an LDD .g file: 8 corners, 12 triangles
|
||||
std::string BoxGeometry(glm::vec3 min, glm::vec3 max) {
|
||||
std::vector<float> positions, normals;
|
||||
for (int i = 0; i < 8; i++) {
|
||||
const glm::vec3 p((i & 1) ? max.x : min.x, (i & 2) ? max.y : min.y, (i & 4) ? max.z : min.z);
|
||||
const auto n = glm::normalize(p - (min + max) * 0.5f);
|
||||
positions.insert(positions.end(), { p.x, p.y, p.z });
|
||||
normals.insert(normals.end(), { n.x, n.y, n.z });
|
||||
}
|
||||
const std::vector<uint32_t> indices = { 0, 2, 1, 1, 2, 3, 4, 5, 6, 5, 7, 6, 0, 1, 4, 1, 5, 4, 2, 6, 3, 3, 6, 7, 0, 4, 2, 2, 4, 6, 1, 3, 5, 3, 7, 5 };
|
||||
std::string out;
|
||||
const int32_t header[4] = { 0x42473031, 8, static_cast<int32_t>(indices.size()), 0 };
|
||||
out.append(reinterpret_cast<const char*>(header), sizeof(header));
|
||||
out.append(reinterpret_cast<const char*>(positions.data()), positions.size() * 4);
|
||||
out.append(reinterpret_cast<const char*>(normals.data()), normals.size() * 4);
|
||||
out.append(reinterpret_cast<const char*>(indices.data()), indices.size() * 4);
|
||||
return out;
|
||||
}
|
||||
|
||||
std::filesystem::path TempFolder(const std::string& name) {
|
||||
auto path = std::filesystem::temp_directory_path() / ("dlu_ugc_test_" + name + "_" + std::to_string(::testing::UnitTest::GetInstance()->random_seed()));
|
||||
std::filesystem::remove_all(path);
|
||||
std::filesystem::create_directories(path);
|
||||
return path;
|
||||
}
|
||||
|
||||
// A res folder with brick 3001 (a 1x1x1 box) and brick 3002 (a big box)
|
||||
std::filesystem::path MakeRes() {
|
||||
const auto res = TempFolder("res");
|
||||
std::filesystem::create_directories(res / "brickprimitives" / "lod0");
|
||||
std::ofstream(res / "brickprimitives" / "lod0" / "3001.g", std::ios::binary) << BoxGeometry(glm::vec3(0.0f), glm::vec3(1.0f));
|
||||
std::ofstream(res / "brickprimitives" / "lod0" / "3002.g", std::ios::binary) << BoxGeometry(glm::vec3(-4.0f), glm::vec3(4.0f));
|
||||
return res;
|
||||
}
|
||||
|
||||
const char* LXFML5 = R"(<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
|
||||
<LXFML versionMajor="5" versionMinor="0"><Bricks>
|
||||
<Brick refID="0" designID="3001"><Part refID="0" designID="3001" materials="21,0"><Bone refID="0" transformation="1,0,0,0,1,0,0,0,1,10,0,0"/></Part></Brick>
|
||||
<Brick refID="1" designID="3001"><Part refID="1" designID="3001" materials="40"><Bone refID="1" transformation="1,0,0,0,1,0,0,0,1,0,5,0"/></Part></Brick>
|
||||
<Brick refID="2" designID="9999"><Part refID="2" designID="9999" materials="1"><Bone refID="2" transformation="1,0,0,0,1,0,0,0,1,0,0,0"/></Part></Brick>
|
||||
</Bricks></LXFML>)";
|
||||
}
|
||||
|
||||
TEST(UgcCompression, GzipRoundTrip) {
|
||||
const std::string data(10000, 'x');
|
||||
const auto gz = ZCompression::Gzip(data);
|
||||
ASSERT_GE(gz.size(), 2u);
|
||||
EXPECT_EQ(static_cast<uint8_t>(gz[0]), 0x1f);
|
||||
EXPECT_EQ(static_cast<uint8_t>(gz[1]), 0x8b);
|
||||
EXPECT_EQ(ZCompression::Gunzip(gz), data);
|
||||
EXPECT_FALSE(ZCompression::Gunzip("not gzip"));
|
||||
}
|
||||
|
||||
TEST(UgcBricks, ParsesGeometryAndRejectsBadData) {
|
||||
const auto geometry = UgcBricks::ParseGeometry(BoxGeometry(glm::vec3(0.0f), glm::vec3(1.0f)));
|
||||
ASSERT_TRUE(geometry);
|
||||
EXPECT_EQ(geometry->positions.size(), 24u);
|
||||
EXPECT_EQ(geometry->indices.size(), 36u);
|
||||
EXPECT_FALSE(UgcBricks::ParseGeometry("10GB"));
|
||||
auto broken = BoxGeometry(glm::vec3(0.0f), glm::vec3(1.0f));
|
||||
broken.resize(broken.size() - 4);
|
||||
EXPECT_FALSE(UgcBricks::ParseGeometry(broken));
|
||||
}
|
||||
|
||||
TEST(UgcBricks, ParsesMaterials) {
|
||||
const auto materials = UgcBricks::ParseMaterials(R"(<Materials><Material MatID="21" Red="222" Green="0" Blue="13" Alpha="255"/><Material MatID="40" Red="238" Green="238" Blue="238" Alpha="150"/></Materials>)");
|
||||
ASSERT_EQ(materials.size(), 2u);
|
||||
EXPECT_EQ(materials.at(21).r, 222);
|
||||
EXPECT_FALSE(materials.at(21).Transparent());
|
||||
EXPECT_TRUE(materials.at(40).Transparent());
|
||||
}
|
||||
|
||||
TEST(UgcBricks, ReadsStoredZipEntries) {
|
||||
// A zip with one stored file, "Materials.xml"
|
||||
const std::string name = "Materials.xml", content = "<Materials/>";
|
||||
std::string zip;
|
||||
const auto u16 = [&zip](uint16_t v) { zip.append(reinterpret_cast<const char*>(&v), 2); };
|
||||
const auto u32 = [&zip](uint32_t v) { zip.append(reinterpret_cast<const char*>(&v), 4); };
|
||||
u32(0x04034b50); u16(20); u16(0); u16(0); u16(0); u16(0); u32(0); u32(content.size()); u32(content.size()); u16(name.size()); u16(0);
|
||||
zip += name + content;
|
||||
const auto central = static_cast<uint32_t>(zip.size());
|
||||
u32(0x02014b50); u16(20); u16(20); u16(0); u16(0); u16(0); u16(0); u32(0); u32(content.size()); u32(content.size()); u16(name.size());
|
||||
u16(0); u16(0); u16(0); u16(0); u32(0); u32(0);
|
||||
zip += name;
|
||||
const auto centralSize = static_cast<uint32_t>(zip.size()) - central;
|
||||
u32(0x06054b50); u16(0); u16(0); u16(1); u16(1); u32(centralSize); u32(central); u16(0);
|
||||
EXPECT_EQ(UgcBricks::ReadZipEntry(zip, "materials.XML"), content);
|
||||
EXPECT_FALSE(UgcBricks::ReadZipEntry(zip, "Other.xml"));
|
||||
}
|
||||
|
||||
TEST(UgcModel, ParsesLxfml5And4) {
|
||||
std::string error;
|
||||
const auto parts = UgcModel::ParseLxfml(LXFML5, error);
|
||||
ASSERT_EQ(parts.size(), 3u);
|
||||
EXPECT_EQ(parts[0].designId, 3001u);
|
||||
EXPECT_EQ(parts[0].materials, (std::vector<uint32_t>{ 21, 21 })); // 0: the part's first material
|
||||
EXPECT_FLOAT_EQ(parts[0].transform[3].x, 10.0f);
|
||||
|
||||
const auto v4 = UgcModel::ParseLxfml(R"(<LXFML versionMajor="4"><Scene><Model><Group ax="0" ay="1" az="0" angle="90" tx="1" ty="0" tz="0">
|
||||
<Part designID="3001" materialID="21" ax="0" ay="1" az="0" angle="0" tx="0" ty="2" tz="0"/></Group></Model></Scene></LXFML>)", error);
|
||||
ASSERT_EQ(v4.size(), 1u);
|
||||
const auto origin = v4[0].transform * glm::vec4(0, 0, 0, 1);
|
||||
EXPECT_NEAR(origin.x, 1.0f, 1e-5f);
|
||||
EXPECT_NEAR(origin.y, 2.0f, 1e-5f);
|
||||
|
||||
EXPECT_TRUE(UgcModel::ParseLxfml("<nope", error).empty());
|
||||
EXPECT_FALSE(error.empty());
|
||||
}
|
||||
|
||||
TEST(UgcModel, BuildsOpaqueAndTransparentMeshes) {
|
||||
UgcBricks::BrickLibrary library(MakeRes(), 0);
|
||||
library.SetMaterials({ { 21, { 222, 0, 13, 255 } }, { 40, { 238, 238, 238, 150 } } });
|
||||
std::string error;
|
||||
const auto model = UgcModel::Build(UgcModel::ParseLxfml(LXFML5, error), library);
|
||||
EXPECT_EQ(model.bricks, 2u);
|
||||
EXPECT_EQ(model.missingDesigns, std::vector<uint32_t>{ 9999 });
|
||||
EXPECT_EQ(model.opaque.TriangleCount(), 12u);
|
||||
EXPECT_EQ(model.transparent.TriangleCount(), 12u);
|
||||
EXPECT_NEAR(model.opaque.colors[0].r, 222.0f / 255.0f, 1e-5f);
|
||||
EXPECT_NEAR(model.transparent.colors[0].a, 150.0f / 255.0f, 1e-5f);
|
||||
EXPECT_NEAR(model.opaque.positions[0].x, 10.0f, 1e-5f);
|
||||
}
|
||||
|
||||
TEST(UgcModel, SplitsBigMeshes) {
|
||||
UgcModel::Mesh mesh;
|
||||
for (uint32_t i = 0; i < 30; i++) {
|
||||
mesh.positions.push_back(glm::vec3(static_cast<float>(i)));
|
||||
mesh.normals.push_back(glm::vec3(0, 1, 0));
|
||||
mesh.colors.push_back(glm::vec4(1.0f));
|
||||
}
|
||||
for (uint32_t i = 0; i + 2 < 30; i += 3) mesh.indices.insert(mesh.indices.end(), { i, i + 1, i + 2 });
|
||||
const auto pieces = UgcModel::Split(mesh, 9, 100);
|
||||
ASSERT_EQ(pieces.size(), 4u); // 10 triangles, 3 fit per piece
|
||||
size_t triangles = 0;
|
||||
for (const auto& piece : pieces) {
|
||||
EXPECT_LE(piece.positions.size(), 9u);
|
||||
triangles += piece.TriangleCount();
|
||||
}
|
||||
EXPECT_EQ(triangles, 10u);
|
||||
}
|
||||
|
||||
TEST(UgcRender, RemovesWhatIsInsideAndDrawsIcons) {
|
||||
UgcBricks::BrickLibrary library(MakeRes(), 0);
|
||||
library.SetMaterials({ { 21, { 222, 0, 13, 255 } } });
|
||||
std::string error;
|
||||
// A small box inside the big one: its faces can't be seen
|
||||
const auto parts = UgcModel::ParseLxfml(R"(<LXFML versionMajor="5"><Bricks>
|
||||
<Brick><Part designID="3002" materials="21"><Bone transformation="1,0,0,0,1,0,0,0,1,0,0,0"/></Part></Brick>
|
||||
<Brick><Part designID="3001" materials="21"><Bone transformation="1,0,0,0,1,0,0,0,1,0,0,0"/></Part></Brick>
|
||||
</Bricks></LXFML>)", error);
|
||||
auto model = UgcModel::Build(parts, library);
|
||||
ASSERT_EQ(model.opaque.TriangleCount(), 24u);
|
||||
const auto result = UgcRender::Optimize(model, UgcRender::OptimizeOptions{ 256, true, true, 0.6f });
|
||||
EXPECT_EQ(result.trianglesRemoved, 12u);
|
||||
EXPECT_EQ(model.opaque.TriangleCount(), 12u);
|
||||
EXPECT_EQ(model.opaque.positions.size(), 8u);
|
||||
|
||||
const auto icon = UgcRender::RenderIcon(model, UgcRender::IconOptions{ 32, 2 });
|
||||
ASSERT_EQ(icon.rgba.size(), 32u * 32u * 4u);
|
||||
EXPECT_EQ(icon.rgba[3], 0); // a corner is background
|
||||
EXPECT_EQ(icon.rgba[(16 * 32 + 16) * 4 + 3], 255); // the middle is the box
|
||||
EXPECT_GT(icon.rgba[(16 * 32 + 16) * 4], icon.rgba[(16 * 32 + 16) * 4 + 1]); // red
|
||||
EXPECT_EQ(UgcRender::SphereDirections().size(), 42u);
|
||||
}
|
||||
|
||||
TEST(UgcFormats, NifReadsBack) {
|
||||
UgcModel::Mesh opaque, transparent;
|
||||
opaque.positions = { { 0, 0, 0 }, { 1, 0, 0 }, { 0, 1, 0 } };
|
||||
opaque.normals = { { 0, 0, 1 }, { 0, 0, 1 }, { 0, 0, 1 } };
|
||||
opaque.colors = { { 1, 0, 0, 1 }, { 1, 0, 0, 1 }, { 1, 0, 0, 1 } };
|
||||
opaque.indices = { 0, 1, 2 };
|
||||
transparent = opaque;
|
||||
for (auto& color : transparent.colors) color.a = 0.5f;
|
||||
const auto nif = UgcFormats::WriteNif("SceneNode_Model", { { "S01_Opaque_Model", &opaque, false }, { "S01_Alpha_Model", &transparent, true } });
|
||||
ASSERT_TRUE(nif.starts_with("Gamebryo File Format, Version 20.3.0.9\n"));
|
||||
std::string error;
|
||||
const auto model = NifFile::Parse(nif, 0, error);
|
||||
ASSERT_TRUE(model) << error;
|
||||
EXPECT_TRUE(model->skipped.empty());
|
||||
ASSERT_EQ(model->meshes.size(), 2u);
|
||||
EXPECT_EQ(model->meshes[0].indices.size(), 3u);
|
||||
EXPECT_EQ(model->meshes[0].colors[0], 255);
|
||||
EXPECT_FALSE(model->meshes[0].material.alphaBlend);
|
||||
EXPECT_TRUE(model->meshes[1].material.alphaBlend);
|
||||
EXPECT_EQ(model->meshes[1].colors[3], 128);
|
||||
EXPECT_EQ(model->meshes[0].material.vertexColorMode, 2);
|
||||
EXPECT_TRUE(model->nodes.contains("SceneNode_Model"));
|
||||
}
|
||||
|
||||
TEST(UgcFormats, ImagesAndChecksums) {
|
||||
UgcRender::Image image{ 2, 2, std::vector<uint8_t>(16, 0) };
|
||||
image.rgba[0] = 10; // red of the first pixel
|
||||
image.rgba[3] = 255;
|
||||
const auto png = UgcFormats::EncodePng(image);
|
||||
EXPECT_TRUE(png.starts_with("\x89PNG\r\n\x1a\n"));
|
||||
const auto dds = UgcFormats::EncodeDds(image);
|
||||
ASSERT_EQ(dds.size(), 128u + 16u);
|
||||
EXPECT_TRUE(dds.starts_with("DDS "));
|
||||
EXPECT_EQ(dds[128 + 2], 10); // stored BGRA
|
||||
EXPECT_EQ(UgcFormats::Md5Hex("abc"), "900150983cd24fb0d6963f7d28e17f72");
|
||||
EXPECT_NE(UgcFormats::ChecksumXml("abc").find("<Checksum><MD5>900150983cd24fb0d6963f7d28e17f72</MD5><Filesize>3</Filesize></Checksum>"), std::string::npos);
|
||||
}
|
||||
|
||||
TEST(UgcModular, ParsesTheCdClientData) {
|
||||
EXPECT_EQ(UgcModular::ParseModuleLots("1:4713+1:4714+1:4715"), (std::vector<uint32_t>{ 4713, 4714, 4715 }));
|
||||
EXPECT_EQ(UgcModular::ParseModuleLots("1:8129;1:x;1:8130"), (std::vector<uint32_t>{ 8129, 8130 }));
|
||||
const auto build = UgcModular::ParseBuild(R"(<ModularBuild><topology><numberOfParts value="3" /><rootPart value="2" />
|
||||
<connection myPartid="2" myLocation="CP_A1" connectingPart="1" /><connection myPartid="1" myLocation="CP_B2" connectingPart="0"/></topology>
|
||||
<Placement><AdditionalModelRotation><Rotation w="0.707" x="0" y="-0.707" z="0" /></AdditionalModelRotation></Placement></ModularBuild>)");
|
||||
ASSERT_TRUE(build);
|
||||
EXPECT_EQ(build->rootPart, 2u);
|
||||
ASSERT_EQ(build->connections.size(), 2u);
|
||||
EXPECT_EQ(build->connections[1].location, "CP_B2");
|
||||
const auto connections = UgcModular::ParseModuleConnections(R"(<ModuleInfo moduleLOT="4714"><connection name="CP_B2"><translation x="0" y="0" z="5.2" /></connection></ModuleInfo>)");
|
||||
EXPECT_FLOAT_EQ(connections.at("CP_B2").z, 5.2f);
|
||||
EXPECT_FALSE(UgcModular::ParseBuild("<ModularBuild/>"));
|
||||
}
|
||||
|
||||
TEST(UgcModular, PutsPartsOnTheirAttachPoints) {
|
||||
const auto triangle = [] {
|
||||
NifFile::Model nif;
|
||||
NifFile::Mesh mesh;
|
||||
mesh.positions = { 0, 0, 0, 1, 0, 0, 0, 1, 0 };
|
||||
mesh.indices = { 0, 1, 2 };
|
||||
nif.meshes.push_back(mesh);
|
||||
return nif;
|
||||
};
|
||||
UgcModular::BuildInfo build;
|
||||
build.rootPart = 2;
|
||||
build.connections = { { 2, "CP_A1", 1 }, { 1, "CP_B2", 0 } };
|
||||
std::vector<UgcModular::Module> modules(3);
|
||||
modules[0].partCode = 2; // bottom: node CP_A1 at y 5
|
||||
modules[0].nif = triangle();
|
||||
modules[0].nif.nodes["CP_A1"].translation = { 0, 5, 0 };
|
||||
modules[1].partCode = 1; // middle: no node, a connection offset of y 3 in the CDClient
|
||||
modules[1].nif = triangle();
|
||||
modules[1].connections["CP_B2"] = glm::vec3(0, 3, 0);
|
||||
modules[2].partCode = 0; // top: its own CP_B2 node at y 1 lines up with the middle's
|
||||
modules[2].nif = triangle();
|
||||
modules[2].nif.nodes["CP_B2"].translation = { 0, 1, 0 };
|
||||
std::string warnings;
|
||||
const auto model = UgcModular::Assemble(build, modules, warnings);
|
||||
EXPECT_TRUE(warnings.empty()) << warnings;
|
||||
ASSERT_EQ(model.opaque.positions.size(), 9u);
|
||||
EXPECT_FLOAT_EQ(model.opaque.positions[0].y, 0.0f);
|
||||
EXPECT_FLOAT_EQ(model.opaque.positions[3].y, 5.0f);
|
||||
EXPECT_FLOAT_EQ(model.opaque.positions[6].y, 7.0f); // 5 + 3 - 1
|
||||
}
|
||||
|
||||
TEST(UgcStorage, WritesListsAndEvicts) {
|
||||
UgcStorage storage(TempFolder("storage"));
|
||||
std::string error;
|
||||
ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODEL, 1001, { { "icon.png", std::string(100, 'a') } }, error)) << error;
|
||||
ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODULAR, 2002, { { "icon.png", std::string(100, 'b') } }, error)) << error;
|
||||
ASSERT_TRUE(storage.Write(UgcStorage::Kind::MODEL, 1001, { { "icon.png", std::string(50, 'c') } }, error)) << error; // replaced
|
||||
EXPECT_TRUE(storage.File(UgcStorage::Kind::MODEL, 1001, "icon.png"));
|
||||
EXPECT_FALSE(storage.File(UgcStorage::Kind::MODEL, 1001, "../../etc/passwd"));
|
||||
EXPECT_EQ(storage.List().size(), 2u);
|
||||
std::filesystem::last_write_time(storage.Folder(UgcStorage::Kind::MODULAR, 2002), std::filesystem::file_time_type::clock::now() - std::chrono::hours(1));
|
||||
const auto removed = storage.Evict(60);
|
||||
ASSERT_EQ(removed.size(), 1u);
|
||||
EXPECT_EQ(removed[0].id, 2002);
|
||||
EXPECT_TRUE(storage.File(UgcStorage::Kind::MODEL, 1001, "icon.png"));
|
||||
std::filesystem::remove_all(storage.GetRoot());
|
||||
}
|
||||
@@ -38,7 +38,6 @@ set(DWEBTESTS_SOURCES
|
||||
"WorkerPoolTests.cpp"
|
||||
"OnceCacheTests.cpp"
|
||||
"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/WorkerPool.cpp"
|
||||
"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/NifFile.cpp"
|
||||
"BackupFilesTests.cpp"
|
||||
"SecurityFixesTests.cpp"
|
||||
"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/BackupFiles.cpp"
|
||||
|
||||
Reference in New Issue
Block a user