Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-27 23:17:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ugc): icon light matched to the game, debounce, shared car icons, purge, icon editor

Browse Source 
Brighter icons: a world light, a fill from the camera, a highlight, exposure
and contrast; with the defaults the icons' mean luminance matches the game's
own model icons (118 against 120 on a scratch set). Every icon parameter is
listed once (UgcIconParams: key, setting, range, default); the settings,
the dashboard's settings entries and the icon editor come from that list.
Presets per kind (player models, each car or rocket build type from
ModularBuildComponent) and overrides per model or module combination are in
ugc_icon_settings.

Saved models wait ugc_debounce_seconds (sharedconfig) after the owner's last
save before they're made (ugc.process_after); a client asking for one, the
owner leaving the world or a reset ends the wait.

Cars and rockets: one icon per combination of modules (sorted LOTs), shared
by every build of it; builds of a combination made already are marked made
right away, the client's per-blueprint downloads serve the shared files.

The dashboard can delete one item's files, purge by filter or all, preview
icons with any values on the UGC server (/admin routes, master password),
save presets and overrides, and draw a kind's icons again (icons only).

Migrations dlu/mysql/86 and dlu/sqlite/69. Not done yet: the dashboard
editor's lighting controls in the page script (routes are there), docs for
it, the empty-model state, /ugc?item= links, the shared fetch helper; the
storage size and property loading bugs are next.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-27 13:51:04 -05:00
parent 0c1253cd16
commit 9d2ad053fd
42 changed files with 1576 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
dUgcServer/CMakeLists.txt
View File

@@ -2,6 +2,7 @@
set(DUGC_SOURCES
"UgcBricks.cpp"
"UgcFormats.cpp"
"UgcIconParams.cpp"
"UgcJobs.cpp"
"UgcModel.cpp"
"UgcModular.cpp"

1
dUgcServer/UgcCdClient.cpp
View File

@@ -34,6 +34,7 @@ namespace UgcCdClient {
module.partCode = static_cast<uint32_t>(moduleRow.getIntField("partCode", 0));
module.moduleXml = moduleRow.getStringField("xml", "");
if (!buildType) buildType = moduleRow.getIntField("buildType", 0);
input.buildType = *buildType;
auto renderQuery = CDClientDatabase::CreatePreppedStmt(
"SELECT rc.render_asset FROM ComponentsRegistry cr JOIN RenderComponent rc ON rc.id = cr.component_id "

88
dUgcServer/UgcIconParams.cpp Normal file
View File

@@ -0,0 +1,88 @@
#include "UgcIconParams.h"
#include <algorithm>
#include <cmath>
#include "GeneralUtils.h"
#include "json.hpp"
namespace {
using Options = UgcRender::IconOptions;
}
namespace UgcIconParams {
const std::vector<Param>& List() {
// Framing from LU Toolbox's icon renderer (its UGC render add-on's BrickBuild scene); the light matched to the
// brightness of the game's own model icons (res/textures/ui/inventory/models), docs/UgcServer.md
static const std::vector<Param> list = {
{ "yaw", "icon_yaw", "Angle around", "degrees", -180, 180, 1, 53.36f, "The camera's angle around the model, from its front.", [](Options& o, float v) { o.yawDegrees = v; } },
{ "pitch", "icon_pitch", "Angle above", "degrees", -89, 89, 1, 19.54f, "", [](Options& o, float v) { o.pitchDegrees = v; } },
{ "fov", "icon_fov", "Field of view", "degrees", 5, 90, 0.5f, 39.6f, "", [](Options& o, float v) { o.fovDegrees = v; } },
{ "margin", "icon_margin", "Border (zoom)", "", 0.5f, 3, 0.01f, 1.03f, "1 fills the icon, more leaves a border.", [](Options& o, float v) { o.margin = v; } },
{ "offsetX", "icon_offset_x", "Shift right", "", -0.5f, 0.5f, 0.01f, 0.0f, "Share of the icon's width.", [](Options& o, float v) { o.offsetX = v; } },
{ "offsetY", "icon_offset_y", "Shift up", "", -0.5f, 0.5f, 0.01f, 0.0f, "Share of the icon's height.", [](Options& o, float v) { o.offsetY = v; } },
{ "sunYaw", "icon_sun_yaw", "Sun around", "degrees", -180, 180, 1, 21.0f, "", [](Options& o, float v) { o.sunYawDegrees = v; } },
{ "sunPitch", "icon_sun_pitch", "Sun above", "degrees", -10, 90, 1, 50.3f, "", [](Options& o, float v) { o.sunPitchDegrees = v; } },
{ "sunStrength", "icon_sun_strength", "Sun strength", "", 0, 10, 0.05f, 2.0f, "", [](Options& o, float v) { o.sunStrength = v; } },
{ "ambient", "icon_ambient", "World light", "", 0, 2, 0.01f, 1.0f, "What every face gets.", [](Options& o, float v) { o.ambient = v; } },
{ "fill", "icon_fill", "Fill light", "", 0, 5, 0.05f, 0.8f, "A light from the camera.", [](Options& o, float v) { o.fill = v; } },
{ "specular", "icon_specular", "Highlights", "", 0, 2, 0.05f, 0.2f, "The sun's highlight on the plastic.", [](Options& o, float v) { o.specular = v; } },
{ "shininess", "icon_shininess", "Highlight tightness", "", 1, 200, 1, 40.0f, "", [](Options& o, float v) { o.shininess = v; } },
{ "exposure", "icon_exposure", "Exposure", "", 0.1f, 4, 0.05f, 2.6f, "Brightness of everything.", [](Options& o, float v) { o.exposure = v; } },
{ "contrast", "icon_contrast", "Contrast", "", 0.5f, 1.5f, 0.01f, 1.0f, "", [](Options& o, float v) { o.contrast = v; } },
{ "shadows", "icon_shadows", "Shadows", "", 0, 1, 0.05f, 0.4f, "How much the sun's shadows darken.", [](Options& o, float v) { o.shadows = v; } },
{ "aoStrength", "icon_ao_strength", "Ambient occlusion", "", 0, 1, 0.05f, 0.0f,
"Occlusion worked out for the icon. Player models' meshes have theirs baked in already.", [](Options& o, float v) { o.ao.strength = v; o.ao.enabled = v > 0.0f; } },
};
return list;
}
const Param* Find(std::string_view key) {
const auto& list = List();
const auto it = std::find_if(list.begin(), list.end(), [key](const Param& p) { return p.key == key; });
return it != list.end() ? &*it : nullptr;
}
Values Parse(std::string_view text) {
Values values;
const auto json = nlohmann::json::parse(text, nullptr, false);
if (!json.is_object()) return values;
for (const auto& param : List()) {
const auto it = json.find(param.key);
if (it == json.end() || !it->is_number()) continue;
const auto value = it->get<float>();
if (std::isfinite(value)) values[param.key] = std::clamp(value, param.min, param.max);
}
return values;
}
std::string ToJson(const Values& values) {
nlohmann::json json = nlohmann::json::object();
for (const auto& [key, value] : values) {
if (Find(key)) json[key] = value;
}
return json.dump();
}
void Apply(UgcRender::IconOptions& options, const Values& values) {
for (const auto& [key, value] : values) {
if (const auto* param = Find(key)) param->apply(options, std::clamp(value, param->min, param->max));
}
}
UgcRender::IconOptions FromSettings(const std::function<std::optional<std::string>(const std::string&)>& get) {
UgcRender::IconOptions options;
for (const auto& param : List()) {
const auto text = get(param.setting);
const auto value = text ? GeneralUtils::TryParse<float>(*text) : std::nullopt;
param.apply(options, std::clamp(value.value_or(param.defaultValue), param.min, param.max));
}
return options;
}
std::string KindTarget(const std::string& kind) { return "kind:" + kind; }
std::string ModelTarget(int64_t id) { return "model:" + std::to_string(id); }
std::string CombinationTarget(const std::string& key) { return "combo:" + key; }
std::string ModelKind() { return "model"; }
std::string BuildKind(int32_t buildType) { return "build" + std::to_string(buildType); }
}

56
dUgcServer/UgcIconParams.h Normal file
View File

@@ -0,0 +1,56 @@
#pragma once
#include <functional>
#include <map>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
#include "UgcRender.h"
/**
* Everything about an icon's framing and lighting that can be set, listed once: its name (in presets and overrides),
* its setting (the shipped default, in ugcconfig.ini and the dashboard's settings), label, unit, range and default.
* The UGC server reads the settings through it, the dashboard builds its settings entries and the icon editor's
* controls from it, and presets and overrides are checked against it. Values for an icon come, each over the one
* before: the settings (icon_*), the kind's preset (player models, or a car or rocket build type) and the item's or
* module combination's own override, both stored in the database (ugc_icon_settings). Pure.
*/
namespace UgcIconParams {
struct Param {
std::string key; // in presets and overrides, e.g. "sunStrength"
std::string setting; // ugcconfig.ini, e.g. "icon_sun_strength"
std::string label;
std::string unit;
float min{};
float max{};
float step{};
float defaultValue{};
std::string description;
std::function<void(UgcRender::IconOptions&, float)> apply;
};
const std::vector<Param>& List();
const Param* Find(std::string_view key);
using Values = std::map<std::string, float>;
// Known keys only, numbers only, clamped to their ranges
Values Parse(std::string_view json);
std::string ToJson(const Values& values);
// Values over the options' own
void Apply(UgcRender::IconOptions& options, const Values& values);
// The options from the settings: `get` gives a setting's value (nullopt or unparsable: the parameter's default)
UgcRender::IconOptions FromSettings(const std::function<std::optional<std::string>(const std::string&)>& get);
// Where presets and overrides are stored (ugc_icon_settings.target)
std::string KindTarget(const std::string& kind); // "kind:model", "kind:build6"
std::string ModelTarget(int64_t id); // "model:<id>"
std::string CombinationTarget(const std::string& key); // "combo:<key>"
// The kind of a player model, and of a car or rocket of build type `buildType` (ModuleComponent.buildType)
std::string ModelKind();
std::string BuildKind(int32_t buildType);
}

32
dUgcServer/UgcJobs.cpp
View File

@@ -1,6 +1,7 @@
#include "UgcJobs.h"
#include <chrono>
#include <cmath>
#include <sstream>
#include "json.hpp"
@@ -66,7 +67,14 @@ namespace UgcJobs {
}
}
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed) {
bool IconFromNif(const std::string& nif, const UgcRender::IconOptions& options, UgcStorage::Files& files, std::string& error) {
const auto readBack = NifFile::Parse(nif, 0, error);
if (!readBack) return false;
AddIcon(files, UgcModel::FromNif(*readBack), options);
return true;
}
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed, const UgcIconParams::Values& iconValues) {
Outcome outcome;
const auto started = std::chrono::steady_clock::now();
const auto lxfml = LxfmlFromBlob(blob);
@@ -169,18 +177,15 @@ namespace UgcJobs {
}
// The icon is drawn from the .nif just made (its most detailed LOD, read back like any client .nif), so it
// shows what the game shows: the colors with their variation, hidden faces removed, the baked lighting. The
// lighting is in its vertex colors already, so the icon adds no occlusion of its own.
// shows what the game shows: the colors with their variation, hidden faces removed, the baked lighting.
const auto iconStart = std::chrono::steady_clock::now();
auto iconOptions = settings.icon;
UgcIconParams::Apply(iconOptions, iconValues);
std::string nifError;
const auto readBack = NifFile::Parse(nif, 0, nifError);
if (!readBack) {
if (!IconFromNif(nif, iconOptions, outcome.files, nifError)) {
outcome.error = "the .nif made can't be read back for the icon: " + nifError;
return outcome;
}
auto iconOptions = settings.icon;
iconOptions.ao.enabled = false;
AddIcon(outcome.files, UgcModel::FromNif(*readBack), iconOptions);
const double iconMs = Since(iconStart);
stats["ms"] = { { "build", std::lround(buildMs) }, { "hiddenSurfaces", std::lround(hsrMs) }, { "ambientOcclusion", std::lround(aoMs) },
@@ -228,10 +233,19 @@ namespace UgcJobs {
outcome.error = "the modules have no triangles";
return outcome;
}
auto options = settings.modularIcon;
auto options = ModularIconOptions(input, settings);
options.modelRotation = build->additionalRotation * options.modelRotation;
AddIcon(outcome.files, model, options);
outcome.files["combo.json"] = nlohmann::json{ { "key", input.key }, { "buildType", input.buildType } }.dump();
outcome.ok = true;
return outcome;
}
UgcRender::IconOptions ModularIconOptions(const ModularInput& input, const Settings& settings) {
auto options = settings.icon;
UgcIconParams::Apply(options, input.iconValues);
return options;
}
}

24
dUgcServer/UgcJobs.h
View File

@@ -2,11 +2,14 @@
#include <cstdint>
#include <filesystem>
#include <map>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
#include "UgcBricks.h"
#include "UgcIconParams.h"
#include "UgcRender.h"
#include "UgcStorage.h"
@@ -27,8 +30,7 @@ namespace UgcJobs {
bool combineTransparent{ false }; // one shape for all transparent bricks, else one per brick (Combine Transparent)
UgcRender::OptimizeOptions optimize; // hidden surface removal
UgcRender::AoOptions ao; // Bake Lighting (AO Only)
UgcRender::IconOptions icon; // player models: drawn from their .nif (LOD 0); its ao is not used
UgcRender::IconOptions modularIcon;
UgcRender::IconOptions icon; // from the icon_* settings (UgcIconParams); presets and overrides go over it
uint32_t maxBricks{}; // a model with more fails; 0: no limit
};
@@ -49,7 +51,13 @@ namespace UgcJobs {
// A player model: the optimized .nif, its icon and its LXFML for download, and stats.json. `seed` picks the color
// variation's random numbers (the model's id, so making it again gives the same colors).
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed = 0);
// `iconValues`: the player models' preset and this model's override (UgcIconParams), over settings.icon.
Outcome ProcessModel(const std::string& blob, UgcBricks::BrickLibrary& library, const Settings& settings, uint64_t seed = 0,
const UgcIconParams::Values& iconValues = {});
// A model's icon files (icon.png, icon.dds download) drawn from its .nif (LOD 0); false (and `error`) when the .nif
// can't be read
bool IconFromNif(const std::string& nif, const UgcRender::IconOptions& options, UgcStorage::Files& files, std::string& error);
// How many bricks (parts) an LXFML has, counted cheaply (for the memory estimate before a job starts)
size_t CountParts(std::string_view lxfml);
@@ -66,9 +74,15 @@ namespace UgcJobs {
struct ModularInput {
std::vector<ModuleInput> modules;
std::string buildXml; // ModularBuildComponent.xml of the modules' build type
std::string buildXml; // ModularBuildComponent.xml of the modules' build type
int32_t buildType{}; // 3 rocket, 6 car
std::string key; // the combination (UgcModularKey::Normalize)
UgcIconParams::Values iconValues; // the build type's preset and the combination's override (DB), over the settings
};
// A modular build's icon, from its modules' meshes put together
// The icon options for a car or rocket: the settings, then its build type's preset and the combination's override
UgcRender::IconOptions ModularIconOptions(const ModularInput& input, const Settings& settings);
// A modular build's icon (icon.png, icon.dds download, combo.json), from its modules' meshes put together
Outcome ProcessModular(const ModularInput& input, const std::filesystem::path& res, const Settings& settings);
}

298
dUgcServer/UgcProcessor.cpp
View File

@@ -3,11 +3,15 @@
#include "CDClientDatabase.h"
#include "Database.h"
#include "Logger.h"
#include "UgcBricks.h"
#include "UgcCdClient.h"
#include "UgcKeys.h"
#include "UgcThrottle.h"
#include "json.hpp"
#include <algorithm>
#include <ctime>
#include <filesystem>
#include <fstream>
#include <thread>
#if defined(__linux__)
@@ -148,12 +152,63 @@ void UgcProcessor::Worker() {
}
UgcThrottle::Begin();
if (job.preview) {
// A preview: the icon goes back to whoever asked, nothing is stored or recorded
HTTPReply reply;
try {
UgcJobs::Outcome outcome{ false, "cancelled" };
if (!job.preview.Cancelled() && job.kind == Kind::MODULAR) {
outcome = UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings);
} else if (!job.preview.Cancelled()) {
// A player model's icon from its stored .nif
const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif");
const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt;
auto options = settings.icon;
UgcIconParams::Apply(options, job.iconValues);
outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, outcome.files, outcome.error);
if (!nif) outcome.error = "the model has no stored .nif yet";
}
if (outcome.ok && outcome.files.contains("icon.png")) {
reply.status = eHTTPStatusCode::OK;
reply.contentType = eContentType::IMAGE_PNG;
reply.message = std::move(outcome.files["icon.png"]);
reply.headers.push_back("Cache-Control: no-store");
} else {
reply.status = eHTTPStatusCode::UNPROCESSABLE_ENTITY;
reply.contentType = eContentType::TEXT_PLAIN;
reply.message = outcome.error;
}
} catch (const std::exception& ex) {
reply.status = eHTTPStatusCode::INTERNAL_SERVER_ERROR;
reply.contentType = eContentType::TEXT_PLAIN;
reply.message = ex.what();
}
job.preview.Send(std::move(reply));
{
std::lock_guard lock(m_Mutex);
m_Active--;
m_MemoryInUse -= std::min(m_MemoryInUse, job.memory);
}
m_Wake.notify_all();
continue;
}
const auto start = std::chrono::steady_clock::now();
Done done{ job.kind, job.id, job.attempts };
done.iconOnly = job.iconOnly;
try {
done.outcome = job.kind == Kind::MODEL
? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast<uint64_t>(job.id))
: UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings);
if (job.iconOnly) {
// Only the icon, from the .nif made before
const auto path = m_Storage.File(Kind::MODEL, job.id, "model.nif");
const auto nif = path ? UgcBricks::ReadFile(*path) : std::nullopt;
auto options = settings.icon;
UgcIconParams::Apply(options, job.iconValues);
done.outcome.ok = nif && UgcJobs::IconFromNif(*nif, options, done.outcome.files, done.outcome.error);
if (!nif) done.outcome.error = "no stored .nif";
} else {
done.outcome = job.kind == Kind::MODEL
? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast<uint64_t>(job.id), job.iconValues)
: UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings);
}
} catch (const std::exception& ex) {
done.outcome.ok = false;
done.outcome.error = std::string("crashed: ") + ex.what();
@@ -162,7 +217,7 @@ void UgcProcessor::Worker() {
job.blob.shrink_to_fit();
if (done.outcome.ok) {
std::string error;
const auto bytes = m_Storage.Write(job.kind, job.id, done.outcome.files, error);
const auto bytes = job.iconOnly ? m_Storage.Update(job.kind, job.id, done.outcome.files, error) : m_Storage.Write(job.kind, job.id, done.outcome.files, error);
if (bytes) {
done.bytes = *bytes;
} else {
@@ -219,26 +274,51 @@ void UgcProcessor::Poll() {
Job job{ Kind::MODEL, model.id, model.attempts, UgcJobs::LxfmlFromBlob(model.lxfml) };
if (job.blob.empty()) job.blob = std::move(model.lxfml); // the worker reports it can't be read
job.parts = UgcJobs::CountParts(job.blob);
job.iconValues = IconValues(UgcIconParams::ModelKind(), UgcIconParams::ModelTarget(model.id));
job.memory = UgcJobs::EstimateMemory(job.parts, settings);
jobs.push_back(std::move(job));
}
// Cars and rockets: one icon per combination of modules, shared by every build of it
for (auto& build : Database::Get()->GetModularBuildsToProcess(limit)) {
if (m_InFlight.contains({ Kind::MODULAR, build.id })) continue;
Job job{ Kind::MODULAR, build.id, build.attempts };
const auto key = UgcModularKey::Normalize(build.modules);
if (key.empty()) {
Record(Done{ Kind::MODULAR, build.id, build.attempts, UgcJobs::Outcome{ false, "no modules in \"" + build.modules + "\"" } });
continue;
}
const auto combo = UgcModularKey::StorageId(key);
m_ComboOf[build.id] = combo;
// Made already for another build of the same modules
if (!m_ComboJobs.contains(combo) && m_Storage.File(Kind::MODULAR, combo, "icon.png")) {
UgcJobs::Outcome reused{ true };
reused.note = "the same modules as a build made before (" + key + ")";
m_Reused++;
Record(Done{ Kind::MODULAR, build.id, build.attempts, std::move(reused) });
continue;
}
m_InFlight.insert({ Kind::MODULAR, build.id });
m_ComboRows[combo].emplace_back(build.id, build.attempts);
if (m_ComboJobs.contains(combo)) continue;
Job job{ Kind::MODULAR, combo, build.attempts };
job.memory = UgcJobs::EstimateMemory(64, settings);
std::string error;
if (!UgcCdClient::GatherModular(build.modules, job.modular, error)) {
// Nothing a worker could do: record it right away
m_InFlight.erase({ Kind::MODULAR, build.id });
m_ComboRows.erase(combo);
Record(Done{ Kind::MODULAR, build.id, build.attempts, UgcJobs::Outcome{ false, error } });
continue;
}
job.modular.key = key;
job.modular.iconValues = IconValues(UgcIconParams::BuildKind(job.modular.buildType), UgcIconParams::CombinationTarget(key));
m_ComboJobs.insert(combo);
jobs.push_back(std::move(job));
}
if (jobs.empty()) return;
{
std::lock_guard lock(m_Mutex);
for (auto& job : jobs) {
m_InFlight.insert({ job.kind, job.id });
if (job.kind == Kind::MODEL) m_InFlight.insert({ job.kind, job.id });
m_Jobs.push_back(std::move(job));
}
}
@@ -277,8 +357,33 @@ void UgcProcessor::Collect() {
finished.swap(m_Done);
}
for (const auto& done : finished) {
m_InFlight.erase({ done.kind, done.id });
Record(done);
if (done.kind == Kind::MODEL) {
m_InFlight.erase({ done.kind, done.id });
if (!done.iconOnly) {
Record(done);
continue;
}
m_Log.push_back({ Kind::MODEL, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? "icon drawn again" : done.outcome.error, UnixNow() });
while (m_Log.size() > LOG_LENGTH) m_Log.pop_front();
continue;
}
// A combination: every build waiting for it gets its outcome (none when only its icon was drawn again)
m_ComboJobs.erase(done.id);
if (done.outcome.ok) m_StoredBytes += done.bytes;
auto rows = std::move(m_ComboRows[done.id]);
m_ComboRows.erase(done.id);
for (const auto& [row, attempts] : rows) {
m_InFlight.erase({ Kind::MODULAR, row });
Done forRow = done;
forRow.id = row;
forRow.attempts = attempts;
forRow.bytes = 0;
Record(forRow);
}
if (rows.empty()) {
m_Log.push_back({ Kind::MODULAR, done.id, done.outcome.ok, done.milliseconds, done.outcome.ok ? "icon drawn again" : done.outcome.error, UnixNow() });
while (m_Log.size() > LOG_LENGTH) m_Log.pop_front();
}
}
// Something finished: look for more now rather than at the next interval
if (!finished.empty()) m_NextPoll = std::min(m_NextPoll, std::chrono::steady_clock::now() + std::chrono::milliseconds(100));
@@ -307,10 +412,22 @@ void UgcProcessor::Update() {
std::erase_if(m_Recent, [now](const auto& item) { return now - item.second.first > RECENT_ANSWER_TIME; });
}
LWOOBJID UgcProcessor::StorageId(Kind kind, LWOOBJID id) {
if (kind == Kind::MODEL) return id;
if (const auto it = m_ComboOf.find(id); it != m_ComboOf.end()) return it->second;
const auto info = Database::Get()->GetModularBuildProcessInfo(id);
const auto key = info ? UgcModularKey::Normalize(info->details) : std::string();
const LWOOBJID combo = key.empty() ? 0 : UgcModularKey::StorageId(key);
if (combo != 0) m_ComboOf[id] = combo;
return combo;
}
UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) {
if (m_InFlight.contains({ kind, id })) return Availability::QUEUED;
if (m_Storage.File(kind, id, "icon.png")) {
m_Storage.Touch(kind, id);
const auto storageId = StorageId(kind, id);
if (kind == Kind::MODULAR && m_ComboJobs.contains(storageId)) return Availability::QUEUED;
if (storageId != 0 && m_Storage.File(kind, storageId, "icon.png")) {
m_Storage.Touch(kind, storageId);
return Availability::READY;
}
if (const auto it = m_Recent.find({ kind, id }); it != m_Recent.end()) return it->second.second;
@@ -323,6 +440,8 @@ UgcProcessor::Availability UgcProcessor::Request(Kind kind, LWOOBJID id) {
if (kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(id, false);
else Database::Get()->ResetModularBuildProcessing(id, false);
}
// Someone wants it now: no need to wait out the quiet period after its save
if (kind == Kind::MODEL) Database::Get()->ExpediteUgcModel(id);
m_NextPoll = std::chrono::steady_clock::now();
answer = Availability::QUEUED;
}
@@ -345,6 +464,7 @@ nlohmann::json UgcProcessor::Status() const {
status["memoryWaits"] = m_Waiting;
}
status["workers"] = m_Threads.size();
status["reused"] = m_Reused;
const auto throttle = UgcThrottle::GetStats();
status["usage"] = {
{ "cpuPercent", std::lround(m_CpuPercent) }, // of one core
@@ -372,3 +492,161 @@ nlohmann::json UgcProcessor::Status() const {
}
return status;
}
UgcIconParams::Values UgcProcessor::IconValues(const std::string& kind, const std::string& itemTarget) {
UgcIconParams::Values values;
if (const auto preset = Database::Get()->GetUgcIconSettings(UgcIconParams::KindTarget(kind))) values = UgcIconParams::Parse(*preset);
if (const auto own = Database::Get()->GetUgcIconSettings(itemTarget)) {
for (const auto& [key, value] : UgcIconParams::Parse(*own)) values[key] = value;
}
return values;
}
bool UgcProcessor::QueuePreview(Kind kind, LWOOBJID id, const std::string& modules, const UgcIconParams::Values& values, DeferredReply reply, std::string& error) {
Job job{ kind, id, 0 };
if (kind == Kind::MODULAR) {
if (!UgcCdClient::GatherModular(modules, job.modular, error)) return false;
job.modular.key = UgcModularKey::Normalize(modules);
job.modular.iconValues = values;
} else {
if (!m_Storage.File(Kind::MODEL, id, "model.nif")) {
error = "the model has no stored .nif yet";
return false;
}
job.iconValues = values;
}
job.preview = std::move(reply);
{
std::lock_guard lock(m_Mutex);
job.memory = UgcJobs::EstimateMemory(64, m_Settings);
m_Jobs.push_front(std::move(job));
}
m_Wake.notify_all();
return true;
}
size_t UgcProcessor::RegenerateIcons(const std::string& kind) {
std::vector<Job> jobs;
UgcJobs::Settings settings;
{
std::lock_guard lock(m_Mutex);
settings = m_Settings;
}
const bool models = kind == UgcIconParams::ModelKind();
for (const auto& entry : m_Storage.List()) {
if (models) {
if (entry.kind != Kind::MODEL || m_InFlight.contains({ Kind::MODEL, entry.id }) || !m_Storage.File(Kind::MODEL, entry.id, "model.nif")) continue;
Job job{ Kind::MODEL, entry.id, 0 };
job.iconOnly = true;
job.iconValues = IconValues(kind, UgcIconParams::ModelTarget(entry.id));
job.memory = UgcJobs::EstimateMemory(64, settings);
m_InFlight.insert({ Kind::MODEL, entry.id });
jobs.push_back(std::move(job));
continue;
}
if (entry.kind != Kind::MODULAR || m_ComboJobs.contains(entry.id)) continue;
const auto path = m_Storage.File(Kind::MODULAR, entry.id, "combo.json");
const auto text = path ? UgcBricks::ReadFile(*path) : std::nullopt;
const auto combo = text ? nlohmann::json::parse(*text, nullptr, false) : nlohmann::json();
if (!combo.is_object() || UgcIconParams::BuildKind(combo.value("buildType", -1)) != kind) continue;
const auto key = combo.value("key", std::string());
Job job{ Kind::MODULAR, entry.id, 0 };
std::string error;
// The key's LOTs written as an ldf_config ("4713-4714" -> "4713+4714")
std::string modules = key;
std::replace(modules.begin(), modules.end(), '-', '+');
if (key.empty() || !UgcCdClient::GatherModular(modules, job.modular, error)) continue;
job.modular.key = key;
job.modular.iconValues = IconValues(kind, UgcIconParams::CombinationTarget(key));
job.memory = UgcJobs::EstimateMemory(64, settings);
m_ComboJobs.insert(entry.id);
jobs.push_back(std::move(job));
}
const auto count = jobs.size();
{
std::lock_guard lock(m_Mutex);
for (auto& job : jobs) m_Jobs.push_back(std::move(job));
}
m_Wake.notify_all();
return count;
}
UgcProcessor::DeleteResult UgcProcessor::Delete(const DeleteRequest& request) {
DeleteResult result;
const auto now = std::filesystem::file_time_type::clock::now();
const auto days = [](int64_t count) { return std::chrono::duration_cast<std::filesystem::file_time_type::duration>(std::chrono::hours(24 * count)); };
// What to delete: the files' ids, and the rows they came from (when they're known)
std::map<LWOOBJID, std::vector<LWOOBJID>> targets; // storage id -> rows
if (request.all) {
for (const auto& entry : m_Storage.List()) {
if (entry.kind == request.kind) targets[entry.id];
}
}
for (const auto id : request.ids) {
const auto storageId = StorageId(request.kind, id);
if (storageId != 0) targets[storageId].push_back(id);
}
std::vector<LWOOBJID> deletedRows;
for (const auto& [storageId, rows] : targets) {
const bool busy = request.kind == Kind::MODEL ? m_InFlight.contains({ Kind::MODEL, storageId }) : m_ComboJobs.contains(storageId);
if (busy) {
result.busy++;
continue;
}
const auto folder = m_Storage.Folder(request.kind, storageId);
std::error_code error;
if (!std::filesystem::exists(folder, error)) {
deletedRows.insert(deletedRows.end(), rows.begin(), rows.end());
continue;
}
if (request.unusedDays > 0) {
const auto used = std::filesystem::last_write_time(folder, error);
if (!error && now - used < days(request.unusedDays)) continue;
}
if (request.olderThanDays > 0) {
const auto made = std::filesystem::last_write_time(folder / "icon.png", error);
if (!error && now - made < days(request.olderThanDays)) continue;
}
uint64_t bytes = 0;
for (const auto& file : std::filesystem::directory_iterator(folder, error)) bytes += file.is_regular_file(error) ? file.file_size(error) : 0;
m_Storage.Remove(request.kind, storageId);
result.deleted++;
result.bytes += bytes;
m_StoredBytes -= std::min(m_StoredBytes, bytes);
m_Recent.clear();
deletedRows.insert(deletedRows.end(), rows.begin(), rows.end());
if (request.kind == Kind::MODEL) deletedRows.push_back(storageId);
}
std::sort(deletedRows.begin(), deletedRows.end());
deletedRows.erase(std::unique(deletedRows.begin(), deletedRows.end()), deletedRows.end());
// What happens to the rows
if (request.after == eAfterDelete::NOW) {
if (request.all) {
if (request.kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(std::nullopt, false);
else Database::Get()->ResetModularBuildProcessing(std::nullopt, false);
} else {
for (const auto row : deletedRows) {
if (request.kind == Kind::MODEL) Database::Get()->ResetUgcModelProcessing(row, false);
else Database::Get()->ResetModularBuildProcessing(row, false);
}
}
m_NextPoll = std::chrono::steady_clock::now();
result.notes.push_back("They're queued to be made again now.");
} else if (request.after == eAfterDelete::GONE) {
for (const auto row : deletedRows) {
if (request.kind == Kind::MODEL) Database::Get()->SetUgcModelProcessed(row, IUgc::eProcessState::FAILED, m_Config.maxAttempts, "Deleted from the dashboard", false);
else Database::Get()->SetModularBuildProcessed(row, IUgc::eProcessState::FAILED, m_Config.maxAttempts, "Deleted from the dashboard");
}
result.notes.push_back("They're marked deleted and won't be made again unless made again from the dashboard.");
} else {
result.notes.push_back("They'll be made again when a game client asks for them.");
}
if (request.kind == Kind::MODULAR && result.deleted > 0) {
result.notes.push_back("Car and rocket icons are shared by every build of the same modules: the other builds' icons are made again when asked for.");
}
if (result.busy > 0) result.notes.push_back(std::to_string(result.busy) + " being made right now were left alone.");
return result;
}

54
dUgcServer/UgcProcessor.h
View File

@@ -12,9 +12,11 @@
#include <thread>
#include <vector>
#include "DeferredReply.h"
#include "dCommonVars.h"
#include "json_fwd.hpp"
#include "UgcBricks.h"
#include "UgcIconParams.h"
#include "UgcJobs.h"
#include "UgcStorage.h"
@@ -69,6 +71,46 @@ public:
// Main thread: whether an item's files can be served, queuing it to be made when it exists but they're missing
Availability Request(Kind kind, LWOOBJID id);
// Main thread: the id an item's files are stored under. A car or rocket's are its combination's (every build of
// the same modules shares one icon); 0 when a build's modules can't be told.
LWOOBJID StorageId(Kind kind, LWOOBJID id);
/**
* Main thread: renders an icon with `values` (UgcIconParams, over the settings) without storing it, on a worker
* (within the budgets, ahead of the queue); `reply` gets the PNG. A player model's is drawn from its stored .nif, a
* car or rocket's from `modules`. False (and `error`) when there's nothing to draw from.
*/
bool QueuePreview(Kind kind, LWOOBJID id, const std::string& modules, const UgcIconParams::Values& values, DeferredReply reply, std::string& error);
// Main thread: queues every stored icon of a kind ("model", or "build<type>" for cars and rockets) to be drawn again
// with the current settings, presets and overrides; only icons (models' from their stored .nif). How many.
size_t RegenerateIcons(const std::string& kind);
enum class eAfterDelete {
ON_DEMAND, // rows stay made; the files are made again when something asks for them
NOW, // rows go back to waiting, so they're made again soon
GONE, // rows are marked failed ("deleted"), so they aren't made again until someone asks for that
};
struct DeleteRequest {
Kind kind{};
std::vector<LWOOBJID> ids; // models, or builds (their combinations are deleted); empty with `all`
bool all{};
int64_t olderThanDays{}; // only files made longer ago than this (0: any)
int64_t unusedDays{}; // only files not asked for in this long (0: any)
eAfterDelete after{ eAfterDelete::ON_DEMAND };
};
struct DeleteResult {
size_t deleted{};
uint64_t bytes{};
size_t busy{}; // being made right now, left alone
std::vector<std::string> notes;
};
// Main thread: deletes stored files. Items being made are skipped, so a worker never writes what is being deleted.
DeleteResult Delete(const DeleteRequest& request);
// Main thread: what the server is doing, for /status
nlohmann::json Status() const;
@@ -98,6 +140,9 @@ private:
UgcJobs::ModularInput modular; // modular builds
uint64_t memory{}; // estimated bytes it needs
size_t parts{};
DeferredReply preview; // an icon preview: answered with the PNG, nothing stored
bool iconOnly{}; // a model's icon drawn again from its stored .nif
UgcIconParams::Values iconValues; // models: the preset and override (UgcIconParams)
};
struct Done {
@@ -107,9 +152,12 @@ private:
UgcJobs::Outcome outcome;
uint64_t bytes{};
double milliseconds{};
bool iconOnly{};
};
void Poll();
// Main thread: the icon values for a kind and an item (the kind's preset, then the item's override)
UgcIconParams::Values IconValues(const std::string& kind, const std::string& itemTarget);
void Collect();
void Record(const Done& done);
void Worker();
@@ -137,7 +185,11 @@ private:
std::vector<std::thread> m_Threads;
// Main thread only
std::set<std::pair<Kind, LWOOBJID>> m_InFlight;
std::set<std::pair<Kind, LWOOBJID>> m_InFlight; // models, and builds waiting for their combination
std::set<LWOOBJID> m_ComboJobs; // combinations being made
std::map<LWOOBJID, std::vector<std::pair<LWOOBJID, uint32_t>>> m_ComboRows; // combination -> builds (id, attempts) waiting for it
std::map<LWOOBJID, LWOOBJID> m_ComboOf; // build -> combination (StorageId)
uint64_t m_Reused{}; // builds whose combination was made already
std::chrono::steady_clock::time_point m_NextPoll{};
std::chrono::steady_clock::time_point m_NextEviction{};
std::map<std::pair<Kind, LWOOBJID>, std::pair<std::chrono::steady_clock::time_point, Availability>> m_Recent; // answers for missing files

28
dUgcServer/UgcRender.cpp
View File

@@ -430,7 +430,7 @@ namespace UgcRender {
const auto project = [&](const glm::vec3& position) {
const auto clip = viewProjection * glm::vec4(position, 1.0f);
const float w = clip.w > 1e-6f ? clip.w : 1e-6f;
return glm::vec3((0.5f + (clip.x / w - centerX) * scale * 0.5f) * n, (0.5f - (clip.y / w - centerY) * scale * 0.5f) * n, clip.z / w);
return glm::vec3((0.5f + options.offsetX + (clip.x / w - centerX) * scale * 0.5f) * n, (0.5f - options.offsetY - (clip.y / w - centerY) * scale * 0.5f) * n, clip.z / w);
};
// Linear, premultiplied
@@ -449,7 +449,7 @@ namespace UgcRender {
const int shadowSize = 1024;
const OrthoView sunView(center, radius * 1.05f, light, shadowSize);
std::vector<float> shadowDepth;
if (options.shadows) {
if (options.shadows > 0.0f) {
shadowDepth.assign(static_cast<size_t>(shadowSize) * shadowSize, INF);
const auto& mesh = model.opaque;
std::vector<glm::vec3> screen(mesh.positions.size());
@@ -473,8 +473,11 @@ namespace UgcRender {
if (x < 0 || y < 0 || x >= shadowSize || y >= shadowSize || p.z <= shadowDepth[static_cast<size_t>(y) * shadowSize + x] + shadowBias) lit += 1.0f;
}
}
return lit / 9.0f;
const float shadowed = 1.0f - lit / 9.0f;
return 1.0f - std::clamp(options.shadows, 0.0f, 1.0f) * shadowed;
};
const glm::vec3 toCamera = glm::normalize(dir);
const glm::vec3 halfway = glm::normalize(light + toCamera);
const auto shade = [&](const UgcModel::Mesh& mesh, bool isOpaque, uint32_t i0, uint32_t i1, uint32_t i2, float w0, float w1, float w2) {
glm::vec3 normal(0.0f, 1.0f, 0.0f);
@@ -490,10 +493,15 @@ namespace UgcRender {
const float strength = std::clamp(options.ao.strength, 0.0f, 1.0f);
const auto position = mesh.positions[i0] * w0 + mesh.positions[i1] * w1 + mesh.positions[i2] * w2;
const float direct = std::max(0.0f, glm::dot(normal, light));
// Diffuse: the world's light (radiance `ambient`) and the sun's (irradiance `sunStrength`, over pi)
const float sun = direct > 0.0f ? sunlit(position + normal * shadowBias) : 0.0f;
// Diffuse: the world's light (radiance `ambient`), a fill from the camera and the sun's (irradiances, over pi)
const float lighting = options.ambient * (1.0f - strength * (1.0f - occlusion)) +
options.sunStrength / 3.14159265f * direct * (direct > 0.0f ? sunlit(position + normal * shadowBias) : 0.0f);
return glm::vec4(ToLinear(base.r) * lighting, ToLinear(base.g) * lighting, ToLinear(base.b) * lighting, std::clamp(base.a, 0.0f, 1.0f));
(options.fill * std::max(0.0f, glm::dot(normal, toCamera)) + options.sunStrength * direct * sun) / 3.14159265f;
// The sun's highlight (Blinn-Phong), white, on top of the color
const float highlight = direct > 0.0f ? options.specular * options.sunStrength / 3.14159265f * sun * std::pow(std::max(0.0f, glm::dot(normal, halfway)), std::max(options.shininess, 1.0f)) : 0.0f;
const float exposure = std::max(options.exposure, 0.0f);
return glm::vec4((ToLinear(base.r) * lighting + highlight) * exposure, (ToLinear(base.g) * lighting + highlight) * exposure,
(ToLinear(base.b) * lighting + highlight) * exposure, std::clamp(base.a, 0.0f, 1.0f));
};
// Opaque first, with the depth buffer
@@ -546,9 +554,11 @@ namespace UgcRender {
sum /= samples;
uint8_t* out = &image.rgba[(static_cast<size_t>(y) * size + x) * 4];
if (sum.a <= 0.0f) continue;
out[0] = static_cast<uint8_t>(std::lround(ToSrgb(sum.r / sum.a) * 255.0f));
out[1] = static_cast<uint8_t>(std::lround(ToSrgb(sum.g / sum.a) * 255.0f));
out[2] = static_cast<uint8_t>(std::lround(ToSrgb(sum.b / sum.a) * 255.0f));
// sRGB, then the contrast around its middle grey
const auto tone = [&options](float linear) { return std::clamp(0.5f + (ToSrgb(linear) - 0.5f) * options.contrast, 0.0f, 1.0f); };
out[0] = static_cast<uint8_t>(std::lround(tone(sum.r / sum.a) * 255.0f));
out[1] = static_cast<uint8_t>(std::lround(tone(sum.g / sum.a) * 255.0f));
out[2] = static_cast<uint8_t>(std::lround(tone(sum.b / sum.a) * 255.0f));
out[3] = static_cast<uint8_t>(std::lround(std::clamp(sum.a, 0.0f, 1.0f) * 255.0f));
}
}

18
dUgcServer/UgcRender.h
View File

@@ -28,9 +28,8 @@ namespace UgcRender {
};
/**
* The icon, set up like LU Toolbox's icon renderer (the BrickBuild scene of its UGC render add-on): a 50 mm lens
* (39.6 degree field of view) looking from 53.4 degrees around and 19.5 above, framed at 1.03, a sun of strength
* 2.5 from 21 around and 50.3 above with soft shadows, and a grey (0.192) world light, darkened by ambient occlusion.
* How an icon is drawn. The values that can be set (settings icon_*, presets and overrides) are listed once, with
* their ranges and shipped defaults, in UgcIconParams; the ones here are only what the struct starts with.
*/
struct IconOptions {
int size{ 128 };
@@ -39,13 +38,20 @@ namespace UgcRender {
float pitchDegrees{ 19.54f }; // camera above the model
float fovDegrees{ 39.6f };
float margin{ 1.03f }; // 1 fills the icon, more leaves a border
float offsetX{}; // the model moved right by this share of the icon's width (after framing)
float offsetY{}; // and up by this share of its height
glm::mat4 modelRotation{ 1.0f }; // applied to the model before the camera looks at it
float sunYawDegrees{ 21.0f };
float sunPitchDegrees{ 50.3f };
float sunStrength{ 2.5f };
float ambient{ 0.192f };
bool shadows{ true };
AoOptions ao{ true, 5.0f, 32, 1.0f, 0.0f };
float ambient{ 0.192f }; // the world light (radiance) every face gets
float fill{ 0.0f }; // a light from the camera (irradiance), what lifts the sides facing the viewer
float specular{ 0.0f }; // the sun's highlight on the plastic
float shininess{ 60.0f }; // how tight the highlight is
float exposure{ 1.0f }; // everything times this before it's written as sRGB
float contrast{ 1.0f }; // around the middle grey of the sRGB result
float shadows{ 1.0f }; // how much the sun's shadows darken, 0 to 1
AoOptions ao{ false, 5.0f, 32, 1.0f, 0.0f };
};
// The model drawn from the icon's camera, framed to fit, on a transparent background. `opaqueAo`: the opaque mesh's

98
dUgcServer/UgcServer.cpp
View File

@@ -31,6 +31,7 @@
#include "UgcBricks.h"
#include "UgcCdClient.h"
#include "UgcIconParams.h"
#include "UgcJobs.h"
#include "UgcModel.h"
#include "UgcProcessor.h"
@@ -47,6 +48,7 @@ namespace Game {
namespace {
dServer* g_Server = nullptr;
std::string g_AdminKey; // the master password: the dashboard's admin requests carry it (X-Ugc-Admin-Key)
UgcProcessor* g_Processor = nullptr;
UgcStorage* g_Storage = nullptr;
@@ -89,21 +91,13 @@ namespace {
settings.ao.samples = std::clamp(Setting<int32_t>("ao_samples", 64), 1, 1024);
settings.ao.strength = Setting<float>("ao_strength", 1.0f);
settings.ao.glowStrength = Setting<float>("glow_strength", 6.0f);
// The icon's framing and light: the icon_* settings, listed with their defaults in UgcIconParams
settings.icon = UgcIconParams::FromSettings([](const std::string& key) -> std::optional<std::string> {
const auto value = Game::config->GetValue(key);
return value.empty() ? std::nullopt : std::optional(value);
});
settings.icon.size = Setting<int32_t>("icon_size", 128);
settings.icon.yawDegrees = Setting<float>("icon_yaw", 53.36f);
settings.icon.pitchDegrees = Setting<float>("icon_pitch", 19.54f);
settings.icon.fovDegrees = Setting<float>("icon_fov", 39.6f);
settings.icon.margin = Setting<float>("icon_margin", 1.03f);
settings.icon.sunYawDegrees = Setting<float>("icon_sun_yaw", 21.0f);
settings.icon.sunPitchDegrees = Setting<float>("icon_sun_pitch", 50.3f);
settings.icon.sunStrength = Setting<float>("icon_sun_strength", 2.5f);
settings.icon.ambient = Setting<float>("icon_ambient", 0.192f);
settings.icon.shadows = Setting<int32_t>("icon_shadows", 1) != 0;
settings.icon.ao.enabled = Setting<int32_t>("icon_ao", 1) != 0;
settings.icon.ao.distance = settings.ao.distance;
settings.modularIcon = settings.icon;
settings.modularIcon.yawDegrees = Setting<float>("modular_icon_yaw", 53.36f);
settings.modularIcon.pitchDegrees = Setting<float>("modular_icon_pitch", 19.54f);
settings.maxBricks = Setting<uint32_t>("max_model_bricks", 0);
return settings;
}
@@ -156,7 +150,8 @@ namespace {
NotFound(reply);
return;
}
const auto path = g_Storage->File(kind, id, name);
// A car or rocket's files are its combination's (shared by every build of the same modules)
const auto path = g_Storage->File(kind, g_Processor->StorageId(kind, id), name);
if (!path) {
NotFound(reply);
return;
@@ -237,7 +232,7 @@ namespace {
const auto id = GeneralUtils::TryParse<uint64_t>(segments[2]);
const auto kind = segments[1] == "model" ? std::optional(UgcStorage::Kind::MODEL) : segments[1] == "modular" ? std::optional(UgcStorage::Kind::MODULAR) : std::nullopt;
const auto& name = segments[3];
static const std::set<std::string> PREVIEWS = { "icon.png", "model.nif", "model.noao.nif", "stats.json",
static const std::set<std::string> PREVIEWS = { "icon.png", "model.nif", "model.noao.nif", "stats.json", "combo.json",
"previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json" };
if (!id || !kind || !PREVIEWS.contains(name)) return;
reply.headers.clear();
@@ -250,6 +245,78 @@ namespace {
}
} });
// For the dashboard only (it sends the master password): icon previews, drawing icons again, deleting files
const auto admin = [](HTTPReply& reply, const HTTPContext& context, std::optional<nlohmann::json>& body) {
const auto& key = context.GetHeader("X-Ugc-Admin-Key");
bool same = !g_AdminKey.empty() && key.size() == g_AdminKey.size();
unsigned char diff = 0;
for (size_t i = 0; same && i < key.size(); i++) diff |= static_cast<unsigned char>(key[i] ^ g_AdminKey[i]);
reply.contentType = eContentType::APPLICATION_JSON;
reply.headers.push_back("Cache-Control: no-store");
if (!same || diff != 0) {
reply.status = eHTTPStatusCode::FORBIDDEN;
reply.message = R"({"success":false,"error":"forbidden"})";
return false;
}
body = nlohmann::json::parse(context.body.empty() ? std::string("{}") : context.body, nullptr, false);
if (!body->is_object()) {
reply.status = eHTTPStatusCode::BAD_REQUEST;
reply.message = R"({"success":false,"error":"the body isn't a JSON object"})";
return false;
}
return true;
};
Game::web.RegisterHTTPRoute({ .path = "/admin/preview", .method = eHTTPMethod::POST, .middleware = {},
.handle = [admin](HTTPReply& reply, const HTTPContext& context) {
std::optional<nlohmann::json> body;
if (!admin(reply, context, body)) return;
const auto values = UgcIconParams::Parse(body->value("values", nlohmann::json::object()).dump());
const auto kind = body->value("kind", std::string("modular")) == "model" ? UgcStorage::Kind::MODEL : UgcStorage::Kind::MODULAR;
const auto id = GeneralUtils::TryParse<LWOOBJID>(body->value("id", std::string("0"))).value_or(0);
auto deferred = Web::Defer(reply, context);
std::string error;
if (!g_Processor->QueuePreview(kind, id, body->value("modules", std::string()), values, deferred, error)) {
HTTPReply out;
out.status = eHTTPStatusCode::BAD_REQUEST;
out.contentType = eContentType::APPLICATION_JSON;
out.message = nlohmann::json{ { "success", false }, { "error", error } }.dump();
deferred.Send(std::move(out));
}
} });
Game::web.RegisterHTTPRoute({ .path = "/admin/regenerate-icons", .method = eHTTPMethod::POST, .middleware = {},
.handle = [admin](HTTPReply& reply, const HTTPContext& context) {
std::optional<nlohmann::json> body;
if (!admin(reply, context, body)) return;
const auto queued = g_Processor->RegenerateIcons(body->value("kind", std::string()));
reply.status = eHTTPStatusCode::OK;
reply.message = nlohmann::json{ { "success", true }, { "queued", queued } }.dump();
} });
Game::web.RegisterHTTPRoute({ .path = "/admin/delete", .method = eHTTPMethod::POST, .middleware = {},
.handle = [admin](HTTPReply& reply, const HTTPContext& context) {
std::optional<nlohmann::json> body;
if (!admin(reply, context, body)) return;
UgcProcessor::DeleteRequest request;
request.kind = body->value("kind", std::string("model")) == "modular" ? UgcStorage::Kind::MODULAR : UgcStorage::Kind::MODEL;
if (const auto ids = body->find("ids"); ids != body->end() && ids->is_array()) {
for (const auto& id : *ids) {
const auto parsed = GeneralUtils::TryParse<LWOOBJID>(id.is_string() ? id.get<std::string>() : id.dump());
if (parsed) request.ids.push_back(*parsed);
}
}
request.all = body->value("all", false);
request.olderThanDays = body->value("olderThanDays", int64_t{ 0 });
request.unusedDays = body->value("unusedDays", int64_t{ 0 });
const auto after = body->value("after", std::string("on_demand"));
request.after = after == "now" ? UgcProcessor::eAfterDelete::NOW : after == "gone" ? UgcProcessor::eAfterDelete::GONE : UgcProcessor::eAfterDelete::ON_DEMAND;
const auto result = g_Processor->Delete(request);
LOG("Deleted the files of %zu item(s) (%llu bytes) for the dashboard", result.deleted, static_cast<unsigned long long>(result.bytes));
reply.status = eHTTPStatusCode::OK;
reply.message = nlohmann::json{ { "success", true }, { "deleted", result.deleted }, { "bytes", result.bytes }, { "busy", result.busy }, { "notes", result.notes } }.dump();
} });
Game::web.RegisterHTTPRoute({ .path = "/status", .method = eHTTPMethod::GET, .middleware = {},
.handle = [](HTTPReply& reply, const HTTPContext&) {
reply.status = eHTTPStatusCode::OK;
@@ -361,6 +428,7 @@ int main(int argc, char** argv) {
masterPort = masterInfo->port;
masterPassword = masterInfo->password;
}
g_AdminKey = masterPassword;
// The master starts it again when this link drops
g_Server = new dServer(masterIP, Setting<uint32_t>("net_port", 2012), 0, 16, false, false, Game::logger, masterIP, masterPort,

39
dUgcServer/UgcStorage.cpp
View File

@@ -10,7 +10,7 @@ namespace {
"model.nif", "model.nif.gz", "model.nif.checksum",
"model.lxfml.gz", "model.lxfml.checksum",
"icon.dds.gz", "icon.dds.checksum", "icon.png",
"model.noao.nif", "stats.json",
"model.noao.nif", "stats.json", "combo.json",
"previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json",
};
@@ -88,6 +88,43 @@ std::optional<uint64_t> UgcStorage::Write(Kind kind, LWOOBJID id, const Files& f
return bytes;
}
std::optional<uint64_t> UgcStorage::Update(Kind kind, LWOOBJID id, const Files& files, std::string& error) const {
const auto folder = Folder(kind, id);
std::error_code code;
if (!std::filesystem::is_directory(folder, code)) {
error = "no files to update in " + folder.string();
return std::nullopt;
}
uint64_t bytes = 0;
for (const auto* name : KEPT_FILES) {
if (files.contains(name) && std::filesystem::exists(folder / name, code)) {
std::filesystem::copy_file(folder / name, folder / (std::string("previous.") + name), std::filesystem::copy_options::overwrite_existing, code);
code.clear();
}
}
for (const auto& [name, data] : files) {
if (!IsKnownFile(name)) continue;
const auto aside = folder / (".tmp-" + name + "-" + RandomSuffix());
{
std::ofstream out(aside, std::ios::binary | std::ios::trunc);
out.write(data.data(), static_cast<std::streamsize>(data.size()));
if (!out) {
error = "could not write " + aside.string();
std::filesystem::remove(aside, code);
return std::nullopt;
}
}
std::filesystem::rename(aside, folder / name, code);
if (code) {
error = "could not replace " + (folder / name).string() + ": " + code.message();
std::filesystem::remove(aside, code);
return std::nullopt;
}
bytes += data.size();
}
return bytes;
}
void UgcStorage::Remove(Kind kind, LWOOBJID id) const {
std::error_code error;
std::filesystem::remove_all(Folder(kind, id), error);

5
dUgcServer/UgcStorage.h
View File

@@ -33,6 +33,11 @@ public:
// Replaces an item's files with `files`; the bytes written, nullopt (and `error`) on failure
std::optional<uint64_t> Write(Kind kind, LWOOBJID id, const Files& files, std::string& error) const;
// Replaces some of an item's files in place (each written aside and renamed over the old one), keeping the others;
// the ones kept for comparing (icon.png...) are copied to previous.* first. The bytes written, nullopt when the item
// has no folder or a write fails.
std::optional<uint64_t> Update(Kind kind, LWOOBJID id, const Files& files, std::string& error) const;
// Deletes an item's files
void Remove(Kind kind, LWOOBJID id) const;