Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-02 10:53:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: build a zone's 3D data on worker threads

Browse Source 
Opening a zone the first time built its terrain, scene objects and manifests
and ran ImageMagick on the web thread, stalling the dashboard for seconds.

- Workers: the shared pool plus Workers::Reply (answer at once when built,
  else from a worker via Web::Defer).
- terrain_chunks/terrain_layers/scene/paths/scenery/flairs (world3d, property
  and showcase routes) and terrain textures go through it; results are built
  once in OnceCaches, the .raw is read once per zone for chunks, layers and
  flairs, deflated bodies are cached thread-safely.
- ImageMagick conversions are deduplicated and written under a temporary name.
- Workers don't query the CDClient, read settings or call mongoose: ZoneTable,
  render components, flairs, object names, LOT kinds and terrain texture names
  are read at startup; client_location is read once; base64 is plain C++.
- Logger writes one line at a time (mutex; localtime's buffer is shared).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-26 23:57:48 -05:00
parent 20f70ac0de
commit 17689cd663
20 changed files with 895 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
dDashboardServer/DashboardServer.cpp
View File

@@ -60,6 +60,11 @@
#include "ModeratorHelper.h"
#include "LiveWorld.h"
#include "Scenery.h"
#include "Workers.h"
#include "ReportRoutes.h"
#include "WorldView.h"
#include "ClientAssets.h"
#include "DashboardRoutes.h"
#include "LiveEventRoutes.h"
#include "ChallengeRoutes.h"
#include "Inspector.h"
@@ -411,6 +416,19 @@ int main(int argc, char** argv) {
);
Game::server = g_Server;
// What the worker threads read from the CDClient and the settings, read now on this thread: workers never query
// the CDClient, read settings or touch the network (RakNet and mongoose are the main thread's)
{
const auto start = std::chrono::steady_clock::now();
ClientAssets::Preload();
PreloadZoneData();
Scenery::Preload();
WorldView::Preload();
ZoneNames();
LOG("Read the client data for the 3D views in %lld ms", static_cast<long long>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - start).count()));
}
Workers::Start();
// Initialize web server
if (!Game::web.Startup(ourIP, ourPort)) {
LOG("Failed to start web server on %s:%d", ourIP.c_str(), ourPort);
@@ -570,8 +588,8 @@ int main(int argc, char** argv) {
}
// Cleanup: the conversion threads first (they answer deferred requests), then the web server's connections
Scenery::Shutdown();
// Cleanup: the worker threads first (they answer deferred requests), then the web server's connections
Workers::Stop();
Game::web.Shutdown();
Inspector::Shutdown();
EmailService::Shutdown();

57
dDashboardServer/routes/APIRoutes.cpp
View File

@@ -13,6 +13,7 @@
#include "CharacterTools.h"
#include "ClientAssets.h"
#include "Scenery.h"
#include "Workers.h"
#include "LiveWorld.h"
#include "ServerState.h"
#include "WSRoutes.h"
@@ -1091,12 +1092,14 @@ namespace {
if (!propId) return;
const auto info = AuthorizedProperty(context, *propId, reply);
if (!info) return;
const auto terrain = ZoneTerrainJson(info->zoneId);
if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone (is client_location set?)");
reply.status = eHTTPStatusCode::OK;
reply.contentType = eContentType::APPLICATION_JSON;
reply.message = *terrain;
reply.headers.push_back("Cache-Control: private, max-age=86400");
Workers::Reply(reply, context, ZoneTerrainJsonReady(info->zoneId), [zone = info->zoneId](HTTPReply& out) {
const auto terrain = ZoneTerrainJson(zone);
if (!terrain) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone (is client_location set?)");
out.status = eHTTPStatusCode::OK;
out.contentType = eContentType::APPLICATION_JSON;
out.message = *terrain;
out.headers.push_back("Cache-Control: private, max-age=86400");
});
});
Route(eHTTPMethod::GET, "/api/properties/:id/terrain_chunks", 0,
@@ -1106,12 +1109,14 @@ namespace {
if (!propId) return;
const auto info = AuthorizedProperty(context, *propId, reply);
if (!info) return;
const auto terrain = ZoneTerrainChunksJson(info->zoneId);
if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone (is client_location set?)");
reply.status = eHTTPStatusCode::OK;
reply.contentType = eContentType::APPLICATION_JSON;
reply.message = *terrain;
reply.headers.push_back("Cache-Control: private, max-age=86400");
Workers::Reply(reply, context, ZoneTerrainChunksReady(info->zoneId), [zone = info->zoneId](HTTPReply& out) {
const auto terrain = ZoneTerrainChunksJson(zone);
if (!terrain) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone (is client_location set?)");
out.status = eHTTPStatusCode::OK;
out.contentType = eContentType::APPLICATION_JSON;
out.message = *terrain;
out.headers.push_back("Cache-Control: private, max-age=86400");
});
});
Route(eHTTPMethod::GET, "/api/properties/:id/scenery", 0,
@@ -1121,24 +1126,28 @@ namespace {
if (!propId) return;
const auto info = AuthorizedProperty(context, *propId, reply);
if (!info) return;
const auto scenery = Scenery::ZoneJson(info->zoneId);
if (!scenery) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No scenery for this zone (is client_location set?)");
reply.status = eHTTPStatusCode::OK;
reply.contentType = eContentType::APPLICATION_JSON;
reply.message = *scenery;
reply.headers.push_back("Cache-Control: private, max-age=86400");
Workers::Reply(reply, context, Scenery::ZoneReady(info->zoneId), [zone = info->zoneId](HTTPReply& out) {
const auto scenery = Scenery::ZoneJson(zone);
if (!scenery) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No scenery for this zone (is client_location set?)");
out.status = eHTTPStatusCode::OK;
out.contentType = eContentType::APPLICATION_JSON;
out.message = *scenery;
out.headers.push_back("Cache-Control: private, max-age=86400");
});
});
Route(eHTTPMethod::GET, "/api/terrain_textures/:id", 0, "A terrain texture (mapTextureResource ID) as PNG, for the property 3D view",
[](HTTPReply& reply, const HTTPContext& context) {
const auto textureId = RequireId<uint32_t>(context, 2, reply);
if (!textureId) return;
const auto png = TerrainTextureFile(*textureId);
if (!png) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "Texture not found");
reply.status = eHTTPStatusCode::OK;
reply.contentType = eContentType::IMAGE_PNG;
reply.message = *png;
reply.headers.push_back("Cache-Control: private, max-age=604800");
Workers::Reply(reply, context, TerrainTextureReady(*textureId), [id = *textureId](HTTPReply& out) {
const auto png = TerrainTextureFile(id);
if (!png) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "Texture not found");
out.status = eHTTPStatusCode::OK;
out.contentType = eContentType::IMAGE_PNG;
out.message = *png;
out.headers.push_back("Cache-Control: private, max-age=604800");
});
});
Route(eHTTPMethod::GET, "/api/properties/:id/boundary", 0, "Where a property's owner may build: the outline from the zone file and the height limit (needs client_location)",

1
dDashboardServer/routes/CMakeLists.txt
View File

@@ -43,6 +43,7 @@ set(DASHBOARDROUTES_SOURCES
"NifFile.cpp"
"Scenery.cpp"
"WorkerPool.cpp"
"Workers.cpp"
"AccountModeration.cpp"
"ModerationTools.cpp"
"ModeratorHelper.cpp"

86
dDashboardServer/routes/ClientAssets.cpp
View File

@@ -7,6 +7,7 @@
#include <set>
#include <unordered_map>
#include "OnceCache.h"
#include "RouteUtils.h"
#include "CDClientDatabase.h"
#include "Game.h"
@@ -26,11 +27,14 @@ namespace {
std::map<LOT, nlohmann::json> g_ItemCache;
std::map<LOT, std::vector<int>> g_ItemSetsByLot; // lot -> set ids containing it
bool g_ItemSetsLoaded = false;
std::set<std::string> g_FailedConversions;
// Read once (client_location only changes with a restart), so worker threads never read the settings
std::filesystem::path ClientRes() {
const auto client = Game::config->GetValue("client_location");
return client.empty() ? std::filesystem::path{} : std::filesystem::path(client) / "res";
static const std::filesystem::path res = [] {
const auto client = Game::config ? Game::config->GetValue("client_location") : std::string{};
return client.empty() ? std::filesystem::path{} : std::filesystem::path(client) / "res";
}();
return res;
}
// Lowercase, forward slashes, no leading "../" or "res/"
@@ -195,11 +199,8 @@ namespace {
// Every object's name and displayName, read once: the CDClient has no index on Objects.id, so looking a LOT up
// scans the whole table, and pages name thousands of items at once
const std::unordered_map<LOT, ObjectNames>& ObjectNameTable() {
static std::unordered_map<LOT, ObjectNames> table;
static bool loaded = false;
if (loaded) return table;
loaded = true;
std::unordered_map<LOT, ObjectNames> ReadObjectNames() {
std::unordered_map<LOT, ObjectNames> table;
try {
auto row = CDClientDatabase::ExecuteQuery("SELECT id, name, displayName FROM Objects;");
for (; !row.eof(); row.nextRow()) {
@@ -211,6 +212,23 @@ namespace {
}
return table;
}
// Read at startup (ClientAssets::Preload), so worker threads only read it and never query the CDClient
const std::unordered_map<LOT, ObjectNames>& ObjectNameTable() {
static const auto table = ReadObjectNames();
return table;
}
// Where TextureAsPng keeps a texture it converted
std::filesystem::path PngCachePath(const std::string& normalized, uint32_t maxSize, bool opaque) {
std::string cacheName = normalized + "_" + std::to_string(maxSize) + (opaque ? "_opaque" : "") + ".png";
std::replace(cacheName.begin(), cacheName.end(), '/', '_');
std::replace(cacheName.begin(), cacheName.end(), ' ', '_');
return std::filesystem::path("dDashboardServer") / "icon_cache" / cacheName;
}
// ImageMagick conversions by cache file name: whether it worked (a failed one isn't tried again)
OnceCache<std::string, bool> g_Conversions;
}
namespace ClientAssets {
@@ -305,28 +323,39 @@ namespace ClientAssets {
if (normalized.ends_with(".png")) return ReadFile(source);
if (!normalized.ends_with(".dds")) return std::nullopt;
std::string cacheName = normalized + "_" + std::to_string(maxSize) + (opaque ? "_opaque" : "") + ".png";
std::replace(cacheName.begin(), cacheName.end(), '/', '_');
std::replace(cacheName.begin(), cacheName.end(), ' ', '_');
const auto cacheDir = std::filesystem::path("dDashboardServer") / "icon_cache";
const auto target = cacheDir / cacheName;
if (!std::filesystem::exists(target)) {
// ImageMagick runs without a shell (an argument vector); odd characters are still refused as a precaution
if (g_FailedConversions.contains(cacheName) || Process::HasShellMetacharacters(source.string())) return std::nullopt;
std::filesystem::create_directories(cacheDir, ec);
const std::string size = std::to_string(maxSize) + "x" + std::to_string(maxSize) + ">";
std::vector<std::string> arguments{ "magick", source.string() };
if (opaque) { arguments.push_back("-alpha"); arguments.push_back("off"); }
arguments.insert(arguments.end(), { "-resize", size, target.string() });
if (Process::Run(arguments) != 0) {
LOG_DEBUG("Texture conversion failed for %s (is ImageMagick installed?)", normalized.c_str());
g_FailedConversions.insert(cacheName);
return std::nullopt;
}
const auto target = PngCachePath(normalized, maxSize, opaque);
const auto cacheDir = target.parent_path();
const auto cacheName = target.filename().string();
// ImageMagick runs without a shell (an argument vector); odd characters are still refused as a precaution
if (!std::filesystem::exists(target, ec) && !Process::HasShellMetacharacters(source.string())) {
// Once per file, however many threads ask at once; written under another name and renamed, so a reader
// never sees half a file
const auto converted = g_Conversions.Get(cacheName, [&] {
if (std::filesystem::exists(target, ec)) return true;
std::filesystem::create_directories(cacheDir, ec);
const auto partial = cacheDir / (cacheName + ".part.png");
const std::string size = std::to_string(maxSize) + "x" + std::to_string(maxSize) + ">";
std::vector<std::string> arguments{ "magick", source.string() };
if (opaque) { arguments.push_back("-alpha"); arguments.push_back("off"); }
arguments.insert(arguments.end(), { "-resize", size, partial.string() });
std::error_code renameError;
if (Process::Run(arguments) != 0 || (std::filesystem::rename(partial, target, renameError), renameError)) {
LOG_DEBUG("Texture conversion failed for %s (is ImageMagick installed?)", normalized.c_str());
return false;
}
return true;
});
if (!converted) return std::nullopt;
}
return ReadFile(target);
}
bool TextureAsPngReady(const std::string& assetPath, uint32_t maxSize, bool opaque) {
const auto normalized = NormalizeAssetPath(assetPath);
std::error_code ec;
return normalized.ends_with(".png") || std::filesystem::exists(PngCachePath(normalized, maxSize, opaque), ec);
}
std::optional<std::string> ReadResFile(const std::string& relativePath) {
const auto res = ClientRes();
if (res.empty() || !IsSafeAssetPath(NormalizeAssetPath(relativePath))) return std::nullopt;
@@ -342,6 +371,11 @@ namespace ClientAssets {
return path;
}
void Preload() {
ClientRes();
ObjectNameTable();
}
std::filesystem::path ResFolder() {
return ClientRes();
}

14
dDashboardServer/routes/ClientAssets.h
View File

@@ -13,6 +13,12 @@
* client's res folder (client_location). Textures are converted to PNG with ImageMagick and cached.
*/
namespace ClientAssets {
/**
* Read what is read once (the res folder from client_location, the objects' names) at startup, on the main
* thread, so the dashboard's worker threads only ever read it: they must not query the CDClient or read settings.
*/
void Preload();
std::string ItemName(LOT lot);
// An object's raw Objects.name (not localized), or nullopt when the LOT isn't in the CDClient
@@ -27,10 +33,14 @@ namespace ClientAssets {
// Relative path inside res/ using only safe characters and no traversal
bool IsSafeAssetPath(const std::string& path);
// A DDS or PNG texture from res/ as PNG, scaled down to fit maxSize. `opaque` drops the alpha channel first, for
// A DDS or PNG texture from res/ as PNG, scaled down to fit maxSize (converted once, even when asked for from
// several threads at once). `opaque` drops the alpha channel first, for
// textures whose alpha isn't transparency (terrain textures have alpha 0 everywhere; kept, they'd come out black)
std::optional<std::string> TextureAsPng(const std::string& assetPath, uint32_t maxSize, bool opaque = false);
// Whether TextureAsPng has the texture converted already (so it's quick)
bool TextureAsPngReady(const std::string& assetPath, uint32_t maxSize, bool opaque = false);
// Raw bytes of a file inside res/ (matched ignoring case)
std::optional<std::string> ReadResFile(const std::string& relativePath);
@@ -40,7 +50,7 @@ namespace ClientAssets {
// The client's res/ folder (empty when client_location isn't set)
std::filesystem::path ResFolder();
// ResolveResFile inside a res/ folder from ResFolder: reads no settings, so worker threads may call it
// ResolveResFile inside a res/ folder from ResFolder
std::optional<std::filesystem::path> ResolveResFile(const std::string& relativePath, const std::filesystem::path& res);
// The res/-relative path of a file called `fileName` (ignoring case) anywhere under res/<folder>

31
dDashboardServer/routes/DashboardRoutes.cpp
View File

@@ -37,24 +37,21 @@ namespace {
return name.empty() ? "Zone " + std::to_string(zoneId) : name;
}
nlohmann::json g_ZoneNamesCache;
bool g_ZoneNamesCached = false;
// Built once (worker threads read it too, so never lazily twice)
const nlohmann::json& GetZoneNamesJson() {
if (g_ZoneNamesCached) return g_ZoneNamesCache;
g_ZoneNamesCache["0"] = "Character Select";
auto keys = Locale::GetPhraseIdsWithPrefix("ZoneTable_");
for (const auto& key : keys) {
if (key.find("_DisplayDescription") == std::string::npos) continue;
auto start = std::string("ZoneTable_").length();
auto end = key.find("_DisplayDescription");
auto idStr = key.substr(start, end - start);
const auto& name = Locale::GetPhrase(key);
if (!name.empty()) g_ZoneNamesCache[idStr] = name;
}
g_ZoneNamesCached = true;
return g_ZoneNamesCache;
static const nlohmann::json names = [] {
nlohmann::json names;
names["0"] = "Character Select";
for (const auto& key : Locale::GetPhraseIdsWithPrefix("ZoneTable_")) {
if (key.find("_DisplayDescription") == std::string::npos) continue;
const auto start = std::string("ZoneTable_").length();
const auto end = key.find("_DisplayDescription");
const auto& name = Locale::GetPhrase(key);
if (!name.empty()) names[key.substr(start, end - start)] = name;
}
return names;
}();
return names;
}
nlohmann::json ParseCharacterXml(const std::string& xml) {

80
dDashboardServer/routes/OnceCache.h Normal file
View File

@@ -0,0 +1,80 @@
#pragma once
#include <chrono>
#include <exception>
#include <functional>
#include <future>
#include <map>
#include <mutex>
/**
* A value per key, built the first time it is asked for and kept: for zone data built from the client's files (a
* zone's terrain, its scene objects), which worker threads build while the web thread may ask for it too. Any thread.
* Asked for again while it is being built, the caller waits for that build instead of starting another. A build that
* throws is forgotten (the next caller tries again) and the exception reaches everyone waiting for it.
*/
template<typename Key, typename Value>
class OnceCache {
public:
// Whether the value is built (never waits)
bool Ready(const Key& key) const {
std::lock_guard lock(m_Mutex);
const auto it = m_Entries.find(key);
return it != m_Entries.end() && it->second.wait_for(std::chrono::seconds(0)) == std::future_status::ready;
}
/**
* The value, built with `build` when nobody has yet. The reference stays valid: values are never replaced or
* removed once built.
*/
const Value& Get(const Key& key, const std::function<Value()>& build) {
std::shared_future<Value> future;
std::promise<Value> promise;
bool mine = false;
{
std::lock_guard lock(m_Mutex);
const auto it = m_Entries.find(key);
if (it != m_Entries.end()) {
future = it->second;
} else {
future = promise.get_future().share();
m_Entries.emplace(key, future);
mine = true;
}
}
if (mine) {
try {
promise.set_value(build());
} catch (...) {
{
std::lock_guard lock(m_Mutex);
m_Entries.erase(key);
}
promise.set_exception(std::current_exception());
}
}
// The map keeps its own copy of the future, so what get() refers to outlives this call (unless the build threw,
// in which case get() throws)
return WaitFor(key, future);
}
size_t Size() const {
std::lock_guard lock(m_Mutex);
return m_Entries.size();
}
private:
const Value& WaitFor(const Key& key, const std::shared_future<Value>& future) {
future.wait();
std::lock_guard lock(m_Mutex);
const auto it = m_Entries.find(key);
if (it == m_Entries.end()) {
future.get(); // the build failed: rethrows its exception
std::terminate(); // unreachable: a successful build is never removed
}
return it->second.get();
}
mutable std::mutex m_Mutex;
std::map<Key, std::shared_future<Value>> m_Entries;
};

211
dDashboardServer/routes/ReportRoutes.cpp
View File

@@ -1,4 +1,7 @@
#include "ReportRoutes.h"
#include "OnceCache.h"
#include <deque>
#include <future>
#include "ZonePaths.h"
#include <algorithm>
@@ -387,18 +390,72 @@ namespace {
std::string luzPath; // relative to res/maps
};
std::optional<ZoneInfo> GetZoneInfo(uint32_t zoneId) {
static std::map<uint32_t, std::optional<ZoneInfo>> cache;
if (const auto it = cache.find(zoneId); it != cache.end()) return it->second;
auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT zoneName, DisplayDescription FROM ZoneTable WHERE zoneID = ?;");
stmt.bind(1, static_cast<int>(zoneId));
auto result = stmt.execQuery();
std::optional<ZoneInfo> info;
if (!result.eof()) {
const std::string description = result.getStringField("DisplayDescription", "");
info = ZoneInfo{ description.empty() ? "Zone " + std::to_string(zoneId) : description, result.getStringField("zoneName", "") };
// Standard base64 (plain C++, no mongoose: zone data is built on worker threads)
std::string Base64(std::string_view bytes) {
static constexpr char ALPHABET[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
std::string out;
out.reserve((bytes.size() + 2) / 3 * 4);
size_t i = 0;
for (; i + 2 < bytes.size(); i += 3) {
const uint32_t n = (static_cast<uint8_t>(bytes[i]) << 16) | (static_cast<uint8_t>(bytes[i + 1]) << 8) | static_cast<uint8_t>(bytes[i + 2]);
for (const int shift : { 18, 12, 6, 0 }) out += ALPHABET[(n >> shift) & 63];
}
return cache[zoneId] = info;
if (i < bytes.size()) {
uint32_t n = static_cast<uint8_t>(bytes[i]) << 16;
if (i + 1 < bytes.size()) n |= static_cast<uint8_t>(bytes[i + 1]) << 8;
out += ALPHABET[(n >> 18) & 63];
out += ALPHABET[(n >> 12) & 63];
out += i + 1 < bytes.size() ? ALPHABET[(n >> 6) & 63] : '=';
out += '=';
}
return out;
}
std::map<uint32_t, ZoneInfo> ReadZoneTable() {
std::map<uint32_t, ZoneInfo> zones;
try {
auto result = CDClientDatabase::ExecuteQuery("SELECT zoneID, zoneName, DisplayDescription FROM ZoneTable;");
for (; !result.eof(); result.nextRow()) {
const std::string description = result.getStringField("DisplayDescription", "");
const auto zoneId = static_cast<uint32_t>(result.getIntField("zoneID", 0));
zones.try_emplace(zoneId, ZoneInfo{ description.empty() ? "Zone " + std::to_string(zoneId) : description, result.getStringField("zoneName", "") });
}
} catch (const std::exception& ex) {
LOG("Could not read the zone table: %s", ex.what());
}
return zones;
}
// ZoneTable, read at startup (PreloadZoneData) so worker threads never query the CDClient
const std::map<uint32_t, ZoneInfo>& ZoneTable() {
static const auto zones = ReadZoneTable();
return zones;
}
std::optional<ZoneInfo> GetZoneInfo(uint32_t zoneId) {
const auto& zones = ZoneTable();
const auto it = zones.find(zoneId);
if (it == zones.end()) return std::nullopt;
return it->second;
}
// mapTextureResource: texture id -> file name (lowercase), read at startup
const std::map<uint32_t, std::string>& TerrainTextureNames() {
static const auto names = [] {
std::map<uint32_t, std::string> names;
try {
auto result = CDClientDatabase::ExecuteQuery("SELECT id, texturepath FROM mapTextureResource;");
for (; !result.eof(); result.nextRow()) {
std::string file = result.getStringField(1, "");
std::transform(file.begin(), file.end(), file.begin(), ::tolower);
names.try_emplace(static_cast<uint32_t>(result.getIntField(0, 0)), file);
}
} catch (const std::exception& ex) {
LOG("Could not read the terrain textures: %s", ex.what());
}
return names;
}();
return names;
}
/**
@@ -457,10 +514,9 @@ namespace {
}
// Terrain for a zone as JSON with base64 16-bit heights, built once per zone
std::optional<std::string> TerrainJson(uint32_t zoneId) {
static std::map<uint32_t, std::optional<std::string>> cache;
if (const auto it = cache.find(zoneId); it != cache.end()) return it->second;
OnceCache<uint32_t, std::optional<std::string>> g_TerrainJson;
std::optional<std::string> BuildTerrainJson(uint32_t zoneId) {
std::optional<std::string> json;
const auto zone = GetZoneInfo(zoneId);
if (zone) {
@@ -469,8 +525,7 @@ namespace {
const auto grid = raw ? TerrainMap::Parse(*raw, 1024) : std::nullopt;
if (grid) {
const auto packed = TerrainMap::Quantize(*grid);
std::string encoded(packed.size() * 4 / 3 + 8, '\0');
encoded.resize(mg_base64_encode(reinterpret_cast<const unsigned char*>(packed.data()), packed.size(), encoded.data(), encoded.size()));
const auto encoded = Base64({ reinterpret_cast<const char*>(packed.data()), packed.size() * sizeof(packed[0]) });
nlohmann::json terrain{
{"zone", zoneId}, {"minX", grid->minX}, {"minZ", grid->minZ}, {"step", grid->step},
{"width", grid->width}, {"height", grid->height}, {"minY", grid->minY}, {"maxY", grid->maxY}, {"heights", encoded}
@@ -479,7 +534,11 @@ namespace {
json = terrain.dump();
}
}
return cache[zoneId] = json;
return json;
}
std::optional<std::string> TerrainJson(uint32_t zoneId) {
return g_TerrainJson.Get(zoneId, [zoneId] { return BuildTerrainJson(zoneId); });
}
std::string DayText(uint32_t day) {
@@ -534,11 +593,6 @@ std::optional<std::string> ZoneTerrainJson(uint32_t zoneId) {
}
namespace {
std::string Base64(std::string_view bytes) {
std::string encoded(bytes.size() * 4 / 3 + 8, '\0');
encoded.resize(mg_base64_encode(reinterpret_cast<const unsigned char*>(bytes.data()), bytes.size(), encoded.data(), encoded.size()));
return encoded;
}
}
std::optional<Raw::Raw> ZoneRaw(uint32_t zoneId) {
@@ -546,17 +600,70 @@ std::optional<Raw::Raw> ZoneRaw(uint32_t zoneId) {
return zone ? ReadZoneRaw(*zone) : std::nullopt;
}
std::shared_ptr<const Raw::Raw> ZoneRawShared(uint32_t zoneId) {
// The last few zones: their chunks, layers and flairs are built one after the other when a zone is first viewed
constexpr size_t KEEP = 2;
static std::mutex mutex;
static std::deque<std::pair<uint32_t, std::shared_future<std::shared_ptr<const Raw::Raw>>>> recent;
std::promise<std::shared_ptr<const Raw::Raw>> promise;
std::shared_future<std::shared_ptr<const Raw::Raw>> future;
{
std::lock_guard lock(mutex);
const auto it = std::find_if(recent.begin(), recent.end(), [zoneId](const auto& entry) { return entry.first == zoneId; });
if (it != recent.end()) return it->second.get(); // may wait for the thread reading it
future = promise.get_future().share();
recent.emplace_back(zoneId, future);
if (recent.size() > KEEP) recent.pop_front();
}
std::shared_ptr<const Raw::Raw> raw;
try {
if (auto read = ZoneRaw(zoneId)) raw = std::make_shared<const Raw::Raw>(std::move(*read));
} catch (const std::exception& ex) {
LOG("Could not read the terrain of zone %u: %s", zoneId, ex.what());
}
promise.set_value(raw);
return raw;
}
namespace {
std::string_view BytesOf(const std::vector<uint8_t>& bytes) {
return { reinterpret_cast<const char*>(bytes.data()), bytes.size() };
}
}
namespace {
OnceCache<uint32_t, std::optional<std::string>> g_TerrainChunks;
OnceCache<uint32_t, std::optional<std::string>> g_TerrainLayers;
OnceCache<uint32_t, std::string> g_TerrainTexturePaths; // id -> path under res/ ("" when the client lacks it)
}
bool ZoneTerrainJsonReady(uint32_t zoneId) {
return g_TerrainJson.Ready(zoneId);
}
bool ZoneTerrainChunksReady(uint32_t zoneId) {
return g_TerrainChunks.Ready(zoneId);
}
bool ZoneTerrainLayersReady(uint32_t zoneId) {
return g_TerrainLayers.Ready(zoneId);
}
std::optional<std::string> ZoneLuzPath(uint32_t zoneId) {
const auto zone = GetZoneInfo(zoneId);
if (!zone || !zone->luzPath.ends_with(".luz")) return std::nullopt;
return zone->luzPath;
}
void PreloadZoneData() {
ZoneTable();
TerrainTextureNames();
}
std::optional<std::string> ZoneTerrainChunksJson(uint32_t zoneId) {
static std::map<uint32_t, std::optional<std::string>> cache;
if (const auto it = cache.find(zoneId); it != cache.end()) return it->second;
return g_TerrainChunks.Get(zoneId, [zoneId]() -> std::optional<std::string> {
std::optional<std::string> json;
if (const auto raw = ZoneRaw(zoneId)) {
if (const auto raw = ZoneRawShared(zoneId)) {
nlohmann::json out{ {"zone", zoneId}, {"chunks", nlohmann::json::array()} };
std::set<uint32_t> textures;
for (const auto& chunk : raw->chunks) {
@@ -575,15 +682,15 @@ std::optional<std::string> ZoneTerrainChunksJson(uint32_t zoneId) {
out["textures"] = textures;
json = out.dump();
}
return cache[zoneId] = json;
return json;
});
}
std::optional<std::string> ZoneTerrainLayersJson(uint32_t zoneId) {
static std::map<uint32_t, std::optional<std::string>> cache;
if (const auto it = cache.find(zoneId); it != cache.end()) return it->second;
return g_TerrainLayers.Get(zoneId, [zoneId]() -> std::optional<std::string> {
std::optional<std::string> json;
const auto zone = GetZoneInfo(zoneId);
const auto raw = zone ? ReadZoneRaw(*zone) : std::nullopt;
const auto raw = zone ? ZoneRawShared(zoneId) : nullptr;
if (raw) {
// Scene names from the .luz: the terrain's scene map holds the ids of its general scenes (audio scenes share them)
std::map<uint32_t, std::string> names;
@@ -615,30 +722,42 @@ std::optional<std::string> ZoneTerrainLayersJson(uint32_t zoneId) {
for (const auto& chunk : raw->chunks) flairs += chunk.flairs.size();
json = nlohmann::json{ {"zone", zoneId}, {"version", raw->version}, {"scenes", scenes}, {"chunks", chunks}, {"flairs", flairs} }.dump();
}
return cache[zoneId] = json;
return json;
});
}
namespace {
// Where terrain texture `textureId` is under res/, found once ("" when the client lacks it)
const std::string& TerrainTexturePath(uint32_t textureId) {
return g_TerrainTexturePaths.Get(textureId, [textureId] {
const auto& names = TerrainTextureNames();
const auto name = names.find(textureId);
if (name == names.end()) return std::string{};
if (ClientAssets::ResolveResFile("textures/env/" + name->second)) return "textures/env/" + name->second;
return ClientAssets::FindResFile("textures", name->second).value_or("");
});
}
// Tiled 4 times per chunk: 512 pixels is plenty and keeps the page light
constexpr uint32_t TERRAIN_TEXTURE_SIZE = 512;
}
std::optional<std::string> TerrainTextureFile(uint32_t textureId) {
static std::map<uint32_t, std::string> paths; // id -> path under res/, found once
if (!paths.contains(textureId)) {
auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT texturepath FROM mapTextureResource WHERE id = ? LIMIT 1;");
stmt.bind(1, static_cast<int>(textureId));
auto result = stmt.execQuery();
if (result.eof()) return std::nullopt;
std::string file = result.getStringField(0);
std::transform(file.begin(), file.end(), file.begin(), ::tolower);
std::string found;
if (ClientAssets::ReadResFile("textures/env/" + file)) found = "textures/env/" + file;
else if (const auto listing = ClientAssets::FindResFile("textures", file)) found = *listing;
paths[textureId] = found;
}
if (paths[textureId].empty()) return std::nullopt;
// Tiled 4 times per chunk: 512 pixels is plenty and keeps the page light
return ClientAssets::TextureAsPng(paths[textureId], 512, true);
const auto& path = TerrainTexturePath(textureId);
if (path.empty()) return std::nullopt;
return ClientAssets::TextureAsPng(path, TERRAIN_TEXTURE_SIZE, true);
}
bool TerrainTextureReady(uint32_t textureId) {
if (!g_TerrainTexturePaths.Ready(textureId)) return false;
const auto& path = TerrainTexturePath(textureId);
return path.empty() || ClientAssets::TextureAsPngReady(path, TERRAIN_TEXTURE_SIZE, true);
}
std::optional<nlohmann::json> ZonePropertyAreasJson(uint32_t zoneId) {
static std::mutex mutex;
static std::map<uint32_t, std::optional<nlohmann::json>> cache;
std::lock_guard lock(mutex);
if (const auto it = cache.find(zoneId); it != cache.end()) return it->second;
std::optional<nlohmann::json> json;
const auto zone = GetZoneInfo(zoneId);

19
dDashboardServer/routes/ReportRoutes.h
View File

@@ -22,6 +22,25 @@ std::optional<std::string> ZoneTerrainJson(uint32_t zoneId);
#include "Raw.h"
/**
* The zone data below is built on the dashboard's worker threads too, so it is thread safe: results are built once
* (OnceCache) and the CDClient tables it needs are read at startup by PreloadZoneData (workers never query the
* CDClient). The ...Ready functions say whether a result is built, so a route can answer it at once.
*/
void PreloadZoneData();
bool ZoneTerrainJsonReady(uint32_t zoneId);
bool ZoneTerrainChunksReady(uint32_t zoneId);
bool ZoneTerrainLayersReady(uint32_t zoneId);
bool TerrainTextureReady(uint32_t textureId);
// A zone's .luz file (relative to res/maps) from ZoneTable
std::optional<std::string> ZoneLuzPath(uint32_t zoneId);
#include <memory>
// ZoneRaw, shared: the last few zones read are kept, and a zone asked for by several threads at once is read once
std::shared_ptr<const Raw::Raw> ZoneRawShared(uint32_t zoneId);
// A zone's terrain file (.raw) read whole, as its .luz names it; nullopt without client files or when it's damaged
std::optional<Raw::Raw> ZoneRaw(uint32_t zoneId);

327
dDashboardServer/routes/Scenery.cpp
View File

@@ -19,9 +19,11 @@
#include "NifFile.h"
#include "ReportRoutes.h"
#include "RouteUtils.h"
#include "OnceCache.h"
#include "TtlCache.h"
#include "Web.h"
#include "WorkerPool.h"
#include "Workers.h"
#include "WorldScene.h"
#include "ZonePaths.h"
@@ -78,13 +80,10 @@ namespace {
std::unordered_map<std::string, std::vector<std::string>> byName;
};
const FileIndex& Files() {
static std::optional<FileIndex> index;
if (index) return *index;
index.emplace();
const auto client = Game::config->GetValue("client_location");
if (client.empty()) return *index;
const auto res = std::filesystem::path(client) / "res";
FileIndex IndexFiles() {
FileIndex index;
const auto res = ClientAssets::ResFolder();
if (res.empty()) return index;
std::error_code ec;
for (const char* folder : { "mesh", "textures", "animations" }) {
// Unpacked clients keep their original case, so the top folder is matched ignoring case too
@@ -93,12 +92,18 @@ namespace {
for (auto it = std::filesystem::recursive_directory_iterator(top.path(), ec); !ec && it != std::filesystem::recursive_directory_iterator(); it.increment(ec)) {
if (!it->is_regular_file(ec)) continue;
const auto relative = Lower(std::filesystem::relative(it->path(), res, ec).generic_string());
index->paths.insert(relative);
index->byName[Lower(it->path().filename().string())].push_back(relative);
index.paths.insert(relative);
index.byName[Lower(it->path().filename().string())].push_back(relative);
}
}
}
return *index;
return index;
}
// Built at startup (Scenery::Preload); read only afterwards
const FileIndex& Files() {
static const FileIndex index = IndexFiles();
return index;
}
// A texture a model names: next to the model if it's there (the usual case), else the file of that name sharing
@@ -140,24 +145,27 @@ namespace {
};
// Every LOT with a render component, read once (Objects has no index on id, so it is read whole too)
const std::unordered_map<uint32_t, RenderInfo>& RenderInfos() {
static std::optional<std::unordered_map<uint32_t, RenderInfo>> infos;
if (infos) return *infos;
infos.emplace();
std::unordered_map<uint32_t, RenderInfo> ReadRenderInfos() {
std::unordered_map<uint32_t, RenderInfo> infos;
try {
auto row = CDClientDatabase::ExecuteQuery(
"SELECT cr.id, rc.render_asset FROM ComponentsRegistry cr JOIN RenderComponent rc ON rc.id = cr.component_id "
"WHERE cr.component_type = " + std::to_string(RENDER_COMPONENT) + ";");
for (; !row.eof(); row.nextRow()) infos->try_emplace(static_cast<uint32_t>(row.getIntField(0)), RenderInfo{ row.getStringField(1, ""), "" });
for (; !row.eof(); row.nextRow()) infos.try_emplace(static_cast<uint32_t>(row.getIntField(0)), RenderInfo{ row.getStringField(1, ""), "" });
auto types = CDClientDatabase::ExecuteQuery("SELECT id, type FROM Objects;");
for (; !types.eof(); types.nextRow()) {
const auto it = infos->find(static_cast<uint32_t>(types.getIntField(0)));
if (it != infos->end() && it->second.type.empty()) it->second.type = types.getStringField(1, "");
const auto it = infos.find(static_cast<uint32_t>(types.getIntField(0)));
if (it != infos.end() && it->second.type.empty()) it->second.type = types.getStringField(1, "");
}
} catch (const std::exception& ex) {
LOG("Could not read the render components of objects: %s", ex.what());
}
return *infos;
return infos;
}
const std::unordered_map<uint32_t, RenderInfo>& RenderInfos() {
static const auto infos = ReadRenderInfos();
return infos;
}
struct Model {
@@ -176,30 +184,33 @@ namespace {
if (info == infos.end()) return {};
const auto draw = WorldScene::ClientDraws(object, info->second.type);
if (draw == WorldScene::eClientDraw::NO_MODEL) return {};
static std::mutex mutex;
static std::unordered_map<std::string, std::string> resolved;
const auto& stored = object.nifName.empty() ? info->second.asset : object.nifName;
auto it = resolved.find(stored);
if (it == resolved.end()) it = resolved.emplace(stored, ResolveModel(stored)).first;
return { it->second, draw == WorldScene::eClientDraw::HIDDEN };
{
std::lock_guard lock(mutex);
if (const auto it = resolved.find(stored); it != resolved.end()) return { it->second, draw == WorldScene::eClientDraw::HIDDEN };
}
auto path = ResolveModel(stored);
std::lock_guard lock(mutex);
resolved.try_emplace(stored, path);
return { std::move(path), draw == WorldScene::eClientDraw::HIDDEN };
}
std::optional<std::string> LuzPath(uint32_t zone) {
auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT zoneName FROM ZoneTable WHERE zoneID = ? LIMIT 1;");
stmt.bind(1, static_cast<int>(zone));
auto result = stmt.execQuery();
if (result.eof()) return std::nullopt;
std::string path = result.getStringField(0, "");
if (!path.ends_with(".luz")) return std::nullopt;
return path;
return ZoneLuzPath(zone);
}
double Round(float value, double scale) { return std::round(static_cast<double>(value) * scale) / scale; }
/**
* A zone's models and scenery manifest, built once (g_Zones). Once shared, assets, index, flairModels and the
* warmed flags are guarded by g_ZoneMutex: the flairs' models join assets when their manifest is built.
*/
struct ZoneScenery {
std::vector<std::string> assets; // res paths of models, indexed by the manifests
std::map<std::string, size_t> index; // res path -> its index in assets
std::string json;
std::optional<std::string> flairs; // the flairs' manifest, built when first asked for (their models join assets)
std::unordered_set<std::string> flairModels; // res paths of the flairs' models, converted ahead of others
bool warmedScenery{}; // WarmUp queued the scenery's models
bool warmedFlairs{}; // and the flairs'
@@ -211,14 +222,12 @@ namespace {
}
};
std::optional<ZoneScenery>& Zone(uint32_t zoneId) {
static std::map<uint32_t, std::optional<ZoneScenery>> cache;
if (const auto it = cache.find(zoneId); it != cache.end()) return it->second;
auto& entry = cache[zoneId];
std::mutex g_ZoneMutex;
std::shared_ptr<ZoneScenery> BuildZone(uint32_t zoneId) {
const auto luzPath = LuzPath(zoneId);
const auto luz = luzPath ? ClientAssets::ReadResFile("maps/" + *luzPath) : std::nullopt;
if (!luz) return entry;
if (!luz) return nullptr;
const auto folder = luzPath->substr(0, luzPath->find_last_of('/') + 1);
ZoneScenery scenery;
@@ -247,22 +256,36 @@ namespace {
{"zone", zoneId}, {"sky", sky}, {"assets", scenery.assets},
{"objects", { {"asset", assetOf}, {"pos", positions}, {"rot", rotations}, {"scale", scales}, {"hidden", hidden} }}
}.dump();
entry = std::move(scenery);
return entry;
return std::make_shared<ZoneScenery>(std::move(scenery));
}
OnceCache<uint32_t, std::shared_ptr<ZoneScenery>> g_Zones;
// The zone's scenery, built when first asked for (any thread); nullptr without client files
std::shared_ptr<ZoneScenery> Zone(uint32_t zoneId) {
return g_Zones.Get(zoneId, [zoneId] { return BuildZone(zoneId); });
}
// The same, only when it is built already (never waits)
std::shared_ptr<ZoneScenery> ZoneIfBuilt(uint32_t zoneId) {
return g_Zones.Ready(zoneId) ? Zone(zoneId) : nullptr;
}
// FlairTable: flair id -> the model's res path (empty when the client lacks it), read once
const std::unordered_map<uint32_t, std::string>& FlairModels() {
static std::optional<std::unordered_map<uint32_t, std::string>> models;
if (models) return *models;
models.emplace();
std::unordered_map<uint32_t, std::string> ReadFlairModels() {
std::unordered_map<uint32_t, std::string> models;
try {
auto row = CDClientDatabase::ExecuteQuery("SELECT id, asset FROM FlairTable;");
for (; !row.eof(); row.nextRow()) models->try_emplace(static_cast<uint32_t>(row.getIntField(0)), ResolveModel(row.getStringField(1, "")));
for (; !row.eof(); row.nextRow()) models.try_emplace(static_cast<uint32_t>(row.getIntField(0)), ResolveModel(row.getStringField(1, "")));
} catch (const std::exception& ex) {
LOG("Could not read the flairs: %s", ex.what());
}
return *models;
return models;
}
const std::unordered_map<uint32_t, std::string>& FlairModels() {
static const auto models = ReadFlairModels();
return models;
}
/**
@@ -275,18 +298,16 @@ namespace {
* The flairs' manifest (as the scenery's, plus a tint per flair), from the zone's terrain file. A flair's color
* tints its model; 63 is full strength (the files use 0 to 63 for most flairs, a little more for brighter ones).
*/
const std::optional<std::string>& Flairs(uint32_t zoneId, ZoneScenery& scenery) {
if (scenery.flairs) return scenery.flairs;
const auto raw = ZoneRaw(zoneId);
std::optional<std::string> BuildFlairs(uint32_t zoneId, ZoneScenery& scenery) {
const auto raw = ZoneRawShared(zoneId);
const auto& models = FlairModels();
nlohmann::json assetOf = nlohmann::json::array(), positions = nlohmann::json::array(), rotations = nlohmann::json::array(),
scales = nlohmann::json::array(), colors = nlohmann::json::array();
std::vector<const std::string*> modelOf;
nlohmann::json positions = nlohmann::json::array(), rotations = nlohmann::json::array(), scales = nlohmann::json::array(), colors = nlohmann::json::array();
for (const auto& chunk : raw ? raw->chunks : std::vector<Raw::Chunk>{}) {
for (const auto& flair : chunk.flairs) {
const auto model = models.find(flair.id);
if (model == models.end() || model->second.empty() || !std::isfinite(flair.position.x)) continue;
assetOf.push_back(scenery.IndexOf(model->second));
scenery.flairModels.insert(model->second);
modelOf.push_back(&model->second);
for (const auto value : { flair.position.x, flair.position.y, flair.position.z }) positions.push_back(Round(value, 100.0));
// Radians about x, y and z, applied in that order
const double c1 = std::cos(flair.rotation.x / 2), c2 = std::cos(flair.rotation.y / 2), c3 = std::cos(flair.rotation.z / 2);
@@ -298,11 +319,32 @@ namespace {
for (const auto value : { flair.colorR, flair.colorG, flair.colorB }) colors.push_back(value);
}
}
scenery.flairs = nlohmann::json{
{"zone", zoneId}, {"sky", -1}, {"assets", scenery.assets}, {"distance", FLAIR_DISTANCE}, {"colorScale", 1.0 / 63.0},
// The flairs' models join the zone's list, which mesh requests read meanwhile
nlohmann::json assetOf = nlohmann::json::array();
std::vector<std::string> assets;
{
std::lock_guard lock(g_ZoneMutex);
for (const auto* model : modelOf) {
assetOf.push_back(scenery.IndexOf(*model));
scenery.flairModels.insert(*model);
}
assets = scenery.assets;
}
return nlohmann::json{
{"zone", zoneId}, {"sky", -1}, {"assets", assets}, {"distance", FLAIR_DISTANCE}, {"colorScale", 1.0 / 63.0},
{"objects", { {"asset", assetOf}, {"pos", positions}, {"rot", rotations}, {"scale", scales}, {"color", colors} }}
}.dump();
return scenery.flairs;
}
OnceCache<uint32_t, std::optional<std::string>> g_Flairs;
/**
* The flairs' manifest (as the scenery's, plus a tint per flair), from the zone's terrain file. A flair's color
* tints its model; 63 is full strength (the files use 0 to 63 for most flairs, a little more for brighter ones).
* Built when first asked for (any thread).
*/
const std::optional<std::string>& Flairs(uint32_t zoneId, ZoneScenery& scenery) {
return g_Flairs.Get(zoneId, [zoneId, &scenery] { return BuildFlairs(zoneId, scenery); });
}
constexpr uint32_t FORMAT_VERSION = 1; // bump when NifFile's output changes, so cached files are rebuilt
@@ -518,7 +560,7 @@ namespace {
// ---- Converting on worker threads, so a big model never holds up the web server's one thread ----
WorkerPool g_Pool;
WorkerPool& Pool() { return Workers::Pool(); }
constexpr uintmax_t SMALL_MODEL_BYTES = 256 * 1024; // .nif files this small convert in the pool's fast lane
constexpr uintmax_t LARGE_MODEL_BYTES = 4 * 1024 * 1024; // and this big wait behind everything smaller
@@ -550,8 +592,11 @@ namespace {
}
// Flairs (small, and drawn around the camera) and small models first; big ones behind the rest
WorkerPool::ePriority PriorityOf(const ZoneScenery& zone, const std::string& path, const std::filesystem::path& file) {
if (zone.flairModels.contains(path)) return WorkerPool::ePriority::URGENT;
WorkerPool::ePriority PriorityOf(const ZoneScenery* zone, const std::string& path, const std::filesystem::path& file) {
if (zone) {
std::lock_guard lock(g_ZoneMutex);
if (zone->flairModels.contains(path)) return WorkerPool::ePriority::URGENT;
}
std::error_code ec;
const auto size = std::filesystem::file_size(file, ec);
if (ec || size <= SMALL_MODEL_BYTES) return WorkerPool::ePriority::URGENT;
@@ -563,11 +608,10 @@ namespace {
/**
* Convert models of a zone ahead of time (onto the disk cache), at the LOD its viewer last asked for, while
* someone views it. The lowest priority: only when nothing else waits. Smallest first; `front` puts these before
* the zone's other queued ones (the flairs). Web thread.
* the zone's other queued ones (the flairs). Any thread.
*/
void WarmUp(uint32_t zoneId, const std::vector<std::string>& paths, bool front) {
if (!g_Pool.Running() || paths.empty()) return;
Files(); // built here: workers only read it
if (!Pool().Running() || paths.empty()) return;
struct Item {
std::string path;
std::filesystem::path file;
@@ -576,9 +620,10 @@ namespace {
std::vector<Item> items;
std::set<std::string> seen;
std::error_code ec;
const auto res = ClientAssets::ResFolder();
for (const auto& path : paths) {
if (!seen.insert(path).second) continue;
const auto file = ClientAssets::ResolveResFile(path);
const auto file = ClientAssets::ResolveResFile(path, res);
if (file) items.push_back({ path, *file, std::filesystem::file_size(*file, ec) });
}
std::stable_sort(items.begin(), items.end(), [](const Item& a, const Item& b) { return a.size < b.size; });
@@ -587,10 +632,10 @@ namespace {
// Queued at the front in reverse, so they still run smallest first
if (front) std::reverse(items.begin(), items.end());
for (auto& item : items) {
g_Pool.Submit(WorkerPool::ePriority::BACKGROUND, [zoneId, group, path = std::move(item.path), file = std::move(item.file)] {
Pool().Submit(WorkerPool::ePriority::BACKGROUND, [zoneId, group, path = std::move(item.path), file = std::move(item.file)] {
const auto activity = Viewed(zoneId);
if (!activity || g_Disk.Total() >= WARM_DISK_BYTES) {
g_Pool.Cancel(group);
Pool().Cancel(group);
return;
}
const auto key = ModelKey(path, activity->lod);
@@ -601,15 +646,26 @@ namespace {
}
}
// Warm the zone's models when its manifest is asked for (again after nobody viewed it for a while)
/**
* Warm the zone's models when its manifest is asked for (again after nobody viewed it for a while). Planning it
* (finding the files) is left to a worker, so a manifest in memory is still answered at once.
*/
void WarmScenery(uint32_t zoneId, ZoneScenery& zone, bool flairs) {
if (!Viewed(zoneId)) zone.warmedScenery = zone.warmedFlairs = false;
std::vector<std::string> paths;
{
const bool idle = !Viewed(zoneId);
std::lock_guard lock(g_ZoneMutex);
if (idle) zone.warmedScenery = zone.warmedFlairs = false;
auto& warmed = flairs ? zone.warmedFlairs : zone.warmedScenery;
if (!warmed) {
warmed = true;
if (flairs) paths.assign(zone.flairModels.begin(), zone.flairModels.end());
else paths = zone.assets;
}
}
Touch(zoneId);
auto& warmed = flairs ? zone.warmedFlairs : zone.warmedScenery;
if (warmed) return;
warmed = true;
if (flairs) WarmUp(zoneId, { zone.flairModels.begin(), zone.flairModels.end() }, true);
else WarmUp(zoneId, zone.assets, false);
if (paths.empty() || !Pool().Running()) return;
Pool().Submit(WorkerPool::ePriority::NORMAL, [zoneId, paths = std::move(paths), flairs] { WarmUp(zoneId, paths, flairs); });
}
// The texture `name` of model `path` (textures[slot] of its encoded form) as a DDS file. Any thread.
@@ -636,87 +692,135 @@ namespace {
return dds;
}
// The model path of `asset` in the zone's manifests, or a 404 reply
const std::string* AssetPath(HTTPReply& reply, uint32_t zoneId, uint32_t asset) {
auto& zone = Zone(zoneId);
// The flairs' models join the list when their manifest is first built (a browser may still have it cached)
if (zone && asset >= zone->assets.size()) Flairs(zoneId, *zone);
if (!zone || asset >= zone->assets.size()) {
JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No such model in this zone");
return nullptr;
// The model path of `asset` in the zone's manifests, building what's missing (any thread); nullopt: no such model
std::optional<std::string> AssetPath(uint32_t zoneId, uint32_t asset) {
const auto zone = Zone(zoneId);
if (!zone) return std::nullopt;
{
std::lock_guard lock(g_ZoneMutex);
if (asset < zone->assets.size()) return zone->assets[asset];
}
return &zone->assets[asset];
// The flairs' models join the list when their manifest is first built (a browser may still have it cached)
Flairs(zoneId, *zone);
std::lock_guard lock(g_ZoneMutex);
if (asset < zone->assets.size()) return zone->assets[asset];
return std::nullopt;
}
// The same without building anything (never waits): nullopt when it isn't known yet
std::optional<std::string> AssetPathIfBuilt(uint32_t zoneId, uint32_t asset) {
const auto zone = ZoneIfBuilt(zoneId);
if (!zone) return std::nullopt;
std::lock_guard lock(g_ZoneMutex);
if (asset < zone->assets.size()) return zone->assets[asset];
return std::nullopt;
}
/**
* Where model `asset` is, and how urgent converting it is, when that is known without building anything: for
* the web thread, which only hands the work on
*/
struct Known {
std::optional<std::string> path;
std::optional<std::filesystem::path> file;
WorkerPool::ePriority priority{ WorkerPool::ePriority::NORMAL };
};
Known KnownAsset(uint32_t zoneId, uint32_t asset) {
Known known;
known.path = AssetPathIfBuilt(zoneId, asset);
if (!known.path) return known;
known.file = ClientAssets::ResolveResFile(*known.path, ClientAssets::ResFolder());
if (known.file) known.priority = PriorityOf(ZoneIfBuilt(zoneId).get(), *known.path, *known.file);
return known;
}
}
namespace Scenery {
void Preload() {
Files();
RenderInfos();
FlairModels();
}
std::optional<std::string> ZoneJson(uint32_t zoneId) {
auto& zone = Zone(zoneId);
const auto zone = Zone(zoneId);
if (!zone) return std::nullopt;
WarmScenery(zoneId, *zone, false);
return zone->json;
}
bool ZoneReady(uint32_t zoneId) {
return g_Zones.Ready(zoneId);
}
bool HasModel(const WorldScene::Object& object) {
const auto model = ModelFor(object);
return !model.path.empty() && !model.hidden;
}
std::optional<std::string> FlairsJson(uint32_t zoneId) {
auto& zone = Zone(zoneId);
const auto zone = Zone(zoneId);
if (!zone) return std::nullopt;
const auto& flairs = Flairs(zoneId, *zone);
WarmScenery(zoneId, *zone, true);
return flairs;
}
bool FlairsReady(uint32_t zoneId) {
return g_Zones.Ready(zoneId) && g_Flairs.Ready(zoneId);
}
void ReplyMesh(HTTPReply& reply, const HTTPContext& context, uint32_t zoneId, uint32_t asset, uint32_t lod) {
const auto* found = AssetPath(reply, zoneId, asset);
if (!found) return;
const auto path = *found;
lod = std::min(lod, MAX_LOD);
Touch(zoneId, lod);
if (const auto cached = g_Models.Get(ModelKey(path, lod))) return Binary(reply, *cached);
const auto file = ClientAssets::ResolveResFile(path);
if (!file) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "Could not read this model");
Files(); // built here: workers only read it
auto known = KnownAsset(zoneId, asset);
if (known.path) {
if (const auto cached = g_Models.Get(ModelKey(*known.path, lod))) return Binary(reply, *cached);
}
const auto res = ClientAssets::ResFolder();
const auto deferred = Web::Defer(reply, context);
g_Pool.Submit(PriorityOf(*Zone(zoneId), path, *file), [deferred, path, lod, file = *file] {
Pool().Submit(known.priority, [deferred, zoneId, asset, lod, res, known = std::move(known)] {
if (deferred.Cancelled()) return;
HTTPReply out;
const auto encoded = Encoded(path, lod, file);
if (encoded) Binary(out, *encoded);
else JsonError(out, eHTTPStatusCode::NOT_FOUND, "Could not read this model");
const auto path = known.path ? known.path : AssetPath(zoneId, asset);
const auto file = known.file ? known.file : path ? ClientAssets::ResolveResFile(*path, res) : std::nullopt;
if (!path) {
JsonError(out, eHTTPStatusCode::NOT_FOUND, "No such model in this zone");
} else if (const auto encoded = file ? Encoded(*path, lod, *file) : nullptr) {
Binary(out, *encoded);
} else {
JsonError(out, eHTTPStatusCode::NOT_FOUND, "Could not read this model");
}
deferred.Send(std::move(out));
});
}
void ReplyTexture(HTTPReply& reply, const HTTPContext& context, uint32_t zoneId, uint32_t asset, uint32_t slot, uint32_t lod) {
const auto* found = AssetPath(reply, zoneId, asset);
if (!found) return;
const auto path = *found;
lod = std::min(lod, MAX_LOD);
Touch(zoneId, lod);
const auto file = ClientAssets::ResolveResFile(path);
if (!file) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No such texture");
Files();
auto known = KnownAsset(zoneId, asset);
// Quick when the model is converted already and the texture is a file of its own, or one kept from its model
auto priority = WorkerPool::ePriority::URGENT;
if (const auto cached = g_Models.Get(ModelKey(path, lod))) {
const auto textures = TexturesOf(*cached);
if (slot < textures.size() && textures[slot].starts_with('#') && !g_Embedded.Get(path + textures[slot])) priority = PriorityOf(*Zone(zoneId), path, *file);
} else {
priority = PriorityOf(*Zone(zoneId), path, *file);
if (known.path) {
if (const auto cached = g_Models.Get(ModelKey(*known.path, lod))) {
const auto textures = TexturesOf(*cached);
if (slot >= textures.size() || !textures[slot].starts_with('#') || g_Embedded.Get(*known.path + textures[slot])) known.priority = WorkerPool::ePriority::URGENT;
}
}
const auto res = ClientAssets::ResFolder();
const auto deferred = Web::Defer(reply, context);
g_Pool.Submit(priority, [deferred, path, lod, slot, file = *file, res = ClientAssets::ResFolder()] {
Pool().Submit(known.priority, [deferred, zoneId, asset, lod, slot, res, known = std::move(known)] {
if (deferred.Cancelled()) return;
HTTPReply out;
const auto encoded = Encoded(path, lod, file);
const auto path = known.path ? known.path : AssetPath(zoneId, asset);
const auto file = known.file ? known.file : path ? ClientAssets::ResolveResFile(*path, res) : std::nullopt;
const auto encoded = path && file ? Encoded(*path, lod, *file) : nullptr;
const auto textures = encoded ? TexturesOf(*encoded) : std::vector<std::string>{};
if (slot >= textures.size() || textures[slot].empty()) {
if (!path) {
JsonError(out, eHTTPStatusCode::NOT_FOUND, "No such model in this zone");
} else if (slot >= textures.size() || textures[slot].empty()) {
JsonError(out, eHTTPStatusCode::NOT_FOUND, "No such texture");
} else if (auto dds = TextureBytes(path, file, textures, textures[slot], res)) {
} else if (auto dds = TextureBytes(*path, *file, textures, textures[slot], res)) {
Binary(out, std::move(*dds));
} else {
JsonError(out, eHTTPStatusCode::NOT_FOUND, "Could not read this texture");
@@ -725,18 +829,7 @@ namespace Scenery {
});
}
void Shutdown() {
g_Pool.Stop();
}
void RegisterRoutes() {
auto threads = Game::config ? Game::config->GetValue<uint32_t>("scenery_workers", 0) : 0;
if (threads == 0) threads = static_cast<uint32_t>(WorkerPool::DefaultThreads(std::thread::hardware_concurrency()));
threads = std::clamp<uint32_t>(threads, 2, 16);
// Converting ahead of time never takes more than half of the threads besides the fast lane
g_Pool.Start(threads, std::max<size_t>(1, (threads - 1) / 2));
LOG("Converting scenery models with %u threads", threads);
Route(eHTTPMethod::GET, "/api/scenery/:zone/mesh/:asset", 0,
"Model `asset` of a zone's scenery (see the scenery routes of properties and /world3d), converted from the client's .nif. Query: ?lod=0 (most detailed) to 3",
[](HTTPReply& reply, const HTTPContext& context) {

21
dDashboardServer/routes/Scenery.h
View File

@@ -21,13 +21,20 @@ namespace WorldScene {
* bounded in-memory cache; textures are sent as the client's own DDS files (textures stored inside a .nif are
* wrapped as DDS), which the browser decodes itself, so nothing needs ImageMagick.
*
* Everything here is thread safe: the manifests are built on worker threads too (the routes use Workers::Reply).
* Converting a big model takes a while, so models and textures not in memory are answered from a few worker threads
* (Web::Defer, WorkerPool; scenery_workers in dashboardconfig.ini) and the web server keeps answering meanwhile.
* (Web::Defer, Workers; scenery_workers in dashboardconfig.ini) and the web server keeps answering meanwhile.
* Flairs and small models go first, in a lane of their own; the same model asked for twice at once is converted
* once. When a zone's manifest is asked for, its models are converted ahead of time onto the disk cache while
* someone views the zone.
*/
namespace Scenery {
/**
* Read what the builders need from the CDClient (render components, flairs) and index the client's model files,
* at startup: the manifests are built on worker threads, which never query the CDClient.
*/
void Preload();
/**
* The zone's manifest as JSON: {zone, sky (asset index or -1), assets: [res path, ...],
* objects: {asset: [...], pos: [x, y, z, ...], rot: [x, y, z, w, ...], scale: [...], hidden: [0 or 1, ...]}}, where
@@ -36,6 +43,9 @@ namespace Scenery {
*/
std::optional<std::string> ZoneJson(uint32_t zoneId);
// Whether ZoneJson is built (so answering it is quick)
bool ZoneReady(uint32_t zoneId);
/**
* The zone's flairs (the grass, flowers and small rocks its terrain file strews over it, models from FlairTable) as a
* manifest in ZoneJson's form, whose models the same mesh and texture routes serve, plus: distance (how far from the
@@ -44,6 +54,9 @@ namespace Scenery {
*/
std::optional<std::string> FlairsJson(uint32_t zoneId);
// Whether FlairsJson is built
bool FlairsReady(uint32_t zoneId);
// Whether the client draws a model for this scene object (its render component's, or its nif_name)
bool HasModel(const WorldScene::Object& object);
@@ -54,10 +67,8 @@ namespace Scenery {
// Reply with texture `slot` (the model's "textures" list at that `lod`) of model `asset`, as a DDS file. Deferred.
void ReplyTexture(HTTPReply& reply, const HTTPContext& context, uint32_t zoneId, uint32_t asset, uint32_t slot, uint32_t lod);
// /api/scenery/:zone/mesh/:asset and /api/scenery/:zone/texture/:asset/:slot, for anyone signed in; starts the
// conversion threads
// /api/scenery/:zone/mesh/:asset and /api/scenery/:zone/texture/:asset/:slot, for anyone signed in
void RegisterRoutes();
// Stop the conversion threads (queued work is dropped)
void Shutdown();
}

28
dDashboardServer/routes/Showcase.cpp
View File

@@ -1,4 +1,5 @@
#include "Showcase.h"
#include "Workers.h"
#include <chrono>
#include <map>
@@ -164,16 +165,21 @@ namespace {
if (!cacheControl.empty()) reply.headers.push_back("Cache-Control: " + cacheControl);
}
// Terrain and the build area come from the client's files and are the same for every property in a zone
/**
* Terrain and the build area come from the client's files and are the same for every property in a zone.
* `ready` says whether what `fetch` needs is built; when it isn't, fetch runs on a worker thread (Workers::Reply),
* so it must be thread safe. The property is looked up here, on the web thread.
*/
template<typename Fetch>
void ZoneFileRoute(const std::string& path, const std::string& description, Fetch fetch) {
Route(eHTTPMethod::GET, path, PUBLIC, description, [fetch](HTTPReply& reply, const HTTPContext& context) {
void ZoneFileRoute(const std::string& path, const std::string& description, Fetch fetch, std::function<bool(uint32_t zone)> ready = nullptr) {
Route(eHTTPMethod::GET, path, PUBLIC, description, [fetch, ready](HTTPReply& reply, const HTTPContext& context) {
if (!Allowed(context, reply, Kind::DATA)) return;
const auto id = PathId<LWOOBJID>(context.path, 2);
if (!id) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid ID");
const auto info = ShownProperty(*id, reply);
if (!info) return;
fetch(reply, *info);
if (!ready) return fetch(reply, *info);
Workers::Reply(reply, context, ready(info->zoneId), [fetch, info = *info](HTTPReply& out) { fetch(out, info); });
});
}
}
@@ -251,14 +257,14 @@ void RegisterShowcaseRoutes() {
const auto terrain = ZoneTerrainChunksJson(info.zoneId);
if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
RawReply(reply, eContentType::APPLICATION_JSON, *terrain, "private, max-age=86400");
});
}, ZoneTerrainChunksReady);
ZoneFileRoute("/api/showcase/:id/terrain", "A height grid of a showcased property's zone (needs client_location)",
[](HTTPReply& reply, const IProperty::Info& info) {
const auto terrain = ZoneTerrainJson(info.zoneId);
if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
RawReply(reply, eContentType::APPLICATION_JSON, *terrain, "private, max-age=86400");
});
}, ZoneTerrainJsonReady);
ZoneFileRoute("/api/showcase/:id/boundary", "Where a showcased property's owner may build (needs client_location)",
[](HTTPReply& reply, const IProperty::Info& info) {
@@ -311,9 +317,11 @@ void RegisterShowcaseRoutes() {
if (!Allowed(context, reply, Kind::ASSET)) return;
const auto textureId = PathId<uint32_t>(context.path, 3);
if (!textureId) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid ID");
const auto png = TerrainTextureFile(*textureId);
if (!png) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "Texture not found");
RawReply(reply, eContentType::IMAGE_PNG, *png, "public, max-age=604800");
Workers::Reply(reply, context, TerrainTextureReady(*textureId), [id = *textureId](HTTPReply& out) {
const auto png = TerrainTextureFile(id);
if (!png) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "Texture not found");
RawReply(out, eContentType::IMAGE_PNG, *png, "public, max-age=604800");
});
});
ZoneFileRoute("/api/showcase/:id/scenery", "Everything the game draws around a showcased property: its zone's scene objects with their models and the sky (needs client_location)",
@@ -321,7 +329,7 @@ void RegisterShowcaseRoutes() {
const auto scenery = Scenery::ZoneJson(info.zoneId);
if (!scenery) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No scenery for this zone");
RawReply(reply, eContentType::APPLICATION_JSON, *scenery, "private, max-age=86400");
});
}, Scenery::ZoneReady);
// Scenery models and textures of the zones properties are in (the only ones the showcase shows)
const auto propertyZone = [](HTTPReply& reply, const HTTPContext& context) -> std::optional<uint32_t> {

53
dDashboardServer/routes/Workers.cpp Normal file
View File

@@ -0,0 +1,53 @@
#include "Workers.h"
#include <algorithm>
#include <exception>
#include <thread>
#include "RouteUtils.h"
#include "Web.h"
#include "Game.h"
#include "Logger.h"
#include "dConfig.h"
namespace {
WorkerPool g_Pool;
}
namespace Workers {
void Start() {
auto threads = Game::config ? Game::config->GetValue<uint32_t>("scenery_workers", 0) : 0;
if (threads == 0) threads = static_cast<uint32_t>(WorkerPool::DefaultThreads(std::thread::hardware_concurrency()));
threads = std::clamp<uint32_t>(threads, 2, 16);
// Work ahead of time (converting a zone's models) never takes more than half of the threads besides the fast lane
g_Pool.Start(threads, std::max<size_t>(1, (threads - 1) / 2));
LOG("Dashboard worker threads: %u", threads);
}
void Stop() {
g_Pool.Stop();
}
WorkerPool& Pool() {
return g_Pool;
}
void Reply(HTTPReply& reply, const HTTPContext& context, bool ready, std::function<void(HTTPReply&)> fill, WorkerPool::ePriority priority) {
if (ready || !g_Pool.Running()) return fill(reply);
const auto deferred = Web::Defer(reply, context);
const auto path = context.path;
g_Pool.Submit(priority, [deferred, path, fill = std::move(fill)] {
if (deferred.Cancelled()) return;
HTTPReply out;
try {
fill(out);
} catch (const std::exception& ex) {
LOG("Error handling GET %s: %s", path.c_str(), ex.what());
out = HTTPReply{};
RouteUtils::JsonError(out, eHTTPStatusCode::INTERNAL_SERVER_ERROR, "Internal server error");
}
deferred.Send(std::move(out));
});
}
}

32
dDashboardServer/routes/Workers.h Normal file
View File

@@ -0,0 +1,32 @@
#pragma once
#include <functional>
#include "WorkerPool.h"
struct HTTPReply;
struct HTTPContext;
/**
* The dashboard's worker threads (one WorkerPool, scenery_workers threads) for slow work routes would otherwise do
* on the web server's one thread: converting the client's models, building a zone's terrain and scene data the
* first time it is viewed, converting textures with ImageMagick. What they run must be thread safe: the zone data
* builders keep their results in OnceCaches and read the client's files through ClientAssets.
*/
namespace Workers {
// Start the threads (scenery_workers: 0 picks WorkerPool::DefaultThreads)
void Start();
// Stop them; queued work is dropped (answer deferred requests before Web::Shutdown)
void Stop();
WorkerPool& Pool();
/**
* Answer a request with what `fill` writes to the reply: right away when `ready` (what it needs is built, so it's
* quick), else from a worker thread (Web::Defer). `fill` runs on either thread, so it must be thread safe and must
* not keep a reference to the request.
*/
void Reply(HTTPReply& reply, const HTTPContext& context, bool ready, std::function<void(HTTPReply&)> fill,
WorkerPool::ePriority priority = WorkerPool::ePriority::NORMAL);
}

140
dDashboardServer/routes/WorldView.cpp
View File

@@ -11,6 +11,8 @@
#include "Permissions.h"
#include "ServerState.h"
#include "Background.h"
#include "OnceCache.h"
#include "Workers.h"
#include <chrono>
#include <cmath>
@@ -83,20 +85,13 @@ namespace {
}
std::optional<std::string> LuzPath(uint32_t zone) {
auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT zoneName FROM ZoneTable WHERE zoneID = ? LIMIT 1;");
stmt.bind(1, static_cast<int>(zone));
auto result = stmt.execQuery();
if (result.eof()) return std::nullopt;
std::string path = result.getStringField(0, "");
if (!path.ends_with(".luz")) return std::nullopt;
return path;
return ZoneLuzPath(zone);
}
// LOT -> index into WorldScene::KINDS for every LOT with one of those components, read once
const std::map<uint32_t, uint8_t>& LotKinds() {
static std::optional<std::map<uint32_t, uint8_t>> kinds;
if (kinds) return *kinds;
kinds.emplace();
// LOT -> index into WorldScene::KINDS for every LOT with one of those components, read at startup (Preload): the
// zone data is built on worker threads, which never query the CDClient
std::map<uint32_t, uint8_t> ReadLotKinds() {
std::map<uint32_t, uint8_t> kinds;
std::map<uint32_t, std::vector<uint32_t>> components;
try {
auto result = CDClientDatabase::ExecuteQuery("SELECT id, component_type FROM ComponentsRegistry;");
@@ -106,9 +101,14 @@ namespace {
}
for (const auto& [lot, types] : components) {
const auto kind = WorldScene::Classify(types);
if (kind < WorldScene::KINDS.size()) (*kinds)[lot] = static_cast<uint8_t>(kind);
if (kind < WorldScene::KINDS.size()) kinds[lot] = static_cast<uint8_t>(kind);
}
return *kinds;
return kinds;
}
const std::map<uint32_t, uint8_t>& LotKinds() {
static const auto kinds = ReadLotKinds();
return kinds;
}
double Round(float value) { return std::round(static_cast<double>(value) * 100.0) / 100.0; }
@@ -118,15 +118,12 @@ namespace {
* its .luz (the scene list) and its scene files. Objects are columns (lot, kind, flags, scene, x/y/z) to keep big zones
* small: flags 1 = spawner, 2 = client only, 4 = drawn with a model by the scenery layer.
*/
std::optional<std::string> SceneJson(uint32_t zone) {
static std::map<uint32_t, std::optional<std::string>> cache;
if (const auto it = cache.find(zone); it != cache.end()) return it->second;
std::optional<std::string> BuildSceneJson(uint32_t zone) {
const auto luzPath = LuzPath(zone);
const auto luz = luzPath ? ClientAssets::ReadResFile("maps/" + *luzPath) : std::nullopt;
std::string error;
const auto zoneFile = luz ? ZonePaths::ReadHeader(*luz, error) : std::nullopt;
if (!zoneFile) return cache[zone] = std::nullopt;
if (!zoneFile) return std::nullopt;
const auto& lotKinds = LotKinds();
const auto other = static_cast<uint8_t>(WorldScene::KINDS.size());
@@ -167,19 +164,24 @@ namespace {
{"objects", { {"lot", lots}, {"kind", kindOf}, {"flags", flags}, {"scene", sceneOf}, {"pos", positions}, {"names", names} }}, {"lotNames", lotNames},
{"spawnPoints", ZoneSpawnPointsJson(zone).value_or(nlohmann::json::array())}
};
return cache[zone] = json.dump();
return json.dump();
}
OnceCache<uint32_t, std::optional<std::string>> g_Scenes;
OnceCache<uint32_t, std::optional<std::string>> g_Paths;
// Built on a worker thread the first time (any thread)
const std::optional<std::string>& SceneJson(uint32_t zone) {
return g_Scenes.Get(zone, [zone] { return BuildSceneJson(zone); });
}
// A zone's paths, which need the whole .luz (they are most of it): loaded when a path layer is first shown
std::optional<std::string> PathsJson(uint32_t zone) {
static std::map<uint32_t, std::optional<std::string>> cache;
if (const auto it = cache.find(zone); it != cache.end()) return it->second;
std::optional<std::string> BuildPathsJson(uint32_t zone) {
const auto luzPath = LuzPath(zone);
const auto luz = luzPath ? ClientAssets::ReadResFile("maps/" + *luzPath) : std::nullopt;
std::string error;
const auto zoneFile = luz ? ZonePaths::Read(*luz, error) : std::nullopt;
if (!zoneFile) return cache[zone] = std::nullopt;
if (!zoneFile) return std::nullopt;
nlohmann::json pathTypes = nlohmann::json::array();
for (const auto type : magic_enum::enum_values<PathType>()) pathTypes.push_back({ {"value", static_cast<int>(type)}, {"name", GameLabels::Name(type)} });
@@ -194,7 +196,11 @@ namespace {
if (points.empty()) continue;
paths.push_back({ {"name", path.pathName}, {"type", static_cast<int>(path.pathType)}, {"loop", path.pathBehavior == PathBehavior::Loop}, {"points", points} });
}
return cache[zone] = nlohmann::json{ {"zone", zone}, {"pathTypes", pathTypes}, {"paths", paths} }.dump();
return nlohmann::json{ {"zone", zone}, {"pathTypes", pathTypes}, {"paths", paths} }.dump();
}
const std::optional<std::string>& PathsJson(uint32_t zone) {
return g_Paths.Get(zone, [zone] { return BuildPathsJson(zone); });
}
std::string CharacterName(LWOOBJID id) {
@@ -211,22 +217,28 @@ namespace {
reply.headers.push_back(std::string("Cache-Control: ") + cache);
}
using Deflated = OnceCache<uint32_t, std::optional<std::string>>;
bool TakesDeflate(const HTTPContext& context) {
return context.GetHeader("Accept-Encoding").find("deflate") != std::string::npos;
}
/**
* A big cached JSON body, deflated once per zone when the browser takes it: a zone's terrain chunks are mostly
* base64 maps that shrink about tenfold (Avant Gardens: 19 MB to 1.4 MB).
* base64 maps that shrink about tenfold (Avant Gardens: 19 MB to 1.4 MB). Any thread.
*/
void DeflatedJson(HTTPReply& reply, const HTTPContext& context, std::map<uint32_t, std::string>& deflated, uint32_t zone, const std::string& json, const char* cache) {
if (context.GetHeader("Accept-Encoding").find("deflate") == std::string::npos || json.size() > UINT32_MAX / 2) return RawJson(reply, json, cache);
auto it = deflated.find(zone);
if (it == deflated.end()) {
void DeflatedJson(HTTPReply& reply, bool deflate, Deflated& deflated, uint32_t zone, const std::string& json, const char* cache) {
if (!deflate || json.size() > UINT32_MAX / 2) return RawJson(reply, json, cache);
const auto& out = deflated.Get(zone, [&json]() -> std::optional<std::string> {
std::string out(ZCompression::GetMaxCompressedLength(static_cast<uint32_t>(json.size())), '\0');
const auto size = ZCompression::Compress(reinterpret_cast<const uint8_t*>(json.data()), static_cast<uint32_t>(json.size()),
reinterpret_cast<uint8_t*>(out.data()), static_cast<uint32_t>(out.size()));
if (size <= 0) return RawJson(reply, json, cache);
if (size <= 0) return std::nullopt;
out.resize(static_cast<size_t>(size));
it = deflated.emplace(zone, std::move(out)).first;
}
RawJson(reply, it->second, cache);
return out;
});
if (!out) return RawJson(reply, json, cache);
RawJson(reply, *out, cache);
reply.headers.push_back("Content-Encoding: deflate");
reply.headers.push_back("Vary: Accept-Encoding");
}
@@ -237,6 +249,10 @@ namespace {
}
namespace WorldView {
void Preload() {
LotKinds();
}
void RecordPositions(const PlayerPositions& positions) {
if (Setting("position_history", 1) == 0) {
g_Buffer.clear();
@@ -286,9 +302,11 @@ namespace WorldView {
[](HTTPReply& reply, const HTTPContext& context) {
const auto zone = PathId<uint32_t>(context.path, 2);
if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone");
const auto json = SceneJson(*zone);
if (!json) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)");
RawJson(reply, *json, "private, max-age=3600");
Workers::Reply(reply, context, g_Scenes.Ready(*zone), [zone = *zone](HTTPReply& out) {
const auto& json = SceneJson(zone);
if (!json) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)");
RawJson(out, *json, "private, max-age=3600");
});
});
Route(eHTTPMethod::GET, "/api/world3d/:zone/paths", Perm("players_view"),
@@ -296,9 +314,11 @@ namespace WorldView {
[](HTTPReply& reply, const HTTPContext& context) {
const auto zone = PathId<uint32_t>(context.path, 2);
if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone");
const auto json = PathsJson(*zone);
if (!json) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)");
RawJson(reply, *json, "private, max-age=3600");
Workers::Reply(reply, context, g_Paths.Ready(*zone), [zone = *zone](HTTPReply& out) {
const auto& json = PathsJson(zone);
if (!json) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)");
RawJson(out, *json, "private, max-age=3600");
});
});
Route(eHTTPMethod::GET, "/api/world3d/:zone/terrain_layers", Perm("players_view"),
@@ -306,10 +326,13 @@ namespace WorldView {
[](HTTPReply& reply, const HTTPContext& context) {
const auto zone = PathId<uint32_t>(context.path, 2);
if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone");
const auto layers = ZoneTerrainLayersJson(*zone);
if (!layers) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
static std::map<uint32_t, std::string> deflated;
DeflatedJson(reply, context, deflated, *zone, *layers, "private, max-age=86400");
static Deflated deflated;
const auto deflate = TakesDeflate(context);
Workers::Reply(reply, context, ZoneTerrainLayersReady(*zone) && (!deflate || deflated.Ready(*zone)), [zone = *zone, deflate](HTTPReply& out) {
const auto layers = ZoneTerrainLayersJson(zone);
if (!layers) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
DeflatedJson(out, deflate, deflated, zone, *layers, "private, max-age=86400");
});
});
Route(eHTTPMethod::GET, "/api/world3d/:zone/flairs", Perm("players_view"),
@@ -317,9 +340,11 @@ namespace WorldView {
[](HTTPReply& reply, const HTTPContext& context) {
const auto zone = PathId<uint32_t>(context.path, 2);
if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone");
const auto flairs = Scenery::FlairsJson(*zone);
if (!flairs) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
RawJson(reply, *flairs, "private, max-age=3600");
Workers::Reply(reply, context, Scenery::FlairsReady(*zone), [zone = *zone](HTTPReply& out) {
const auto flairs = Scenery::FlairsJson(zone);
if (!flairs) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
RawJson(out, *flairs, "private, max-age=3600");
});
});
Route(eHTTPMethod::GET, "/api/world3d/:zone/terrain_chunks", Perm("players_view"),
@@ -327,10 +352,13 @@ namespace WorldView {
[](HTTPReply& reply, const HTTPContext& context) {
const auto zone = PathId<uint32_t>(context.path, 2);
if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone");
const auto terrain = ZoneTerrainChunksJson(*zone);
if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
static std::map<uint32_t, std::string> deflated;
DeflatedJson(reply, context, deflated, *zone, *terrain, "private, max-age=86400");
static Deflated deflated;
const auto deflate = TakesDeflate(context);
Workers::Reply(reply, context, ZoneTerrainChunksReady(*zone) && (!deflate || deflated.Ready(*zone)), [zone = *zone, deflate](HTTPReply& out) {
const auto terrain = ZoneTerrainChunksJson(zone);
if (!terrain) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
DeflatedJson(out, deflate, deflated, zone, *terrain, "private, max-age=86400");
});
});
Route(eHTTPMethod::GET, "/api/world3d/:zone/scenery", Perm("players_view"),
@@ -338,9 +366,11 @@ namespace WorldView {
[](HTTPReply& reply, const HTTPContext& context) {
const auto zone = PathId<uint32_t>(context.path, 2);
if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone");
const auto scenery = Scenery::ZoneJson(*zone);
if (!scenery) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)");
RawJson(reply, *scenery, "private, max-age=3600");
Workers::Reply(reply, context, Scenery::ZoneReady(*zone), [zone = *zone](HTTPReply& out) {
const auto scenery = Scenery::ZoneJson(zone);
if (!scenery) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)");
RawJson(out, *scenery, "private, max-age=3600");
});
});
Route(eHTTPMethod::GET, "/api/world3d/history/instances", Perm("players_history"),

3
dDashboardServer/routes/WorldView.h
View File

@@ -10,6 +10,9 @@ struct PlayerPositions;
namespace WorldView {
void RegisterRoutes();
// Read the CDClient data the zone data builders need (at startup: they run on worker threads)
void Preload();
// A world server reported its players' positions: keep some of them for replays (throttled, written in batches)
void RecordPositions(const PlayerPositions& positions);
}