diff --git a/dCommon/Logger.cpp b/dCommon/Logger.cpp index 0f31f5419..e3b9838c5 100644 --- a/dCommon/Logger.cpp +++ b/dCommon/Logger.cpp @@ -3,8 +3,15 @@ #include #include #include +#include #include +namespace { + // Servers with worker threads (the dashboard) log from several threads: one line is written at a time, and + // localtime's buffer is shared + std::mutex g_LogMutex; +} + Writer::~Writer() { // Flush before we close Flush(); @@ -49,6 +56,7 @@ Logger::Logger(const std::string& outpath, bool logToConsole, bool logDebugState } void Logger::vLog(const char* format, va_list args) { + std::lock_guard lock(g_LogMutex); time_t t = time(NULL); struct tm* time = localtime(&t); char timeStr[70]; @@ -78,6 +86,7 @@ void Logger::LogDebug(const char* className, const char* format, ...) { } void Logger::Flush() { + std::lock_guard lock(g_LogMutex); for (const auto& writer : m_Writers) { writer->Flush(); } diff --git a/dDashboardServer/DashboardServer.cpp b/dDashboardServer/DashboardServer.cpp index a705e434b..dff2edc31 100644 --- a/dDashboardServer/DashboardServer.cpp +++ b/dDashboardServer/DashboardServer.cpp @@ -60,6 +60,11 @@ #include "ModeratorHelper.h" #include "LiveWorld.h" #include "Scenery.h" +#include "Workers.h" +#include "ReportRoutes.h" +#include "WorldView.h" +#include "ClientAssets.h" +#include "DashboardRoutes.h" #include "LiveEventRoutes.h" #include "ChallengeRoutes.h" #include "Inspector.h" @@ -411,6 +416,19 @@ int main(int argc, char** argv) { ); Game::server = g_Server; + // What the worker threads read from the CDClient and the settings, read now on this thread: workers never query + // the CDClient, read settings or touch the network (RakNet and mongoose are the main thread's) + { + const auto start = std::chrono::steady_clock::now(); + ClientAssets::Preload(); + PreloadZoneData(); + Scenery::Preload(); + WorldView::Preload(); + ZoneNames(); + LOG("Read the client data for the 3D views in %lld ms", static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count())); + } + Workers::Start(); + // Initialize web server if (!Game::web.Startup(ourIP, ourPort)) { LOG("Failed to start web server on %s:%d", ourIP.c_str(), ourPort); @@ -570,8 +588,8 @@ int main(int argc, char** argv) { } - // Cleanup: the conversion threads first (they answer deferred requests), then the web server's connections - Scenery::Shutdown(); + // Cleanup: the worker threads first (they answer deferred requests), then the web server's connections + Workers::Stop(); Game::web.Shutdown(); Inspector::Shutdown(); EmailService::Shutdown(); diff --git a/dDashboardServer/routes/APIRoutes.cpp b/dDashboardServer/routes/APIRoutes.cpp index 90d0d98cd..836f4d913 100644 --- a/dDashboardServer/routes/APIRoutes.cpp +++ b/dDashboardServer/routes/APIRoutes.cpp @@ -13,6 +13,7 @@ #include "CharacterTools.h" #include "ClientAssets.h" #include "Scenery.h" +#include "Workers.h" #include "LiveWorld.h" #include "ServerState.h" #include "WSRoutes.h" @@ -1091,12 +1092,14 @@ namespace { if (!propId) return; const auto info = AuthorizedProperty(context, *propId, reply); if (!info) return; - const auto terrain = ZoneTerrainJson(info->zoneId); - if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone (is client_location set?)"); - reply.status = eHTTPStatusCode::OK; - reply.contentType = eContentType::APPLICATION_JSON; - reply.message = *terrain; - reply.headers.push_back("Cache-Control: private, max-age=86400"); + Workers::Reply(reply, context, ZoneTerrainJsonReady(info->zoneId), [zone = info->zoneId](HTTPReply& out) { + const auto terrain = ZoneTerrainJson(zone); + if (!terrain) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone (is client_location set?)"); + out.status = eHTTPStatusCode::OK; + out.contentType = eContentType::APPLICATION_JSON; + out.message = *terrain; + out.headers.push_back("Cache-Control: private, max-age=86400"); + }); }); Route(eHTTPMethod::GET, "/api/properties/:id/terrain_chunks", 0, @@ -1106,12 +1109,14 @@ namespace { if (!propId) return; const auto info = AuthorizedProperty(context, *propId, reply); if (!info) return; - const auto terrain = ZoneTerrainChunksJson(info->zoneId); - if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone (is client_location set?)"); - reply.status = eHTTPStatusCode::OK; - reply.contentType = eContentType::APPLICATION_JSON; - reply.message = *terrain; - reply.headers.push_back("Cache-Control: private, max-age=86400"); + Workers::Reply(reply, context, ZoneTerrainChunksReady(info->zoneId), [zone = info->zoneId](HTTPReply& out) { + const auto terrain = ZoneTerrainChunksJson(zone); + if (!terrain) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone (is client_location set?)"); + out.status = eHTTPStatusCode::OK; + out.contentType = eContentType::APPLICATION_JSON; + out.message = *terrain; + out.headers.push_back("Cache-Control: private, max-age=86400"); + }); }); Route(eHTTPMethod::GET, "/api/properties/:id/scenery", 0, @@ -1121,24 +1126,28 @@ namespace { if (!propId) return; const auto info = AuthorizedProperty(context, *propId, reply); if (!info) return; - const auto scenery = Scenery::ZoneJson(info->zoneId); - if (!scenery) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No scenery for this zone (is client_location set?)"); - reply.status = eHTTPStatusCode::OK; - reply.contentType = eContentType::APPLICATION_JSON; - reply.message = *scenery; - reply.headers.push_back("Cache-Control: private, max-age=86400"); + Workers::Reply(reply, context, Scenery::ZoneReady(info->zoneId), [zone = info->zoneId](HTTPReply& out) { + const auto scenery = Scenery::ZoneJson(zone); + if (!scenery) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No scenery for this zone (is client_location set?)"); + out.status = eHTTPStatusCode::OK; + out.contentType = eContentType::APPLICATION_JSON; + out.message = *scenery; + out.headers.push_back("Cache-Control: private, max-age=86400"); + }); }); Route(eHTTPMethod::GET, "/api/terrain_textures/:id", 0, "A terrain texture (mapTextureResource ID) as PNG, for the property 3D view", [](HTTPReply& reply, const HTTPContext& context) { const auto textureId = RequireId(context, 2, reply); if (!textureId) return; - const auto png = TerrainTextureFile(*textureId); - if (!png) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "Texture not found"); - reply.status = eHTTPStatusCode::OK; - reply.contentType = eContentType::IMAGE_PNG; - reply.message = *png; - reply.headers.push_back("Cache-Control: private, max-age=604800"); + Workers::Reply(reply, context, TerrainTextureReady(*textureId), [id = *textureId](HTTPReply& out) { + const auto png = TerrainTextureFile(id); + if (!png) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "Texture not found"); + out.status = eHTTPStatusCode::OK; + out.contentType = eContentType::IMAGE_PNG; + out.message = *png; + out.headers.push_back("Cache-Control: private, max-age=604800"); + }); }); Route(eHTTPMethod::GET, "/api/properties/:id/boundary", 0, "Where a property's owner may build: the outline from the zone file and the height limit (needs client_location)", diff --git a/dDashboardServer/routes/CMakeLists.txt b/dDashboardServer/routes/CMakeLists.txt index fd3ffe32a..57f15a916 100644 --- a/dDashboardServer/routes/CMakeLists.txt +++ b/dDashboardServer/routes/CMakeLists.txt @@ -43,6 +43,7 @@ set(DASHBOARDROUTES_SOURCES "NifFile.cpp" "Scenery.cpp" "WorkerPool.cpp" + "Workers.cpp" "AccountModeration.cpp" "ModerationTools.cpp" "ModeratorHelper.cpp" diff --git a/dDashboardServer/routes/ClientAssets.cpp b/dDashboardServer/routes/ClientAssets.cpp index 824d1eeb0..5beb1551c 100644 --- a/dDashboardServer/routes/ClientAssets.cpp +++ b/dDashboardServer/routes/ClientAssets.cpp @@ -7,6 +7,7 @@ #include #include +#include "OnceCache.h" #include "RouteUtils.h" #include "CDClientDatabase.h" #include "Game.h" @@ -26,11 +27,14 @@ namespace { std::map g_ItemCache; std::map> g_ItemSetsByLot; // lot -> set ids containing it bool g_ItemSetsLoaded = false; - std::set g_FailedConversions; + // Read once (client_location only changes with a restart), so worker threads never read the settings std::filesystem::path ClientRes() { - const auto client = Game::config->GetValue("client_location"); - return client.empty() ? std::filesystem::path{} : std::filesystem::path(client) / "res"; + static const std::filesystem::path res = [] { + const auto client = Game::config ? Game::config->GetValue("client_location") : std::string{}; + return client.empty() ? std::filesystem::path{} : std::filesystem::path(client) / "res"; + }(); + return res; } // Lowercase, forward slashes, no leading "../" or "res/" @@ -195,11 +199,8 @@ namespace { // Every object's name and displayName, read once: the CDClient has no index on Objects.id, so looking a LOT up // scans the whole table, and pages name thousands of items at once - const std::unordered_map& ObjectNameTable() { - static std::unordered_map table; - static bool loaded = false; - if (loaded) return table; - loaded = true; + std::unordered_map ReadObjectNames() { + std::unordered_map table; try { auto row = CDClientDatabase::ExecuteQuery("SELECT id, name, displayName FROM Objects;"); for (; !row.eof(); row.nextRow()) { @@ -211,6 +212,23 @@ namespace { } return table; } + + // Read at startup (ClientAssets::Preload), so worker threads only read it and never query the CDClient + const std::unordered_map& ObjectNameTable() { + static const auto table = ReadObjectNames(); + return table; + } + + // Where TextureAsPng keeps a texture it converted + std::filesystem::path PngCachePath(const std::string& normalized, uint32_t maxSize, bool opaque) { + std::string cacheName = normalized + "_" + std::to_string(maxSize) + (opaque ? "_opaque" : "") + ".png"; + std::replace(cacheName.begin(), cacheName.end(), '/', '_'); + std::replace(cacheName.begin(), cacheName.end(), ' ', '_'); + return std::filesystem::path("dDashboardServer") / "icon_cache" / cacheName; + } + + // ImageMagick conversions by cache file name: whether it worked (a failed one isn't tried again) + OnceCache g_Conversions; } namespace ClientAssets { @@ -305,28 +323,39 @@ namespace ClientAssets { if (normalized.ends_with(".png")) return ReadFile(source); if (!normalized.ends_with(".dds")) return std::nullopt; - std::string cacheName = normalized + "_" + std::to_string(maxSize) + (opaque ? "_opaque" : "") + ".png"; - std::replace(cacheName.begin(), cacheName.end(), '/', '_'); - std::replace(cacheName.begin(), cacheName.end(), ' ', '_'); - const auto cacheDir = std::filesystem::path("dDashboardServer") / "icon_cache"; - const auto target = cacheDir / cacheName; - if (!std::filesystem::exists(target)) { - // ImageMagick runs without a shell (an argument vector); odd characters are still refused as a precaution - if (g_FailedConversions.contains(cacheName) || Process::HasShellMetacharacters(source.string())) return std::nullopt; - std::filesystem::create_directories(cacheDir, ec); - const std::string size = std::to_string(maxSize) + "x" + std::to_string(maxSize) + ">"; - std::vector arguments{ "magick", source.string() }; - if (opaque) { arguments.push_back("-alpha"); arguments.push_back("off"); } - arguments.insert(arguments.end(), { "-resize", size, target.string() }); - if (Process::Run(arguments) != 0) { - LOG_DEBUG("Texture conversion failed for %s (is ImageMagick installed?)", normalized.c_str()); - g_FailedConversions.insert(cacheName); - return std::nullopt; - } + const auto target = PngCachePath(normalized, maxSize, opaque); + const auto cacheDir = target.parent_path(); + const auto cacheName = target.filename().string(); + // ImageMagick runs without a shell (an argument vector); odd characters are still refused as a precaution + if (!std::filesystem::exists(target, ec) && !Process::HasShellMetacharacters(source.string())) { + // Once per file, however many threads ask at once; written under another name and renamed, so a reader + // never sees half a file + const auto converted = g_Conversions.Get(cacheName, [&] { + if (std::filesystem::exists(target, ec)) return true; + std::filesystem::create_directories(cacheDir, ec); + const auto partial = cacheDir / (cacheName + ".part.png"); + const std::string size = std::to_string(maxSize) + "x" + std::to_string(maxSize) + ">"; + std::vector arguments{ "magick", source.string() }; + if (opaque) { arguments.push_back("-alpha"); arguments.push_back("off"); } + arguments.insert(arguments.end(), { "-resize", size, partial.string() }); + std::error_code renameError; + if (Process::Run(arguments) != 0 || (std::filesystem::rename(partial, target, renameError), renameError)) { + LOG_DEBUG("Texture conversion failed for %s (is ImageMagick installed?)", normalized.c_str()); + return false; + } + return true; + }); + if (!converted) return std::nullopt; } return ReadFile(target); } + bool TextureAsPngReady(const std::string& assetPath, uint32_t maxSize, bool opaque) { + const auto normalized = NormalizeAssetPath(assetPath); + std::error_code ec; + return normalized.ends_with(".png") || std::filesystem::exists(PngCachePath(normalized, maxSize, opaque), ec); + } + std::optional ReadResFile(const std::string& relativePath) { const auto res = ClientRes(); if (res.empty() || !IsSafeAssetPath(NormalizeAssetPath(relativePath))) return std::nullopt; @@ -342,6 +371,11 @@ namespace ClientAssets { return path; } + void Preload() { + ClientRes(); + ObjectNameTable(); + } + std::filesystem::path ResFolder() { return ClientRes(); } diff --git a/dDashboardServer/routes/ClientAssets.h b/dDashboardServer/routes/ClientAssets.h index 79b264b80..d24d4ee89 100644 --- a/dDashboardServer/routes/ClientAssets.h +++ b/dDashboardServer/routes/ClientAssets.h @@ -13,6 +13,12 @@ * client's res folder (client_location). Textures are converted to PNG with ImageMagick and cached. */ namespace ClientAssets { + /** + * Read what is read once (the res folder from client_location, the objects' names) at startup, on the main + * thread, so the dashboard's worker threads only ever read it: they must not query the CDClient or read settings. + */ + void Preload(); + std::string ItemName(LOT lot); // An object's raw Objects.name (not localized), or nullopt when the LOT isn't in the CDClient @@ -27,10 +33,14 @@ namespace ClientAssets { // Relative path inside res/ using only safe characters and no traversal bool IsSafeAssetPath(const std::string& path); - // A DDS or PNG texture from res/ as PNG, scaled down to fit maxSize. `opaque` drops the alpha channel first, for + // A DDS or PNG texture from res/ as PNG, scaled down to fit maxSize (converted once, even when asked for from + // several threads at once). `opaque` drops the alpha channel first, for // textures whose alpha isn't transparency (terrain textures have alpha 0 everywhere; kept, they'd come out black) std::optional TextureAsPng(const std::string& assetPath, uint32_t maxSize, bool opaque = false); + // Whether TextureAsPng has the texture converted already (so it's quick) + bool TextureAsPngReady(const std::string& assetPath, uint32_t maxSize, bool opaque = false); + // Raw bytes of a file inside res/ (matched ignoring case) std::optional ReadResFile(const std::string& relativePath); @@ -40,7 +50,7 @@ namespace ClientAssets { // The client's res/ folder (empty when client_location isn't set) std::filesystem::path ResFolder(); - // ResolveResFile inside a res/ folder from ResFolder: reads no settings, so worker threads may call it + // ResolveResFile inside a res/ folder from ResFolder std::optional ResolveResFile(const std::string& relativePath, const std::filesystem::path& res); // The res/-relative path of a file called `fileName` (ignoring case) anywhere under res/ diff --git a/dDashboardServer/routes/DashboardRoutes.cpp b/dDashboardServer/routes/DashboardRoutes.cpp index 7119a5325..7b8e6fbb5 100644 --- a/dDashboardServer/routes/DashboardRoutes.cpp +++ b/dDashboardServer/routes/DashboardRoutes.cpp @@ -37,24 +37,21 @@ namespace { return name.empty() ? "Zone " + std::to_string(zoneId) : name; } - nlohmann::json g_ZoneNamesCache; - bool g_ZoneNamesCached = false; - + // Built once (worker threads read it too, so never lazily twice) const nlohmann::json& GetZoneNamesJson() { - if (g_ZoneNamesCached) return g_ZoneNamesCache; - - g_ZoneNamesCache["0"] = "Character Select"; - auto keys = Locale::GetPhraseIdsWithPrefix("ZoneTable_"); - for (const auto& key : keys) { - if (key.find("_DisplayDescription") == std::string::npos) continue; - auto start = std::string("ZoneTable_").length(); - auto end = key.find("_DisplayDescription"); - auto idStr = key.substr(start, end - start); - const auto& name = Locale::GetPhrase(key); - if (!name.empty()) g_ZoneNamesCache[idStr] = name; - } - g_ZoneNamesCached = true; - return g_ZoneNamesCache; + static const nlohmann::json names = [] { + nlohmann::json names; + names["0"] = "Character Select"; + for (const auto& key : Locale::GetPhraseIdsWithPrefix("ZoneTable_")) { + if (key.find("_DisplayDescription") == std::string::npos) continue; + const auto start = std::string("ZoneTable_").length(); + const auto end = key.find("_DisplayDescription"); + const auto& name = Locale::GetPhrase(key); + if (!name.empty()) names[key.substr(start, end - start)] = name; + } + return names; + }(); + return names; } nlohmann::json ParseCharacterXml(const std::string& xml) { diff --git a/dDashboardServer/routes/OnceCache.h b/dDashboardServer/routes/OnceCache.h new file mode 100644 index 000000000..55426d627 --- /dev/null +++ b/dDashboardServer/routes/OnceCache.h @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +/** + * A value per key, built the first time it is asked for and kept: for zone data built from the client's files (a + * zone's terrain, its scene objects), which worker threads build while the web thread may ask for it too. Any thread. + * Asked for again while it is being built, the caller waits for that build instead of starting another. A build that + * throws is forgotten (the next caller tries again) and the exception reaches everyone waiting for it. + */ +template +class OnceCache { +public: + // Whether the value is built (never waits) + bool Ready(const Key& key) const { + std::lock_guard lock(m_Mutex); + const auto it = m_Entries.find(key); + return it != m_Entries.end() && it->second.wait_for(std::chrono::seconds(0)) == std::future_status::ready; + } + + /** + * The value, built with `build` when nobody has yet. The reference stays valid: values are never replaced or + * removed once built. + */ + const Value& Get(const Key& key, const std::function& build) { + std::shared_future future; + std::promise promise; + bool mine = false; + { + std::lock_guard lock(m_Mutex); + const auto it = m_Entries.find(key); + if (it != m_Entries.end()) { + future = it->second; + } else { + future = promise.get_future().share(); + m_Entries.emplace(key, future); + mine = true; + } + } + if (mine) { + try { + promise.set_value(build()); + } catch (...) { + { + std::lock_guard lock(m_Mutex); + m_Entries.erase(key); + } + promise.set_exception(std::current_exception()); + } + } + // The map keeps its own copy of the future, so what get() refers to outlives this call (unless the build threw, + // in which case get() throws) + return WaitFor(key, future); + } + + size_t Size() const { + std::lock_guard lock(m_Mutex); + return m_Entries.size(); + } + +private: + const Value& WaitFor(const Key& key, const std::shared_future& future) { + future.wait(); + std::lock_guard lock(m_Mutex); + const auto it = m_Entries.find(key); + if (it == m_Entries.end()) { + future.get(); // the build failed: rethrows its exception + std::terminate(); // unreachable: a successful build is never removed + } + return it->second.get(); + } + + mutable std::mutex m_Mutex; + std::map> m_Entries; +}; diff --git a/dDashboardServer/routes/ReportRoutes.cpp b/dDashboardServer/routes/ReportRoutes.cpp index 6f623993a..4b1e5eb66 100644 --- a/dDashboardServer/routes/ReportRoutes.cpp +++ b/dDashboardServer/routes/ReportRoutes.cpp @@ -1,4 +1,7 @@ #include "ReportRoutes.h" +#include "OnceCache.h" +#include +#include #include "ZonePaths.h" #include @@ -387,18 +390,72 @@ namespace { std::string luzPath; // relative to res/maps }; - std::optional GetZoneInfo(uint32_t zoneId) { - static std::map> cache; - if (const auto it = cache.find(zoneId); it != cache.end()) return it->second; - auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT zoneName, DisplayDescription FROM ZoneTable WHERE zoneID = ?;"); - stmt.bind(1, static_cast(zoneId)); - auto result = stmt.execQuery(); - std::optional info; - if (!result.eof()) { - const std::string description = result.getStringField("DisplayDescription", ""); - info = ZoneInfo{ description.empty() ? "Zone " + std::to_string(zoneId) : description, result.getStringField("zoneName", "") }; + // Standard base64 (plain C++, no mongoose: zone data is built on worker threads) + std::string Base64(std::string_view bytes) { + static constexpr char ALPHABET[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string out; + out.reserve((bytes.size() + 2) / 3 * 4); + size_t i = 0; + for (; i + 2 < bytes.size(); i += 3) { + const uint32_t n = (static_cast(bytes[i]) << 16) | (static_cast(bytes[i + 1]) << 8) | static_cast(bytes[i + 2]); + for (const int shift : { 18, 12, 6, 0 }) out += ALPHABET[(n >> shift) & 63]; } - return cache[zoneId] = info; + if (i < bytes.size()) { + uint32_t n = static_cast(bytes[i]) << 16; + if (i + 1 < bytes.size()) n |= static_cast(bytes[i + 1]) << 8; + out += ALPHABET[(n >> 18) & 63]; + out += ALPHABET[(n >> 12) & 63]; + out += i + 1 < bytes.size() ? ALPHABET[(n >> 6) & 63] : '='; + out += '='; + } + return out; + } + + std::map ReadZoneTable() { + std::map zones; + try { + auto result = CDClientDatabase::ExecuteQuery("SELECT zoneID, zoneName, DisplayDescription FROM ZoneTable;"); + for (; !result.eof(); result.nextRow()) { + const std::string description = result.getStringField("DisplayDescription", ""); + const auto zoneId = static_cast(result.getIntField("zoneID", 0)); + zones.try_emplace(zoneId, ZoneInfo{ description.empty() ? "Zone " + std::to_string(zoneId) : description, result.getStringField("zoneName", "") }); + } + } catch (const std::exception& ex) { + LOG("Could not read the zone table: %s", ex.what()); + } + return zones; + } + + // ZoneTable, read at startup (PreloadZoneData) so worker threads never query the CDClient + const std::map& ZoneTable() { + static const auto zones = ReadZoneTable(); + return zones; + } + + std::optional GetZoneInfo(uint32_t zoneId) { + const auto& zones = ZoneTable(); + const auto it = zones.find(zoneId); + if (it == zones.end()) return std::nullopt; + return it->second; + } + + // mapTextureResource: texture id -> file name (lowercase), read at startup + const std::map& TerrainTextureNames() { + static const auto names = [] { + std::map names; + try { + auto result = CDClientDatabase::ExecuteQuery("SELECT id, texturepath FROM mapTextureResource;"); + for (; !result.eof(); result.nextRow()) { + std::string file = result.getStringField(1, ""); + std::transform(file.begin(), file.end(), file.begin(), ::tolower); + names.try_emplace(static_cast(result.getIntField(0, 0)), file); + } + } catch (const std::exception& ex) { + LOG("Could not read the terrain textures: %s", ex.what()); + } + return names; + }(); + return names; } /** @@ -457,10 +514,9 @@ namespace { } // Terrain for a zone as JSON with base64 16-bit heights, built once per zone - std::optional TerrainJson(uint32_t zoneId) { - static std::map> cache; - if (const auto it = cache.find(zoneId); it != cache.end()) return it->second; + OnceCache> g_TerrainJson; + std::optional BuildTerrainJson(uint32_t zoneId) { std::optional json; const auto zone = GetZoneInfo(zoneId); if (zone) { @@ -469,8 +525,7 @@ namespace { const auto grid = raw ? TerrainMap::Parse(*raw, 1024) : std::nullopt; if (grid) { const auto packed = TerrainMap::Quantize(*grid); - std::string encoded(packed.size() * 4 / 3 + 8, '\0'); - encoded.resize(mg_base64_encode(reinterpret_cast(packed.data()), packed.size(), encoded.data(), encoded.size())); + const auto encoded = Base64({ reinterpret_cast(packed.data()), packed.size() * sizeof(packed[0]) }); nlohmann::json terrain{ {"zone", zoneId}, {"minX", grid->minX}, {"minZ", grid->minZ}, {"step", grid->step}, {"width", grid->width}, {"height", grid->height}, {"minY", grid->minY}, {"maxY", grid->maxY}, {"heights", encoded} @@ -479,7 +534,11 @@ namespace { json = terrain.dump(); } } - return cache[zoneId] = json; + return json; + } + + std::optional TerrainJson(uint32_t zoneId) { + return g_TerrainJson.Get(zoneId, [zoneId] { return BuildTerrainJson(zoneId); }); } std::string DayText(uint32_t day) { @@ -534,11 +593,6 @@ std::optional ZoneTerrainJson(uint32_t zoneId) { } namespace { - std::string Base64(std::string_view bytes) { - std::string encoded(bytes.size() * 4 / 3 + 8, '\0'); - encoded.resize(mg_base64_encode(reinterpret_cast(bytes.data()), bytes.size(), encoded.data(), encoded.size())); - return encoded; - } } std::optional ZoneRaw(uint32_t zoneId) { @@ -546,17 +600,70 @@ std::optional ZoneRaw(uint32_t zoneId) { return zone ? ReadZoneRaw(*zone) : std::nullopt; } +std::shared_ptr ZoneRawShared(uint32_t zoneId) { + // The last few zones: their chunks, layers and flairs are built one after the other when a zone is first viewed + constexpr size_t KEEP = 2; + static std::mutex mutex; + static std::deque>>> recent; + std::promise> promise; + std::shared_future> future; + { + std::lock_guard lock(mutex); + const auto it = std::find_if(recent.begin(), recent.end(), [zoneId](const auto& entry) { return entry.first == zoneId; }); + if (it != recent.end()) return it->second.get(); // may wait for the thread reading it + future = promise.get_future().share(); + recent.emplace_back(zoneId, future); + if (recent.size() > KEEP) recent.pop_front(); + } + std::shared_ptr raw; + try { + if (auto read = ZoneRaw(zoneId)) raw = std::make_shared(std::move(*read)); + } catch (const std::exception& ex) { + LOG("Could not read the terrain of zone %u: %s", zoneId, ex.what()); + } + promise.set_value(raw); + return raw; +} + namespace { std::string_view BytesOf(const std::vector& bytes) { return { reinterpret_cast(bytes.data()), bytes.size() }; } } +namespace { + OnceCache> g_TerrainChunks; + OnceCache> g_TerrainLayers; + OnceCache g_TerrainTexturePaths; // id -> path under res/ ("" when the client lacks it) +} + +bool ZoneTerrainJsonReady(uint32_t zoneId) { + return g_TerrainJson.Ready(zoneId); +} + +bool ZoneTerrainChunksReady(uint32_t zoneId) { + return g_TerrainChunks.Ready(zoneId); +} + +bool ZoneTerrainLayersReady(uint32_t zoneId) { + return g_TerrainLayers.Ready(zoneId); +} + +std::optional ZoneLuzPath(uint32_t zoneId) { + const auto zone = GetZoneInfo(zoneId); + if (!zone || !zone->luzPath.ends_with(".luz")) return std::nullopt; + return zone->luzPath; +} + +void PreloadZoneData() { + ZoneTable(); + TerrainTextureNames(); +} + std::optional ZoneTerrainChunksJson(uint32_t zoneId) { - static std::map> cache; - if (const auto it = cache.find(zoneId); it != cache.end()) return it->second; + return g_TerrainChunks.Get(zoneId, [zoneId]() -> std::optional { std::optional json; - if (const auto raw = ZoneRaw(zoneId)) { + if (const auto raw = ZoneRawShared(zoneId)) { nlohmann::json out{ {"zone", zoneId}, {"chunks", nlohmann::json::array()} }; std::set textures; for (const auto& chunk : raw->chunks) { @@ -575,15 +682,15 @@ std::optional ZoneTerrainChunksJson(uint32_t zoneId) { out["textures"] = textures; json = out.dump(); } - return cache[zoneId] = json; + return json; + }); } std::optional ZoneTerrainLayersJson(uint32_t zoneId) { - static std::map> cache; - if (const auto it = cache.find(zoneId); it != cache.end()) return it->second; + return g_TerrainLayers.Get(zoneId, [zoneId]() -> std::optional { std::optional json; const auto zone = GetZoneInfo(zoneId); - const auto raw = zone ? ReadZoneRaw(*zone) : std::nullopt; + const auto raw = zone ? ZoneRawShared(zoneId) : nullptr; if (raw) { // Scene names from the .luz: the terrain's scene map holds the ids of its general scenes (audio scenes share them) std::map names; @@ -615,30 +722,42 @@ std::optional ZoneTerrainLayersJson(uint32_t zoneId) { for (const auto& chunk : raw->chunks) flairs += chunk.flairs.size(); json = nlohmann::json{ {"zone", zoneId}, {"version", raw->version}, {"scenes", scenes}, {"chunks", chunks}, {"flairs", flairs} }.dump(); } - return cache[zoneId] = json; + return json; + }); +} + +namespace { + // Where terrain texture `textureId` is under res/, found once ("" when the client lacks it) + const std::string& TerrainTexturePath(uint32_t textureId) { + return g_TerrainTexturePaths.Get(textureId, [textureId] { + const auto& names = TerrainTextureNames(); + const auto name = names.find(textureId); + if (name == names.end()) return std::string{}; + if (ClientAssets::ResolveResFile("textures/env/" + name->second)) return "textures/env/" + name->second; + return ClientAssets::FindResFile("textures", name->second).value_or(""); + }); + } + + // Tiled 4 times per chunk: 512 pixels is plenty and keeps the page light + constexpr uint32_t TERRAIN_TEXTURE_SIZE = 512; } std::optional TerrainTextureFile(uint32_t textureId) { - static std::map paths; // id -> path under res/, found once - if (!paths.contains(textureId)) { - auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT texturepath FROM mapTextureResource WHERE id = ? LIMIT 1;"); - stmt.bind(1, static_cast(textureId)); - auto result = stmt.execQuery(); - if (result.eof()) return std::nullopt; - std::string file = result.getStringField(0); - std::transform(file.begin(), file.end(), file.begin(), ::tolower); - std::string found; - if (ClientAssets::ReadResFile("textures/env/" + file)) found = "textures/env/" + file; - else if (const auto listing = ClientAssets::FindResFile("textures", file)) found = *listing; - paths[textureId] = found; - } - if (paths[textureId].empty()) return std::nullopt; - // Tiled 4 times per chunk: 512 pixels is plenty and keeps the page light - return ClientAssets::TextureAsPng(paths[textureId], 512, true); + const auto& path = TerrainTexturePath(textureId); + if (path.empty()) return std::nullopt; + return ClientAssets::TextureAsPng(path, TERRAIN_TEXTURE_SIZE, true); +} + +bool TerrainTextureReady(uint32_t textureId) { + if (!g_TerrainTexturePaths.Ready(textureId)) return false; + const auto& path = TerrainTexturePath(textureId); + return path.empty() || ClientAssets::TextureAsPngReady(path, TERRAIN_TEXTURE_SIZE, true); } std::optional ZonePropertyAreasJson(uint32_t zoneId) { + static std::mutex mutex; static std::map> cache; + std::lock_guard lock(mutex); if (const auto it = cache.find(zoneId); it != cache.end()) return it->second; std::optional json; const auto zone = GetZoneInfo(zoneId); diff --git a/dDashboardServer/routes/ReportRoutes.h b/dDashboardServer/routes/ReportRoutes.h index 71c10af10..7c21a7c3d 100644 --- a/dDashboardServer/routes/ReportRoutes.h +++ b/dDashboardServer/routes/ReportRoutes.h @@ -22,6 +22,25 @@ std::optional ZoneTerrainJson(uint32_t zoneId); #include "Raw.h" +/** + * The zone data below is built on the dashboard's worker threads too, so it is thread safe: results are built once + * (OnceCache) and the CDClient tables it needs are read at startup by PreloadZoneData (workers never query the + * CDClient). The ...Ready functions say whether a result is built, so a route can answer it at once. + */ +void PreloadZoneData(); +bool ZoneTerrainJsonReady(uint32_t zoneId); +bool ZoneTerrainChunksReady(uint32_t zoneId); +bool ZoneTerrainLayersReady(uint32_t zoneId); +bool TerrainTextureReady(uint32_t textureId); + +// A zone's .luz file (relative to res/maps) from ZoneTable +std::optional ZoneLuzPath(uint32_t zoneId); + +#include + +// ZoneRaw, shared: the last few zones read are kept, and a zone asked for by several threads at once is read once +std::shared_ptr ZoneRawShared(uint32_t zoneId); + // A zone's terrain file (.raw) read whole, as its .luz names it; nullopt without client files or when it's damaged std::optional ZoneRaw(uint32_t zoneId); diff --git a/dDashboardServer/routes/Scenery.cpp b/dDashboardServer/routes/Scenery.cpp index 5913a20ca..706181af5 100644 --- a/dDashboardServer/routes/Scenery.cpp +++ b/dDashboardServer/routes/Scenery.cpp @@ -19,9 +19,11 @@ #include "NifFile.h" #include "ReportRoutes.h" #include "RouteUtils.h" +#include "OnceCache.h" #include "TtlCache.h" #include "Web.h" #include "WorkerPool.h" +#include "Workers.h" #include "WorldScene.h" #include "ZonePaths.h" @@ -78,13 +80,10 @@ namespace { std::unordered_map> byName; }; - const FileIndex& Files() { - static std::optional index; - if (index) return *index; - index.emplace(); - const auto client = Game::config->GetValue("client_location"); - if (client.empty()) return *index; - const auto res = std::filesystem::path(client) / "res"; + FileIndex IndexFiles() { + FileIndex index; + const auto res = ClientAssets::ResFolder(); + if (res.empty()) return index; std::error_code ec; for (const char* folder : { "mesh", "textures", "animations" }) { // Unpacked clients keep their original case, so the top folder is matched ignoring case too @@ -93,12 +92,18 @@ namespace { for (auto it = std::filesystem::recursive_directory_iterator(top.path(), ec); !ec && it != std::filesystem::recursive_directory_iterator(); it.increment(ec)) { if (!it->is_regular_file(ec)) continue; const auto relative = Lower(std::filesystem::relative(it->path(), res, ec).generic_string()); - index->paths.insert(relative); - index->byName[Lower(it->path().filename().string())].push_back(relative); + index.paths.insert(relative); + index.byName[Lower(it->path().filename().string())].push_back(relative); } } } - return *index; + return index; + } + + // Built at startup (Scenery::Preload); read only afterwards + const FileIndex& Files() { + static const FileIndex index = IndexFiles(); + return index; } // A texture a model names: next to the model if it's there (the usual case), else the file of that name sharing @@ -140,24 +145,27 @@ namespace { }; // Every LOT with a render component, read once (Objects has no index on id, so it is read whole too) - const std::unordered_map& RenderInfos() { - static std::optional> infos; - if (infos) return *infos; - infos.emplace(); + std::unordered_map ReadRenderInfos() { + std::unordered_map infos; try { auto row = CDClientDatabase::ExecuteQuery( "SELECT cr.id, rc.render_asset FROM ComponentsRegistry cr JOIN RenderComponent rc ON rc.id = cr.component_id " "WHERE cr.component_type = " + std::to_string(RENDER_COMPONENT) + ";"); - for (; !row.eof(); row.nextRow()) infos->try_emplace(static_cast(row.getIntField(0)), RenderInfo{ row.getStringField(1, ""), "" }); + for (; !row.eof(); row.nextRow()) infos.try_emplace(static_cast(row.getIntField(0)), RenderInfo{ row.getStringField(1, ""), "" }); auto types = CDClientDatabase::ExecuteQuery("SELECT id, type FROM Objects;"); for (; !types.eof(); types.nextRow()) { - const auto it = infos->find(static_cast(types.getIntField(0))); - if (it != infos->end() && it->second.type.empty()) it->second.type = types.getStringField(1, ""); + const auto it = infos.find(static_cast(types.getIntField(0))); + if (it != infos.end() && it->second.type.empty()) it->second.type = types.getStringField(1, ""); } } catch (const std::exception& ex) { LOG("Could not read the render components of objects: %s", ex.what()); } - return *infos; + return infos; + } + + const std::unordered_map& RenderInfos() { + static const auto infos = ReadRenderInfos(); + return infos; } struct Model { @@ -176,30 +184,33 @@ namespace { if (info == infos.end()) return {}; const auto draw = WorldScene::ClientDraws(object, info->second.type); if (draw == WorldScene::eClientDraw::NO_MODEL) return {}; + static std::mutex mutex; static std::unordered_map resolved; const auto& stored = object.nifName.empty() ? info->second.asset : object.nifName; - auto it = resolved.find(stored); - if (it == resolved.end()) it = resolved.emplace(stored, ResolveModel(stored)).first; - return { it->second, draw == WorldScene::eClientDraw::HIDDEN }; + { + std::lock_guard lock(mutex); + if (const auto it = resolved.find(stored); it != resolved.end()) return { it->second, draw == WorldScene::eClientDraw::HIDDEN }; + } + auto path = ResolveModel(stored); + std::lock_guard lock(mutex); + resolved.try_emplace(stored, path); + return { std::move(path), draw == WorldScene::eClientDraw::HIDDEN }; } std::optional LuzPath(uint32_t zone) { - auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT zoneName FROM ZoneTable WHERE zoneID = ? LIMIT 1;"); - stmt.bind(1, static_cast(zone)); - auto result = stmt.execQuery(); - if (result.eof()) return std::nullopt; - std::string path = result.getStringField(0, ""); - if (!path.ends_with(".luz")) return std::nullopt; - return path; + return ZoneLuzPath(zone); } double Round(float value, double scale) { return std::round(static_cast(value) * scale) / scale; } + /** + * A zone's models and scenery manifest, built once (g_Zones). Once shared, assets, index, flairModels and the + * warmed flags are guarded by g_ZoneMutex: the flairs' models join assets when their manifest is built. + */ struct ZoneScenery { std::vector assets; // res paths of models, indexed by the manifests std::map index; // res path -> its index in assets std::string json; - std::optional flairs; // the flairs' manifest, built when first asked for (their models join assets) std::unordered_set flairModels; // res paths of the flairs' models, converted ahead of others bool warmedScenery{}; // WarmUp queued the scenery's models bool warmedFlairs{}; // and the flairs' @@ -211,14 +222,12 @@ namespace { } }; - std::optional& Zone(uint32_t zoneId) { - static std::map> cache; - if (const auto it = cache.find(zoneId); it != cache.end()) return it->second; - auto& entry = cache[zoneId]; + std::mutex g_ZoneMutex; + std::shared_ptr BuildZone(uint32_t zoneId) { const auto luzPath = LuzPath(zoneId); const auto luz = luzPath ? ClientAssets::ReadResFile("maps/" + *luzPath) : std::nullopt; - if (!luz) return entry; + if (!luz) return nullptr; const auto folder = luzPath->substr(0, luzPath->find_last_of('/') + 1); ZoneScenery scenery; @@ -247,22 +256,36 @@ namespace { {"zone", zoneId}, {"sky", sky}, {"assets", scenery.assets}, {"objects", { {"asset", assetOf}, {"pos", positions}, {"rot", rotations}, {"scale", scales}, {"hidden", hidden} }} }.dump(); - entry = std::move(scenery); - return entry; + return std::make_shared(std::move(scenery)); + } + + OnceCache> g_Zones; + + // The zone's scenery, built when first asked for (any thread); nullptr without client files + std::shared_ptr Zone(uint32_t zoneId) { + return g_Zones.Get(zoneId, [zoneId] { return BuildZone(zoneId); }); + } + + // The same, only when it is built already (never waits) + std::shared_ptr ZoneIfBuilt(uint32_t zoneId) { + return g_Zones.Ready(zoneId) ? Zone(zoneId) : nullptr; } // FlairTable: flair id -> the model's res path (empty when the client lacks it), read once - const std::unordered_map& FlairModels() { - static std::optional> models; - if (models) return *models; - models.emplace(); + std::unordered_map ReadFlairModels() { + std::unordered_map models; try { auto row = CDClientDatabase::ExecuteQuery("SELECT id, asset FROM FlairTable;"); - for (; !row.eof(); row.nextRow()) models->try_emplace(static_cast(row.getIntField(0)), ResolveModel(row.getStringField(1, ""))); + for (; !row.eof(); row.nextRow()) models.try_emplace(static_cast(row.getIntField(0)), ResolveModel(row.getStringField(1, ""))); } catch (const std::exception& ex) { LOG("Could not read the flairs: %s", ex.what()); } - return *models; + return models; + } + + const std::unordered_map& FlairModels() { + static const auto models = ReadFlairModels(); + return models; } /** @@ -275,18 +298,16 @@ namespace { * The flairs' manifest (as the scenery's, plus a tint per flair), from the zone's terrain file. A flair's color * tints its model; 63 is full strength (the files use 0 to 63 for most flairs, a little more for brighter ones). */ - const std::optional& Flairs(uint32_t zoneId, ZoneScenery& scenery) { - if (scenery.flairs) return scenery.flairs; - const auto raw = ZoneRaw(zoneId); + std::optional BuildFlairs(uint32_t zoneId, ZoneScenery& scenery) { + const auto raw = ZoneRawShared(zoneId); const auto& models = FlairModels(); - nlohmann::json assetOf = nlohmann::json::array(), positions = nlohmann::json::array(), rotations = nlohmann::json::array(), - scales = nlohmann::json::array(), colors = nlohmann::json::array(); + std::vector modelOf; + nlohmann::json positions = nlohmann::json::array(), rotations = nlohmann::json::array(), scales = nlohmann::json::array(), colors = nlohmann::json::array(); for (const auto& chunk : raw ? raw->chunks : std::vector{}) { for (const auto& flair : chunk.flairs) { const auto model = models.find(flair.id); if (model == models.end() || model->second.empty() || !std::isfinite(flair.position.x)) continue; - assetOf.push_back(scenery.IndexOf(model->second)); - scenery.flairModels.insert(model->second); + modelOf.push_back(&model->second); for (const auto value : { flair.position.x, flair.position.y, flair.position.z }) positions.push_back(Round(value, 100.0)); // Radians about x, y and z, applied in that order const double c1 = std::cos(flair.rotation.x / 2), c2 = std::cos(flair.rotation.y / 2), c3 = std::cos(flair.rotation.z / 2); @@ -298,11 +319,32 @@ namespace { for (const auto value : { flair.colorR, flair.colorG, flair.colorB }) colors.push_back(value); } } - scenery.flairs = nlohmann::json{ - {"zone", zoneId}, {"sky", -1}, {"assets", scenery.assets}, {"distance", FLAIR_DISTANCE}, {"colorScale", 1.0 / 63.0}, + // The flairs' models join the zone's list, which mesh requests read meanwhile + nlohmann::json assetOf = nlohmann::json::array(); + std::vector assets; + { + std::lock_guard lock(g_ZoneMutex); + for (const auto* model : modelOf) { + assetOf.push_back(scenery.IndexOf(*model)); + scenery.flairModels.insert(*model); + } + assets = scenery.assets; + } + return nlohmann::json{ + {"zone", zoneId}, {"sky", -1}, {"assets", assets}, {"distance", FLAIR_DISTANCE}, {"colorScale", 1.0 / 63.0}, {"objects", { {"asset", assetOf}, {"pos", positions}, {"rot", rotations}, {"scale", scales}, {"color", colors} }} }.dump(); - return scenery.flairs; + } + + OnceCache> g_Flairs; + + /** + * The flairs' manifest (as the scenery's, plus a tint per flair), from the zone's terrain file. A flair's color + * tints its model; 63 is full strength (the files use 0 to 63 for most flairs, a little more for brighter ones). + * Built when first asked for (any thread). + */ + const std::optional& Flairs(uint32_t zoneId, ZoneScenery& scenery) { + return g_Flairs.Get(zoneId, [zoneId, &scenery] { return BuildFlairs(zoneId, scenery); }); } constexpr uint32_t FORMAT_VERSION = 1; // bump when NifFile's output changes, so cached files are rebuilt @@ -518,7 +560,7 @@ namespace { // ---- Converting on worker threads, so a big model never holds up the web server's one thread ---- - WorkerPool g_Pool; + WorkerPool& Pool() { return Workers::Pool(); } constexpr uintmax_t SMALL_MODEL_BYTES = 256 * 1024; // .nif files this small convert in the pool's fast lane constexpr uintmax_t LARGE_MODEL_BYTES = 4 * 1024 * 1024; // and this big wait behind everything smaller @@ -550,8 +592,11 @@ namespace { } // Flairs (small, and drawn around the camera) and small models first; big ones behind the rest - WorkerPool::ePriority PriorityOf(const ZoneScenery& zone, const std::string& path, const std::filesystem::path& file) { - if (zone.flairModels.contains(path)) return WorkerPool::ePriority::URGENT; + WorkerPool::ePriority PriorityOf(const ZoneScenery* zone, const std::string& path, const std::filesystem::path& file) { + if (zone) { + std::lock_guard lock(g_ZoneMutex); + if (zone->flairModels.contains(path)) return WorkerPool::ePriority::URGENT; + } std::error_code ec; const auto size = std::filesystem::file_size(file, ec); if (ec || size <= SMALL_MODEL_BYTES) return WorkerPool::ePriority::URGENT; @@ -563,11 +608,10 @@ namespace { /** * Convert models of a zone ahead of time (onto the disk cache), at the LOD its viewer last asked for, while * someone views it. The lowest priority: only when nothing else waits. Smallest first; `front` puts these before - * the zone's other queued ones (the flairs). Web thread. + * the zone's other queued ones (the flairs). Any thread. */ void WarmUp(uint32_t zoneId, const std::vector& paths, bool front) { - if (!g_Pool.Running() || paths.empty()) return; - Files(); // built here: workers only read it + if (!Pool().Running() || paths.empty()) return; struct Item { std::string path; std::filesystem::path file; @@ -576,9 +620,10 @@ namespace { std::vector items; std::set seen; std::error_code ec; + const auto res = ClientAssets::ResFolder(); for (const auto& path : paths) { if (!seen.insert(path).second) continue; - const auto file = ClientAssets::ResolveResFile(path); + const auto file = ClientAssets::ResolveResFile(path, res); if (file) items.push_back({ path, *file, std::filesystem::file_size(*file, ec) }); } std::stable_sort(items.begin(), items.end(), [](const Item& a, const Item& b) { return a.size < b.size; }); @@ -587,10 +632,10 @@ namespace { // Queued at the front in reverse, so they still run smallest first if (front) std::reverse(items.begin(), items.end()); for (auto& item : items) { - g_Pool.Submit(WorkerPool::ePriority::BACKGROUND, [zoneId, group, path = std::move(item.path), file = std::move(item.file)] { + Pool().Submit(WorkerPool::ePriority::BACKGROUND, [zoneId, group, path = std::move(item.path), file = std::move(item.file)] { const auto activity = Viewed(zoneId); if (!activity || g_Disk.Total() >= WARM_DISK_BYTES) { - g_Pool.Cancel(group); + Pool().Cancel(group); return; } const auto key = ModelKey(path, activity->lod); @@ -601,15 +646,26 @@ namespace { } } - // Warm the zone's models when its manifest is asked for (again after nobody viewed it for a while) + /** + * Warm the zone's models when its manifest is asked for (again after nobody viewed it for a while). Planning it + * (finding the files) is left to a worker, so a manifest in memory is still answered at once. + */ void WarmScenery(uint32_t zoneId, ZoneScenery& zone, bool flairs) { - if (!Viewed(zoneId)) zone.warmedScenery = zone.warmedFlairs = false; + std::vector paths; + { + const bool idle = !Viewed(zoneId); + std::lock_guard lock(g_ZoneMutex); + if (idle) zone.warmedScenery = zone.warmedFlairs = false; + auto& warmed = flairs ? zone.warmedFlairs : zone.warmedScenery; + if (!warmed) { + warmed = true; + if (flairs) paths.assign(zone.flairModels.begin(), zone.flairModels.end()); + else paths = zone.assets; + } + } Touch(zoneId); - auto& warmed = flairs ? zone.warmedFlairs : zone.warmedScenery; - if (warmed) return; - warmed = true; - if (flairs) WarmUp(zoneId, { zone.flairModels.begin(), zone.flairModels.end() }, true); - else WarmUp(zoneId, zone.assets, false); + if (paths.empty() || !Pool().Running()) return; + Pool().Submit(WorkerPool::ePriority::NORMAL, [zoneId, paths = std::move(paths), flairs] { WarmUp(zoneId, paths, flairs); }); } // The texture `name` of model `path` (textures[slot] of its encoded form) as a DDS file. Any thread. @@ -636,87 +692,135 @@ namespace { return dds; } - // The model path of `asset` in the zone's manifests, or a 404 reply - const std::string* AssetPath(HTTPReply& reply, uint32_t zoneId, uint32_t asset) { - auto& zone = Zone(zoneId); - // The flairs' models join the list when their manifest is first built (a browser may still have it cached) - if (zone && asset >= zone->assets.size()) Flairs(zoneId, *zone); - if (!zone || asset >= zone->assets.size()) { - JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No such model in this zone"); - return nullptr; + // The model path of `asset` in the zone's manifests, building what's missing (any thread); nullopt: no such model + std::optional AssetPath(uint32_t zoneId, uint32_t asset) { + const auto zone = Zone(zoneId); + if (!zone) return std::nullopt; + { + std::lock_guard lock(g_ZoneMutex); + if (asset < zone->assets.size()) return zone->assets[asset]; } - return &zone->assets[asset]; + // The flairs' models join the list when their manifest is first built (a browser may still have it cached) + Flairs(zoneId, *zone); + std::lock_guard lock(g_ZoneMutex); + if (asset < zone->assets.size()) return zone->assets[asset]; + return std::nullopt; + } + + // The same without building anything (never waits): nullopt when it isn't known yet + std::optional AssetPathIfBuilt(uint32_t zoneId, uint32_t asset) { + const auto zone = ZoneIfBuilt(zoneId); + if (!zone) return std::nullopt; + std::lock_guard lock(g_ZoneMutex); + if (asset < zone->assets.size()) return zone->assets[asset]; + return std::nullopt; + } + + /** + * Where model `asset` is, and how urgent converting it is, when that is known without building anything: for + * the web thread, which only hands the work on + */ + struct Known { + std::optional path; + std::optional file; + WorkerPool::ePriority priority{ WorkerPool::ePriority::NORMAL }; + }; + + Known KnownAsset(uint32_t zoneId, uint32_t asset) { + Known known; + known.path = AssetPathIfBuilt(zoneId, asset); + if (!known.path) return known; + known.file = ClientAssets::ResolveResFile(*known.path, ClientAssets::ResFolder()); + if (known.file) known.priority = PriorityOf(ZoneIfBuilt(zoneId).get(), *known.path, *known.file); + return known; } } namespace Scenery { + void Preload() { + Files(); + RenderInfos(); + FlairModels(); + } + std::optional ZoneJson(uint32_t zoneId) { - auto& zone = Zone(zoneId); + const auto zone = Zone(zoneId); if (!zone) return std::nullopt; WarmScenery(zoneId, *zone, false); return zone->json; } + bool ZoneReady(uint32_t zoneId) { + return g_Zones.Ready(zoneId); + } + bool HasModel(const WorldScene::Object& object) { const auto model = ModelFor(object); return !model.path.empty() && !model.hidden; } std::optional FlairsJson(uint32_t zoneId) { - auto& zone = Zone(zoneId); + const auto zone = Zone(zoneId); if (!zone) return std::nullopt; const auto& flairs = Flairs(zoneId, *zone); WarmScenery(zoneId, *zone, true); return flairs; } + bool FlairsReady(uint32_t zoneId) { + return g_Zones.Ready(zoneId) && g_Flairs.Ready(zoneId); + } + void ReplyMesh(HTTPReply& reply, const HTTPContext& context, uint32_t zoneId, uint32_t asset, uint32_t lod) { - const auto* found = AssetPath(reply, zoneId, asset); - if (!found) return; - const auto path = *found; lod = std::min(lod, MAX_LOD); Touch(zoneId, lod); - if (const auto cached = g_Models.Get(ModelKey(path, lod))) return Binary(reply, *cached); - const auto file = ClientAssets::ResolveResFile(path); - if (!file) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "Could not read this model"); - Files(); // built here: workers only read it + auto known = KnownAsset(zoneId, asset); + if (known.path) { + if (const auto cached = g_Models.Get(ModelKey(*known.path, lod))) return Binary(reply, *cached); + } + const auto res = ClientAssets::ResFolder(); const auto deferred = Web::Defer(reply, context); - g_Pool.Submit(PriorityOf(*Zone(zoneId), path, *file), [deferred, path, lod, file = *file] { + Pool().Submit(known.priority, [deferred, zoneId, asset, lod, res, known = std::move(known)] { if (deferred.Cancelled()) return; HTTPReply out; - const auto encoded = Encoded(path, lod, file); - if (encoded) Binary(out, *encoded); - else JsonError(out, eHTTPStatusCode::NOT_FOUND, "Could not read this model"); + const auto path = known.path ? known.path : AssetPath(zoneId, asset); + const auto file = known.file ? known.file : path ? ClientAssets::ResolveResFile(*path, res) : std::nullopt; + if (!path) { + JsonError(out, eHTTPStatusCode::NOT_FOUND, "No such model in this zone"); + } else if (const auto encoded = file ? Encoded(*path, lod, *file) : nullptr) { + Binary(out, *encoded); + } else { + JsonError(out, eHTTPStatusCode::NOT_FOUND, "Could not read this model"); + } deferred.Send(std::move(out)); }); } void ReplyTexture(HTTPReply& reply, const HTTPContext& context, uint32_t zoneId, uint32_t asset, uint32_t slot, uint32_t lod) { - const auto* found = AssetPath(reply, zoneId, asset); - if (!found) return; - const auto path = *found; lod = std::min(lod, MAX_LOD); Touch(zoneId, lod); - const auto file = ClientAssets::ResolveResFile(path); - if (!file) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No such texture"); - Files(); + auto known = KnownAsset(zoneId, asset); // Quick when the model is converted already and the texture is a file of its own, or one kept from its model - auto priority = WorkerPool::ePriority::URGENT; - if (const auto cached = g_Models.Get(ModelKey(path, lod))) { - const auto textures = TexturesOf(*cached); - if (slot < textures.size() && textures[slot].starts_with('#') && !g_Embedded.Get(path + textures[slot])) priority = PriorityOf(*Zone(zoneId), path, *file); - } else { - priority = PriorityOf(*Zone(zoneId), path, *file); + if (known.path) { + if (const auto cached = g_Models.Get(ModelKey(*known.path, lod))) { + const auto textures = TexturesOf(*cached); + if (slot >= textures.size() || !textures[slot].starts_with('#') || g_Embedded.Get(*known.path + textures[slot])) known.priority = WorkerPool::ePriority::URGENT; + } } + const auto res = ClientAssets::ResFolder(); const auto deferred = Web::Defer(reply, context); - g_Pool.Submit(priority, [deferred, path, lod, slot, file = *file, res = ClientAssets::ResFolder()] { + Pool().Submit(known.priority, [deferred, zoneId, asset, lod, slot, res, known = std::move(known)] { if (deferred.Cancelled()) return; HTTPReply out; - const auto encoded = Encoded(path, lod, file); + const auto path = known.path ? known.path : AssetPath(zoneId, asset); + const auto file = known.file ? known.file : path ? ClientAssets::ResolveResFile(*path, res) : std::nullopt; + const auto encoded = path && file ? Encoded(*path, lod, *file) : nullptr; const auto textures = encoded ? TexturesOf(*encoded) : std::vector{}; - if (slot >= textures.size() || textures[slot].empty()) { + if (!path) { + JsonError(out, eHTTPStatusCode::NOT_FOUND, "No such model in this zone"); + } else if (slot >= textures.size() || textures[slot].empty()) { JsonError(out, eHTTPStatusCode::NOT_FOUND, "No such texture"); - } else if (auto dds = TextureBytes(path, file, textures, textures[slot], res)) { + } else if (auto dds = TextureBytes(*path, *file, textures, textures[slot], res)) { Binary(out, std::move(*dds)); } else { JsonError(out, eHTTPStatusCode::NOT_FOUND, "Could not read this texture"); @@ -725,18 +829,7 @@ namespace Scenery { }); } - void Shutdown() { - g_Pool.Stop(); - } - void RegisterRoutes() { - auto threads = Game::config ? Game::config->GetValue("scenery_workers", 0) : 0; - if (threads == 0) threads = static_cast(WorkerPool::DefaultThreads(std::thread::hardware_concurrency())); - threads = std::clamp(threads, 2, 16); - // Converting ahead of time never takes more than half of the threads besides the fast lane - g_Pool.Start(threads, std::max(1, (threads - 1) / 2)); - LOG("Converting scenery models with %u threads", threads); - Route(eHTTPMethod::GET, "/api/scenery/:zone/mesh/:asset", 0, "Model `asset` of a zone's scenery (see the scenery routes of properties and /world3d), converted from the client's .nif. Query: ?lod=0 (most detailed) to 3", [](HTTPReply& reply, const HTTPContext& context) { diff --git a/dDashboardServer/routes/Scenery.h b/dDashboardServer/routes/Scenery.h index 4e3ae0b8c..909488bc4 100644 --- a/dDashboardServer/routes/Scenery.h +++ b/dDashboardServer/routes/Scenery.h @@ -21,13 +21,20 @@ namespace WorldScene { * bounded in-memory cache; textures are sent as the client's own DDS files (textures stored inside a .nif are * wrapped as DDS), which the browser decodes itself, so nothing needs ImageMagick. * + * Everything here is thread safe: the manifests are built on worker threads too (the routes use Workers::Reply). * Converting a big model takes a while, so models and textures not in memory are answered from a few worker threads - * (Web::Defer, WorkerPool; scenery_workers in dashboardconfig.ini) and the web server keeps answering meanwhile. + * (Web::Defer, Workers; scenery_workers in dashboardconfig.ini) and the web server keeps answering meanwhile. * Flairs and small models go first, in a lane of their own; the same model asked for twice at once is converted * once. When a zone's manifest is asked for, its models are converted ahead of time onto the disk cache while * someone views the zone. */ namespace Scenery { + /** + * Read what the builders need from the CDClient (render components, flairs) and index the client's model files, + * at startup: the manifests are built on worker threads, which never query the CDClient. + */ + void Preload(); + /** * The zone's manifest as JSON: {zone, sky (asset index or -1), assets: [res path, ...], * objects: {asset: [...], pos: [x, y, z, ...], rot: [x, y, z, w, ...], scale: [...], hidden: [0 or 1, ...]}}, where @@ -36,6 +43,9 @@ namespace Scenery { */ std::optional ZoneJson(uint32_t zoneId); + // Whether ZoneJson is built (so answering it is quick) + bool ZoneReady(uint32_t zoneId); + /** * The zone's flairs (the grass, flowers and small rocks its terrain file strews over it, models from FlairTable) as a * manifest in ZoneJson's form, whose models the same mesh and texture routes serve, plus: distance (how far from the @@ -44,6 +54,9 @@ namespace Scenery { */ std::optional FlairsJson(uint32_t zoneId); + // Whether FlairsJson is built + bool FlairsReady(uint32_t zoneId); + // Whether the client draws a model for this scene object (its render component's, or its nif_name) bool HasModel(const WorldScene::Object& object); @@ -54,10 +67,8 @@ namespace Scenery { // Reply with texture `slot` (the model's "textures" list at that `lod`) of model `asset`, as a DDS file. Deferred. void ReplyTexture(HTTPReply& reply, const HTTPContext& context, uint32_t zoneId, uint32_t asset, uint32_t slot, uint32_t lod); - // /api/scenery/:zone/mesh/:asset and /api/scenery/:zone/texture/:asset/:slot, for anyone signed in; starts the - // conversion threads + // /api/scenery/:zone/mesh/:asset and /api/scenery/:zone/texture/:asset/:slot, for anyone signed in void RegisterRoutes(); - // Stop the conversion threads (queued work is dropped) - void Shutdown(); + } diff --git a/dDashboardServer/routes/Showcase.cpp b/dDashboardServer/routes/Showcase.cpp index 312a83f8e..c4afc300b 100644 --- a/dDashboardServer/routes/Showcase.cpp +++ b/dDashboardServer/routes/Showcase.cpp @@ -1,4 +1,5 @@ #include "Showcase.h" +#include "Workers.h" #include #include @@ -164,16 +165,21 @@ namespace { if (!cacheControl.empty()) reply.headers.push_back("Cache-Control: " + cacheControl); } - // Terrain and the build area come from the client's files and are the same for every property in a zone + /** + * Terrain and the build area come from the client's files and are the same for every property in a zone. + * `ready` says whether what `fetch` needs is built; when it isn't, fetch runs on a worker thread (Workers::Reply), + * so it must be thread safe. The property is looked up here, on the web thread. + */ template - void ZoneFileRoute(const std::string& path, const std::string& description, Fetch fetch) { - Route(eHTTPMethod::GET, path, PUBLIC, description, [fetch](HTTPReply& reply, const HTTPContext& context) { + void ZoneFileRoute(const std::string& path, const std::string& description, Fetch fetch, std::function ready = nullptr) { + Route(eHTTPMethod::GET, path, PUBLIC, description, [fetch, ready](HTTPReply& reply, const HTTPContext& context) { if (!Allowed(context, reply, Kind::DATA)) return; const auto id = PathId(context.path, 2); if (!id) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid ID"); const auto info = ShownProperty(*id, reply); if (!info) return; - fetch(reply, *info); + if (!ready) return fetch(reply, *info); + Workers::Reply(reply, context, ready(info->zoneId), [fetch, info = *info](HTTPReply& out) { fetch(out, info); }); }); } } @@ -251,14 +257,14 @@ void RegisterShowcaseRoutes() { const auto terrain = ZoneTerrainChunksJson(info.zoneId); if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone"); RawReply(reply, eContentType::APPLICATION_JSON, *terrain, "private, max-age=86400"); - }); + }, ZoneTerrainChunksReady); ZoneFileRoute("/api/showcase/:id/terrain", "A height grid of a showcased property's zone (needs client_location)", [](HTTPReply& reply, const IProperty::Info& info) { const auto terrain = ZoneTerrainJson(info.zoneId); if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone"); RawReply(reply, eContentType::APPLICATION_JSON, *terrain, "private, max-age=86400"); - }); + }, ZoneTerrainJsonReady); ZoneFileRoute("/api/showcase/:id/boundary", "Where a showcased property's owner may build (needs client_location)", [](HTTPReply& reply, const IProperty::Info& info) { @@ -311,9 +317,11 @@ void RegisterShowcaseRoutes() { if (!Allowed(context, reply, Kind::ASSET)) return; const auto textureId = PathId(context.path, 3); if (!textureId) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid ID"); - const auto png = TerrainTextureFile(*textureId); - if (!png) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "Texture not found"); - RawReply(reply, eContentType::IMAGE_PNG, *png, "public, max-age=604800"); + Workers::Reply(reply, context, TerrainTextureReady(*textureId), [id = *textureId](HTTPReply& out) { + const auto png = TerrainTextureFile(id); + if (!png) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "Texture not found"); + RawReply(out, eContentType::IMAGE_PNG, *png, "public, max-age=604800"); + }); }); ZoneFileRoute("/api/showcase/:id/scenery", "Everything the game draws around a showcased property: its zone's scene objects with their models and the sky (needs client_location)", @@ -321,7 +329,7 @@ void RegisterShowcaseRoutes() { const auto scenery = Scenery::ZoneJson(info.zoneId); if (!scenery) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No scenery for this zone"); RawReply(reply, eContentType::APPLICATION_JSON, *scenery, "private, max-age=86400"); - }); + }, Scenery::ZoneReady); // Scenery models and textures of the zones properties are in (the only ones the showcase shows) const auto propertyZone = [](HTTPReply& reply, const HTTPContext& context) -> std::optional { diff --git a/dDashboardServer/routes/Workers.cpp b/dDashboardServer/routes/Workers.cpp new file mode 100644 index 000000000..1cef75d3d --- /dev/null +++ b/dDashboardServer/routes/Workers.cpp @@ -0,0 +1,53 @@ +#include "Workers.h" + +#include +#include +#include + +#include "RouteUtils.h" +#include "Web.h" + +#include "Game.h" +#include "Logger.h" +#include "dConfig.h" + +namespace { + WorkerPool g_Pool; +} + +namespace Workers { + void Start() { + auto threads = Game::config ? Game::config->GetValue("scenery_workers", 0) : 0; + if (threads == 0) threads = static_cast(WorkerPool::DefaultThreads(std::thread::hardware_concurrency())); + threads = std::clamp(threads, 2, 16); + // Work ahead of time (converting a zone's models) never takes more than half of the threads besides the fast lane + g_Pool.Start(threads, std::max(1, (threads - 1) / 2)); + LOG("Dashboard worker threads: %u", threads); + } + + void Stop() { + g_Pool.Stop(); + } + + WorkerPool& Pool() { + return g_Pool; + } + + void Reply(HTTPReply& reply, const HTTPContext& context, bool ready, std::function fill, WorkerPool::ePriority priority) { + if (ready || !g_Pool.Running()) return fill(reply); + const auto deferred = Web::Defer(reply, context); + const auto path = context.path; + g_Pool.Submit(priority, [deferred, path, fill = std::move(fill)] { + if (deferred.Cancelled()) return; + HTTPReply out; + try { + fill(out); + } catch (const std::exception& ex) { + LOG("Error handling GET %s: %s", path.c_str(), ex.what()); + out = HTTPReply{}; + RouteUtils::JsonError(out, eHTTPStatusCode::INTERNAL_SERVER_ERROR, "Internal server error"); + } + deferred.Send(std::move(out)); + }); + } +} diff --git a/dDashboardServer/routes/Workers.h b/dDashboardServer/routes/Workers.h new file mode 100644 index 000000000..c38c6e2fe --- /dev/null +++ b/dDashboardServer/routes/Workers.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +#include "WorkerPool.h" + +struct HTTPReply; +struct HTTPContext; + +/** + * The dashboard's worker threads (one WorkerPool, scenery_workers threads) for slow work routes would otherwise do + * on the web server's one thread: converting the client's models, building a zone's terrain and scene data the + * first time it is viewed, converting textures with ImageMagick. What they run must be thread safe: the zone data + * builders keep their results in OnceCaches and read the client's files through ClientAssets. + */ +namespace Workers { + // Start the threads (scenery_workers: 0 picks WorkerPool::DefaultThreads) + void Start(); + + // Stop them; queued work is dropped (answer deferred requests before Web::Shutdown) + void Stop(); + + WorkerPool& Pool(); + + /** + * Answer a request with what `fill` writes to the reply: right away when `ready` (what it needs is built, so it's + * quick), else from a worker thread (Web::Defer). `fill` runs on either thread, so it must be thread safe and must + * not keep a reference to the request. + */ + void Reply(HTTPReply& reply, const HTTPContext& context, bool ready, std::function fill, + WorkerPool::ePriority priority = WorkerPool::ePriority::NORMAL); +} diff --git a/dDashboardServer/routes/WorldView.cpp b/dDashboardServer/routes/WorldView.cpp index ea01e1e11..d3cc2a60e 100644 --- a/dDashboardServer/routes/WorldView.cpp +++ b/dDashboardServer/routes/WorldView.cpp @@ -11,6 +11,8 @@ #include "Permissions.h" #include "ServerState.h" #include "Background.h" +#include "OnceCache.h" +#include "Workers.h" #include #include @@ -83,20 +85,13 @@ namespace { } std::optional LuzPath(uint32_t zone) { - auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT zoneName FROM ZoneTable WHERE zoneID = ? LIMIT 1;"); - stmt.bind(1, static_cast(zone)); - auto result = stmt.execQuery(); - if (result.eof()) return std::nullopt; - std::string path = result.getStringField(0, ""); - if (!path.ends_with(".luz")) return std::nullopt; - return path; + return ZoneLuzPath(zone); } - // LOT -> index into WorldScene::KINDS for every LOT with one of those components, read once - const std::map& LotKinds() { - static std::optional> kinds; - if (kinds) return *kinds; - kinds.emplace(); + // LOT -> index into WorldScene::KINDS for every LOT with one of those components, read at startup (Preload): the + // zone data is built on worker threads, which never query the CDClient + std::map ReadLotKinds() { + std::map kinds; std::map> components; try { auto result = CDClientDatabase::ExecuteQuery("SELECT id, component_type FROM ComponentsRegistry;"); @@ -106,9 +101,14 @@ namespace { } for (const auto& [lot, types] : components) { const auto kind = WorldScene::Classify(types); - if (kind < WorldScene::KINDS.size()) (*kinds)[lot] = static_cast(kind); + if (kind < WorldScene::KINDS.size()) kinds[lot] = static_cast(kind); } - return *kinds; + return kinds; + } + + const std::map& LotKinds() { + static const auto kinds = ReadLotKinds(); + return kinds; } double Round(float value) { return std::round(static_cast(value) * 100.0) / 100.0; } @@ -118,15 +118,12 @@ namespace { * its .luz (the scene list) and its scene files. Objects are columns (lot, kind, flags, scene, x/y/z) to keep big zones * small: flags 1 = spawner, 2 = client only, 4 = drawn with a model by the scenery layer. */ - std::optional SceneJson(uint32_t zone) { - static std::map> cache; - if (const auto it = cache.find(zone); it != cache.end()) return it->second; - + std::optional BuildSceneJson(uint32_t zone) { const auto luzPath = LuzPath(zone); const auto luz = luzPath ? ClientAssets::ReadResFile("maps/" + *luzPath) : std::nullopt; std::string error; const auto zoneFile = luz ? ZonePaths::ReadHeader(*luz, error) : std::nullopt; - if (!zoneFile) return cache[zone] = std::nullopt; + if (!zoneFile) return std::nullopt; const auto& lotKinds = LotKinds(); const auto other = static_cast(WorldScene::KINDS.size()); @@ -167,19 +164,24 @@ namespace { {"objects", { {"lot", lots}, {"kind", kindOf}, {"flags", flags}, {"scene", sceneOf}, {"pos", positions}, {"names", names} }}, {"lotNames", lotNames}, {"spawnPoints", ZoneSpawnPointsJson(zone).value_or(nlohmann::json::array())} }; - return cache[zone] = json.dump(); + return json.dump(); + } + + OnceCache> g_Scenes; + OnceCache> g_Paths; + + // Built on a worker thread the first time (any thread) + const std::optional& SceneJson(uint32_t zone) { + return g_Scenes.Get(zone, [zone] { return BuildSceneJson(zone); }); } // A zone's paths, which need the whole .luz (they are most of it): loaded when a path layer is first shown - std::optional PathsJson(uint32_t zone) { - static std::map> cache; - if (const auto it = cache.find(zone); it != cache.end()) return it->second; - + std::optional BuildPathsJson(uint32_t zone) { const auto luzPath = LuzPath(zone); const auto luz = luzPath ? ClientAssets::ReadResFile("maps/" + *luzPath) : std::nullopt; std::string error; const auto zoneFile = luz ? ZonePaths::Read(*luz, error) : std::nullopt; - if (!zoneFile) return cache[zone] = std::nullopt; + if (!zoneFile) return std::nullopt; nlohmann::json pathTypes = nlohmann::json::array(); for (const auto type : magic_enum::enum_values()) pathTypes.push_back({ {"value", static_cast(type)}, {"name", GameLabels::Name(type)} }); @@ -194,7 +196,11 @@ namespace { if (points.empty()) continue; paths.push_back({ {"name", path.pathName}, {"type", static_cast(path.pathType)}, {"loop", path.pathBehavior == PathBehavior::Loop}, {"points", points} }); } - return cache[zone] = nlohmann::json{ {"zone", zone}, {"pathTypes", pathTypes}, {"paths", paths} }.dump(); + return nlohmann::json{ {"zone", zone}, {"pathTypes", pathTypes}, {"paths", paths} }.dump(); + } + + const std::optional& PathsJson(uint32_t zone) { + return g_Paths.Get(zone, [zone] { return BuildPathsJson(zone); }); } std::string CharacterName(LWOOBJID id) { @@ -211,22 +217,28 @@ namespace { reply.headers.push_back(std::string("Cache-Control: ") + cache); } + using Deflated = OnceCache>; + + bool TakesDeflate(const HTTPContext& context) { + return context.GetHeader("Accept-Encoding").find("deflate") != std::string::npos; + } + /** * A big cached JSON body, deflated once per zone when the browser takes it: a zone's terrain chunks are mostly - * base64 maps that shrink about tenfold (Avant Gardens: 19 MB to 1.4 MB). + * base64 maps that shrink about tenfold (Avant Gardens: 19 MB to 1.4 MB). Any thread. */ - void DeflatedJson(HTTPReply& reply, const HTTPContext& context, std::map& deflated, uint32_t zone, const std::string& json, const char* cache) { - if (context.GetHeader("Accept-Encoding").find("deflate") == std::string::npos || json.size() > UINT32_MAX / 2) return RawJson(reply, json, cache); - auto it = deflated.find(zone); - if (it == deflated.end()) { + void DeflatedJson(HTTPReply& reply, bool deflate, Deflated& deflated, uint32_t zone, const std::string& json, const char* cache) { + if (!deflate || json.size() > UINT32_MAX / 2) return RawJson(reply, json, cache); + const auto& out = deflated.Get(zone, [&json]() -> std::optional { std::string out(ZCompression::GetMaxCompressedLength(static_cast(json.size())), '\0'); const auto size = ZCompression::Compress(reinterpret_cast(json.data()), static_cast(json.size()), reinterpret_cast(out.data()), static_cast(out.size())); - if (size <= 0) return RawJson(reply, json, cache); + if (size <= 0) return std::nullopt; out.resize(static_cast(size)); - it = deflated.emplace(zone, std::move(out)).first; - } - RawJson(reply, it->second, cache); + return out; + }); + if (!out) return RawJson(reply, json, cache); + RawJson(reply, *out, cache); reply.headers.push_back("Content-Encoding: deflate"); reply.headers.push_back("Vary: Accept-Encoding"); } @@ -237,6 +249,10 @@ namespace { } namespace WorldView { + void Preload() { + LotKinds(); + } + void RecordPositions(const PlayerPositions& positions) { if (Setting("position_history", 1) == 0) { g_Buffer.clear(); @@ -286,9 +302,11 @@ namespace WorldView { [](HTTPReply& reply, const HTTPContext& context) { const auto zone = PathId(context.path, 2); if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone"); - const auto json = SceneJson(*zone); - if (!json) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)"); - RawJson(reply, *json, "private, max-age=3600"); + Workers::Reply(reply, context, g_Scenes.Ready(*zone), [zone = *zone](HTTPReply& out) { + const auto& json = SceneJson(zone); + if (!json) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)"); + RawJson(out, *json, "private, max-age=3600"); + }); }); Route(eHTTPMethod::GET, "/api/world3d/:zone/paths", Perm("players_view"), @@ -296,9 +314,11 @@ namespace WorldView { [](HTTPReply& reply, const HTTPContext& context) { const auto zone = PathId(context.path, 2); if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone"); - const auto json = PathsJson(*zone); - if (!json) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)"); - RawJson(reply, *json, "private, max-age=3600"); + Workers::Reply(reply, context, g_Paths.Ready(*zone), [zone = *zone](HTTPReply& out) { + const auto& json = PathsJson(zone); + if (!json) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)"); + RawJson(out, *json, "private, max-age=3600"); + }); }); Route(eHTTPMethod::GET, "/api/world3d/:zone/terrain_layers", Perm("players_view"), @@ -306,10 +326,13 @@ namespace WorldView { [](HTTPReply& reply, const HTTPContext& context) { const auto zone = PathId(context.path, 2); if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone"); - const auto layers = ZoneTerrainLayersJson(*zone); - if (!layers) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone"); - static std::map deflated; - DeflatedJson(reply, context, deflated, *zone, *layers, "private, max-age=86400"); + static Deflated deflated; + const auto deflate = TakesDeflate(context); + Workers::Reply(reply, context, ZoneTerrainLayersReady(*zone) && (!deflate || deflated.Ready(*zone)), [zone = *zone, deflate](HTTPReply& out) { + const auto layers = ZoneTerrainLayersJson(zone); + if (!layers) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone"); + DeflatedJson(out, deflate, deflated, zone, *layers, "private, max-age=86400"); + }); }); Route(eHTTPMethod::GET, "/api/world3d/:zone/flairs", Perm("players_view"), @@ -317,9 +340,11 @@ namespace WorldView { [](HTTPReply& reply, const HTTPContext& context) { const auto zone = PathId(context.path, 2); if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone"); - const auto flairs = Scenery::FlairsJson(*zone); - if (!flairs) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone"); - RawJson(reply, *flairs, "private, max-age=3600"); + Workers::Reply(reply, context, Scenery::FlairsReady(*zone), [zone = *zone](HTTPReply& out) { + const auto flairs = Scenery::FlairsJson(zone); + if (!flairs) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone"); + RawJson(out, *flairs, "private, max-age=3600"); + }); }); Route(eHTTPMethod::GET, "/api/world3d/:zone/terrain_chunks", Perm("players_view"), @@ -327,10 +352,13 @@ namespace WorldView { [](HTTPReply& reply, const HTTPContext& context) { const auto zone = PathId(context.path, 2); if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone"); - const auto terrain = ZoneTerrainChunksJson(*zone); - if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone"); - static std::map deflated; - DeflatedJson(reply, context, deflated, *zone, *terrain, "private, max-age=86400"); + static Deflated deflated; + const auto deflate = TakesDeflate(context); + Workers::Reply(reply, context, ZoneTerrainChunksReady(*zone) && (!deflate || deflated.Ready(*zone)), [zone = *zone, deflate](HTTPReply& out) { + const auto terrain = ZoneTerrainChunksJson(zone); + if (!terrain) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone"); + DeflatedJson(out, deflate, deflated, zone, *terrain, "private, max-age=86400"); + }); }); Route(eHTTPMethod::GET, "/api/world3d/:zone/scenery", Perm("players_view"), @@ -338,9 +366,11 @@ namespace WorldView { [](HTTPReply& reply, const HTTPContext& context) { const auto zone = PathId(context.path, 2); if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone"); - const auto scenery = Scenery::ZoneJson(*zone); - if (!scenery) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)"); - RawJson(reply, *scenery, "private, max-age=3600"); + Workers::Reply(reply, context, Scenery::ZoneReady(*zone), [zone = *zone](HTTPReply& out) { + const auto scenery = Scenery::ZoneJson(zone); + if (!scenery) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)"); + RawJson(out, *scenery, "private, max-age=3600"); + }); }); Route(eHTTPMethod::GET, "/api/world3d/history/instances", Perm("players_history"), diff --git a/dDashboardServer/routes/WorldView.h b/dDashboardServer/routes/WorldView.h index ca19e2ceb..f0b779220 100644 --- a/dDashboardServer/routes/WorldView.h +++ b/dDashboardServer/routes/WorldView.h @@ -10,6 +10,9 @@ struct PlayerPositions; namespace WorldView { void RegisterRoutes(); + // Read the CDClient data the zone data builders need (at startup: they run on worker threads) + void Preload(); + // A world server reported its players' positions: keep some of them for replays (throttled, written in batches) void RecordPositions(const PlayerPositions& positions); } diff --git a/docs/Dashboard.md b/docs/Dashboard.md index f68fbb37f..785ce7cb3 100644 --- a/docs/Dashboard.md +++ b/docs/Dashboard.md @@ -906,7 +906,14 @@ manifest is asked for, the zone's models are also converted ahead of time onto t first, at the detail its viewer last used: only when nothing else waits, on at most half of the threads besides the flairs' one, stopping when nobody has viewed the zone for 90 seconds or the disk cache is three quarters full (it never evicts for this). The number of threads is `scenery_workers` in `dashboardconfig.ini` (Settings > Dashboard > -Web server; 0, the default, picks half the CPU cores, 2 to 4; read at startup). Endpoints: +Web server; 0, the default, picks half the CPU cores, 2 to 4; read at startup). + +The same threads build a zone's data the first time it is viewed (its terrain chunks and layers, scene objects, +paths, scenery and flair manifests, deflated bodies) and convert terrain textures with ImageMagick, so opening a new +zone doesn't hold up the rest of the dashboard; what is built already is answered at once. Each is built once, even +when asked for twice at the same time. Workers never query the CDClient, read settings or touch the network: the +tables they need (ZoneTable, render components, flairs, object names, terrain textures) are read at startup. +Endpoints: `/api/properties/:id/scenery`, `/api/world3d/:zone/scenery`, `/api/world3d/:zone/flairs`, `/api/scenery/:zone/mesh/:asset?lod=`, `/api/scenery/:zone/texture/:asset/:slot?lod=`. The world view's other data: `/api/world3d/:zone/scene` (objects and scenes), `/terrain_chunks` and `/terrain_layers` (the terrain file; sent diff --git a/tests/dWebTests/CMakeLists.txt b/tests/dWebTests/CMakeLists.txt index dbeb1546e..4246977ed 100644 --- a/tests/dWebTests/CMakeLists.txt +++ b/tests/dWebTests/CMakeLists.txt @@ -36,6 +36,7 @@ set(DWEBTESTS_SOURCES "NifFileTests.cpp" "DeferredReplyTests.cpp" "WorkerPoolTests.cpp" + "OnceCacheTests.cpp" "${PROJECT_SOURCE_DIR}/dDashboardServer/routes/WorkerPool.cpp" "${PROJECT_SOURCE_DIR}/dDashboardServer/routes/NifFile.cpp" "BackupFilesTests.cpp" diff --git a/tests/dWebTests/OnceCacheTests.cpp b/tests/dWebTests/OnceCacheTests.cpp new file mode 100644 index 000000000..5d3ca311a --- /dev/null +++ b/tests/dWebTests/OnceCacheTests.cpp @@ -0,0 +1,56 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include "OnceCache.h" + +TEST(OnceCacheTests, BuildsOnceForManyThreads) { + OnceCache cache; + std::atomic builds{}; + std::vector threads; + std::vector results(16); + for (size_t i = 0; i < results.size(); i++) { + threads.emplace_back([&, i] { + results[i] = cache.Get(1, [&] { + builds++; + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + return std::string("built"); + }); + }); + } + for (auto& thread : threads) thread.join(); + EXPECT_EQ(builds.load(), 1); + for (const auto& result : results) EXPECT_EQ(result, "built"); + EXPECT_TRUE(cache.Ready(1)); + EXPECT_FALSE(cache.Ready(2)); +} + +TEST(OnceCacheTests, NotReadyWhileBuilding) { + OnceCache cache; + std::atomic release{}, started{}; + std::thread builder([&] { + cache.Get(1, [&] { + started = true; + while (!release) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return 5; + }); + }); + while (!started) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + EXPECT_FALSE(cache.Ready(1)); + release = true; + builder.join(); + EXPECT_TRUE(cache.Ready(1)); + EXPECT_EQ(cache.Get(1, [] { return 0; }), 5); +} + +TEST(OnceCacheTests, FailedBuildIsTriedAgain) { + OnceCache cache; + EXPECT_THROW(cache.Get(1, []() -> int { throw std::runtime_error("no"); }), std::runtime_error); + EXPECT_FALSE(cache.Ready(1)); + EXPECT_EQ(cache.Get(1, [] { return 3; }), 3); +}