feat: build a zone's 3D data on worker threads

Opening a zone the first time built its terrain, scene objects and manifests
and ran ImageMagick on the web thread, stalling the dashboard for seconds.

- Workers: the shared pool plus Workers::Reply (answer at once when built,
  else from a worker via Web::Defer).
- terrain_chunks/terrain_layers/scene/paths/scenery/flairs (world3d, property
  and showcase routes) and terrain textures go through it; results are built
  once in OnceCaches, the .raw is read once per zone for chunks, layers and
  flairs, deflated bodies are cached thread-safely.
- ImageMagick conversions are deduplicated and written under a temporary name.
- Workers don't query the CDClient, read settings or call mongoose: ZoneTable,
  render components, flairs, object names, LOT kinds and terrain texture names
  are read at startup; client_location is read once; base64 is plain C++.
- Logger writes one line at a time (mutex; localtime's buffer is shared).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dCommon/Logger.cpp

@@ -3,8 +3,15 @@
 #include <algorithm>
 #include <ctime>
 #include <filesystem>
+#include <mutex>
 #include <stdarg.h>
 
+namespace {
+	// Servers with worker threads (the dashboard) log from several threads: one line is written at a time, and
+	// localtime's buffer is shared
+	std::mutex g_LogMutex;
+}
+
 Writer::~Writer() {
 	// Flush before we close
 	Flush();
@@ -49,6 +56,7 @@ Logger::Logger(const std::string& outpath, bool logToConsole, bool logDebugState
 }
 
 void Logger::vLog(const char* format, va_list args) {
+	std::lock_guard lock(g_LogMutex);
 	time_t t = time(NULL);
 	struct tm* time = localtime(&t);
 	char timeStr[70];
@@ -78,6 +86,7 @@ void Logger::LogDebug(const char* className, const char* format, ...) {
 }
 
 void Logger::Flush() {
+	std::lock_guard lock(g_LogMutex);
 	for (const auto& writer : m_Writers) {
 		writer->Flush();
 	}

dDashboardServer/DashboardServer.cpp

@@ -60,6 +60,11 @@
 #include "ModeratorHelper.h"
 #include "LiveWorld.h"
 #include "Scenery.h"
+#include "Workers.h"
+#include "ReportRoutes.h"
+#include "WorldView.h"
+#include "ClientAssets.h"
+#include "DashboardRoutes.h"
 #include "LiveEventRoutes.h"
 #include "ChallengeRoutes.h"
 #include "Inspector.h"
@@ -411,6 +416,19 @@ int main(int argc, char** argv) {
 	);
 	Game::server = g_Server;
 
+	// What the worker threads read from the CDClient and the settings, read now on this thread: workers never query
+	// the CDClient, read settings or touch the network (RakNet and mongoose are the main thread's)
+	{
+		const auto start = std::chrono::steady_clock::now();
+		ClientAssets::Preload();
+		PreloadZoneData();
+		Scenery::Preload();
+		WorldView::Preload();
+		ZoneNames();
+		LOG("Read the client data for the 3D views in %lld ms", static_cast<long long>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - start).count()));
+	}
+	Workers::Start();
+
 	// Initialize web server
 	if (!Game::web.Startup(ourIP, ourPort)) {
 		LOG("Failed to start web server on %s:%d", ourIP.c_str(), ourPort);
@@ -570,8 +588,8 @@ int main(int argc, char** argv) {
 
 	}
 
-	// Cleanup: the conversion threads first (they answer deferred requests), then the web server's connections
+	// Cleanup: the worker threads first (they answer deferred requests), then the web server's connections
-	Scenery::Shutdown();
+	Workers::Stop();
 	Game::web.Shutdown();
 	Inspector::Shutdown();
 	EmailService::Shutdown();

dDashboardServer/routes/APIRoutes.cpp

@@ -13,6 +13,7 @@
 #include "CharacterTools.h"
 #include "ClientAssets.h"
 #include "Scenery.h"
+#include "Workers.h"
 #include "LiveWorld.h"
 #include "ServerState.h"
 #include "WSRoutes.h"
@@ -1091,12 +1092,14 @@ namespace {
 				if (!propId) return;
 				const auto info = AuthorizedProperty(context, *propId, reply);
 				if (!info) return;
-				const auto terrain = ZoneTerrainJson(info->zoneId);
+				Workers::Reply(reply, context, ZoneTerrainJsonReady(info->zoneId), [zone = info->zoneId](HTTPReply& out) {
-				if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone (is client_location set?)");
+					const auto terrain = ZoneTerrainJson(zone);
-				reply.status = eHTTPStatusCode::OK;
+					if (!terrain) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone (is client_location set?)");
-				reply.contentType = eContentType::APPLICATION_JSON;
+					out.status = eHTTPStatusCode::OK;
-				reply.message = *terrain;
+					out.contentType = eContentType::APPLICATION_JSON;
-				reply.headers.push_back("Cache-Control: private, max-age=86400");
+					out.message = *terrain;
+					out.headers.push_back("Cache-Control: private, max-age=86400");
+				});
 			});
 
 		Route(eHTTPMethod::GET, "/api/properties/:id/terrain_chunks", 0,
@@ -1106,12 +1109,14 @@ namespace {
 				if (!propId) return;
 				const auto info = AuthorizedProperty(context, *propId, reply);
 				if (!info) return;
-				const auto terrain = ZoneTerrainChunksJson(info->zoneId);
+				Workers::Reply(reply, context, ZoneTerrainChunksReady(info->zoneId), [zone = info->zoneId](HTTPReply& out) {
-				if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone (is client_location set?)");
+					const auto terrain = ZoneTerrainChunksJson(zone);
-				reply.status = eHTTPStatusCode::OK;
+					if (!terrain) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone (is client_location set?)");
-				reply.contentType = eContentType::APPLICATION_JSON;
+					out.status = eHTTPStatusCode::OK;
-				reply.message = *terrain;
+					out.contentType = eContentType::APPLICATION_JSON;
-				reply.headers.push_back("Cache-Control: private, max-age=86400");
+					out.message = *terrain;
+					out.headers.push_back("Cache-Control: private, max-age=86400");
+				});
 			});
 
 		Route(eHTTPMethod::GET, "/api/properties/:id/scenery", 0,
@@ -1121,24 +1126,28 @@ namespace {
 				if (!propId) return;
 				const auto info = AuthorizedProperty(context, *propId, reply);
 				if (!info) return;
-				const auto scenery = Scenery::ZoneJson(info->zoneId);
+				Workers::Reply(reply, context, Scenery::ZoneReady(info->zoneId), [zone = info->zoneId](HTTPReply& out) {
-				if (!scenery) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No scenery for this zone (is client_location set?)");
+					const auto scenery = Scenery::ZoneJson(zone);
-				reply.status = eHTTPStatusCode::OK;
+					if (!scenery) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No scenery for this zone (is client_location set?)");
-				reply.contentType = eContentType::APPLICATION_JSON;
+					out.status = eHTTPStatusCode::OK;
-				reply.message = *scenery;
+					out.contentType = eContentType::APPLICATION_JSON;
-				reply.headers.push_back("Cache-Control: private, max-age=86400");
+					out.message = *scenery;
+					out.headers.push_back("Cache-Control: private, max-age=86400");
+				});
 			});
 
 		Route(eHTTPMethod::GET, "/api/terrain_textures/:id", 0, "A terrain texture (mapTextureResource ID) as PNG, for the property 3D view",
 			[](HTTPReply& reply, const HTTPContext& context) {
 				const auto textureId = RequireId<uint32_t>(context, 2, reply);
 				if (!textureId) return;
-				const auto png = TerrainTextureFile(*textureId);
+				Workers::Reply(reply, context, TerrainTextureReady(*textureId), [id = *textureId](HTTPReply& out) {
-				if (!png) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "Texture not found");
+					const auto png = TerrainTextureFile(id);
-				reply.status = eHTTPStatusCode::OK;
+					if (!png) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "Texture not found");
-				reply.contentType = eContentType::IMAGE_PNG;
+					out.status = eHTTPStatusCode::OK;
-				reply.message = *png;
+					out.contentType = eContentType::IMAGE_PNG;
-				reply.headers.push_back("Cache-Control: private, max-age=604800");
+					out.message = *png;
+					out.headers.push_back("Cache-Control: private, max-age=604800");
+				});
 			});
 
 		Route(eHTTPMethod::GET, "/api/properties/:id/boundary", 0, "Where a property's owner may build: the outline from the zone file and the height limit (needs client_location)",

dDashboardServer/routes/CMakeLists.txt

@@ -43,6 +43,7 @@ set(DASHBOARDROUTES_SOURCES
 	"NifFile.cpp"
 	"Scenery.cpp"
 	"WorkerPool.cpp"
+	"Workers.cpp"
 	"AccountModeration.cpp"
 	"ModerationTools.cpp"
 	"ModeratorHelper.cpp"

dDashboardServer/routes/ClientAssets.cpp

@@ -7,6 +7,7 @@
 #include <set>
 #include <unordered_map>
 
+#include "OnceCache.h"
 #include "RouteUtils.h"
 #include "CDClientDatabase.h"
 #include "Game.h"
@@ -26,11 +27,14 @@ namespace {
 	std::map<LOT, nlohmann::json> g_ItemCache;
 	std::map<LOT, std::vector<int>> g_ItemSetsByLot; // lot -> set ids containing it
 	bool g_ItemSetsLoaded = false;
-	std::set<std::string> g_FailedConversions;
 
+	// Read once (client_location only changes with a restart), so worker threads never read the settings
 	std::filesystem::path ClientRes() {
-		const auto client = Game::config->GetValue("client_location");
+		static const std::filesystem::path res = [] {
-		return client.empty() ? std::filesystem::path{} : std::filesystem::path(client) / "res";
+			const auto client = Game::config ? Game::config->GetValue("client_location") : std::string{};
+			return client.empty() ? std::filesystem::path{} : std::filesystem::path(client) / "res";
+		}();
+		return res;
 	}
 
 	// Lowercase, forward slashes, no leading "../" or "res/"
@@ -195,11 +199,8 @@ namespace {
 
 	// Every object's name and displayName, read once: the CDClient has no index on Objects.id, so looking a LOT up
 	// scans the whole table, and pages name thousands of items at once
-	const std::unordered_map<LOT, ObjectNames>& ObjectNameTable() {
+	std::unordered_map<LOT, ObjectNames> ReadObjectNames() {
-		static std::unordered_map<LOT, ObjectNames> table;
+		std::unordered_map<LOT, ObjectNames> table;
-		static bool loaded = false;
-		if (loaded) return table;
-		loaded = true;
 		try {
 			auto row = CDClientDatabase::ExecuteQuery("SELECT id, name, displayName FROM Objects;");
 			for (; !row.eof(); row.nextRow()) {
@@ -211,6 +212,23 @@ namespace {
 		}
 		return table;
 	}
+
+	// Read at startup (ClientAssets::Preload), so worker threads only read it and never query the CDClient
+	const std::unordered_map<LOT, ObjectNames>& ObjectNameTable() {
+		static const auto table = ReadObjectNames();
+		return table;
+	}
+
+	// Where TextureAsPng keeps a texture it converted
+	std::filesystem::path PngCachePath(const std::string& normalized, uint32_t maxSize, bool opaque) {
+		std::string cacheName = normalized + "_" + std::to_string(maxSize) + (opaque ? "_opaque" : "") + ".png";
+		std::replace(cacheName.begin(), cacheName.end(), '/', '_');
+		std::replace(cacheName.begin(), cacheName.end(), ' ', '_');
+		return std::filesystem::path("dDashboardServer") / "icon_cache" / cacheName;
+	}
+
+	// ImageMagick conversions by cache file name: whether it worked (a failed one isn't tried again)
+	OnceCache<std::string, bool> g_Conversions;
 }
 
 namespace ClientAssets {
@@ -305,28 +323,39 @@ namespace ClientAssets {
 		if (normalized.ends_with(".png")) return ReadFile(source);
 		if (!normalized.ends_with(".dds")) return std::nullopt;
 
-		std::string cacheName = normalized + "_" + std::to_string(maxSize) + (opaque ? "_opaque" : "") + ".png";
+		const auto target = PngCachePath(normalized, maxSize, opaque);
-		std::replace(cacheName.begin(), cacheName.end(), '/', '_');
+		const auto cacheDir = target.parent_path();
-		std::replace(cacheName.begin(), cacheName.end(), ' ', '_');
+		const auto cacheName = target.filename().string();
-		const auto cacheDir = std::filesystem::path("dDashboardServer") / "icon_cache";
+		// ImageMagick runs without a shell (an argument vector); odd characters are still refused as a precaution
-		const auto target = cacheDir / cacheName;
+		if (!std::filesystem::exists(target, ec) && !Process::HasShellMetacharacters(source.string())) {
-		if (!std::filesystem::exists(target)) {
+			// Once per file, however many threads ask at once; written under another name and renamed, so a reader
-			// ImageMagick runs without a shell (an argument vector); odd characters are still refused as a precaution
+			// never sees half a file
-			if (g_FailedConversions.contains(cacheName) || Process::HasShellMetacharacters(source.string())) return std::nullopt;
+			const auto converted = g_Conversions.Get(cacheName, [&] {
-			std::filesystem::create_directories(cacheDir, ec);
+				if (std::filesystem::exists(target, ec)) return true;
-			const std::string size = std::to_string(maxSize) + "x" + std::to_string(maxSize) + ">";
+				std::filesystem::create_directories(cacheDir, ec);
-			std::vector<std::string> arguments{ "magick", source.string() };
+				const auto partial = cacheDir / (cacheName + ".part.png");
-			if (opaque) { arguments.push_back("-alpha"); arguments.push_back("off"); }
+				const std::string size = std::to_string(maxSize) + "x" + std::to_string(maxSize) + ">";
-			arguments.insert(arguments.end(), { "-resize", size, target.string() });
+				std::vector<std::string> arguments{ "magick", source.string() };
-			if (Process::Run(arguments) != 0) {
+				if (opaque) { arguments.push_back("-alpha"); arguments.push_back("off"); }
-				LOG_DEBUG("Texture conversion failed for %s (is ImageMagick installed?)", normalized.c_str());
+				arguments.insert(arguments.end(), { "-resize", size, partial.string() });
-				g_FailedConversions.insert(cacheName);
+				std::error_code renameError;
-				return std::nullopt;
+				if (Process::Run(arguments) != 0 || (std::filesystem::rename(partial, target, renameError), renameError)) {
-			}
+					LOG_DEBUG("Texture conversion failed for %s (is ImageMagick installed?)", normalized.c_str());
+					return false;
+				}
+				return true;
+			});
+			if (!converted) return std::nullopt;
 		}
 		return ReadFile(target);
 	}
 
+	bool TextureAsPngReady(const std::string& assetPath, uint32_t maxSize, bool opaque) {
+		const auto normalized = NormalizeAssetPath(assetPath);
+		std::error_code ec;
+		return normalized.ends_with(".png") || std::filesystem::exists(PngCachePath(normalized, maxSize, opaque), ec);
+	}
+
 	std::optional<std::string> ReadResFile(const std::string& relativePath) {
 		const auto res = ClientRes();
 		if (res.empty() || !IsSafeAssetPath(NormalizeAssetPath(relativePath))) return std::nullopt;
@@ -342,6 +371,11 @@ namespace ClientAssets {
 		return path;
 	}
 
+	void Preload() {
+		ClientRes();
+		ObjectNameTable();
+	}
+
 	std::filesystem::path ResFolder() {
 		return ClientRes();
 	}

dDashboardServer/routes/ClientAssets.h

@@ -13,6 +13,12 @@
  * client's res folder (client_location). Textures are converted to PNG with ImageMagick and cached.
  */
 namespace ClientAssets {
+	/**
+	 * Read what is read once (the res folder from client_location, the objects' names) at startup, on the main
+	 * thread, so the dashboard's worker threads only ever read it: they must not query the CDClient or read settings.
+	 */
+	void Preload();
+
 	std::string ItemName(LOT lot);
 
 	// An object's raw Objects.name (not localized), or nullopt when the LOT isn't in the CDClient
@@ -27,10 +33,14 @@ namespace ClientAssets {
 	// Relative path inside res/ using only safe characters and no traversal
 	bool IsSafeAssetPath(const std::string& path);
 
-	// A DDS or PNG texture from res/ as PNG, scaled down to fit maxSize. `opaque` drops the alpha channel first, for
+	// A DDS or PNG texture from res/ as PNG, scaled down to fit maxSize (converted once, even when asked for from
+	// several threads at once). `opaque` drops the alpha channel first, for
 	// textures whose alpha isn't transparency (terrain textures have alpha 0 everywhere; kept, they'd come out black)
 	std::optional<std::string> TextureAsPng(const std::string& assetPath, uint32_t maxSize, bool opaque = false);
 
+	// Whether TextureAsPng has the texture converted already (so it's quick)
+	bool TextureAsPngReady(const std::string& assetPath, uint32_t maxSize, bool opaque = false);
+
 	// Raw bytes of a file inside res/ (matched ignoring case)
 	std::optional<std::string> ReadResFile(const std::string& relativePath);
 
@@ -40,7 +50,7 @@ namespace ClientAssets {
 	// The client's res/ folder (empty when client_location isn't set)
 	std::filesystem::path ResFolder();
 
-	// ResolveResFile inside a res/ folder from ResFolder: reads no settings, so worker threads may call it
+	// ResolveResFile inside a res/ folder from ResFolder
 	std::optional<std::filesystem::path> ResolveResFile(const std::string& relativePath, const std::filesystem::path& res);
 
 	// The res/-relative path of a file called `fileName` (ignoring case) anywhere under res/<folder>

dDashboardServer/routes/DashboardRoutes.cpp

@@ -37,24 +37,21 @@ namespace {
 		return name.empty() ? "Zone " + std::to_string(zoneId) : name;
 	}
 
-	nlohmann::json g_ZoneNamesCache;
+	// Built once (worker threads read it too, so never lazily twice)
-	bool g_ZoneNamesCached = false;
-
 	const nlohmann::json& GetZoneNamesJson() {
-		if (g_ZoneNamesCached) return g_ZoneNamesCache;
+		static const nlohmann::json names = [] {
-
+			nlohmann::json names;
-		g_ZoneNamesCache["0"] = "Character Select";
+			names["0"] = "Character Select";
-		auto keys = Locale::GetPhraseIdsWithPrefix("ZoneTable_");
+			for (const auto& key : Locale::GetPhraseIdsWithPrefix("ZoneTable_")) {
-		for (const auto& key : keys) {
+				if (key.find("_DisplayDescription") == std::string::npos) continue;
-			if (key.find("_DisplayDescription") == std::string::npos) continue;
+				const auto start = std::string("ZoneTable_").length();
-			auto start = std::string("ZoneTable_").length();
+				const auto end = key.find("_DisplayDescription");
-			auto end = key.find("_DisplayDescription");
+				const auto& name = Locale::GetPhrase(key);
-			auto idStr = key.substr(start, end - start);
+				if (!name.empty()) names[key.substr(start, end - start)] = name;
-			const auto& name = Locale::GetPhrase(key);
+			}
-			if (!name.empty()) g_ZoneNamesCache[idStr] = name;
+			return names;
-		}
+		}();
-		g_ZoneNamesCached = true;
+		return names;
-		return g_ZoneNamesCache;
 	}
 
 	nlohmann::json ParseCharacterXml(const std::string& xml) {

dDashboardServer/routes/OnceCache.h

@@ -0,0 +1,80 @@
+#pragma once
+
+#include <chrono>
+#include <exception>
+#include <functional>
+#include <future>
+#include <map>
+#include <mutex>
+
+/**
+ * A value per key, built the first time it is asked for and kept: for zone data built from the client's files (a
+ * zone's terrain, its scene objects), which worker threads build while the web thread may ask for it too. Any thread.
+ * Asked for again while it is being built, the caller waits for that build instead of starting another. A build that
+ * throws is forgotten (the next caller tries again) and the exception reaches everyone waiting for it.
+ */
+template<typename Key, typename Value>
+class OnceCache {
+public:
+	// Whether the value is built (never waits)
+	bool Ready(const Key& key) const {
+		std::lock_guard lock(m_Mutex);
+		const auto it = m_Entries.find(key);
+		return it != m_Entries.end() && it->second.wait_for(std::chrono::seconds(0)) == std::future_status::ready;
+	}
+
+	/**
+	 * The value, built with `build` when nobody has yet. The reference stays valid: values are never replaced or
+	 * removed once built.
+	 */
+	const Value& Get(const Key& key, const std::function<Value()>& build) {
+		std::shared_future<Value> future;
+		std::promise<Value> promise;
+		bool mine = false;
+		{
+			std::lock_guard lock(m_Mutex);
+			const auto it = m_Entries.find(key);
+			if (it != m_Entries.end()) {
+				future = it->second;
+			} else {
+				future = promise.get_future().share();
+				m_Entries.emplace(key, future);
+				mine = true;
+			}
+		}
+		if (mine) {
+			try {
+				promise.set_value(build());
+			} catch (...) {
+				{
+					std::lock_guard lock(m_Mutex);
+					m_Entries.erase(key);
+				}
+				promise.set_exception(std::current_exception());
+			}
+		}
+		// The map keeps its own copy of the future, so what get() refers to outlives this call (unless the build threw,
+		// in which case get() throws)
+		return WaitFor(key, future);
+	}
+
+	size_t Size() const {
+		std::lock_guard lock(m_Mutex);
+		return m_Entries.size();
+	}
+
+private:
+	const Value& WaitFor(const Key& key, const std::shared_future<Value>& future) {
+		future.wait();
+		std::lock_guard lock(m_Mutex);
+		const auto it = m_Entries.find(key);
+		if (it == m_Entries.end()) {
+			future.get(); // the build failed: rethrows its exception
+			std::terminate(); // unreachable: a successful build is never removed
+		}
+		return it->second.get();
+	}
+
+	mutable std::mutex m_Mutex;
+	std::map<Key, std::shared_future<Value>> m_Entries;
+};

dDashboardServer/routes/ReportRoutes.cpp

@@ -1,4 +1,7 @@
 #include "ReportRoutes.h"
+#include "OnceCache.h"
+#include <deque>
+#include <future>
 #include "ZonePaths.h"
 
 #include <algorithm>
@@ -387,18 +390,72 @@ namespace {
 		std::string luzPath; // relative to res/maps
 	};
 
-	std::optional<ZoneInfo> GetZoneInfo(uint32_t zoneId) {
+	// Standard base64 (plain C++, no mongoose: zone data is built on worker threads)
-		static std::map<uint32_t, std::optional<ZoneInfo>> cache;
+	std::string Base64(std::string_view bytes) {
-		if (const auto it = cache.find(zoneId); it != cache.end()) return it->second;
+		static constexpr char ALPHABET[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-		auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT zoneName, DisplayDescription FROM ZoneTable WHERE zoneID = ?;");
+		std::string out;
-		stmt.bind(1, static_cast<int>(zoneId));
+		out.reserve((bytes.size() + 2) / 3 * 4);
-		auto result = stmt.execQuery();
+		size_t i = 0;
-		std::optional<ZoneInfo> info;
+		for (; i + 2 < bytes.size(); i += 3) {
-		if (!result.eof()) {
+			const uint32_t n = (static_cast<uint8_t>(bytes[i]) << 16) | (static_cast<uint8_t>(bytes[i + 1]) << 8) | static_cast<uint8_t>(bytes[i + 2]);
-			const std::string description = result.getStringField("DisplayDescription", "");
+			for (const int shift : { 18, 12, 6, 0 }) out += ALPHABET[(n >> shift) & 63];
-			info = ZoneInfo{ description.empty() ? "Zone " + std::to_string(zoneId) : description, result.getStringField("zoneName", "") };
 		}
-		return cache[zoneId] = info;
+		if (i < bytes.size()) {
+			uint32_t n = static_cast<uint8_t>(bytes[i]) << 16;
+			if (i + 1 < bytes.size()) n |= static_cast<uint8_t>(bytes[i + 1]) << 8;
+			out += ALPHABET[(n >> 18) & 63];
+			out += ALPHABET[(n >> 12) & 63];
+			out += i + 1 < bytes.size() ? ALPHABET[(n >> 6) & 63] : '=';
+			out += '=';
+		}
+		return out;
+	}
+
+	std::map<uint32_t, ZoneInfo> ReadZoneTable() {
+		std::map<uint32_t, ZoneInfo> zones;
+		try {
+			auto result = CDClientDatabase::ExecuteQuery("SELECT zoneID, zoneName, DisplayDescription FROM ZoneTable;");
+			for (; !result.eof(); result.nextRow()) {
+				const std::string description = result.getStringField("DisplayDescription", "");
+				const auto zoneId = static_cast<uint32_t>(result.getIntField("zoneID", 0));
+				zones.try_emplace(zoneId, ZoneInfo{ description.empty() ? "Zone " + std::to_string(zoneId) : description, result.getStringField("zoneName", "") });
+			}
+		} catch (const std::exception& ex) {
+			LOG("Could not read the zone table: %s", ex.what());
+		}
+		return zones;
+	}
+
+	// ZoneTable, read at startup (PreloadZoneData) so worker threads never query the CDClient
+	const std::map<uint32_t, ZoneInfo>& ZoneTable() {
+		static const auto zones = ReadZoneTable();
+		return zones;
+	}
+
+	std::optional<ZoneInfo> GetZoneInfo(uint32_t zoneId) {
+		const auto& zones = ZoneTable();
+		const auto it = zones.find(zoneId);
+		if (it == zones.end()) return std::nullopt;
+		return it->second;
+	}
+
+	// mapTextureResource: texture id -> file name (lowercase), read at startup
+	const std::map<uint32_t, std::string>& TerrainTextureNames() {
+		static const auto names = [] {
+			std::map<uint32_t, std::string> names;
+			try {
+				auto result = CDClientDatabase::ExecuteQuery("SELECT id, texturepath FROM mapTextureResource;");
+				for (; !result.eof(); result.nextRow()) {
+					std::string file = result.getStringField(1, "");
+					std::transform(file.begin(), file.end(), file.begin(), ::tolower);
+					names.try_emplace(static_cast<uint32_t>(result.getIntField(0, 0)), file);
+				}
+			} catch (const std::exception& ex) {
+				LOG("Could not read the terrain textures: %s", ex.what());
+			}
+			return names;
+		}();
+		return names;
 	}
 
 	/**
@@ -457,10 +514,9 @@ namespace {
 	}
 
 	// Terrain for a zone as JSON with base64 16-bit heights, built once per zone
-	std::optional<std::string> TerrainJson(uint32_t zoneId) {
+	OnceCache<uint32_t, std::optional<std::string>> g_TerrainJson;
-		static std::map<uint32_t, std::optional<std::string>> cache;
-		if (const auto it = cache.find(zoneId); it != cache.end()) return it->second;
 
+	std::optional<std::string> BuildTerrainJson(uint32_t zoneId) {
 		std::optional<std::string> json;
 		const auto zone = GetZoneInfo(zoneId);
 		if (zone) {
@@ -469,8 +525,7 @@ namespace {
 			const auto grid = raw ? TerrainMap::Parse(*raw, 1024) : std::nullopt;
 			if (grid) {
 				const auto packed = TerrainMap::Quantize(*grid);
-				std::string encoded(packed.size() * 4 / 3 + 8, '\0');
+				const auto encoded = Base64({ reinterpret_cast<const char*>(packed.data()), packed.size() * sizeof(packed[0]) });
-				encoded.resize(mg_base64_encode(reinterpret_cast<const unsigned char*>(packed.data()), packed.size(), encoded.data(), encoded.size()));
 				nlohmann::json terrain{
 					{"zone", zoneId}, {"minX", grid->minX}, {"minZ", grid->minZ}, {"step", grid->step},
 					{"width", grid->width}, {"height", grid->height}, {"minY", grid->minY}, {"maxY", grid->maxY}, {"heights", encoded}
@@ -479,7 +534,11 @@ namespace {
 				json = terrain.dump();
 			}
 		}
-		return cache[zoneId] = json;
+		return json;
+	}
+
+	std::optional<std::string> TerrainJson(uint32_t zoneId) {
+		return g_TerrainJson.Get(zoneId, [zoneId] { return BuildTerrainJson(zoneId); });
 	}
 
 	std::string DayText(uint32_t day) {
@@ -534,11 +593,6 @@ std::optional<std::string> ZoneTerrainJson(uint32_t zoneId) {
 }
 
 namespace {
-	std::string Base64(std::string_view bytes) {
-		std::string encoded(bytes.size() * 4 / 3 + 8, '\0');
-		encoded.resize(mg_base64_encode(reinterpret_cast<const unsigned char*>(bytes.data()), bytes.size(), encoded.data(), encoded.size()));
-		return encoded;
-	}
 }
 
 std::optional<Raw::Raw> ZoneRaw(uint32_t zoneId) {
@@ -546,17 +600,70 @@ std::optional<Raw::Raw> ZoneRaw(uint32_t zoneId) {
 	return zone ? ReadZoneRaw(*zone) : std::nullopt;
 }
 
+std::shared_ptr<const Raw::Raw> ZoneRawShared(uint32_t zoneId) {
+	// The last few zones: their chunks, layers and flairs are built one after the other when a zone is first viewed
+	constexpr size_t KEEP = 2;
+	static std::mutex mutex;
+	static std::deque<std::pair<uint32_t, std::shared_future<std::shared_ptr<const Raw::Raw>>>> recent;
+	std::promise<std::shared_ptr<const Raw::Raw>> promise;
+	std::shared_future<std::shared_ptr<const Raw::Raw>> future;
+	{
+		std::lock_guard lock(mutex);
+		const auto it = std::find_if(recent.begin(), recent.end(), [zoneId](const auto& entry) { return entry.first == zoneId; });
+		if (it != recent.end()) return it->second.get(); // may wait for the thread reading it
+		future = promise.get_future().share();
+		recent.emplace_back(zoneId, future);
+		if (recent.size() > KEEP) recent.pop_front();
+	}
+	std::shared_ptr<const Raw::Raw> raw;
+	try {
+		if (auto read = ZoneRaw(zoneId)) raw = std::make_shared<const Raw::Raw>(std::move(*read));
+	} catch (const std::exception& ex) {
+		LOG("Could not read the terrain of zone %u: %s", zoneId, ex.what());
+	}
+	promise.set_value(raw);
+	return raw;
+}
+
 namespace {
 	std::string_view BytesOf(const std::vector<uint8_t>& bytes) {
 		return { reinterpret_cast<const char*>(bytes.data()), bytes.size() };
 	}
 }
 
+namespace {
+	OnceCache<uint32_t, std::optional<std::string>> g_TerrainChunks;
+	OnceCache<uint32_t, std::optional<std::string>> g_TerrainLayers;
+	OnceCache<uint32_t, std::string> g_TerrainTexturePaths; // id -> path under res/ ("" when the client lacks it)
+}
+
+bool ZoneTerrainJsonReady(uint32_t zoneId) {
+	return g_TerrainJson.Ready(zoneId);
+}
+
+bool ZoneTerrainChunksReady(uint32_t zoneId) {
+	return g_TerrainChunks.Ready(zoneId);
+}
+
+bool ZoneTerrainLayersReady(uint32_t zoneId) {
+	return g_TerrainLayers.Ready(zoneId);
+}
+
+std::optional<std::string> ZoneLuzPath(uint32_t zoneId) {
+	const auto zone = GetZoneInfo(zoneId);
+	if (!zone || !zone->luzPath.ends_with(".luz")) return std::nullopt;
+	return zone->luzPath;
+}
+
+void PreloadZoneData() {
+	ZoneTable();
+	TerrainTextureNames();
+}
+
 std::optional<std::string> ZoneTerrainChunksJson(uint32_t zoneId) {
-	static std::map<uint32_t, std::optional<std::string>> cache;
+	return g_TerrainChunks.Get(zoneId, [zoneId]() -> std::optional<std::string> {
-	if (const auto it = cache.find(zoneId); it != cache.end()) return it->second;
 	std::optional<std::string> json;
-	if (const auto raw = ZoneRaw(zoneId)) {
+	if (const auto raw = ZoneRawShared(zoneId)) {
 		nlohmann::json out{ {"zone", zoneId}, {"chunks", nlohmann::json::array()} };
 		std::set<uint32_t> textures;
 		for (const auto& chunk : raw->chunks) {
@@ -575,15 +682,15 @@ std::optional<std::string> ZoneTerrainChunksJson(uint32_t zoneId) {
 		out["textures"] = textures;
 		json = out.dump();
 	}
-	return cache[zoneId] = json;
+	return json;
+	});
 }
 
 std::optional<std::string> ZoneTerrainLayersJson(uint32_t zoneId) {
-	static std::map<uint32_t, std::optional<std::string>> cache;
+	return g_TerrainLayers.Get(zoneId, [zoneId]() -> std::optional<std::string> {
-	if (const auto it = cache.find(zoneId); it != cache.end()) return it->second;
 	std::optional<std::string> json;
 	const auto zone = GetZoneInfo(zoneId);
-	const auto raw = zone ? ReadZoneRaw(*zone) : std::nullopt;
+	const auto raw = zone ? ZoneRawShared(zoneId) : nullptr;
 	if (raw) {
 		// Scene names from the .luz: the terrain's scene map holds the ids of its general scenes (audio scenes share them)
 		std::map<uint32_t, std::string> names;
@@ -615,30 +722,42 @@ std::optional<std::string> ZoneTerrainLayersJson(uint32_t zoneId) {
 		for (const auto& chunk : raw->chunks) flairs += chunk.flairs.size();
 		json = nlohmann::json{ {"zone", zoneId}, {"version", raw->version}, {"scenes", scenes}, {"chunks", chunks}, {"flairs", flairs} }.dump();
 	}
-	return cache[zoneId] = json;
+	return json;
+	});
+}
+
+namespace {
+	// Where terrain texture `textureId` is under res/, found once ("" when the client lacks it)
+	const std::string& TerrainTexturePath(uint32_t textureId) {
+		return g_TerrainTexturePaths.Get(textureId, [textureId] {
+			const auto& names = TerrainTextureNames();
+			const auto name = names.find(textureId);
+			if (name == names.end()) return std::string{};
+			if (ClientAssets::ResolveResFile("textures/env/" + name->second)) return "textures/env/" + name->second;
+			return ClientAssets::FindResFile("textures", name->second).value_or("");
+		});
+	}
+
+	// Tiled 4 times per chunk: 512 pixels is plenty and keeps the page light
+	constexpr uint32_t TERRAIN_TEXTURE_SIZE = 512;
 }
 
 std::optional<std::string> TerrainTextureFile(uint32_t textureId) {
-	static std::map<uint32_t, std::string> paths; // id -> path under res/, found once
+	const auto& path = TerrainTexturePath(textureId);
-	if (!paths.contains(textureId)) {
+	if (path.empty()) return std::nullopt;
-		auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT texturepath FROM mapTextureResource WHERE id = ? LIMIT 1;");
+	return ClientAssets::TextureAsPng(path, TERRAIN_TEXTURE_SIZE, true);
-		stmt.bind(1, static_cast<int>(textureId));
+}
-		auto result = stmt.execQuery();
+
-		if (result.eof()) return std::nullopt;
+bool TerrainTextureReady(uint32_t textureId) {
-		std::string file = result.getStringField(0);
+	if (!g_TerrainTexturePaths.Ready(textureId)) return false;
-		std::transform(file.begin(), file.end(), file.begin(), ::tolower);
+	const auto& path = TerrainTexturePath(textureId);
-		std::string found;
+	return path.empty() || ClientAssets::TextureAsPngReady(path, TERRAIN_TEXTURE_SIZE, true);
-		if (ClientAssets::ReadResFile("textures/env/" + file)) found = "textures/env/" + file;
-		else if (const auto listing = ClientAssets::FindResFile("textures", file)) found = *listing;
-		paths[textureId] = found;
-	}
-	if (paths[textureId].empty()) return std::nullopt;
-	// Tiled 4 times per chunk: 512 pixels is plenty and keeps the page light
-	return ClientAssets::TextureAsPng(paths[textureId], 512, true);
 }
 
 std::optional<nlohmann::json> ZonePropertyAreasJson(uint32_t zoneId) {
+	static std::mutex mutex;
 	static std::map<uint32_t, std::optional<nlohmann::json>> cache;
+	std::lock_guard lock(mutex);
 	if (const auto it = cache.find(zoneId); it != cache.end()) return it->second;
 	std::optional<nlohmann::json> json;
 	const auto zone = GetZoneInfo(zoneId);

dDashboardServer/routes/ReportRoutes.h

@@ -22,6 +22,25 @@ std::optional<std::string> ZoneTerrainJson(uint32_t zoneId);
 
 #include "Raw.h"
 
+/**
+ * The zone data below is built on the dashboard's worker threads too, so it is thread safe: results are built once
+ * (OnceCache) and the CDClient tables it needs are read at startup by PreloadZoneData (workers never query the
+ * CDClient). The ...Ready functions say whether a result is built, so a route can answer it at once.
+ */
+void PreloadZoneData();
+bool ZoneTerrainJsonReady(uint32_t zoneId);
+bool ZoneTerrainChunksReady(uint32_t zoneId);
+bool ZoneTerrainLayersReady(uint32_t zoneId);
+bool TerrainTextureReady(uint32_t textureId);
+
+// A zone's .luz file (relative to res/maps) from ZoneTable
+std::optional<std::string> ZoneLuzPath(uint32_t zoneId);
+
+#include <memory>
+
+// ZoneRaw, shared: the last few zones read are kept, and a zone asked for by several threads at once is read once
+std::shared_ptr<const Raw::Raw> ZoneRawShared(uint32_t zoneId);
+
 // A zone's terrain file (.raw) read whole, as its .luz names it; nullopt without client files or when it's damaged
 std::optional<Raw::Raw> ZoneRaw(uint32_t zoneId);
 

dDashboardServer/routes/Scenery.cpp

@@ -19,9 +19,11 @@
 #include "NifFile.h"
 #include "ReportRoutes.h"
 #include "RouteUtils.h"
+#include "OnceCache.h"
 #include "TtlCache.h"
 #include "Web.h"
 #include "WorkerPool.h"
+#include "Workers.h"
 #include "WorldScene.h"
 #include "ZonePaths.h"
 
@@ -78,13 +80,10 @@ namespace {
 		std::unordered_map<std::string, std::vector<std::string>> byName;
 	};
 
-	const FileIndex& Files() {
+	FileIndex IndexFiles() {
-		static std::optional<FileIndex> index;
+		FileIndex index;
-		if (index) return *index;
+		const auto res = ClientAssets::ResFolder();
-		index.emplace();
+		if (res.empty()) return index;
-		const auto client = Game::config->GetValue("client_location");
-		if (client.empty()) return *index;
-		const auto res = std::filesystem::path(client) / "res";
 		std::error_code ec;
 		for (const char* folder : { "mesh", "textures", "animations" }) {
 			// Unpacked clients keep their original case, so the top folder is matched ignoring case too
@@ -93,12 +92,18 @@ namespace {
 				for (auto it = std::filesystem::recursive_directory_iterator(top.path(), ec); !ec && it != std::filesystem::recursive_directory_iterator(); it.increment(ec)) {
 					if (!it->is_regular_file(ec)) continue;
 					const auto relative = Lower(std::filesystem::relative(it->path(), res, ec).generic_string());
-					index->paths.insert(relative);
+					index.paths.insert(relative);
-					index->byName[Lower(it->path().filename().string())].push_back(relative);
+					index.byName[Lower(it->path().filename().string())].push_back(relative);
 				}
 			}
 		}
-		return *index;
+		return index;
+	}
+
+	// Built at startup (Scenery::Preload); read only afterwards
+	const FileIndex& Files() {
+		static const FileIndex index = IndexFiles();
+		return index;
 	}
 
 	// A texture a model names: next to the model if it's there (the usual case), else the file of that name sharing
@@ -140,24 +145,27 @@ namespace {
 	};
 
 	// Every LOT with a render component, read once (Objects has no index on id, so it is read whole too)
-	const std::unordered_map<uint32_t, RenderInfo>& RenderInfos() {
+	std::unordered_map<uint32_t, RenderInfo> ReadRenderInfos() {
-		static std::optional<std::unordered_map<uint32_t, RenderInfo>> infos;
+		std::unordered_map<uint32_t, RenderInfo> infos;
-		if (infos) return *infos;
-		infos.emplace();
 		try {
 			auto row = CDClientDatabase::ExecuteQuery(
 				"SELECT cr.id, rc.render_asset FROM ComponentsRegistry cr JOIN RenderComponent rc ON rc.id = cr.component_id "
 				"WHERE cr.component_type = " + std::to_string(RENDER_COMPONENT) + ";");
-			for (; !row.eof(); row.nextRow()) infos->try_emplace(static_cast<uint32_t>(row.getIntField(0)), RenderInfo{ row.getStringField(1, ""), "" });
+			for (; !row.eof(); row.nextRow()) infos.try_emplace(static_cast<uint32_t>(row.getIntField(0)), RenderInfo{ row.getStringField(1, ""), "" });
 			auto types = CDClientDatabase::ExecuteQuery("SELECT id, type FROM Objects;");
 			for (; !types.eof(); types.nextRow()) {
-				const auto it = infos->find(static_cast<uint32_t>(types.getIntField(0)));
+				const auto it = infos.find(static_cast<uint32_t>(types.getIntField(0)));
-				if (it != infos->end() && it->second.type.empty()) it->second.type = types.getStringField(1, "");
+				if (it != infos.end() && it->second.type.empty()) it->second.type = types.getStringField(1, "");
 			}
 		} catch (const std::exception& ex) {
 			LOG("Could not read the render components of objects: %s", ex.what());
 		}
-		return *infos;
+		return infos;
+	}
+
+	const std::unordered_map<uint32_t, RenderInfo>& RenderInfos() {
+		static const auto infos = ReadRenderInfos();
+		return infos;
 	}
 
 	struct Model {
@@ -176,30 +184,33 @@ namespace {
 		if (info == infos.end()) return {};
 		const auto draw = WorldScene::ClientDraws(object, info->second.type);
 		if (draw == WorldScene::eClientDraw::NO_MODEL) return {};
+		static std::mutex mutex;
 		static std::unordered_map<std::string, std::string> resolved;
 		const auto& stored = object.nifName.empty() ? info->second.asset : object.nifName;
-		auto it = resolved.find(stored);
+		{
-		if (it == resolved.end()) it = resolved.emplace(stored, ResolveModel(stored)).first;
+			std::lock_guard lock(mutex);
-		return { it->second, draw == WorldScene::eClientDraw::HIDDEN };
+			if (const auto it = resolved.find(stored); it != resolved.end()) return { it->second, draw == WorldScene::eClientDraw::HIDDEN };
+		}
+		auto path = ResolveModel(stored);
+		std::lock_guard lock(mutex);
+		resolved.try_emplace(stored, path);
+		return { std::move(path), draw == WorldScene::eClientDraw::HIDDEN };
 	}
 
 	std::optional<std::string> LuzPath(uint32_t zone) {
-		auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT zoneName FROM ZoneTable WHERE zoneID = ? LIMIT 1;");
+		return ZoneLuzPath(zone);
-		stmt.bind(1, static_cast<int>(zone));
-		auto result = stmt.execQuery();
-		if (result.eof()) return std::nullopt;
-		std::string path = result.getStringField(0, "");
-		if (!path.ends_with(".luz")) return std::nullopt;
-		return path;
 	}
 
 	double Round(float value, double scale) { return std::round(static_cast<double>(value) * scale) / scale; }
 
+	/**
+	 * A zone's models and scenery manifest, built once (g_Zones). Once shared, assets, index, flairModels and the
+	 * warmed flags are guarded by g_ZoneMutex: the flairs' models join assets when their manifest is built.
+	 */
 	struct ZoneScenery {
 		std::vector<std::string> assets;      // res paths of models, indexed by the manifests
 		std::map<std::string, size_t> index;  // res path -> its index in assets
 		std::string json;
-		std::optional<std::string> flairs;    // the flairs' manifest, built when first asked for (their models join assets)
 		std::unordered_set<std::string> flairModels; // res paths of the flairs' models, converted ahead of others
 		bool warmedScenery{};                 // WarmUp queued the scenery's models
 		bool warmedFlairs{};                  // and the flairs'
@@ -211,14 +222,12 @@ namespace {
 		}
 	};
 
-	std::optional<ZoneScenery>& Zone(uint32_t zoneId) {
+	std::mutex g_ZoneMutex;
-		static std::map<uint32_t, std::optional<ZoneScenery>> cache;
-		if (const auto it = cache.find(zoneId); it != cache.end()) return it->second;
-		auto& entry = cache[zoneId];
 
+	std::shared_ptr<ZoneScenery> BuildZone(uint32_t zoneId) {
 		const auto luzPath = LuzPath(zoneId);
 		const auto luz = luzPath ? ClientAssets::ReadResFile("maps/" + *luzPath) : std::nullopt;
-		if (!luz) return entry;
+		if (!luz) return nullptr;
 		const auto folder = luzPath->substr(0, luzPath->find_last_of('/') + 1);
 
 		ZoneScenery scenery;
@@ -247,22 +256,36 @@ namespace {
 			{"zone", zoneId}, {"sky", sky}, {"assets", scenery.assets},
 			{"objects", { {"asset", assetOf}, {"pos", positions}, {"rot", rotations}, {"scale", scales}, {"hidden", hidden} }}
 		}.dump();
-		entry = std::move(scenery);
+		return std::make_shared<ZoneScenery>(std::move(scenery));
-		return entry;
+	}
+
+	OnceCache<uint32_t, std::shared_ptr<ZoneScenery>> g_Zones;
+
+	// The zone's scenery, built when first asked for (any thread); nullptr without client files
+	std::shared_ptr<ZoneScenery> Zone(uint32_t zoneId) {
+		return g_Zones.Get(zoneId, [zoneId] { return BuildZone(zoneId); });
+	}
+
+	// The same, only when it is built already (never waits)
+	std::shared_ptr<ZoneScenery> ZoneIfBuilt(uint32_t zoneId) {
+		return g_Zones.Ready(zoneId) ? Zone(zoneId) : nullptr;
 	}
 
 	// FlairTable: flair id -> the model's res path (empty when the client lacks it), read once
-	const std::unordered_map<uint32_t, std::string>& FlairModels() {
+	std::unordered_map<uint32_t, std::string> ReadFlairModels() {
-		static std::optional<std::unordered_map<uint32_t, std::string>> models;
+		std::unordered_map<uint32_t, std::string> models;
-		if (models) return *models;
-		models.emplace();
 		try {
 			auto row = CDClientDatabase::ExecuteQuery("SELECT id, asset FROM FlairTable;");
-			for (; !row.eof(); row.nextRow()) models->try_emplace(static_cast<uint32_t>(row.getIntField(0)), ResolveModel(row.getStringField(1, "")));
+			for (; !row.eof(); row.nextRow()) models.try_emplace(static_cast<uint32_t>(row.getIntField(0)), ResolveModel(row.getStringField(1, "")));
 		} catch (const std::exception& ex) {
 			LOG("Could not read the flairs: %s", ex.what());
 		}
-		return *models;
+		return models;
+	}
+
+	const std::unordered_map<uint32_t, std::string>& FlairModels() {
+		static const auto models = ReadFlairModels();
+		return models;
 	}
 
 	/**
@@ -275,18 +298,16 @@ namespace {
 	 * The flairs' manifest (as the scenery's, plus a tint per flair), from the zone's terrain file. A flair's color
 	 * tints its model; 63 is full strength (the files use 0 to 63 for most flairs, a little more for brighter ones).
 	 */
-	const std::optional<std::string>& Flairs(uint32_t zoneId, ZoneScenery& scenery) {
+	std::optional<std::string> BuildFlairs(uint32_t zoneId, ZoneScenery& scenery) {
-		if (scenery.flairs) return scenery.flairs;
+		const auto raw = ZoneRawShared(zoneId);
-		const auto raw = ZoneRaw(zoneId);
 		const auto& models = FlairModels();
-		nlohmann::json assetOf = nlohmann::json::array(), positions = nlohmann::json::array(), rotations = nlohmann::json::array(),
+		std::vector<const std::string*> modelOf;
-			scales = nlohmann::json::array(), colors = nlohmann::json::array();
+		nlohmann::json positions = nlohmann::json::array(), rotations = nlohmann::json::array(), scales = nlohmann::json::array(), colors = nlohmann::json::array();
 		for (const auto& chunk : raw ? raw->chunks : std::vector<Raw::Chunk>{}) {
 			for (const auto& flair : chunk.flairs) {
 				const auto model = models.find(flair.id);
 				if (model == models.end() || model->second.empty() || !std::isfinite(flair.position.x)) continue;
-				assetOf.push_back(scenery.IndexOf(model->second));
+				modelOf.push_back(&model->second);
-				scenery.flairModels.insert(model->second);
 				for (const auto value : { flair.position.x, flair.position.y, flair.position.z }) positions.push_back(Round(value, 100.0));
 				// Radians about x, y and z, applied in that order
 				const double c1 = std::cos(flair.rotation.x / 2), c2 = std::cos(flair.rotation.y / 2), c3 = std::cos(flair.rotation.z / 2);
@@ -298,11 +319,32 @@ namespace {
 				for (const auto value : { flair.colorR, flair.colorG, flair.colorB }) colors.push_back(value);
 			}
 		}
-		scenery.flairs = nlohmann::json{
+		// The flairs' models join the zone's list, which mesh requests read meanwhile
-			{"zone", zoneId}, {"sky", -1}, {"assets", scenery.assets}, {"distance", FLAIR_DISTANCE}, {"colorScale", 1.0 / 63.0},
+		nlohmann::json assetOf = nlohmann::json::array();
+		std::vector<std::string> assets;
+		{
+			std::lock_guard lock(g_ZoneMutex);
+			for (const auto* model : modelOf) {
+				assetOf.push_back(scenery.IndexOf(*model));
+				scenery.flairModels.insert(*model);
+			}
+			assets = scenery.assets;
+		}
+		return nlohmann::json{
+			{"zone", zoneId}, {"sky", -1}, {"assets", assets}, {"distance", FLAIR_DISTANCE}, {"colorScale", 1.0 / 63.0},
 			{"objects", { {"asset", assetOf}, {"pos", positions}, {"rot", rotations}, {"scale", scales}, {"color", colors} }}
 		}.dump();
-		return scenery.flairs;
+	}
+
+	OnceCache<uint32_t, std::optional<std::string>> g_Flairs;
+
+	/**
+	 * The flairs' manifest (as the scenery's, plus a tint per flair), from the zone's terrain file. A flair's color
+	 * tints its model; 63 is full strength (the files use 0 to 63 for most flairs, a little more for brighter ones).
+	 * Built when first asked for (any thread).
+	 */
+	const std::optional<std::string>& Flairs(uint32_t zoneId, ZoneScenery& scenery) {
+		return g_Flairs.Get(zoneId, [zoneId, &scenery] { return BuildFlairs(zoneId, scenery); });
 	}
 
 	constexpr uint32_t FORMAT_VERSION = 1; // bump when NifFile's output changes, so cached files are rebuilt
@@ -518,7 +560,7 @@ namespace {
 
 	// ---- Converting on worker threads, so a big model never holds up the web server's one thread ----
 
-	WorkerPool g_Pool;
+	WorkerPool& Pool() { return Workers::Pool(); }
 
 	constexpr uintmax_t SMALL_MODEL_BYTES = 256 * 1024;       // .nif files this small convert in the pool's fast lane
 	constexpr uintmax_t LARGE_MODEL_BYTES = 4 * 1024 * 1024;  // and this big wait behind everything smaller
@@ -550,8 +592,11 @@ namespace {
 	}
 
 	// Flairs (small, and drawn around the camera) and small models first; big ones behind the rest
-	WorkerPool::ePriority PriorityOf(const ZoneScenery& zone, const std::string& path, const std::filesystem::path& file) {
+	WorkerPool::ePriority PriorityOf(const ZoneScenery* zone, const std::string& path, const std::filesystem::path& file) {
-		if (zone.flairModels.contains(path)) return WorkerPool::ePriority::URGENT;
+		if (zone) {
+			std::lock_guard lock(g_ZoneMutex);
+			if (zone->flairModels.contains(path)) return WorkerPool::ePriority::URGENT;
+		}
 		std::error_code ec;
 		const auto size = std::filesystem::file_size(file, ec);
 		if (ec || size <= SMALL_MODEL_BYTES) return WorkerPool::ePriority::URGENT;
@@ -563,11 +608,10 @@ namespace {
 	/**
 	 * Convert models of a zone ahead of time (onto the disk cache), at the LOD its viewer last asked for, while
 	 * someone views it. The lowest priority: only when nothing else waits. Smallest first; `front` puts these before
-	 * the zone's other queued ones (the flairs). Web thread.
+	 * the zone's other queued ones (the flairs). Any thread.
 	 */
 	void WarmUp(uint32_t zoneId, const std::vector<std::string>& paths, bool front) {
-		if (!g_Pool.Running() || paths.empty()) return;
+		if (!Pool().Running() || paths.empty()) return;
-		Files(); // built here: workers only read it
 		struct Item {
 			std::string path;
 			std::filesystem::path file;
@@ -576,9 +620,10 @@ namespace {
 		std::vector<Item> items;
 		std::set<std::string> seen;
 		std::error_code ec;
+		const auto res = ClientAssets::ResFolder();
 		for (const auto& path : paths) {
 			if (!seen.insert(path).second) continue;
-			const auto file = ClientAssets::ResolveResFile(path);
+			const auto file = ClientAssets::ResolveResFile(path, res);
 			if (file) items.push_back({ path, *file, std::filesystem::file_size(*file, ec) });
 		}
 		std::stable_sort(items.begin(), items.end(), [](const Item& a, const Item& b) { return a.size < b.size; });
@@ -587,10 +632,10 @@ namespace {
 		// Queued at the front in reverse, so they still run smallest first
 		if (front) std::reverse(items.begin(), items.end());
 		for (auto& item : items) {
-			g_Pool.Submit(WorkerPool::ePriority::BACKGROUND, [zoneId, group, path = std::move(item.path), file = std::move(item.file)] {
+			Pool().Submit(WorkerPool::ePriority::BACKGROUND, [zoneId, group, path = std::move(item.path), file = std::move(item.file)] {
 				const auto activity = Viewed(zoneId);
 				if (!activity || g_Disk.Total() >= WARM_DISK_BYTES) {
-					g_Pool.Cancel(group);
+					Pool().Cancel(group);
 					return;
 				}
 				const auto key = ModelKey(path, activity->lod);
@@ -601,15 +646,26 @@ namespace {
 		}
 	}
 
-	// Warm the zone's models when its manifest is asked for (again after nobody viewed it for a while)
+	/**
+	 * Warm the zone's models when its manifest is asked for (again after nobody viewed it for a while). Planning it
+	 * (finding the files) is left to a worker, so a manifest in memory is still answered at once.
+	 */
 	void WarmScenery(uint32_t zoneId, ZoneScenery& zone, bool flairs) {
-		if (!Viewed(zoneId)) zone.warmedScenery = zone.warmedFlairs = false;
+		std::vector<std::string> paths;
+		{
+			const bool idle = !Viewed(zoneId);
+			std::lock_guard lock(g_ZoneMutex);
+			if (idle) zone.warmedScenery = zone.warmedFlairs = false;
+			auto& warmed = flairs ? zone.warmedFlairs : zone.warmedScenery;
+			if (!warmed) {
+				warmed = true;
+				if (flairs) paths.assign(zone.flairModels.begin(), zone.flairModels.end());
+				else paths = zone.assets;
+			}
+		}
 		Touch(zoneId);
-		auto& warmed = flairs ? zone.warmedFlairs : zone.warmedScenery;
+		if (paths.empty() || !Pool().Running()) return;
-		if (warmed) return;
+		Pool().Submit(WorkerPool::ePriority::NORMAL, [zoneId, paths = std::move(paths), flairs] { WarmUp(zoneId, paths, flairs); });
-		warmed = true;
-		if (flairs) WarmUp(zoneId, { zone.flairModels.begin(), zone.flairModels.end() }, true);
-		else WarmUp(zoneId, zone.assets, false);
 	}
 
 	// The texture `name` of model `path` (textures[slot] of its encoded form) as a DDS file. Any thread.
@@ -636,87 +692,135 @@ namespace {
 		return dds;
 	}
 
-	// The model path of `asset` in the zone's manifests, or a 404 reply
+	// The model path of `asset` in the zone's manifests, building what's missing (any thread); nullopt: no such model
-	const std::string* AssetPath(HTTPReply& reply, uint32_t zoneId, uint32_t asset) {
+	std::optional<std::string> AssetPath(uint32_t zoneId, uint32_t asset) {
-		auto& zone = Zone(zoneId);
+		const auto zone = Zone(zoneId);
-		// The flairs' models join the list when their manifest is first built (a browser may still have it cached)
+		if (!zone) return std::nullopt;
-		if (zone && asset >= zone->assets.size()) Flairs(zoneId, *zone);
+		{
-		if (!zone || asset >= zone->assets.size()) {
+			std::lock_guard lock(g_ZoneMutex);
-			JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No such model in this zone");
+			if (asset < zone->assets.size()) return zone->assets[asset];
-			return nullptr;
 		}
-		return &zone->assets[asset];
+		// The flairs' models join the list when their manifest is first built (a browser may still have it cached)
+		Flairs(zoneId, *zone);
+		std::lock_guard lock(g_ZoneMutex);
+		if (asset < zone->assets.size()) return zone->assets[asset];
+		return std::nullopt;
+	}
+
+	// The same without building anything (never waits): nullopt when it isn't known yet
+	std::optional<std::string> AssetPathIfBuilt(uint32_t zoneId, uint32_t asset) {
+		const auto zone = ZoneIfBuilt(zoneId);
+		if (!zone) return std::nullopt;
+		std::lock_guard lock(g_ZoneMutex);
+		if (asset < zone->assets.size()) return zone->assets[asset];
+		return std::nullopt;
+	}
+
+	/**
+	 * Where model `asset` is, and how urgent converting it is, when that is known without building anything: for
+	 * the web thread, which only hands the work on
+	 */
+	struct Known {
+		std::optional<std::string> path;
+		std::optional<std::filesystem::path> file;
+		WorkerPool::ePriority priority{ WorkerPool::ePriority::NORMAL };
+	};
+
+	Known KnownAsset(uint32_t zoneId, uint32_t asset) {
+		Known known;
+		known.path = AssetPathIfBuilt(zoneId, asset);
+		if (!known.path) return known;
+		known.file = ClientAssets::ResolveResFile(*known.path, ClientAssets::ResFolder());
+		if (known.file) known.priority = PriorityOf(ZoneIfBuilt(zoneId).get(), *known.path, *known.file);
+		return known;
 	}
 }
 
 namespace Scenery {
+	void Preload() {
+		Files();
+		RenderInfos();
+		FlairModels();
+	}
+
 	std::optional<std::string> ZoneJson(uint32_t zoneId) {
-		auto& zone = Zone(zoneId);
+		const auto zone = Zone(zoneId);
 		if (!zone) return std::nullopt;
 		WarmScenery(zoneId, *zone, false);
 		return zone->json;
 	}
 
+	bool ZoneReady(uint32_t zoneId) {
+		return g_Zones.Ready(zoneId);
+	}
+
 	bool HasModel(const WorldScene::Object& object) {
 		const auto model = ModelFor(object);
 		return !model.path.empty() && !model.hidden;
 	}
 
 	std::optional<std::string> FlairsJson(uint32_t zoneId) {
-		auto& zone = Zone(zoneId);
+		const auto zone = Zone(zoneId);
 		if (!zone) return std::nullopt;
 		const auto& flairs = Flairs(zoneId, *zone);
 		WarmScenery(zoneId, *zone, true);
 		return flairs;
 	}
 
+	bool FlairsReady(uint32_t zoneId) {
+		return g_Zones.Ready(zoneId) && g_Flairs.Ready(zoneId);
+	}
+
 	void ReplyMesh(HTTPReply& reply, const HTTPContext& context, uint32_t zoneId, uint32_t asset, uint32_t lod) {
-		const auto* found = AssetPath(reply, zoneId, asset);
-		if (!found) return;
-		const auto path = *found;
 		lod = std::min(lod, MAX_LOD);
 		Touch(zoneId, lod);
-		if (const auto cached = g_Models.Get(ModelKey(path, lod))) return Binary(reply, *cached);
+		auto known = KnownAsset(zoneId, asset);
-		const auto file = ClientAssets::ResolveResFile(path);
+		if (known.path) {
-		if (!file) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "Could not read this model");
+			if (const auto cached = g_Models.Get(ModelKey(*known.path, lod))) return Binary(reply, *cached);
-		Files(); // built here: workers only read it
+		}
+		const auto res = ClientAssets::ResFolder();
 		const auto deferred = Web::Defer(reply, context);
-		g_Pool.Submit(PriorityOf(*Zone(zoneId), path, *file), [deferred, path, lod, file = *file] {
+		Pool().Submit(known.priority, [deferred, zoneId, asset, lod, res, known = std::move(known)] {
 			if (deferred.Cancelled()) return;
 			HTTPReply out;
-			const auto encoded = Encoded(path, lod, file);
+			const auto path = known.path ? known.path : AssetPath(zoneId, asset);
-			if (encoded) Binary(out, *encoded);
+			const auto file = known.file ? known.file : path ? ClientAssets::ResolveResFile(*path, res) : std::nullopt;
-			else JsonError(out, eHTTPStatusCode::NOT_FOUND, "Could not read this model");
+			if (!path) {
+				JsonError(out, eHTTPStatusCode::NOT_FOUND, "No such model in this zone");
+			} else if (const auto encoded = file ? Encoded(*path, lod, *file) : nullptr) {
+				Binary(out, *encoded);
+			} else {
+				JsonError(out, eHTTPStatusCode::NOT_FOUND, "Could not read this model");
+			}
 			deferred.Send(std::move(out));
 		});
 	}
 
 	void ReplyTexture(HTTPReply& reply, const HTTPContext& context, uint32_t zoneId, uint32_t asset, uint32_t slot, uint32_t lod) {
-		const auto* found = AssetPath(reply, zoneId, asset);
-		if (!found) return;
-		const auto path = *found;
 		lod = std::min(lod, MAX_LOD);
 		Touch(zoneId, lod);
-		const auto file = ClientAssets::ResolveResFile(path);
+		auto known = KnownAsset(zoneId, asset);
-		if (!file) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No such texture");
-		Files();
 		// Quick when the model is converted already and the texture is a file of its own, or one kept from its model
-		auto priority = WorkerPool::ePriority::URGENT;
+		if (known.path) {
-		if (const auto cached = g_Models.Get(ModelKey(path, lod))) {
+			if (const auto cached = g_Models.Get(ModelKey(*known.path, lod))) {
-			const auto textures = TexturesOf(*cached);
+				const auto textures = TexturesOf(*cached);
-			if (slot < textures.size() && textures[slot].starts_with('#') && !g_Embedded.Get(path + textures[slot])) priority = PriorityOf(*Zone(zoneId), path, *file);
+				if (slot >= textures.size() || !textures[slot].starts_with('#') || g_Embedded.Get(*known.path + textures[slot])) known.priority = WorkerPool::ePriority::URGENT;
-		} else {
+			}
-			priority = PriorityOf(*Zone(zoneId), path, *file);
 		}
+		const auto res = ClientAssets::ResFolder();
 		const auto deferred = Web::Defer(reply, context);
-		g_Pool.Submit(priority, [deferred, path, lod, slot, file = *file, res = ClientAssets::ResFolder()] {
+		Pool().Submit(known.priority, [deferred, zoneId, asset, lod, slot, res, known = std::move(known)] {
 			if (deferred.Cancelled()) return;
 			HTTPReply out;
-			const auto encoded = Encoded(path, lod, file);
+			const auto path = known.path ? known.path : AssetPath(zoneId, asset);
+			const auto file = known.file ? known.file : path ? ClientAssets::ResolveResFile(*path, res) : std::nullopt;
+			const auto encoded = path && file ? Encoded(*path, lod, *file) : nullptr;
 			const auto textures = encoded ? TexturesOf(*encoded) : std::vector<std::string>{};
-			if (slot >= textures.size() || textures[slot].empty()) {
+			if (!path) {
+				JsonError(out, eHTTPStatusCode::NOT_FOUND, "No such model in this zone");
+			} else if (slot >= textures.size() || textures[slot].empty()) {
 				JsonError(out, eHTTPStatusCode::NOT_FOUND, "No such texture");
-			} else if (auto dds = TextureBytes(path, file, textures, textures[slot], res)) {
+			} else if (auto dds = TextureBytes(*path, *file, textures, textures[slot], res)) {
 				Binary(out, std::move(*dds));
 			} else {
 				JsonError(out, eHTTPStatusCode::NOT_FOUND, "Could not read this texture");
@@ -725,18 +829,7 @@ namespace Scenery {
 		});
 	}
 
-	void Shutdown() {
-		g_Pool.Stop();
-	}
-
 	void RegisterRoutes() {
-		auto threads = Game::config ? Game::config->GetValue<uint32_t>("scenery_workers", 0) : 0;
-		if (threads == 0) threads = static_cast<uint32_t>(WorkerPool::DefaultThreads(std::thread::hardware_concurrency()));
-		threads = std::clamp<uint32_t>(threads, 2, 16);
-		// Converting ahead of time never takes more than half of the threads besides the fast lane
-		g_Pool.Start(threads, std::max<size_t>(1, (threads - 1) / 2));
-		LOG("Converting scenery models with %u threads", threads);
-
 		Route(eHTTPMethod::GET, "/api/scenery/:zone/mesh/:asset", 0,
 			"Model `asset` of a zone's scenery (see the scenery routes of properties and /world3d), converted from the client's .nif. Query: ?lod=0 (most detailed) to 3",
 			[](HTTPReply& reply, const HTTPContext& context) {

dDashboardServer/routes/Scenery.h

@@ -21,13 +21,20 @@ namespace WorldScene {
  * bounded in-memory cache; textures are sent as the client's own DDS files (textures stored inside a .nif are
  * wrapped as DDS), which the browser decodes itself, so nothing needs ImageMagick.
  *
+ * Everything here is thread safe: the manifests are built on worker threads too (the routes use Workers::Reply).
  * Converting a big model takes a while, so models and textures not in memory are answered from a few worker threads
- * (Web::Defer, WorkerPool; scenery_workers in dashboardconfig.ini) and the web server keeps answering meanwhile.
+ * (Web::Defer, Workers; scenery_workers in dashboardconfig.ini) and the web server keeps answering meanwhile.
  * Flairs and small models go first, in a lane of their own; the same model asked for twice at once is converted
  * once. When a zone's manifest is asked for, its models are converted ahead of time onto the disk cache while
  * someone views the zone.
  */
 namespace Scenery {
+	/**
+	 * Read what the builders need from the CDClient (render components, flairs) and index the client's model files,
+	 * at startup: the manifests are built on worker threads, which never query the CDClient.
+	 */
+	void Preload();
+
 	/**
 	 * The zone's manifest as JSON: {zone, sky (asset index or -1), assets: [res path, ...],
 	 * objects: {asset: [...], pos: [x, y, z, ...], rot: [x, y, z, w, ...], scale: [...], hidden: [0 or 1, ...]}}, where
@@ -36,6 +43,9 @@ namespace Scenery {
 	 */
 	std::optional<std::string> ZoneJson(uint32_t zoneId);
 
+	// Whether ZoneJson is built (so answering it is quick)
+	bool ZoneReady(uint32_t zoneId);
+
 	/**
 	 * The zone's flairs (the grass, flowers and small rocks its terrain file strews over it, models from FlairTable) as a
 	 * manifest in ZoneJson's form, whose models the same mesh and texture routes serve, plus: distance (how far from the
@@ -44,6 +54,9 @@ namespace Scenery {
 	 */
 	std::optional<std::string> FlairsJson(uint32_t zoneId);
 
+	// Whether FlairsJson is built
+	bool FlairsReady(uint32_t zoneId);
+
 	// Whether the client draws a model for this scene object (its render component's, or its nif_name)
 	bool HasModel(const WorldScene::Object& object);
 
@@ -54,10 +67,8 @@ namespace Scenery {
 	// Reply with texture `slot` (the model's "textures" list at that `lod`) of model `asset`, as a DDS file. Deferred.
 	void ReplyTexture(HTTPReply& reply, const HTTPContext& context, uint32_t zoneId, uint32_t asset, uint32_t slot, uint32_t lod);
 
-	// /api/scenery/:zone/mesh/:asset and /api/scenery/:zone/texture/:asset/:slot, for anyone signed in; starts the
+	// /api/scenery/:zone/mesh/:asset and /api/scenery/:zone/texture/:asset/:slot, for anyone signed in
-	// conversion threads
 	void RegisterRoutes();
 
-	// Stop the conversion threads (queued work is dropped)
+
-	void Shutdown();
 }

dDashboardServer/routes/Showcase.cpp

@@ -1,4 +1,5 @@
 #include "Showcase.h"
+#include "Workers.h"
 
 #include <chrono>
 #include <map>
@@ -164,16 +165,21 @@ namespace {
 		if (!cacheControl.empty()) reply.headers.push_back("Cache-Control: " + cacheControl);
 	}
 
-	// Terrain and the build area come from the client's files and are the same for every property in a zone
+	/**
+	 * Terrain and the build area come from the client's files and are the same for every property in a zone.
+	 * `ready` says whether what `fetch` needs is built; when it isn't, fetch runs on a worker thread (Workers::Reply),
+	 * so it must be thread safe. The property is looked up here, on the web thread.
+	 */
 	template<typename Fetch>
-	void ZoneFileRoute(const std::string& path, const std::string& description, Fetch fetch) {
+	void ZoneFileRoute(const std::string& path, const std::string& description, Fetch fetch, std::function<bool(uint32_t zone)> ready = nullptr) {
-		Route(eHTTPMethod::GET, path, PUBLIC, description, [fetch](HTTPReply& reply, const HTTPContext& context) {
+		Route(eHTTPMethod::GET, path, PUBLIC, description, [fetch, ready](HTTPReply& reply, const HTTPContext& context) {
 			if (!Allowed(context, reply, Kind::DATA)) return;
 			const auto id = PathId<LWOOBJID>(context.path, 2);
 			if (!id) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid ID");
 			const auto info = ShownProperty(*id, reply);
 			if (!info) return;
-			fetch(reply, *info);
+			if (!ready) return fetch(reply, *info);
+			Workers::Reply(reply, context, ready(info->zoneId), [fetch, info = *info](HTTPReply& out) { fetch(out, info); });
 		});
 	}
 }
@@ -251,14 +257,14 @@ void RegisterShowcaseRoutes() {
 			const auto terrain = ZoneTerrainChunksJson(info.zoneId);
 			if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
 			RawReply(reply, eContentType::APPLICATION_JSON, *terrain, "private, max-age=86400");
-		});
+		}, ZoneTerrainChunksReady);
 
 	ZoneFileRoute("/api/showcase/:id/terrain", "A height grid of a showcased property's zone (needs client_location)",
 		[](HTTPReply& reply, const IProperty::Info& info) {
 			const auto terrain = ZoneTerrainJson(info.zoneId);
 			if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
 			RawReply(reply, eContentType::APPLICATION_JSON, *terrain, "private, max-age=86400");
-		});
+		}, ZoneTerrainJsonReady);
 
 	ZoneFileRoute("/api/showcase/:id/boundary", "Where a showcased property's owner may build (needs client_location)",
 		[](HTTPReply& reply, const IProperty::Info& info) {
@@ -311,9 +317,11 @@ void RegisterShowcaseRoutes() {
 			if (!Allowed(context, reply, Kind::ASSET)) return;
 			const auto textureId = PathId<uint32_t>(context.path, 3);
 			if (!textureId) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid ID");
-			const auto png = TerrainTextureFile(*textureId);
+			Workers::Reply(reply, context, TerrainTextureReady(*textureId), [id = *textureId](HTTPReply& out) {
-			if (!png) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "Texture not found");
+				const auto png = TerrainTextureFile(id);
-			RawReply(reply, eContentType::IMAGE_PNG, *png, "public, max-age=604800");
+				if (!png) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "Texture not found");
+				RawReply(out, eContentType::IMAGE_PNG, *png, "public, max-age=604800");
+			});
 		});
 
 	ZoneFileRoute("/api/showcase/:id/scenery", "Everything the game draws around a showcased property: its zone's scene objects with their models and the sky (needs client_location)",
@@ -321,7 +329,7 @@ void RegisterShowcaseRoutes() {
 			const auto scenery = Scenery::ZoneJson(info.zoneId);
 			if (!scenery) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No scenery for this zone");
 			RawReply(reply, eContentType::APPLICATION_JSON, *scenery, "private, max-age=86400");
-		});
+		}, Scenery::ZoneReady);
 
 	// Scenery models and textures of the zones properties are in (the only ones the showcase shows)
 	const auto propertyZone = [](HTTPReply& reply, const HTTPContext& context) -> std::optional<uint32_t> {

dDashboardServer/routes/Workers.cpp

@@ -0,0 +1,53 @@
+#include "Workers.h"
+
+#include <algorithm>
+#include <exception>
+#include <thread>
+
+#include "RouteUtils.h"
+#include "Web.h"
+
+#include "Game.h"
+#include "Logger.h"
+#include "dConfig.h"
+
+namespace {
+	WorkerPool g_Pool;
+}
+
+namespace Workers {
+	void Start() {
+		auto threads = Game::config ? Game::config->GetValue<uint32_t>("scenery_workers", 0) : 0;
+		if (threads == 0) threads = static_cast<uint32_t>(WorkerPool::DefaultThreads(std::thread::hardware_concurrency()));
+		threads = std::clamp<uint32_t>(threads, 2, 16);
+		// Work ahead of time (converting a zone's models) never takes more than half of the threads besides the fast lane
+		g_Pool.Start(threads, std::max<size_t>(1, (threads - 1) / 2));
+		LOG("Dashboard worker threads: %u", threads);
+	}
+
+	void Stop() {
+		g_Pool.Stop();
+	}
+
+	WorkerPool& Pool() {
+		return g_Pool;
+	}
+
+	void Reply(HTTPReply& reply, const HTTPContext& context, bool ready, std::function<void(HTTPReply&)> fill, WorkerPool::ePriority priority) {
+		if (ready || !g_Pool.Running()) return fill(reply);
+		const auto deferred = Web::Defer(reply, context);
+		const auto path = context.path;
+		g_Pool.Submit(priority, [deferred, path, fill = std::move(fill)] {
+			if (deferred.Cancelled()) return;
+			HTTPReply out;
+			try {
+				fill(out);
+			} catch (const std::exception& ex) {
+				LOG("Error handling GET %s: %s", path.c_str(), ex.what());
+				out = HTTPReply{};
+				RouteUtils::JsonError(out, eHTTPStatusCode::INTERNAL_SERVER_ERROR, "Internal server error");
+			}
+			deferred.Send(std::move(out));
+		});
+	}
+}

dDashboardServer/routes/Workers.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <functional>
+
+#include "WorkerPool.h"
+
+struct HTTPReply;
+struct HTTPContext;
+
+/**
+ * The dashboard's worker threads (one WorkerPool, scenery_workers threads) for slow work routes would otherwise do
+ * on the web server's one thread: converting the client's models, building a zone's terrain and scene data the
+ * first time it is viewed, converting textures with ImageMagick. What they run must be thread safe: the zone data
+ * builders keep their results in OnceCaches and read the client's files through ClientAssets.
+ */
+namespace Workers {
+	// Start the threads (scenery_workers: 0 picks WorkerPool::DefaultThreads)
+	void Start();
+
+	// Stop them; queued work is dropped (answer deferred requests before Web::Shutdown)
+	void Stop();
+
+	WorkerPool& Pool();
+
+	/**
+	 * Answer a request with what `fill` writes to the reply: right away when `ready` (what it needs is built, so it's
+	 * quick), else from a worker thread (Web::Defer). `fill` runs on either thread, so it must be thread safe and must
+	 * not keep a reference to the request.
+	 */
+	void Reply(HTTPReply& reply, const HTTPContext& context, bool ready, std::function<void(HTTPReply&)> fill,
+		WorkerPool::ePriority priority = WorkerPool::ePriority::NORMAL);
+}

dDashboardServer/routes/WorldView.cpp

@@ -11,6 +11,8 @@
 #include "Permissions.h"
 #include "ServerState.h"
 #include "Background.h"
+#include "OnceCache.h"
+#include "Workers.h"
 
 #include <chrono>
 #include <cmath>
@@ -83,20 +85,13 @@ namespace {
 	}
 
 	std::optional<std::string> LuzPath(uint32_t zone) {
-		auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT zoneName FROM ZoneTable WHERE zoneID = ? LIMIT 1;");
+		return ZoneLuzPath(zone);
-		stmt.bind(1, static_cast<int>(zone));
-		auto result = stmt.execQuery();
-		if (result.eof()) return std::nullopt;
-		std::string path = result.getStringField(0, "");
-		if (!path.ends_with(".luz")) return std::nullopt;
-		return path;
 	}
 
-	// LOT -> index into WorldScene::KINDS for every LOT with one of those components, read once
+	// LOT -> index into WorldScene::KINDS for every LOT with one of those components, read at startup (Preload): the
-	const std::map<uint32_t, uint8_t>& LotKinds() {
+	// zone data is built on worker threads, which never query the CDClient
-		static std::optional<std::map<uint32_t, uint8_t>> kinds;
+	std::map<uint32_t, uint8_t> ReadLotKinds() {
-		if (kinds) return *kinds;
+		std::map<uint32_t, uint8_t> kinds;
-		kinds.emplace();
 		std::map<uint32_t, std::vector<uint32_t>> components;
 		try {
 			auto result = CDClientDatabase::ExecuteQuery("SELECT id, component_type FROM ComponentsRegistry;");
@@ -106,9 +101,14 @@ namespace {
 		}
 		for (const auto& [lot, types] : components) {
 			const auto kind = WorldScene::Classify(types);
-			if (kind < WorldScene::KINDS.size()) (*kinds)[lot] = static_cast<uint8_t>(kind);
+			if (kind < WorldScene::KINDS.size()) kinds[lot] = static_cast<uint8_t>(kind);
 		}
-		return *kinds;
+		return kinds;
+	}
+
+	const std::map<uint32_t, uint8_t>& LotKinds() {
+		static const auto kinds = ReadLotKinds();
+		return kinds;
 	}
 
 	double Round(float value) { return std::round(static_cast<double>(value) * 100.0) / 100.0; }
@@ -118,15 +118,12 @@ namespace {
 	 * its .luz (the scene list) and its scene files. Objects are columns (lot, kind, flags, scene, x/y/z) to keep big zones
 	 * small: flags 1 = spawner, 2 = client only, 4 = drawn with a model by the scenery layer.
 	 */
-	std::optional<std::string> SceneJson(uint32_t zone) {
+	std::optional<std::string> BuildSceneJson(uint32_t zone) {
-		static std::map<uint32_t, std::optional<std::string>> cache;
-		if (const auto it = cache.find(zone); it != cache.end()) return it->second;
-
 		const auto luzPath = LuzPath(zone);
 		const auto luz = luzPath ? ClientAssets::ReadResFile("maps/" + *luzPath) : std::nullopt;
 		std::string error;
 		const auto zoneFile = luz ? ZonePaths::ReadHeader(*luz, error) : std::nullopt;
-		if (!zoneFile) return cache[zone] = std::nullopt;
+		if (!zoneFile) return std::nullopt;
 
 		const auto& lotKinds = LotKinds();
 		const auto other = static_cast<uint8_t>(WorldScene::KINDS.size());
@@ -167,19 +164,24 @@ namespace {
 			{"objects", { {"lot", lots}, {"kind", kindOf}, {"flags", flags}, {"scene", sceneOf}, {"pos", positions}, {"names", names} }}, {"lotNames", lotNames},
 			{"spawnPoints", ZoneSpawnPointsJson(zone).value_or(nlohmann::json::array())}
 		};
-		return cache[zone] = json.dump();
+		return json.dump();
+	}
+
+	OnceCache<uint32_t, std::optional<std::string>> g_Scenes;
+	OnceCache<uint32_t, std::optional<std::string>> g_Paths;
+
+	// Built on a worker thread the first time (any thread)
+	const std::optional<std::string>& SceneJson(uint32_t zone) {
+		return g_Scenes.Get(zone, [zone] { return BuildSceneJson(zone); });
 	}
 
 	// A zone's paths, which need the whole .luz (they are most of it): loaded when a path layer is first shown
-	std::optional<std::string> PathsJson(uint32_t zone) {
+	std::optional<std::string> BuildPathsJson(uint32_t zone) {
-		static std::map<uint32_t, std::optional<std::string>> cache;
-		if (const auto it = cache.find(zone); it != cache.end()) return it->second;
-
 		const auto luzPath = LuzPath(zone);
 		const auto luz = luzPath ? ClientAssets::ReadResFile("maps/" + *luzPath) : std::nullopt;
 		std::string error;
 		const auto zoneFile = luz ? ZonePaths::Read(*luz, error) : std::nullopt;
-		if (!zoneFile) return cache[zone] = std::nullopt;
+		if (!zoneFile) return std::nullopt;
 
 		nlohmann::json pathTypes = nlohmann::json::array();
 		for (const auto type : magic_enum::enum_values<PathType>()) pathTypes.push_back({ {"value", static_cast<int>(type)}, {"name", GameLabels::Name(type)} });
@@ -194,7 +196,11 @@ namespace {
 			if (points.empty()) continue;
 			paths.push_back({ {"name", path.pathName}, {"type", static_cast<int>(path.pathType)}, {"loop", path.pathBehavior == PathBehavior::Loop}, {"points", points} });
 		}
-		return cache[zone] = nlohmann::json{ {"zone", zone}, {"pathTypes", pathTypes}, {"paths", paths} }.dump();
+		return nlohmann::json{ {"zone", zone}, {"pathTypes", pathTypes}, {"paths", paths} }.dump();
+	}
+
+	const std::optional<std::string>& PathsJson(uint32_t zone) {
+		return g_Paths.Get(zone, [zone] { return BuildPathsJson(zone); });
 	}
 
 	std::string CharacterName(LWOOBJID id) {
@@ -211,22 +217,28 @@ namespace {
 		reply.headers.push_back(std::string("Cache-Control: ") + cache);
 	}
 
+	using Deflated = OnceCache<uint32_t, std::optional<std::string>>;
+
+	bool TakesDeflate(const HTTPContext& context) {
+		return context.GetHeader("Accept-Encoding").find("deflate") != std::string::npos;
+	}
+
 	/**
 	 * A big cached JSON body, deflated once per zone when the browser takes it: a zone's terrain chunks are mostly
-	 * base64 maps that shrink about tenfold (Avant Gardens: 19 MB to 1.4 MB).
+	 * base64 maps that shrink about tenfold (Avant Gardens: 19 MB to 1.4 MB). Any thread.
 	 */
-	void DeflatedJson(HTTPReply& reply, const HTTPContext& context, std::map<uint32_t, std::string>& deflated, uint32_t zone, const std::string& json, const char* cache) {
+	void DeflatedJson(HTTPReply& reply, bool deflate, Deflated& deflated, uint32_t zone, const std::string& json, const char* cache) {
-		if (context.GetHeader("Accept-Encoding").find("deflate") == std::string::npos || json.size() > UINT32_MAX / 2) return RawJson(reply, json, cache);
+		if (!deflate || json.size() > UINT32_MAX / 2) return RawJson(reply, json, cache);
-		auto it = deflated.find(zone);
+		const auto& out = deflated.Get(zone, [&json]() -> std::optional<std::string> {
-		if (it == deflated.end()) {
 			std::string out(ZCompression::GetMaxCompressedLength(static_cast<uint32_t>(json.size())), '\0');
 			const auto size = ZCompression::Compress(reinterpret_cast<const uint8_t*>(json.data()), static_cast<uint32_t>(json.size()),
 				reinterpret_cast<uint8_t*>(out.data()), static_cast<uint32_t>(out.size()));
-			if (size <= 0) return RawJson(reply, json, cache);
+			if (size <= 0) return std::nullopt;
 			out.resize(static_cast<size_t>(size));
-			it = deflated.emplace(zone, std::move(out)).first;
+			return out;
-		}
+		});
-		RawJson(reply, it->second, cache);
+		if (!out) return RawJson(reply, json, cache);
+		RawJson(reply, *out, cache);
 		reply.headers.push_back("Content-Encoding: deflate");
 		reply.headers.push_back("Vary: Accept-Encoding");
 	}
@@ -237,6 +249,10 @@ namespace {
 }
 
 namespace WorldView {
+	void Preload() {
+		LotKinds();
+	}
+
 	void RecordPositions(const PlayerPositions& positions) {
 		if (Setting("position_history", 1) == 0) {
 			g_Buffer.clear();
@@ -286,9 +302,11 @@ namespace WorldView {
 			[](HTTPReply& reply, const HTTPContext& context) {
 				const auto zone = PathId<uint32_t>(context.path, 2);
 				if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone");
-				const auto json = SceneJson(*zone);
+				Workers::Reply(reply, context, g_Scenes.Ready(*zone), [zone = *zone](HTTPReply& out) {
-				if (!json) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)");
+					const auto& json = SceneJson(zone);
-				RawJson(reply, *json, "private, max-age=3600");
+					if (!json) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)");
+					RawJson(out, *json, "private, max-age=3600");
+				});
 			});
 
 		Route(eHTTPMethod::GET, "/api/world3d/:zone/paths", Perm("players_view"),
@@ -296,9 +314,11 @@ namespace WorldView {
 			[](HTTPReply& reply, const HTTPContext& context) {
 				const auto zone = PathId<uint32_t>(context.path, 2);
 				if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone");
-				const auto json = PathsJson(*zone);
+				Workers::Reply(reply, context, g_Paths.Ready(*zone), [zone = *zone](HTTPReply& out) {
-				if (!json) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)");
+					const auto& json = PathsJson(zone);
-				RawJson(reply, *json, "private, max-age=3600");
+					if (!json) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)");
+					RawJson(out, *json, "private, max-age=3600");
+				});
 			});
 
 		Route(eHTTPMethod::GET, "/api/world3d/:zone/terrain_layers", Perm("players_view"),
@@ -306,10 +326,13 @@ namespace WorldView {
 			[](HTTPReply& reply, const HTTPContext& context) {
 				const auto zone = PathId<uint32_t>(context.path, 2);
 				if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone");
-				const auto layers = ZoneTerrainLayersJson(*zone);
+				static Deflated deflated;
-				if (!layers) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
+				const auto deflate = TakesDeflate(context);
-				static std::map<uint32_t, std::string> deflated;
+				Workers::Reply(reply, context, ZoneTerrainLayersReady(*zone) && (!deflate || deflated.Ready(*zone)), [zone = *zone, deflate](HTTPReply& out) {
-				DeflatedJson(reply, context, deflated, *zone, *layers, "private, max-age=86400");
+					const auto layers = ZoneTerrainLayersJson(zone);
+					if (!layers) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
+					DeflatedJson(out, deflate, deflated, zone, *layers, "private, max-age=86400");
+				});
 			});
 
 		Route(eHTTPMethod::GET, "/api/world3d/:zone/flairs", Perm("players_view"),
@@ -317,9 +340,11 @@ namespace WorldView {
 			[](HTTPReply& reply, const HTTPContext& context) {
 				const auto zone = PathId<uint32_t>(context.path, 2);
 				if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone");
-				const auto flairs = Scenery::FlairsJson(*zone);
+				Workers::Reply(reply, context, Scenery::FlairsReady(*zone), [zone = *zone](HTTPReply& out) {
-				if (!flairs) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
+					const auto flairs = Scenery::FlairsJson(zone);
-				RawJson(reply, *flairs, "private, max-age=3600");
+					if (!flairs) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
+					RawJson(out, *flairs, "private, max-age=3600");
+				});
 			});
 
 		Route(eHTTPMethod::GET, "/api/world3d/:zone/terrain_chunks", Perm("players_view"),
@@ -327,10 +352,13 @@ namespace WorldView {
 			[](HTTPReply& reply, const HTTPContext& context) {
 				const auto zone = PathId<uint32_t>(context.path, 2);
 				if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone");
-				const auto terrain = ZoneTerrainChunksJson(*zone);
+				static Deflated deflated;
-				if (!terrain) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
+				const auto deflate = TakesDeflate(context);
-				static std::map<uint32_t, std::string> deflated;
+				Workers::Reply(reply, context, ZoneTerrainChunksReady(*zone) && (!deflate || deflated.Ready(*zone)), [zone = *zone, deflate](HTTPReply& out) {
-				DeflatedJson(reply, context, deflated, *zone, *terrain, "private, max-age=86400");
+					const auto terrain = ZoneTerrainChunksJson(zone);
+					if (!terrain) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No terrain for this zone");
+					DeflatedJson(out, deflate, deflated, zone, *terrain, "private, max-age=86400");
+				});
 			});
 
 		Route(eHTTPMethod::GET, "/api/world3d/:zone/scenery", Perm("players_view"),
@@ -338,9 +366,11 @@ namespace WorldView {
 			[](HTTPReply& reply, const HTTPContext& context) {
 				const auto zone = PathId<uint32_t>(context.path, 2);
 				if (!zone) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone");
-				const auto scenery = Scenery::ZoneJson(*zone);
+				Workers::Reply(reply, context, Scenery::ZoneReady(*zone), [zone = *zone](HTTPReply& out) {
-				if (!scenery) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)");
+					const auto scenery = Scenery::ZoneJson(zone);
-				RawJson(reply, *scenery, "private, max-age=3600");
+					if (!scenery) return JsonError(out, eHTTPStatusCode::NOT_FOUND, "No zone file for this zone (is client_location set?)");
+					RawJson(out, *scenery, "private, max-age=3600");
+				});
 			});
 
 		Route(eHTTPMethod::GET, "/api/world3d/history/instances", Perm("players_history"),

dDashboardServer/routes/WorldView.h

@@ -10,6 +10,9 @@ struct PlayerPositions;
 namespace WorldView {
 	void RegisterRoutes();
 
+	// Read the CDClient data the zone data builders need (at startup: they run on worker threads)
+	void Preload();
+
 	// A world server reported its players' positions: keep some of them for replays (throttled, written in batches)
 	void RecordPositions(const PlayerPositions& positions);
 }

docs/Dashboard.md

@@ -906,7 +906,14 @@ manifest is asked for, the zone's models are also converted ahead of time onto t
 first, at the detail its viewer last used: only when nothing else waits, on at most half of the threads besides the
 flairs' one, stopping when nobody has viewed the zone for 90 seconds or the disk cache is three quarters full (it
 never evicts for this). The number of threads is `scenery_workers` in `dashboardconfig.ini` (Settings > Dashboard >
-Web server; 0, the default, picks half the CPU cores, 2 to 4; read at startup). Endpoints:
+Web server; 0, the default, picks half the CPU cores, 2 to 4; read at startup).
+
+The same threads build a zone's data the first time it is viewed (its terrain chunks and layers, scene objects,
+paths, scenery and flair manifests, deflated bodies) and convert terrain textures with ImageMagick, so opening a new
+zone doesn't hold up the rest of the dashboard; what is built already is answered at once. Each is built once, even
+when asked for twice at the same time. Workers never query the CDClient, read settings or touch the network: the
+tables they need (ZoneTable, render components, flairs, object names, terrain textures) are read at startup.
+Endpoints:
 `/api/properties/:id/scenery`, `/api/world3d/:zone/scenery`, `/api/world3d/:zone/flairs`,
 `/api/scenery/:zone/mesh/:asset?lod=`, `/api/scenery/:zone/texture/:asset/:slot?lod=`. The world view's other data:
 `/api/world3d/:zone/scene` (objects and scenes), `/terrain_chunks` and `/terrain_layers` (the terrain file; sent

tests/dWebTests/CMakeLists.txt

@@ -36,6 +36,7 @@ set(DWEBTESTS_SOURCES
 	"NifFileTests.cpp"
 	"DeferredReplyTests.cpp"
 	"WorkerPoolTests.cpp"
+	"OnceCacheTests.cpp"
 	"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/WorkerPool.cpp"
 	"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/NifFile.cpp"
 	"BackupFilesTests.cpp"

tests/dWebTests/OnceCacheTests.cpp

@@ -0,0 +1,56 @@
+#include <gtest/gtest.h>
+
+#include <atomic>
+#include <chrono>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "OnceCache.h"
+
+TEST(OnceCacheTests, BuildsOnceForManyThreads) {
+	OnceCache<int, std::string> cache;
+	std::atomic<int> builds{};
+	std::vector<std::thread> threads;
+	std::vector<std::string> results(16);
+	for (size_t i = 0; i < results.size(); i++) {
+		threads.emplace_back([&, i] {
+			results[i] = cache.Get(1, [&] {
+				builds++;
+				std::this_thread::sleep_for(std::chrono::milliseconds(20));
+				return std::string("built");
+			});
+		});
+	}
+	for (auto& thread : threads) thread.join();
+	EXPECT_EQ(builds.load(), 1);
+	for (const auto& result : results) EXPECT_EQ(result, "built");
+	EXPECT_TRUE(cache.Ready(1));
+	EXPECT_FALSE(cache.Ready(2));
+}
+
+TEST(OnceCacheTests, NotReadyWhileBuilding) {
+	OnceCache<int, int> cache;
+	std::atomic<bool> release{}, started{};
+	std::thread builder([&] {
+		cache.Get(1, [&] {
+			started = true;
+			while (!release) std::this_thread::sleep_for(std::chrono::milliseconds(1));
+			return 5;
+		});
+	});
+	while (!started) std::this_thread::sleep_for(std::chrono::milliseconds(1));
+	EXPECT_FALSE(cache.Ready(1));
+	release = true;
+	builder.join();
+	EXPECT_TRUE(cache.Ready(1));
+	EXPECT_EQ(cache.Get(1, [] { return 0; }), 5);
+}
+
+TEST(OnceCacheTests, FailedBuildIsTriedAgain) {
+	OnceCache<int, int> cache;
+	EXPECT_THROW(cache.Get(1, []() -> int { throw std::runtime_error("no"); }), std::runtime_error);
+	EXPECT_FALSE(cache.Ready(1));
+	EXPECT_EQ(cache.Get(1, [] { return 3; }), 3);
+}