#include "Scenery.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ClientAssets.h" #include "NifFile.h" #include "ReportRoutes.h" #include "RouteUtils.h" #include "OnceCache.h" #include "TtlCache.h" #include "Web.h" #include "WorkerPool.h" #include "Workers.h" #include "WorldScene.h" #include "ZonePaths.h" #include "ZoneScenes.h" #include "CDClientDatabase.h" #include "Game.h" #include "GeneralUtils.h" #include "Logger.h" #include "dConfig.h" #include "eHTTPMethod.h" using namespace RouteUtils; namespace { constexpr size_t MESH_CACHE_BYTES = 64 * 1024 * 1024; constexpr uint32_t MAX_LOD = 3; constexpr uint32_t RENDER_COMPONENT = 2; std::string Lower(std::string text) { std::transform(text.begin(), text.end(), text.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); return text; } /** * `relative` (backslashes or slashes, may climb with "..") inside res/ folder `folder`, lowercase with slashes. * Empty when it climbs out of res/. */ std::string JoinPath(const std::string& folder, const std::string& relative) { std::vector parts; auto text = Lower(folder.empty() ? relative : folder + "/" + relative); std::replace(text.begin(), text.end(), '\\', '/'); for (const auto& part : GeneralUtils::SplitString(text, '/')) { if (part.empty() || part == ".") continue; if (part == "..") { if (parts.empty()) return {}; parts.pop_back(); } else { parts.push_back(part); } } std::string out; for (const auto& part : parts) out += (out.empty() ? "" : "/") + part; if (out.starts_with("res/")) out = out.substr(4); return out; } std::string FolderOf(const std::string& path) { const auto slash = path.find_last_of('/'); return slash == std::string::npos ? std::string{} : path.substr(0, slash); } // Every file under res/mesh, res/textures and res/animations (lowercase, relative to res/), and by file name struct FileIndex { std::unordered_set paths; std::unordered_map> byName; }; FileIndex IndexFiles() { FileIndex index; const auto res = ClientAssets::ResFolder(); if (res.empty()) return index; std::error_code ec; for (const char* folder : { "mesh", "textures", "animations" }) { // Unpacked clients keep their original case, so the top folder is matched ignoring case too for (const auto& top : std::filesystem::directory_iterator(res, ec)) { if (Lower(top.path().filename().string()) != folder || !top.is_directory(ec)) continue; for (auto it = std::filesystem::recursive_directory_iterator(top.path(), ec); !ec && it != std::filesystem::recursive_directory_iterator(); it.increment(ec)) { if (!it->is_regular_file(ec)) continue; const auto relative = Lower(std::filesystem::relative(it->path(), res, ec).generic_string()); index.paths.insert(relative); index.byName[Lower(it->path().filename().string())].push_back(relative); } } } return index; } // Built at startup (Scenery::Preload); read only afterwards const FileIndex& Files() { static const FileIndex index = IndexFiles(); return index; } // A texture a model names: next to the model if it's there (the usual case), else the file of that name sharing // the longest folder prefix with the model. Only DDS files, which the browser decodes. std::string FindTexture(const std::string& modelFolder, const std::string& stored) { const auto path = JoinPath(modelFolder, stored); if (path.empty() || !path.ends_with(".dds")) return {}; const auto& files = Files(); if (files.paths.contains(path)) return path; const auto name = path.substr(path.find_last_of('/') + 1); const auto it = files.byName.find(name); if (it == files.byName.end()) return {}; std::string best; size_t bestShared = 0; for (const auto& candidate : it->second) { const auto shared = static_cast(std::mismatch(candidate.begin(), candidate.end(), modelFolder.begin(), modelFolder.end()).first - candidate.begin()); if (best.empty() || shared > bestShared) { best = candidate; bestShared = shared; } } return best; } // A render asset as stored (a .nif, or a .kfm naming one) as a res path of a .nif that exists; empty otherwise std::string ResolveModel(const std::string& stored) { auto path = JoinPath("", stored); if (path.ends_with(".kfm") && Files().paths.contains(path)) { const auto kfm = ClientAssets::ReadResFile(path); const auto named = kfm ? NifFile::KfmModelPath(*kfm) : std::nullopt; path = named ? JoinPath(FolderOf(path), *named) : std::string{}; } return path.ends_with(".nif") && Files().paths.contains(path) ? path : std::string{}; } struct RenderInfo { std::string asset; // RenderComponent.render_asset as stored std::string type; // Objects.type int32_t shader{ -1 }; // mapShaders.gameValue of RenderComponent.shader_id (-1 fixed function or unknown) bool hidesOnLoad{}; // its client script hides it as soon as it loads (WorldScene::ClientScriptHidesOnLoad) }; // A client script's text, by its ScriptComponent path (backslashes, any case); empty when the client has none std::string ReadClientScript(const std::string& stored) { static const auto index = [] { std::unordered_map files; const auto res = ClientAssets::ResFolder(); if (res.empty()) return files; std::error_code ec; for (const auto& top : std::filesystem::directory_iterator(res, ec)) { if (Lower(top.path().filename().string()) != "scripts" || !top.is_directory(ec)) continue; for (auto it = std::filesystem::recursive_directory_iterator(top.path(), ec); !ec && it != std::filesystem::recursive_directory_iterator(); it.increment(ec)) { if (it->is_regular_file(ec)) files.emplace(Lower(std::filesystem::relative(it->path(), res, ec).generic_string()), it->path()); } } return files; }(); auto key = Lower(stored); std::replace(key.begin(), key.end(), '\\', '/'); const auto it = index.find(key); if (it == index.end()) return {}; std::ifstream in(it->second, std::ios::binary); return std::string(std::istreambuf_iterator(in), std::istreambuf_iterator()); } // mapShaders: id (what RenderComponent.shader_id and multishader tags name) -> gameValue (the shader drawn) std::map g_ShaderValues; // Every LOT with a render component, read once (Objects has no index on id, so it is read whole too) std::unordered_map ReadRenderInfos() { std::unordered_map infos; try { auto row = CDClientDatabase::ExecuteQuery( "SELECT cr.id, rc.render_asset, m.gameValue FROM ComponentsRegistry cr JOIN RenderComponent rc ON rc.id = cr.component_id " "LEFT JOIN mapShaders m ON m.id = rc.shader_id WHERE cr.component_type = " + std::to_string(RENDER_COMPONENT) + ";"); for (; !row.eof(); row.nextRow()) { infos.try_emplace(static_cast(row.getIntField(0)), RenderInfo{ row.getStringField(1, ""), "", row.fieldIsNull(2) ? -1 : row.getIntField(2) }); } auto shaders = CDClientDatabase::ExecuteQuery("SELECT id, gameValue FROM mapShaders;"); for (; !shaders.eof(); shaders.nextRow()) g_ShaderValues.try_emplace(shaders.getIntField(0), shaders.getIntField(1)); // Objects whose client script hides them as soon as they load auto scripts = CDClientDatabase::ExecuteQuery( "SELECT cr.id, sc.client_script_name FROM ComponentsRegistry cr JOIN ScriptComponent sc ON sc.id = cr.component_id " "WHERE cr.component_type = " + std::to_string(static_cast(eReplicaComponentType::SCRIPT)) + " AND sc.client_script_name IS NOT NULL AND sc.client_script_name != '';"); std::unordered_map hides; // by script, read once for (; !scripts.eof(); scripts.nextRow()) { const auto it = infos.find(static_cast(scripts.getIntField(0))); if (it == infos.end()) continue; const std::string script = scripts.getStringField(1, ""); auto [known, added] = hides.try_emplace(script, false); if (added) known->second = WorldScene::ClientScriptHidesOnLoad(ReadClientScript(script)); it->second.hidesOnLoad = it->second.hidesOnLoad || known->second; } auto types = CDClientDatabase::ExecuteQuery("SELECT id, type FROM Objects;"); for (; !types.eof(); types.nextRow()) { const auto it = infos.find(static_cast(types.getIntField(0))); if (it != infos.end() && it->second.type.empty()) it->second.type = types.getStringField(1, ""); } } catch (const std::exception& ex) { LOG("Could not read the render components of objects: %s", ex.what()); } return infos; } const std::unordered_map& RenderInfos() { static const auto infos = ReadRenderInfos(); return infos; } struct Model { std::string path; // res path of the .nif, empty for none bool hidden{}; // the client doesn't draw it (WorldScene::ClientDraws) int32_t shader{ -1 }; // RenderInfo::shader }; /** * The .nif of a scene object (its render component's, or nif_name), and whether the client draws it * (WorldScene::ClientDraws). Primitive models (built from parts at run time) aren't drawn here. */ Model ModelFor(const WorldScene::Object& object) { const auto lot = object.spawner ? object.templateLot : object.lot; const auto& infos = RenderInfos(); const auto info = infos.find(lot); if (info == infos.end()) return {}; auto draw = WorldScene::ClientDraws(object, info->second.type); if (draw == WorldScene::eClientDraw::DRAWN && info->second.hidesOnLoad) draw = WorldScene::eClientDraw::HIDDEN; if (draw == WorldScene::eClientDraw::NO_MODEL) return {}; static std::mutex mutex; static std::unordered_map resolved; const auto& stored = object.nifName.empty() ? info->second.asset : object.nifName; { std::lock_guard lock(mutex); if (const auto it = resolved.find(stored); it != resolved.end()) return { it->second, draw == WorldScene::eClientDraw::HIDDEN, info->second.shader }; } auto path = ResolveModel(stored); std::lock_guard lock(mutex); resolved.try_emplace(stored, path); return { std::move(path), draw == WorldScene::eClientDraw::HIDDEN, info->second.shader }; } /** * How the client draws each model, for the viewers' game shaders (static/js/game-shaders.js): "shaders" gives each * asset's shader (the first object drawing it wins; -1 fixed function or not an object), "shaderTags" a multishader * part's tag -> shader, and "techniques" every shader's technique (NifFile::TechniquesJson, the one table of them). */ void AddShaders(nlohmann::json& manifest, const std::vector& assetShaders) { RenderInfos(); // reads g_ShaderValues manifest["shaders"] = assetShaders; nlohmann::json tags = nlohmann::json::object(); std::set values{ -1, NifFile::LEGO_SHADER }; for (const auto& [id, value] : g_ShaderValues) { tags[std::to_string(id)] = value; values.insert(value); } manifest["shaderTags"] = std::move(tags); manifest["techniques"] = nlohmann::json::parse(NifFile::TechniquesJson({ values.begin(), values.end() })); manifest["multishader"] = NifFile::MULTISHADER; manifest["defaultShader"] = NifFile::LEGO_SHADER; } std::optional LuzPath(uint32_t zone) { return ZoneLuzPath(zone); } double Round(float value, double scale) { return std::round(static_cast(value) * scale) / scale; } /** * Bump when NifFile's output changes: converted models kept on disk are made again, and the manifests' "format" * goes into the viewers' model and texture URLs so browsers don't keep drawing the old ones (they're cached for * a week). The viewers ask for manifests with the format they're written for (scenery-core.js SCENERY_FORMAT, * which must follow this; SceneryCoreJs checks), so a manifest a browser kept for a day from an older server * isn't drawn with newer code. 2: meshes carry their multishader tag; conversions without it drew glom parts with * the LEGO shader. 3: dark textures and the UV set each texture names. 4: the game's shaders draw the models * (manifest "techniques"), vertex colors go to them as stored. 5: an animated material alpha is its highest key. 6: flair tints are bytes over * 255 (they were over 63). */ constexpr uint32_t FORMAT_VERSION = 6; // A zone's lighting (WorldScene::Lighting) for the viewers' shaders nlohmann::json LightingJson(const WorldScene::Lighting& lighting) { const auto triple = [](const std::array& value) { return nlohmann::json{ Round(value[0], 1000.0), Round(value[1], 1000.0), Round(value[2], 1000.0) }; }; return { {"ambient", triple(lighting.ambient)}, {"light", triple(lighting.light)}, {"lightVec", triple(lighting.lightVec)}, {"specular", triple(lighting.specular)}, {"upperHemi", triple(lighting.upperHemi)}, {"fogColor", triple(lighting.fogColor)}, {"fogNear", Round(lighting.fogNear, 10.0)}, {"fogFar", Round(lighting.fogFar, 10.0)} }; } /** * A zone's models and scenery manifest, built once (g_Zones). Once shared, assets, index, flairModels and the * warmed flags are guarded by g_ZoneMutex: the flairs' models join assets when their manifest is built. */ struct ZoneScenery { std::vector assets; // res paths of models, indexed by the manifests std::map index; // res path -> its index in assets std::string json; nlohmann::json lighting; // LightingJson of the zone's lighting, null when its scenes have none std::unordered_set flairModels; // res paths of the flairs' models, converted ahead of others bool warmedScenery{}; // WarmUp queued the scenery's models bool warmedFlairs{}; // and the flairs' size_t IndexOf(const std::string& path) { const auto [it, added] = index.try_emplace(path, assets.size()); if (added) assets.push_back(path); return it->second; } }; std::mutex g_ZoneMutex; /** * The zone's scenes for the viewers' "scenes like the game": each general scene's id, name, the scenes its * transitions connect it to (ZoneScenes::SceneGraph) and its lighting (null when its file has none). */ nlohmann::json ScenesJson(const ZoneFile& zone, const std::map& lightingOf) { const ZoneScenes::SceneGraph graph(zone.scenes, zone.sceneTransitions); nlohmann::json scenes = nlohmann::json::array(); std::set seen; for (const auto& scene : zone.scenes) { if (!seen.insert(scene.id).second) continue; // the audio layers share their scene's id const auto& neighbours = graph.Neighbours(scene.id); const auto lighting = lightingOf.find(scene.id); scenes.push_back({ {"id", scene.id}, {"name", scene.name}, {"neighbours", std::vector(neighbours.begin(), neighbours.end())}, {"lighting", lighting == lightingOf.end() ? nlohmann::json() : lighting->second} }); } return scenes; } /** * The terrain's scene map for the viewers to find the scene at a point as the client does (ZoneScenes::SceneMap; * scenery-core.js sceneAt reads it the same way): per chunk its corner, far corner, cells per side and its cells * (x major) as ZoneScenes::RunLengths, base64. Null without a terrain file. */ nlohmann::json SceneMapJson(uint32_t zoneId) { const auto raw = ZoneRawShared(zoneId); if (!raw) return nullptr; nlohmann::json chunks = nlohmann::json::array(); for (const auto& chunk : raw->chunks) { if (!chunk.IsValidForSceneLookup() || chunk.sceneMap.size() < static_cast(chunk.colorMapResolution) * chunk.colorMapResolution) continue; chunks.push_back({ {"x", chunk.offsetX}, {"z", chunk.offsetZ}, {"maxX", chunk.offsetX + static_cast(chunk.width - 1) * chunk.scaleFactor}, {"maxZ", chunk.offsetZ + static_cast(chunk.height - 1) * chunk.scaleFactor}, {"size", chunk.colorMapResolution}, {"runs", ZoneDataBase64(ZoneScenes::RunLengths(chunk.sceneMap, static_cast(chunk.colorMapResolution) * chunk.colorMapResolution))} }); } return chunks.empty() ? nlohmann::json() : nlohmann::json{ {"chunks", chunks} }; } std::shared_ptr BuildZone(uint32_t zoneId) { const auto luzPath = LuzPath(zoneId); const auto luz = luzPath ? ClientAssets::ReadResFile("maps/" + *luzPath) : std::nullopt; if (!luz) return nullptr; const auto folder = luzPath->substr(0, luzPath->find_last_of('/') + 1); std::string error; const auto zoneFile = ZonePaths::Read(*luz, error); if (!zoneFile) return nullptr; ZoneScenery scenery; nlohmann::json assetOf = nlohmann::json::array(), positions = nlohmann::json::array(), rotations = nlohmann::json::array(), scales = nlohmann::json::array(); nlohmann::json hidden = nlohmann::json::array(), sceneOf = nlohmann::json::array(); std::vector assetShaders; int64_t sky = -1; std::vector> sceneLighting; // each scene's, with how many objects it has std::map lightingOf; // scene id -> its general layer's lighting for (const auto& scene : zoneFile->scenes) { const auto lvl = ClientAssets::ReadResFile("maps/" + folder + scene.filename); if (!lvl) continue; const auto objectsBefore = assetOf.size(); const auto lighting = WorldScene::ReadLighting(*lvl); if (lighting && scene.sceneType == eSceneType::General) lightingOf.try_emplace(scene.id, LightingJson(*lighting)); if (sky < 0) { const auto skydome = JoinPath("", WorldScene::ReadSkydome(*lvl)); if (skydome.ends_with(".nif") && Files().paths.contains(skydome)) sky = static_cast(scenery.IndexOf(skydome)); } for (const auto& object : WorldScene::ReadObjects(*lvl)) { // A spawner is drawn as what it spawns, where the client would show it const auto model = ModelFor(object); if (model.path.empty()) continue; const auto asset = scenery.IndexOf(model.path); assetOf.push_back(asset); if (asset >= assetShaders.size()) assetShaders.resize(asset + 1, -1); if (assetShaders[asset] == -1) assetShaders[asset] = model.shader; hidden.push_back(model.hidden ? 1 : 0); for (const auto value : { object.x, object.y, object.z }) positions.push_back(Round(value, 100.0)); for (const auto value : { object.qx, object.qy, object.qz, object.qw }) rotations.push_back(Round(value, 10000.0)); scales.push_back(Round(object.scale, 1000.0)); sceneOf.push_back(scene.id); } if (lighting) sceneLighting.emplace_back(*lighting, assetOf.size() - objectsBefore); } if (const auto lighting = WorldScene::ZoneLighting(sceneLighting)) scenery.lighting = LightingJson(*lighting); nlohmann::json manifest{ {"zone", zoneId}, {"sky", sky}, {"assets", scenery.assets}, {"lighting", scenery.lighting}, {"format", FORMAT_VERSION}, {"objects", { {"asset", assetOf}, {"pos", positions}, {"rot", rotations}, {"scale", scales}, {"hidden", hidden}, {"scene", sceneOf} }}, {"scenes", ScenesJson(*zoneFile, lightingOf)}, {"sceneMap", SceneMapJson(zoneId)} }; assetShaders.resize(scenery.assets.size(), -1); AddShaders(manifest, assetShaders); scenery.json = manifest.dump(); return std::make_shared(std::move(scenery)); } OnceCache> g_Zones; // The zone's scenery, built when first asked for (any thread); nullptr without client files std::shared_ptr Zone(uint32_t zoneId) { return g_Zones.Get(zoneId, [zoneId] { return BuildZone(zoneId); }); } // The same, only when it is built already (never waits) std::shared_ptr ZoneIfBuilt(uint32_t zoneId) { return g_Zones.Ready(zoneId) ? Zone(zoneId) : nullptr; } // FlairTable: flair id -> the model's res path (empty when the client lacks it), read once std::unordered_map ReadFlairModels() { std::unordered_map models; try { auto row = CDClientDatabase::ExecuteQuery("SELECT id, asset FROM FlairTable;"); for (; !row.eof(); row.nextRow()) models.try_emplace(static_cast(row.getIntField(0)), ResolveModel(row.getStringField(1, ""))); } catch (const std::exception& ex) { LOG("Could not read the flairs: %s", ex.what()); } return models; } const std::unordered_map& FlairModels() { static const auto models = ReadFlairModels(); return models; } /** * How far from the camera flairs are drawn: the client's Flair.fx draws them fully to sqrt(60000) units and fades * them out over the next 15000 of distance squared. */ constexpr double FLAIR_DISTANCE = 274.0; /** * The flairs' manifest (as the scenery's, plus a tint per flair), from the zone's terrain file. A flair's color * tints its model as a byte over 255, times the model's own vertex colors (FlairAssets::AppendRenderBuffer). */ std::optional BuildFlairs(uint32_t zoneId, ZoneScenery& scenery) { const auto raw = ZoneRawShared(zoneId); const auto& models = FlairModels(); std::vector modelOf; nlohmann::json positions = nlohmann::json::array(), rotations = nlohmann::json::array(), scales = nlohmann::json::array(), colors = nlohmann::json::array(); for (const auto& chunk : raw ? raw->chunks : std::vector{}) { for (const auto& flair : chunk.flairs) { const auto model = models.find(flair.id); if (model == models.end() || model->second.empty() || !std::isfinite(flair.position.x)) continue; modelOf.push_back(&model->second); for (const auto value : { flair.position.x, flair.position.y, flair.position.z }) positions.push_back(Round(value, 100.0)); // Radians about x, y and z, applied in that order const double c1 = std::cos(flair.rotation.x / 2), c2 = std::cos(flair.rotation.y / 2), c3 = std::cos(flair.rotation.z / 2); const double s1 = std::sin(flair.rotation.x / 2), s2 = std::sin(flair.rotation.y / 2), s3 = std::sin(flair.rotation.z / 2); for (const auto value : { s1 * c2 * c3 + c1 * s2 * s3, c1 * s2 * c3 - s1 * c2 * s3, c1 * c2 * s3 + s1 * s2 * c3, c1 * c2 * c3 - s1 * s2 * s3 }) { rotations.push_back(Round(static_cast(value), 10000.0)); } scales.push_back(Round(flair.scaleFactor, 1000.0)); for (const auto value : { flair.colorR, flair.colorG, flair.colorB }) colors.push_back(value); } } // The flairs' models join the zone's list, which mesh requests read meanwhile nlohmann::json assetOf = nlohmann::json::array(); std::vector assets; { std::lock_guard lock(g_ZoneMutex); for (const auto* model : modelOf) { assetOf.push_back(scenery.IndexOf(*model)); scenery.flairModels.insert(*model); } assets = scenery.assets; } return nlohmann::json{ {"zone", zoneId}, {"sky", -1}, {"assets", assets}, {"distance", FLAIR_DISTANCE}, {"colorScale", 1.0 / 255.0}, {"lighting", scenery.lighting}, {"format", FORMAT_VERSION}, // Flair.fx for all of them: (0.85 * sun + ambient) * the flair's tint, whatever their facing {"technique", { {"family", "flair"}, {"look", 0}, {"alpha", "opacity"}, {"flags", 0} }}, {"objects", { {"asset", assetOf}, {"pos", positions}, {"rot", rotations}, {"scale", scales}, {"color", colors} }} }.dump(); } OnceCache> g_Flairs; /** * The flairs' manifest (as the scenery's, plus a tint per flair), from the zone's terrain file. A flair's color * tints its model as a byte over 255, times the model's own vertex colors (FlairAssets::AppendRenderBuffer). * Built when first asked for (any thread). */ const std::optional& Flairs(uint32_t zoneId, ZoneScenery& scenery) { return g_Flairs.Get(zoneId, [zoneId, &scenery] { return BuildFlairs(zoneId, scenery); }); } constexpr uintmax_t DISK_CACHE_BYTES = 512ull * 1024 * 1024; const std::filesystem::path CACHE_DIR = std::filesystem::path("dDashboardServer") / "scenery_cache"; uint64_t Fnv1a(const std::string& text) { uint64_t hash = 14695981039346656037ull; for (const auto c : text) hash = (hash ^ static_cast(c)) * 1099511628211ull; return hash; } std::optional ReadWhole(const std::filesystem::path& path) { std::ifstream file(path, std::ios::binary | std::ios::ate); if (!file) return std::nullopt; const auto size = file.tellg(); if (size <= 0) return std::nullopt; std::string data(static_cast(size), '\0'); file.seekg(0); if (!file.read(data.data(), size)) return std::nullopt; return data; } /** * The converted models on disk (CACHE_DIR), kept under DISK_CACHE_BYTES by removing the least recently written * files. Any thread. */ class DiskCache { public: void Store(const std::filesystem::path& target, const std::string& data) { std::lock_guard lock(m_Mutex); std::error_code ec; std::filesystem::create_directories(CACHE_DIR, ec); CountLocked(); if (m_Total + data.size() > DISK_CACHE_BYTES) { std::vector> files; for (const auto& entry : std::filesystem::directory_iterator(CACHE_DIR, ec)) { if (entry.path().extension() == ".bin") files.emplace_back(entry.last_write_time(ec), entry.path()); } std::sort(files.begin(), files.end()); for (const auto& [time, path] : files) { if (m_Total + data.size() <= DISK_CACHE_BYTES * 3 / 4) break; const auto size = std::filesystem::file_size(path, ec); if (std::filesystem::remove(path, ec)) m_Total -= std::min(m_Total, size); } } // Written next to the target and renamed, so a reader never sees half a file std::ostringstream temporary; temporary << target.string() << "." << std::this_thread::get_id() << ".tmp"; { std::ofstream file(temporary.str(), std::ios::binary | std::ios::trunc); if (!file.write(data.data(), static_cast(data.size()))) return; } const auto existed = std::filesystem::exists(target, ec); std::filesystem::rename(temporary.str(), target, ec); if (!ec && !existed) m_Total += data.size(); } // Bytes in the cache uintmax_t Total() { std::lock_guard lock(m_Mutex); CountLocked(); return m_Total; } private: void CountLocked() { if (m_Counted) return; m_Counted = true; std::error_code ec; for (const auto& entry : std::filesystem::directory_iterator(CACHE_DIR, ec)) { if (entry.path().extension() == ".tmp") std::filesystem::remove(entry.path(), ec); // left by a crash else m_Total += entry.is_regular_file(ec) ? entry.file_size(ec) : 0; } } std::mutex m_Mutex; uintmax_t m_Total{}; bool m_Counted{}; }; DiskCache g_Disk; using Bytes = std::shared_ptr; // A TtlCache any thread may use class SharedCache { public: SharedCache(std::chrono::seconds ttl, size_t maxBytes) : m_Cache(ttl, maxBytes) {} Bytes Get(const std::string& key) { std::lock_guard lock(m_Mutex); const auto cached = m_Cache.Get(key); return cached ? *cached : nullptr; } void Put(const std::string& key, Bytes value) { if (!value) return; std::lock_guard lock(m_Mutex); const auto weight = value->size() + 256; m_Cache.Put(key, std::move(value), weight); } private: std::mutex m_Mutex; TtlCache m_Cache; }; SharedCache g_Models(std::chrono::hours(1), MESH_CACHE_BYTES); // "path|lod" -> NifFile::Encode's output SharedCache g_Embedded(std::chrono::hours(1), MESH_CACHE_BYTES); // "path#block" -> DDS of a texture stored in a .nif // Conversions under way ("path|lod"), so a model asked for twice at once is converted once and both get it std::mutex g_ConvertingMutex; std::map> g_Converting; std::string ModelKey(const std::string& path, uint32_t lod) { return path + "|" + std::to_string(lod); } // Where model `path` at `lod` is kept on disk, named after the source file's size and time so a changed client // file is converted again std::filesystem::path DiskPath(const std::string& key, const std::filesystem::path& file) { std::error_code ec; const auto size = std::filesystem::file_size(file, ec); const auto time = std::filesystem::last_write_time(file, ec).time_since_epoch().count(); const auto diskKey = key + "|" + std::to_string(size) + "|" + std::to_string(time) + "|" + std::to_string(FORMAT_VERSION); return CACHE_DIR / (std::to_string(Fnv1a(diskKey)) + ".bin"); } Bytes Convert(const std::string& path, uint32_t lod, const std::filesystem::path& file, const std::filesystem::path& target) { if (auto encoded = ReadWhole(target)) return std::make_shared(std::move(*encoded)); const auto data = ReadWhole(file); if (!data) return nullptr; std::string error; const auto model = NifFile::Parse(*data, lod, error); if (!model) { LOG_DEBUG("Could not read %s: %s", path.c_str(), error.c_str()); return nullptr; } const auto folder = FolderOf(path); // Per mesh: a res path, "#" for one stored in the .nif, or empty; for its base and its dark texture const auto where = [&folder](int32_t embedded, const std::string& file) { if (embedded >= 0) return "#" + std::to_string(embedded); return file.empty() ? std::string{} : FindTexture(folder, file); }; std::vector textures, darkTextures; for (const auto& mesh : model->meshes) { textures.push_back(where(mesh.material.embeddedTexture, mesh.material.texture)); darkTextures.push_back(where(mesh.material.embeddedDarkTexture, mesh.material.darkTexture)); } auto encoded = std::make_shared(NifFile::Encode(*model, textures, darkTextures)); g_Disk.Store(target, *encoded); return encoded; } /** * Model `path` (on disk at `file`) at `lod` in NifFile::Encode's format. Converting a big .nif takes a moment, so * results are kept in memory (MESH_CACHE_BYTES; unless `keep` is false, for conversions ahead of time) and on disk * (DISK_CACHE_BYTES). Any thread; the same model asked for again while it converts waits for that conversion. */ Bytes Encoded(const std::string& path, uint32_t lod, const std::filesystem::path& file, bool keep = true) { const auto key = ModelKey(path, lod); if (auto cached = g_Models.Get(key)) return cached; std::promise promise; std::shared_future converting; bool mine = false; { std::lock_guard lock(g_ConvertingMutex); const auto it = g_Converting.find(key); if (it != g_Converting.end()) { converting = it->second; } else { converting = promise.get_future().share(); g_Converting.emplace(key, converting); mine = true; } } if (!mine) { auto result = converting.get(); if (keep) g_Models.Put(key, result); return result; } Bytes result; try { result = Convert(path, lod, file, DiskPath(key, file)); } catch (const std::exception& ex) { LOG("Could not convert %s: %s", path.c_str(), ex.what()); } if (keep) g_Models.Put(key, result); { std::lock_guard lock(g_ConvertingMutex); g_Converting.erase(key); } promise.set_value(result); return result; } // The "textures" list of an encoded model's header std::vector TexturesOf(const std::string& encoded) { uint32_t length{}; if (encoded.size() < 4) return {}; std::memcpy(&length, encoded.data(), 4); if (length > encoded.size() - 4) return {}; const auto header = nlohmann::json::parse(encoded.substr(4, length), nullptr, false); if (header.is_discarded() || !header.contains("textures") || !header["textures"].is_array()) return {}; std::vector textures; for (const auto& texture : header["textures"]) textures.push_back(texture.is_string() ? texture.get() : std::string{}); return textures; } void Binary(HTTPReply& reply, std::string body) { reply.status = eHTTPStatusCode::OK; reply.contentType = eContentType::APPLICATION_OCTET_STREAM; reply.message = std::move(body); reply.headers.push_back("Cache-Control: private, max-age=604800"); } uint32_t LodOf(const HTTPContext& context) { return std::min(GeneralUtils::TryParse(QueryValue(context.queryString, "lod")).value_or(0), MAX_LOD); } // ---- Converting on worker threads, so a big model never holds up the web server's one thread ---- WorkerPool& Pool() { return Workers::Pool(); } constexpr uintmax_t SMALL_MODEL_BYTES = 256 * 1024; // .nif files this small convert in the pool's fast lane constexpr uintmax_t LARGE_MODEL_BYTES = 4 * 1024 * 1024; // and this big wait behind everything smaller constexpr auto WARM_IDLE = std::chrono::seconds(90); // converting a zone ahead stops once nobody has asked for it this long constexpr uintmax_t WARM_DISK_BYTES = DISK_CACHE_BYTES * 3 / 4; // and when the disk cache is this full (it never evicts for it) constexpr size_t WARM_MAX_MODELS = 4000; // When someone last asked for something of a zone, and the LOD they last asked a model at struct Activity { std::chrono::steady_clock::time_point last; uint32_t lod{ 1 }; // the world view's default detail }; std::mutex g_ActivityMutex; std::map g_Activity; void Touch(uint32_t zoneId, std::optional lod = std::nullopt) { std::lock_guard lock(g_ActivityMutex); auto& activity = g_Activity[zoneId]; activity.last = std::chrono::steady_clock::now(); if (lod) activity.lod = *lod; } // The zone's activity while someone views it, nullopt once nobody has for WARM_IDLE std::optional Viewed(uint32_t zoneId) { std::lock_guard lock(g_ActivityMutex); const auto it = g_Activity.find(zoneId); if (it == g_Activity.end() || std::chrono::steady_clock::now() - it->second.last > WARM_IDLE) return std::nullopt; return it->second; } // Flairs (small, and drawn around the camera) and small models first; big ones behind the rest WorkerPool::ePriority PriorityOf(const ZoneScenery* zone, const std::string& path, const std::filesystem::path& file) { if (zone) { std::lock_guard lock(g_ZoneMutex); if (zone->flairModels.contains(path)) return WorkerPool::ePriority::URGENT; } std::error_code ec; const auto size = std::filesystem::file_size(file, ec); if (ec || size <= SMALL_MODEL_BYTES) return WorkerPool::ePriority::URGENT; return size >= LARGE_MODEL_BYTES ? WorkerPool::ePriority::LARGE : WorkerPool::ePriority::NORMAL; } uint64_t WarmGroup(uint32_t zoneId) { return static_cast(zoneId) + 1; } /** * Convert models of a zone ahead of time (onto the disk cache), at the LOD its viewer last asked for, while * someone views it. The lowest priority: only when nothing else waits. Smallest first; `front` puts these before * the zone's other queued ones (the flairs). Any thread. */ void WarmUp(uint32_t zoneId, const std::vector& paths, bool front) { if (!Pool().Running() || paths.empty()) return; struct Item { std::string path; std::filesystem::path file; uintmax_t size{}; }; std::vector items; std::set seen; std::error_code ec; const auto res = ClientAssets::ResFolder(); for (const auto& path : paths) { if (!seen.insert(path).second) continue; const auto file = ClientAssets::ResolveResFile(path, res); if (file) items.push_back({ path, *file, std::filesystem::file_size(*file, ec) }); } std::stable_sort(items.begin(), items.end(), [](const Item& a, const Item& b) { return a.size < b.size; }); if (items.size() > WARM_MAX_MODELS) items.resize(WARM_MAX_MODELS); const auto group = WarmGroup(zoneId); // Queued at the front in reverse, so they still run smallest first if (front) std::reverse(items.begin(), items.end()); for (auto& item : items) { Pool().Submit(WorkerPool::ePriority::BACKGROUND, [zoneId, group, path = std::move(item.path), file = std::move(item.file)] { const auto activity = Viewed(zoneId); if (!activity || g_Disk.Total() >= WARM_DISK_BYTES) { Pool().Cancel(group); return; } const auto key = ModelKey(path, activity->lod); std::error_code ec; if (g_Models.Get(key) || std::filesystem::exists(DiskPath(key, file), ec)) return; Encoded(path, activity->lod, file, false); }, group, front); } } /** * Warm the zone's models when its manifest is asked for (again after nobody viewed it for a while). Planning it * (finding the files) is left to a worker, so a manifest in memory is still answered at once. */ void WarmScenery(uint32_t zoneId, ZoneScenery& zone, bool flairs) { std::vector paths; { const bool idle = !Viewed(zoneId); std::lock_guard lock(g_ZoneMutex); if (idle) zone.warmedScenery = zone.warmedFlairs = false; auto& warmed = flairs ? zone.warmedFlairs : zone.warmedScenery; if (!warmed) { warmed = true; if (flairs) paths.assign(zone.flairModels.begin(), zone.flairModels.end()); else paths = zone.assets; } } Touch(zoneId); if (paths.empty() || !Pool().Running()) return; Pool().Submit(WorkerPool::ePriority::NORMAL, [zoneId, paths = std::move(paths), flairs] { WarmUp(zoneId, paths, flairs); }); } // The texture `name` of model `path` (textures[slot] of its encoded form) as a DDS file. Any thread. std::optional TextureBytes(const std::string& path, const std::filesystem::path& modelFile, const std::vector& textures, const std::string& name, const std::filesystem::path& res) { if (!name.starts_with('#')) { const auto file = ClientAssets::ResolveResFile(name, res); return file ? ReadWhole(*file) : std::nullopt; } // Stored inside the model: reading a big .nif again for each of its textures would be slow, so they're kept if (const auto cached = g_Embedded.Get(path + name)) return *cached; const auto data = ReadWhole(modelFile); const auto block = GeneralUtils::TryParse(name.substr(1)); if (!data || !block) return std::nullopt; // Every texture of the file at once, since the browser asks for them together for (const auto& other : textures) { if (other == name) continue; const auto otherBlock = other.starts_with('#') ? GeneralUtils::TryParse(other.substr(1)) : std::nullopt; auto file = otherBlock ? NifFile::EmbeddedTexture(*data, *otherBlock) : std::nullopt; if (file) g_Embedded.Put(path + other, std::make_shared(std::move(*file))); } auto dds = NifFile::EmbeddedTexture(*data, *block); if (dds) g_Embedded.Put(path + name, std::make_shared(*dds)); return dds; } // The model path of `asset` in the zone's manifests, building what's missing (any thread); nullopt: no such model std::optional AssetPath(uint32_t zoneId, uint32_t asset) { const auto zone = Zone(zoneId); if (!zone) return std::nullopt; { std::lock_guard lock(g_ZoneMutex); if (asset < zone->assets.size()) return zone->assets[asset]; } // The flairs' models join the list when their manifest is first built (a browser may still have it cached) Flairs(zoneId, *zone); std::lock_guard lock(g_ZoneMutex); if (asset < zone->assets.size()) return zone->assets[asset]; return std::nullopt; } // The same without building anything (never waits): nullopt when it isn't known yet std::optional AssetPathIfBuilt(uint32_t zoneId, uint32_t asset) { const auto zone = ZoneIfBuilt(zoneId); if (!zone) return std::nullopt; std::lock_guard lock(g_ZoneMutex); if (asset < zone->assets.size()) return zone->assets[asset]; return std::nullopt; } /** * Where model `asset` is, and how urgent converting it is, when that is known without building anything: for * the web thread, which only hands the work on */ struct Known { std::optional path; std::optional file; WorkerPool::ePriority priority{ WorkerPool::ePriority::NORMAL }; }; Known KnownAsset(uint32_t zoneId, uint32_t asset) { Known known; known.path = AssetPathIfBuilt(zoneId, asset); if (!known.path) return known; known.file = ClientAssets::ResolveResFile(*known.path, ClientAssets::ResFolder()); if (known.file) known.priority = PriorityOf(ZoneIfBuilt(zoneId).get(), *known.path, *known.file); return known; } } namespace Scenery { void Preload() { Files(); RenderInfos(); FlairModels(); } std::optional ZoneJson(uint32_t zoneId) { const auto zone = Zone(zoneId); if (!zone) return std::nullopt; WarmScenery(zoneId, *zone, false); return zone->json; } std::vector MultishaderLooks(const NifFile::Model& model) { std::vector looks; for (const auto& mesh : model.meshes) { std::optional shader; if (const auto it = g_ShaderValues.find(mesh.material.shaderTag); it != g_ShaderValues.end()) shader = it->second; looks.push_back(NifFile::ShaderLookFor(NifFile::MultishaderPart(shader))); } return looks; } bool ZoneReady(uint32_t zoneId) { return g_Zones.Ready(zoneId); } bool HasModel(const WorldScene::Object& object) { const auto model = ModelFor(object); return !model.path.empty() && !model.hidden; } std::optional FlairsJson(uint32_t zoneId) { const auto zone = Zone(zoneId); if (!zone) return std::nullopt; const auto& flairs = Flairs(zoneId, *zone); WarmScenery(zoneId, *zone, true); return flairs; } bool FlairsReady(uint32_t zoneId) { return g_Zones.Ready(zoneId) && g_Flairs.Ready(zoneId); } void ReplyMesh(HTTPReply& reply, const HTTPContext& context, uint32_t zoneId, uint32_t asset, uint32_t lod) { lod = std::min(lod, MAX_LOD); Touch(zoneId, lod); auto known = KnownAsset(zoneId, asset); if (known.path) { if (const auto cached = g_Models.Get(ModelKey(*known.path, lod))) return Binary(reply, *cached); } const auto res = ClientAssets::ResFolder(); const auto deferred = Web::Defer(reply, context); Pool().Submit(known.priority, [deferred, zoneId, asset, lod, res, known = std::move(known)] { if (deferred.Cancelled()) return; HTTPReply out; const auto path = known.path ? known.path : AssetPath(zoneId, asset); const auto file = known.file ? known.file : path ? ClientAssets::ResolveResFile(*path, res) : std::nullopt; if (!path) { JsonError(out, eHTTPStatusCode::NOT_FOUND, "No such model in this zone"); } else if (const auto encoded = file ? Encoded(*path, lod, *file) : nullptr) { Binary(out, *encoded); } else { JsonError(out, eHTTPStatusCode::NOT_FOUND, "Could not read this model"); } deferred.Send(std::move(out)); }); } void ReplyTexture(HTTPReply& reply, const HTTPContext& context, uint32_t zoneId, uint32_t asset, uint32_t slot, uint32_t lod) { lod = std::min(lod, MAX_LOD); Touch(zoneId, lod); auto known = KnownAsset(zoneId, asset); // Quick when the model is converted already and the texture is a file of its own, or one kept from its model if (known.path) { if (const auto cached = g_Models.Get(ModelKey(*known.path, lod))) { const auto textures = TexturesOf(*cached); if (slot >= textures.size() || !textures[slot].starts_with('#') || g_Embedded.Get(*known.path + textures[slot])) known.priority = WorkerPool::ePriority::URGENT; } } const auto res = ClientAssets::ResFolder(); const auto deferred = Web::Defer(reply, context); Pool().Submit(known.priority, [deferred, zoneId, asset, lod, slot, res, known = std::move(known)] { if (deferred.Cancelled()) return; HTTPReply out; const auto path = known.path ? known.path : AssetPath(zoneId, asset); const auto file = known.file ? known.file : path ? ClientAssets::ResolveResFile(*path, res) : std::nullopt; const auto encoded = path && file ? Encoded(*path, lod, *file) : nullptr; const auto textures = encoded ? TexturesOf(*encoded) : std::vector{}; if (!path) { JsonError(out, eHTTPStatusCode::NOT_FOUND, "No such model in this zone"); } else if (slot >= textures.size() || textures[slot].empty()) { JsonError(out, eHTTPStatusCode::NOT_FOUND, "No such texture"); } else if (auto dds = TextureBytes(*path, *file, textures, textures[slot], res)) { Binary(out, std::move(*dds)); } else { JsonError(out, eHTTPStatusCode::NOT_FOUND, "Could not read this texture"); } deferred.Send(std::move(out)); }); } /** * The environment textures the client's shaders load themselves, by the name the viewers ask for them: the default * reflection cube (LEGOPPLighting, ClearPlastic) and Metallic.fx's cubes and noise. */ static const std::map& EnvironmentTextures() { static const std::map textures{ { "reflection", "textures/env/default_reflection.dds" }, { "polished", "textures/metal/metal_reflection_polished.dds" }, { "brushed", "textures/metal/metal_reflection_brushed.dds" }, { "brushedNoise", "textures/metal/metal_reflection_brushed_noise.dds" } }; return textures; } void RegisterRoutes() { Route(eHTTPMethod::GET, "/api/scenery/env/:name", 0, "An environment texture the client's shaders load themselves, as a DDS file: reflection (the default reflection cube), polished, brushed (the metal cubes) or brushedNoise", [](HTTPReply& reply, const HTTPContext& context) { const std::string name(PathSegment(context.path, 3)); const auto& textures = EnvironmentTextures(); const auto it = textures.find(name); if (it == textures.end()) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No such environment texture"); // Read once: a few megabytes the views ask for with every zone static std::mutex mutex; static std::map> cache; std::shared_ptr bytes; { std::lock_guard lock(mutex); if (const auto cached = cache.find(name); cached != cache.end()) bytes = cached->second; } if (!bytes) { auto read = ClientAssets::ReadResFile(it->second); if (!read) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "The client has no such texture"); bytes = std::make_shared(std::move(*read)); std::lock_guard lock(mutex); cache.try_emplace(name, bytes); } Binary(reply, *bytes); }); Route(eHTTPMethod::GET, "/api/scenery/:zone/mesh/:asset", 0, "Model `asset` of a zone's scenery (see the scenery routes of properties and /world3d), converted from the client's .nif. Query: ?lod=0 (most detailed) to 3", [](HTTPReply& reply, const HTTPContext& context) { const auto zone = PathId(context.path, 2); const auto asset = PathId(context.path, 4); if (!zone || !asset) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone or model"); ReplyMesh(reply, context, *zone, *asset, LodOf(context)); }); Route(eHTTPMethod::GET, "/api/scenery/:zone/texture/:asset/:slot", 0, "Texture `slot` of scenery model `asset` (its \"textures\" list) as a DDS file. Query: ?lod= as for the model", [](HTTPReply& reply, const HTTPContext& context) { const auto zone = PathId(context.path, 2); const auto asset = PathId(context.path, 4); const auto slot = PathId(context.path, 5); if (!zone || !asset || !slot) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid zone, model or texture"); ReplyTexture(reply, context, *zone, *asset, *slot, LodOf(context)); }); } }