feat(world): record the zone files a world loads, with sizes and hashes

ZoneFileLog keeps the .luz, .lvl, .lutriggers, terrain .raw and navmesh a
world read while loading its zone: absolute path of the loose file (or
the name in the pack), size and an FNV-1a hash of the bytes read.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dCommon/CMakeLists.txt

@@ -23,6 +23,7 @@ set(DCOMMON_SOURCES
 		"FdbMappedFile.cpp"
 		"FdbReader.cpp"
 		"FdbSnapshot.cpp"
+		"ZoneFileLog.cpp"
 		"JSONUtils.cpp"
 		"TinyXmlUtils.cpp"
 		"Sd0.cpp"

dCommon/ZoneFileLog.cpp

@@ -0,0 +1,69 @@
+#include "ZoneFileLog.h"
+
+#include <algorithm>
+
+#include "AssetManager.h"
+#include "FdbSnapshot.h"
+
+namespace {
+	std::vector<ZoneFileLog::Entry> g_Entries;
+
+	std::string Absolute(const std::filesystem::path& path) {
+		std::error_code code;
+		const auto absolute = std::filesystem::absolute(path, code);
+		return (code ? path : absolute).lexically_normal().generic_string();
+	}
+}
+
+const char* ZoneFileLog::KindName(eKind kind) {
+	switch (kind) {
+	case eKind::ZONE: return "zone";
+	case eKind::SCENE: return "scene";
+	case eKind::TRIGGERS: return "triggers";
+	case eKind::TERRAIN: return "terrain";
+	case eKind::NAVMESH: return "navmesh";
+	case eKind::OTHER: return "other";
+	}
+	return "other";
+}
+
+void ZoneFileLog::Record(eKind kind, const std::string& path, bool packed, std::string_view bytes) {
+	Entry entry;
+	entry.kind = kind;
+	entry.packed = packed;
+	entry.size = bytes.size();
+	entry.hash = FdbSnapshot::Hash(reinterpret_cast<const uint8_t*>(bytes.data()), bytes.size());
+	entry.path = path;
+	const auto existing = std::find_if(g_Entries.begin(), g_Entries.end(), [&path](const Entry& e) { return e.path == path; });
+	if (existing != g_Entries.end()) *existing = std::move(entry);
+	else g_Entries.push_back(std::move(entry));
+}
+
+void ZoneFileLog::RecordAsset(const AssetManager* assets, eKind kind, const std::string& name, const AssetStream& stream) {
+	const auto loose = assets ? assets->GetLoosePath(name) : std::nullopt;
+	Record(kind, loose ? Absolute(*loose) : name, !loose, stream.Bytes());
+}
+
+void ZoneFileLog::RecordFile(eKind kind, const std::filesystem::path& path) {
+	std::error_code code;
+	const auto size = std::filesystem::file_size(path, code);
+	if (code) return;
+	const auto hash = FdbSnapshot::HashFile(path);
+	if (!hash) return;
+	Entry entry;
+	entry.kind = kind;
+	entry.size = static_cast<uint64_t>(size);
+	entry.hash = *hash;
+	entry.path = Absolute(path);
+	const auto existing = std::find_if(g_Entries.begin(), g_Entries.end(), [&entry](const Entry& e) { return e.path == entry.path; });
+	if (existing != g_Entries.end()) *existing = std::move(entry);
+	else g_Entries.push_back(std::move(entry));
+}
+
+const std::vector<ZoneFileLog::Entry>& ZoneFileLog::Entries() {
+	return g_Entries;
+}
+
+void ZoneFileLog::Clear() {
+	g_Entries.clear();
+}

dCommon/ZoneFileLog.h

@@ -0,0 +1,57 @@
+#ifndef ZONEFILELOG_H
+#define ZONEFILELOG_H
+
+#include <cstdint>
+#include <filesystem>
+#include <string>
+#include <string_view>
+#include <vector>
+
+class AssetManager;
+struct AssetStream;
+
+/**
+ * The zone data files a world server loaded (docs/WorldHotReload.md): the .luz, every .lvl scene and its .lutriggers,
+ * the terrain .raw and the navmesh, each with its size and a hash of the bytes the world read. The world sends the
+ * list to master once it is ready (WORLD_FILES), and master watches those files for changes.
+ *
+ * Main thread only (the world records while it loads its zone).
+ */
+namespace ZoneFileLog {
+	enum class eKind : uint8_t {
+		ZONE = 0,     // .luz
+		SCENE,        // .lvl
+		TRIGGERS,     // .lutriggers
+		TERRAIN,      // .raw
+		NAVMESH,      // navmeshes/<zone>.bin
+		OTHER,
+	};
+
+	const char* KindName(eKind kind);
+
+	struct Entry {
+		eKind kind{ eKind::OTHER };
+		bool packed{};       // read from the client's packs: path is the name in the pack, and nothing watches it
+		uint64_t size{};
+		uint64_t hash{};     // FdbSnapshot::Hash of the bytes read
+		std::string path;    // absolute path of a loose file
+
+		bool operator==(const Entry& other) const = default;
+	};
+
+	// A file the world read; a path already recorded is recorded again (the last read wins)
+	void Record(eKind kind, const std::string& path, bool packed, std::string_view bytes);
+
+	// A file read from the client's res folder through the asset manager: the loose file's absolute path, or the
+	// name in the pack
+	void RecordAsset(const AssetManager* assets, eKind kind, const std::string& name, const AssetStream& stream);
+
+	// A file read straight from disk (the navmesh): hashed here, not recorded when it can't be read
+	void RecordFile(eKind kind, const std::filesystem::path& path);
+
+	const std::vector<Entry>& Entries();
+
+	void Clear();
+};
+
+#endif // ZONEFILELOG_H

dCommon/dClient/AssetManager.cpp

@@ -154,6 +154,17 @@ bool AssetManager::GetFile(std::string fixedName, char** data, uint32_t* len) co
 	return success;
 }
 
+std::optional<std::filesystem::path> AssetManager::GetLoosePath(std::string fixedName) const {
+	// The same name fixing as GetFile
+	std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); });
+	std::replace(fixedName.begin(), fixedName.end(), '\\', '/');
+	if (this->m_AssetBundleType == eAssetBundleType::Unpacked) GeneralUtils::ReplaceInString(fixedName, "brickmodels", "BrickModels");
+	const auto path = m_ResPath / fixedName;
+	std::error_code code;
+	if (!std::filesystem::exists(path, code)) return std::nullopt;
+	return path;
+}
+
 AssetStream AssetManager::GetFile(const char* name) const {
 	char* buf; uint32_t len;
 

dCommon/dClient/AssetManager.h

@@ -4,6 +4,8 @@
 #include <vector>
 #include <unordered_map>
 #include <filesystem>
+#include <optional>
+#include <string_view>
 
 #include "Pack.h"
 #include "PackIndex.h"
@@ -44,6 +46,12 @@ struct AssetMemoryBuffer : std::streambuf {
 			setg(eback(), eback() + off, egptr());
 		return gptr() - eback();
 	}
+
+	// Every byte of the file (empty when it wasn't found)
+	std::string_view Bytes() const {
+		if (!m_Success) return {};
+		return std::string_view(eback(), static_cast<size_t>(egptr() - eback()));
+	}
 };
 
 struct AssetStream : std::istream {
@@ -56,6 +64,11 @@ struct AssetStream : std::istream {
 	operator bool() {
 		return reinterpret_cast<AssetMemoryBuffer*>(rdbuf())->m_Success;
 	}
+
+	// Every byte of the file, whatever has been read of it so far
+	std::string_view Bytes() const {
+		return reinterpret_cast<const AssetMemoryBuffer*>(rdbuf())->Bytes();
+	}
 };
 
 class AssetManager {
@@ -78,6 +91,10 @@ public:
 	[[nodiscard]]
 	bool GetFile(std::string name, char** data, uint32_t* len) const;
 
+	// Where GetFile reads name from when it is a loose file in res; nullopt when it comes from a pack or isn't there
+	[[nodiscard]]
+	std::optional<std::filesystem::path> GetLoosePath(std::string name) const;
+
 	[[nodiscard]]
 	AssetStream GetFile(const char* name) const;
 

dNavigation/dNavMesh.cpp

@@ -12,6 +12,7 @@
 #include "dZoneManager.h"
 #include "DluAssert.h"
 #include "DetourExtensions.h"
+#include "ZoneFileLog.h"
 
 dNavMesh::dNavMesh(uint32_t zoneId) {
 	m_ZoneId = zoneId;
@@ -60,6 +61,8 @@ void dNavMesh::LoadNavmesh() {
 	if (!fp.fp) {
 		return;
 	}
+	// Master watches the files a world loaded (docs/WorldHotReload.md); a broken one too, so fixing it reloads the world
+	ZoneFileLog::RecordFile(ZoneFileLog::eKind::NAVMESH, path);
 
 	// Read header.
 	NavMeshSetHeader header;

dZoneManager/Level.cpp

@@ -15,6 +15,7 @@
 #include "CDClientManager.h"
 #include "AssetManager.h"
 #include "LevelFile.h"
+#include "ZoneFileLog.h"
 #include "ClientVersion.h"
 #include "dConfig.h"
 #include <ranges>
@@ -29,6 +30,7 @@ Level::Level(Zone* parentZone, const std::string& filepath, int32_t sceneID) {
 		LOG("Failed to load %s", filepath.c_str());
 		return;
 	}
+	ZoneFileLog::RecordAsset(Game::assetManager, ZoneFileLog::eKind::SCENE, filepath, stream);
 	
 	// The file itself is read by LevelFile (dCommon), which the dashboard uses too
 	LevelFile levelFile;

dZoneManager/Zone.cpp

@@ -21,6 +21,7 @@
 #include "eTriggerEventType.h"
 #include "eWaypointCommandType.h"
 #include "dNavMesh.h"
+#include "ZoneFileLog.h"
 
 Zone::Zone(const LWOZONEID zoneID) :
 	m_ZoneID(zoneID) {
@@ -54,6 +55,8 @@ void Zone::LoadZoneIntoMemory() {
 	}
 
 	if (file) {
+		// Master watches the files a world loaded (docs/WorldHotReload.md)
+		ZoneFileLog::RecordAsset(Game::assetManager, ZoneFileLog::eKind::ZONE, m_ZoneFilePath, file);
 		// The file itself is read by ZoneFile (dCommon), which the dashboard uses too
 		ZoneFile zoneFile;
 		zoneFile.Read(file);
@@ -227,6 +230,7 @@ bool Zone::LoadSceneMap() {
 		LOG("Could not open the terrain file %s for its scene map", (m_ZonePath + m_ZoneRawPath).c_str());
 		return false;
 	}
+	ZoneFileLog::RecordAsset(Game::assetManager, ZoneFileLog::eKind::TERRAIN, m_ZonePath + m_ZoneRawPath, file);
 	Raw::Raw raw;
 	if (!Raw::ReadRaw(file, raw)) {
 		LOG("Could not read the terrain file %s for its scene map", (m_ZonePath + m_ZoneRawPath).c_str());
@@ -257,6 +261,7 @@ void Zone::LoadScene(ZoneScene&& zoneScene) {
 
 void Zone::LoadLUTriggers(std::string triggerFile, SceneRef& scene) {
 	auto file = Game::assetManager->GetFile((m_ZonePath + triggerFile).c_str());
+	if (file) ZoneFileLog::RecordAsset(Game::assetManager, ZoneFileLog::eKind::TRIGGERS, m_ZonePath + triggerFile, file);
 
 	std::stringstream data;
 	data << file.rdbuf();