fix: read and keep a zone file's scenes as the client does

LuzFile::ReadLUZFile, LuzReader::ReadScenes and ZoneLoader::ReadZoneFile
(1.10.64):
- a version older than 20 is read as 20;
- before version 30 a scene is a bare u32 SceneTable ID: in ID order,
  each one with a SceneTable row gets the next index (-1 past 255) as
  its ID and the row's sceneName as its file; the others are left out;
- from 30 to 32 the file has no scene IDs: a scene's ID is its index in
  the file (they all had ID 0, colliding);
- a later scene with the same ID and layer replaces an earlier one, a
  scene ID with no layer 0 scene is left out, and before version 41
  only the layer 0 scenes are loaded.

No 1.10.64 zone file has a duplicate scene, a scene without a layer 0
scene or another layer before version 41, so what live zones load is
unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dZoneManager/Zone.cpp

@@ -11,6 +11,7 @@
 #include "dConfig.h"
 
 #include "AssetManager.h"
+#include "CDClientDatabase.h"
 #include "CDClientManager.h"
 #include "CDZoneTableTable.h"
 #include "Spawner.h"
@@ -30,6 +31,13 @@ Zone::Zone(const LWOZONEID zoneID) :
 	m_CheckSum = 0;
 	m_WorldID = 0;
 	m_SceneCount = 0;
+	m_SceneTableLookup = [](uint32_t sceneTableID) -> std::optional<std::string> {
+		auto query = CDClientDatabase::CreatePreppedStmt("SELECT sceneName FROM SceneTable WHERE sceneID = ?;");
+		query.bind(1, static_cast<int32_t>(sceneTableID));
+		auto result = query.execQuery();
+		if (result.eof() || result.fieldIsNull("sceneName")) return std::nullopt;
+		return std::string(result.getStringField("sceneName"));
+	};
 }
 
 Zone::~Zone() {
@@ -56,6 +64,8 @@ void Zone::LoadZoneIntoMemory() {
 
 	if (file) {
 		BinaryIO::BinaryRead(file, m_FileFormatVersion);
+		// The client reads anything older than 20 as 20 (LuzFile::ReadLUZFile)
+		if (m_FileFormatVersion < Zone::FileFormatVersion::Oldest) m_FileFormatVersion = Zone::FileFormatVersion::Oldest;
 
 		uint32_t mapRevision = 0;
 		if (m_FileFormatVersion >= Zone::FileFormatVersion::Alpha) BinaryIO::BinaryRead(file, mapRevision);
@@ -78,8 +88,9 @@ void Zone::LoadZoneIntoMemory() {
 		} else BinaryIO::BinaryRead(file, m_SceneCount);
 
 		for (uint32_t i = 0; i < m_SceneCount; ++i) {
-			LoadScene(file);
+			LoadScene(file, i);
 		}
+		KeepLoadedScenes();
 
 		// Zone boundary lines: a count byte, then the lines
 		uint8_t numBoundaries = 0;
@@ -268,21 +279,29 @@ void Zone::AddRevision(LWOSCENEID sceneID, uint32_t revision) {
 	}
 }
 
-void Zone::LoadScene(std::istream& file) {
+void Zone::LoadScene(std::istream& file, uint32_t index) {
 	SceneRef scene;
 	scene.level = nullptr;
-	LWOSCENEID lwoSceneID(LWOZONEID_INVALID, 0);
+
+	// Before PrePreAlpha a scene is only its SceneTable ID; KeepLoadedScenes finds its file (LuzReader::ReadScenes)
+	if (m_FileFormatVersion < Zone::FileFormatVersion::PrePreAlpha) {
+		BinaryIO::BinaryRead(file, scene.sceneTableID);
+		scene.id = scene.sceneTableID;
+		m_Scenes[LWOSCENEID(scene.id, 0)] = std::move(scene);
+		return;
+	}
 
 	BinaryIO::ReadString<uint8_t>(file, scene.filename, BinaryIO::ReadType::String);
 
 	std::string luTriggersPath = scene.filename.substr(0, scene.filename.size() - 4) + ".lutriggers";
 	if (Game::assetManager->HasFile((m_ZonePath + luTriggersPath).c_str())) LoadLUTriggers(luTriggersPath, scene);
 
-	if (m_FileFormatVersion >= Zone::FileFormatVersion::LatePreAlpha || m_FileFormatVersion < Zone::FileFormatVersion::PrePreAlpha) {
+	// Until LatePreAlpha the file has no scene IDs: a scene's ID is its index (-1 past 255) in the file
+	scene.id = index < 256 ? index : UINT32_MAX;
+	LWOSCENEID lwoSceneID(scene.id, 0);
+	if (m_FileFormatVersion >= Zone::FileFormatVersion::LatePreAlpha) {
 		BinaryIO::BinaryRead(file, scene.id);
 		lwoSceneID.SetSceneID(scene.id);
-	}
-	if (m_FileFormatVersion >= Zone::FileFormatVersion::LatePreAlpha) {
 		BinaryIO::BinaryRead(file, scene.sceneType);
 		lwoSceneID.SetLayerID(static_cast<uint32_t>(scene.sceneType));
 
@@ -306,6 +325,37 @@ void Zone::LoadScene(std::istream& file) {
 	m_Scenes[lwoSceneID] = std::move(scene);
 }
 
+void Zone::KeepLoadedScenes() {
+	if (m_FileFormatVersion < Zone::FileFormatVersion::PrePreAlpha) {
+		// ZoneLoader::ReadZoneFile: the scenes go by their SceneTable ID; each one with a row there gets the next index
+		// (0, 1, ...; -1 past 255) as its ID and the row's sceneName as its file, the others are left out
+		std::map<LWOSCENEID, SceneRef> resolved;
+		uint32_t index = 0;
+		for (auto& scene : m_Scenes | std::views::values) {
+			const auto sceneName = m_SceneTableLookup(scene.sceneTableID);
+			if (!sceneName) continue;
+			scene.id = index < 256 ? index : UINT32_MAX;
+			scene.filename = *sceneName;
+			scene.name = *sceneName;
+			const std::string luTriggersPath = scene.filename.substr(0, scene.filename.size() - 4) + ".lutriggers";
+			if (Game::assetManager->HasFile((m_ZonePath + luTriggersPath).c_str())) LoadLUTriggers(luTriggersPath, scene);
+			resolved[LWOSCENEID(scene.id, 0)] = std::move(scene);
+			index++;
+		}
+		m_Scenes = std::move(resolved);
+		return;
+	}
+
+	// LuzReader::ReadScenes keeps the scenes by scene ID and layer, a later one replacing an earlier one (as m_Scenes
+	// does). ZoneLoader::ReadZoneFile then goes through them by scene ID: a scene ID with no layer 0 (General) scene is
+	// left out, and before Latest only the layer 0 scene is loaded
+	std::erase_if(m_Scenes, [this](const auto& entry) {
+		const auto& [sceneID, scene] = entry;
+		if (!m_Scenes.contains(LWOSCENEID(sceneID.GetSceneID(), 0))) return true;
+		return sceneID.GetLayerID() != 0 && m_FileFormatVersion < Zone::FileFormatVersion::Latest;
+	});
+}
+
 void Zone::LoadLUTriggers(std::string triggerFile, SceneRef& scene) {
 	auto file = Game::assetManager->GetFile((m_ZonePath + triggerFile).c_str());
 

dZoneManager/Zone.h

@@ -7,6 +7,8 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <functional>
+#include <optional>
 #include "Raw.h"
 
 namespace LUTriggers {
@@ -31,6 +33,7 @@ enum class eSceneType : uint32_t {
 struct SceneRef {
 	std::string filename;
 	uint32_t id{};
+	uint32_t sceneTableID{}; // before PrePreAlpha: the scene's row in the CDClient SceneTable
 	eSceneType sceneType{};
 	std::string name;
 	NiPoint3 unknown1;
@@ -204,6 +207,7 @@ struct Path {
 class Zone {
 public:
 	enum class FileFormatVersion : uint32_t { //Times are guessed.
+		Oldest = 20, // the client reads anything older as this
 		PrePreAlpha = 30,
 		PreAlpha = 32,
 		LatePreAlpha = 33,
@@ -231,6 +235,10 @@ public:
 	LUTriggers::Trigger* GetTrigger(uint32_t sceneID, uint32_t triggerID) const;
 	const Path* GetPath(std::string name) const;
 
+	// The CDClient SceneTable's sceneName for a scene of a file from before PrePreAlpha; the database's by default
+	using SceneTableLookup = std::function<std::optional<std::string>(uint32_t sceneTableID)>;
+	void SetSceneTableLookup(SceneTableLookup lookup) { m_SceneTableLookup = std::move(lookup); }
+
 	uint32_t GetWorldID() const { return m_WorldID; }
 	[[nodiscard]] std::string GetZoneName() const { return m_ZoneName; }
 	std::string GetZoneRawPath() const { return m_ZoneRawPath; }
@@ -276,7 +284,9 @@ private:
 
 	std::map<LWOSCENEID, uint32_t> m_MapRevisions; //rhs is the revision!
 	//private ("helper") functions:
-	void LoadScene(std::istream& file);
+	SceneTableLookup m_SceneTableLookup;
+	void LoadScene(std::istream& file, uint32_t index);
+	void KeepLoadedScenes();
 	void LoadLUTriggers(std::string triggerFile, SceneRef& scene);
 	void LoadSceneTransition(std::istream& file);
 	SceneTransitionInfo LoadSceneTransitionInfo(std::istream& file);

tests/dGameTests/dZoneManagerTests/CMakeLists.txt

@@ -1,5 +1,6 @@
 SET(DZONEMANAGER_TESTS
 	"RawTests.cpp"
+	"ZoneFileVersionTests.cpp"
 	"ZoneSceneTests.cpp")
 
 # Get the folder name and prepend it to the files above

tests/dGameTests/dZoneManagerTests/ZoneFileVersionTests.cpp

@@ -0,0 +1,157 @@
+#include "GameDependencies.h"
+
+#include <filesystem>
+#include <fstream>
+#include <sstream>
+
+#include "AssetManager.h"
+#include "CDZoneTableTable.h"
+#include "Zone.h"
+#include "dZoneManager.h"
+
+// The zone table's storage (CDZoneTableTable.cpp), to add the test zone
+namespace CDZoneTableTable {
+	extern Table entries;
+}
+
+namespace {
+	template<typename T>
+	void Write(std::ostream& stream, const T& value) {
+		stream.write(reinterpret_cast<const char*>(&value), sizeof(T));
+	}
+
+	void WriteString8(std::ostream& stream, const std::string& value) {
+		Write<uint8_t>(stream, static_cast<uint8_t>(value.size()));
+		stream.write(value.data(), value.size());
+	}
+
+	struct TestScene {
+		std::string filename;
+		uint32_t id = 0;
+		uint32_t layer = 0;
+	};
+
+	// A zone file of a version from 31 on with the given scenes and no transitions or paths
+	std::string WriteZone(uint32_t version, const std::vector<TestScene>& scenes) {
+		std::ostringstream stream;
+		Write(stream, version);
+		if (version >= 36) Write<uint32_t>(stream, 1); // revision
+		Write<uint32_t>(stream, 1);
+		if (version >= 38) {
+			Write(stream, NiPoint3(0.0f, 0.0f, 0.0f)); // spawn position
+			for (const float value : { 1.0f, 0.0f, 0.0f, 0.0f }) Write(stream, value); // spawn rotation
+		}
+		if (version >= 38) Write<uint32_t>(stream, static_cast<uint32_t>(scenes.size()));
+		else Write<uint8_t>(stream, static_cast<uint8_t>(scenes.size()));
+		for (const auto& scene : scenes) {
+			WriteString8(stream, scene.filename);
+			if (version >= 33) {
+				Write(stream, scene.id);
+				Write(stream, scene.layer);
+				WriteString8(stream, scene.filename);
+				for (const uint8_t color : { 1, 2, 3 }) Write(stream, color);
+			}
+		}
+		Write<uint8_t>(stream, 0); // boundary lines
+		WriteString8(stream, "test.raw");
+		WriteString8(stream, "Test Zone");
+		WriteString8(stream, "A zone for tests");
+		if (version >= 32) Write<uint32_t>(stream, 0); // transitions
+		if (version >= 35) {
+			Write<uint32_t>(stream, 8); // path chunk length
+			Write<uint32_t>(stream, 1); // path chunk version
+			Write<uint32_t>(stream, 0); // paths
+		}
+		return stream.str();
+	}
+}
+
+// Loads a written .luz through the zone manager
+class ZoneFileVersionTest : public GameDependenciesTest {
+protected:
+	static constexpr LWOMAPID TEST_MAP = 9993;
+	std::filesystem::path root;
+
+	void SetUp() override {
+		SetUpDependencies();
+		root = std::filesystem::temp_directory_path() / (std::string("dlu_zone_version_test_") + ::testing::UnitTest::GetInstance()->current_test_info()->name());
+		std::filesystem::create_directories(root / "res" / "maps" / "testzone");
+		std::ofstream(root / "res" / "CDServer.sqlite").close(); // marks an unpacked client
+		Game::assetManager = new AssetManager(root);
+
+		CDZoneTable zone{};
+		zone.zoneID = TEST_MAP;
+		zone.zoneName = "testzone/test.luz";
+		CDZoneTableTable::entries[TEST_MAP] = zone;
+	}
+
+	void TearDown() override {
+		CDZoneTableTable::entries.erase(TEST_MAP);
+		delete Game::assetManager;
+		Game::assetManager = nullptr;
+		std::filesystem::remove_all(root);
+		TearDownDependencies();
+	}
+
+	Zone* LoadZone(const std::string& data, Zone::SceneTableLookup lookup = nullptr) const {
+		std::ofstream(root / "res" / "maps" / "testzone" / "test.luz", std::ios::binary).write(data.data(), data.size());
+		Game::zoneManager->LoadZone(LWOZONEID(TEST_MAP, 0, 0));
+		Zone* zone = Game::zoneManager->GetZoneMut();
+		if (lookup) zone->SetSceneTableLookup(std::move(lookup));
+		zone->LoadZoneIntoMemory();
+		return zone;
+	}
+
+	static std::vector<std::pair<LWOSCENEID, std::string>> Scenes(const Zone* zone) {
+		std::vector<std::pair<LWOSCENEID, std::string>> scenes;
+		for (const auto& [id, scene] : zone->GetScenes()) scenes.emplace_back(id, scene.filename);
+		return scenes;
+	}
+};
+
+// Until version 33 a zone file has no scene IDs: each scene's ID is its index in the file
+TEST_F(ZoneFileVersionTest, OldScenesAreNumberedByTheirIndex) {
+	const Zone* zone = LoadZone(WriteZone(31, { { "a.lvl" }, { "b.lvl" } }));
+	const std::vector<std::pair<LWOSCENEID, std::string>> expected = { { LWOSCENEID(0, 0), "a.lvl" }, { LWOSCENEID(1, 0), "b.lvl" } };
+	EXPECT_EQ(Scenes(zone), expected);
+	EXPECT_EQ(zone->GetZoneName(), "Test Zone");
+}
+
+// A later scene with the same ID and layer replaces an earlier one; a scene ID without a layer 0 scene isn't loaded,
+// and before version 41 only the layer 0 scenes are
+TEST_F(ZoneFileVersionTest, KeepsTheScenesTheClientLoads) {
+	const std::vector<TestScene> scenes = {
+		{ "first.lvl", 1, 0 }, { "audio.lvl", 1, 1 }, { "orphan_audio.lvl", 2, 1 }, { "second.lvl", 1, 0 }, { "three.lvl", 3, 0 },
+	};
+	const std::vector<std::pair<LWOSCENEID, std::string>> latest = {
+		{ LWOSCENEID(1, 0), "second.lvl" }, { LWOSCENEID(1, 1), "audio.lvl" }, { LWOSCENEID(3, 0), "three.lvl" },
+	};
+	EXPECT_EQ(Scenes(LoadZone(WriteZone(41, scenes))), latest);
+
+	const std::vector<std::pair<LWOSCENEID, std::string>> older = { { LWOSCENEID(1, 0), "second.lvl" }, { LWOSCENEID(3, 0), "three.lvl" } };
+	EXPECT_EQ(Scenes(LoadZone(WriteZone(40, scenes))), older);
+}
+
+// Before version 30 a scene is a SceneTable ID: in ID order, each one with a row gets the next index as its ID and the
+// row's scene name as its file; the others aren't loaded. A file older than version 20 is read as 20
+TEST_F(ZoneFileVersionTest, OldestScenesComeFromTheSceneTable) {
+	for (const uint32_t version : { 25u, 10u }) {
+		std::ostringstream stream;
+		Write(stream, version);
+		Write<uint32_t>(stream, 1); // world ID
+		Write<uint8_t>(stream, 3);
+		for (const uint32_t sceneTableID : { 7u, 9u, 3u }) Write(stream, sceneTableID);
+		Write<uint8_t>(stream, 0); // boundary lines
+		WriteString8(stream, "test.raw");
+		Write<uint16_t>(stream, 0); // trailing bytes: older readers read a name and description here
+		const Zone* zone = LoadZone(stream.str(), [](uint32_t sceneTableID) -> std::optional<std::string> {
+			if (sceneTableID == 3) return "three.lvl";
+			if (sceneTableID == 7) return "seven.lvl";
+			return std::nullopt;
+		});
+
+		const std::vector<std::pair<LWOSCENEID, std::string>> expected = { { LWOSCENEID(0, 0), "three.lvl" }, { LWOSCENEID(1, 0), "seven.lvl" } };
+		EXPECT_EQ(Scenes(zone), expected) << version;
+		EXPECT_EQ(zone->GetZoneRawPath(), "test.raw") << version;
+	}
+}

tests/dGameTests/dZoneManagerTests/ZoneSceneTests.cpp

@@ -51,14 +51,14 @@ protected:
 		file.write(data.data(), data.size());
 	}
 
-	// Scenes 0 (global), 1, 2 and 3 (general) and 5 (audio, layer 1); transitions 1 <-> 2 and 2 <-> 3
+	// Scenes 0 (global), 1, 2 and 3 (general) and 3 (audio, layer 1); transitions 1 <-> 2 and 2 <-> 3
 	void WriteZone(const std::string& rawName = "test.raw", uint8_t boundaryCount = 2, uint32_t version = 41) const {
 		const std::vector<TestScene> scenes = {
 			{ "global.lvl", 0, 0, "Global", 10, 20, 30 },
 			{ "one.lvl", 1, 0, "Scene One", 255, 0, 0 },
 			{ "two.lvl", 2, 0, "Scene Two", 0, 255, 0 },
 			{ "three.lvl", 3, 0, "Scene Three", 0, 0, 255 },
-			{ "audio.lvl", 5, 1, "Audio", 0, 0, 0 },
+			{ "audio.lvl", 3, 1, "Audio", 0, 0, 0 },
 		};
 		WriteFile("test.luz", WriteLuz(TEST_MAP, scenes, boundaryCount, rawName, {
 			{ { 1, 0, NiPoint3(1.0f, 0.0f, 0.0f) }, { 2, 0, NiPoint3(2.0f, 0.0f, 0.0f) } },
@@ -123,7 +123,7 @@ TEST_F(ZoneSceneTest, ReadsScenesBoundariesAndTransitions) {
 	EXPECT_FLOAT_EQ(global->color.m_Red, 10 / 255.0f);
 	EXPECT_FLOAT_EQ(global->color.m_Green, 20 / 255.0f);
 	EXPECT_FLOAT_EQ(global->color.m_Blue, 30 / 255.0f);
-	const SceneRef* audio = zone->GetScene(LWOSCENEID(5, 1));
+	const SceneRef* audio = zone->GetScene(LWOSCENEID(3, 1));
 	ASSERT_NE(audio, nullptr);
 	EXPECT_EQ(audio->sceneType, eSceneType::Audio);
 	EXPECT_EQ(zone->GetScene(LWOSCENEID(4, 0)), nullptr);
@@ -151,7 +151,7 @@ TEST_F(ZoneSceneTest, ReadsOlderZoneVersions) {
 		WriteZone("test.raw", 2, version);
 		WriteFile("test.raw", MakeRaw());
 		const Zone* zone = LoadZone();
-		EXPECT_EQ(zone->GetScenes().size(), 5) << version;
+		EXPECT_EQ(zone->GetScenes().size(), 4) << version; // before Latest only layer 0 scenes are loaded
 		EXPECT_EQ(zone->GetBoundaries().size(), 2) << version;
 		EXPECT_EQ(zone->GetZoneRawPath(), "test.raw") << version;
 		EXPECT_EQ(zone->GetZoneName(), "Test Zone") << version;
@@ -185,7 +185,7 @@ TEST_F(ZoneSceneTest, SceneGraphFromTransitions) {
 
 	const auto& graph = Game::zoneManager->GetSceneAdjacencyList();
 	EXPECT_EQ(graph.size(), 4); // the audio scene is left out
-	EXPECT_FALSE(graph.contains(LWOSCENEID(5, 1)));
+	EXPECT_FALSE(graph.contains(LWOSCENEID(3, 1)));
 
 	const auto one = Game::zoneManager->GetAdjacentScenes(LWOSCENEID(1, 0));
 	EXPECT_EQ(one.size(), 2);