fix: read and keep a zone file's scenes as the client does

LuzFile::ReadLUZFile, LuzReader::ReadScenes and ZoneLoader::ReadZoneFile
(1.10.64):
- a version older than 20 is read as 20;
- before version 30 a scene is a bare u32 SceneTable ID: in ID order,
  each one with a SceneTable row gets the next index (-1 past 255) as
  its ID and the row's sceneName as its file; the others are left out;
- from 30 to 32 the file has no scene IDs: a scene's ID is its index in
  the file (they all had ID 0, colliding);
- a later scene with the same ID and layer replaces an earlier one, a
  scene ID with no layer 0 scene is left out, and before version 41
  only the layer 0 scenes are loaded.

No 1.10.64 zone file has a duplicate scene, a scene without a layer 0
scene or another layer before version 41, so what live zones load is
unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dZoneManager/Zone.cpp

@@ -11,6 +11,7 @@
 #include "dConfig.h"
 
 #include "AssetManager.h"
+#include "CDClientDatabase.h"
 #include "CDClientManager.h"
 #include "CDZoneTableTable.h"
 #include "Spawner.h"
@@ -30,6 +31,13 @@ Zone::Zone(const LWOZONEID zoneID) :
 	m_CheckSum = 0;
 	m_WorldID = 0;
 	m_SceneCount = 0;
+	m_SceneTableLookup = [](uint32_t sceneTableID) -> std::optional<std::string> {
+		auto query = CDClientDatabase::CreatePreppedStmt("SELECT sceneName FROM SceneTable WHERE sceneID = ?;");
+		query.bind(1, static_cast<int32_t>(sceneTableID));
+		auto result = query.execQuery();
+		if (result.eof() || result.fieldIsNull("sceneName")) return std::nullopt;
+		return std::string(result.getStringField("sceneName"));
+	};
 }
 
 Zone::~Zone() {
@@ -56,6 +64,8 @@ void Zone::LoadZoneIntoMemory() {
 
 	if (file) {
 		BinaryIO::BinaryRead(file, m_FileFormatVersion);
+		// The client reads anything older than 20 as 20 (LuzFile::ReadLUZFile)
+		if (m_FileFormatVersion < Zone::FileFormatVersion::Oldest) m_FileFormatVersion = Zone::FileFormatVersion::Oldest;
 
 		uint32_t mapRevision = 0;
 		if (m_FileFormatVersion >= Zone::FileFormatVersion::Alpha) BinaryIO::BinaryRead(file, mapRevision);
@@ -78,8 +88,9 @@ void Zone::LoadZoneIntoMemory() {
 		} else BinaryIO::BinaryRead(file, m_SceneCount);
 
 		for (uint32_t i = 0; i < m_SceneCount; ++i) {
-			LoadScene(file);
+			LoadScene(file, i);
 		}
+		KeepLoadedScenes();
 
 		// Zone boundary lines: a count byte, then the lines
 		uint8_t numBoundaries = 0;
@@ -268,21 +279,29 @@ void Zone::AddRevision(LWOSCENEID sceneID, uint32_t revision) {
 	}
 }
 
-void Zone::LoadScene(std::istream& file) {
+void Zone::LoadScene(std::istream& file, uint32_t index) {
 	SceneRef scene;
 	scene.level = nullptr;
-	LWOSCENEID lwoSceneID(LWOZONEID_INVALID, 0);
+
+	// Before PrePreAlpha a scene is only its SceneTable ID; KeepLoadedScenes finds its file (LuzReader::ReadScenes)
+	if (m_FileFormatVersion < Zone::FileFormatVersion::PrePreAlpha) {
+		BinaryIO::BinaryRead(file, scene.sceneTableID);
+		scene.id = scene.sceneTableID;
+		m_Scenes[LWOSCENEID(scene.id, 0)] = std::move(scene);
+		return;
+	}
 
 	BinaryIO::ReadString<uint8_t>(file, scene.filename, BinaryIO::ReadType::String);
 
 	std::string luTriggersPath = scene.filename.substr(0, scene.filename.size() - 4) + ".lutriggers";
 	if (Game::assetManager->HasFile((m_ZonePath + luTriggersPath).c_str())) LoadLUTriggers(luTriggersPath, scene);
 
-	if (m_FileFormatVersion >= Zone::FileFormatVersion::LatePreAlpha || m_FileFormatVersion < Zone::FileFormatVersion::PrePreAlpha) {
+	// Until LatePreAlpha the file has no scene IDs: a scene's ID is its index (-1 past 255) in the file
+	scene.id = index < 256 ? index : UINT32_MAX;
+	LWOSCENEID lwoSceneID(scene.id, 0);
+	if (m_FileFormatVersion >= Zone::FileFormatVersion::LatePreAlpha) {
 		BinaryIO::BinaryRead(file, scene.id);
 		lwoSceneID.SetSceneID(scene.id);
-	}
-	if (m_FileFormatVersion >= Zone::FileFormatVersion::LatePreAlpha) {
 		BinaryIO::BinaryRead(file, scene.sceneType);
 		lwoSceneID.SetLayerID(static_cast<uint32_t>(scene.sceneType));
 
@@ -306,6 +325,37 @@ void Zone::LoadScene(std::istream& file) {
 	m_Scenes[lwoSceneID] = std::move(scene);
 }
 
+void Zone::KeepLoadedScenes() {
+	if (m_FileFormatVersion < Zone::FileFormatVersion::PrePreAlpha) {
+		// ZoneLoader::ReadZoneFile: the scenes go by their SceneTable ID; each one with a row there gets the next index
+		// (0, 1, ...; -1 past 255) as its ID and the row's sceneName as its file, the others are left out
+		std::map<LWOSCENEID, SceneRef> resolved;
+		uint32_t index = 0;
+		for (auto& scene : m_Scenes | std::views::values) {
+			const auto sceneName = m_SceneTableLookup(scene.sceneTableID);
+			if (!sceneName) continue;
+			scene.id = index < 256 ? index : UINT32_MAX;
+			scene.filename = *sceneName;
+			scene.name = *sceneName;
+			const std::string luTriggersPath = scene.filename.substr(0, scene.filename.size() - 4) + ".lutriggers";
+			if (Game::assetManager->HasFile((m_ZonePath + luTriggersPath).c_str())) LoadLUTriggers(luTriggersPath, scene);
+			resolved[LWOSCENEID(scene.id, 0)] = std::move(scene);
+			index++;
+		}
+		m_Scenes = std::move(resolved);
+		return;
+	}
+
+	// LuzReader::ReadScenes keeps the scenes by scene ID and layer, a later one replacing an earlier one (as m_Scenes
+	// does). ZoneLoader::ReadZoneFile then goes through them by scene ID: a scene ID with no layer 0 (General) scene is
+	// left out, and before Latest only the layer 0 scene is loaded
+	std::erase_if(m_Scenes, [this](const auto& entry) {
+		const auto& [sceneID, scene] = entry;
+		if (!m_Scenes.contains(LWOSCENEID(sceneID.GetSceneID(), 0))) return true;
+		return sceneID.GetLayerID() != 0 && m_FileFormatVersion < Zone::FileFormatVersion::Latest;
+	});
+}
+
 void Zone::LoadLUTriggers(std::string triggerFile, SceneRef& scene) {
 	auto file = Game::assetManager->GetFile((m_ZonePath + triggerFile).c_str());
 

dZoneManager/Zone.h

@@ -7,6 +7,8 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <functional>
+#include <optional>
 #include "Raw.h"
 
 namespace LUTriggers {
@@ -31,6 +33,7 @@ enum class eSceneType : uint32_t {
 struct SceneRef {
 	std::string filename;
 	uint32_t id{};
+	uint32_t sceneTableID{}; // before PrePreAlpha: the scene's row in the CDClient SceneTable
 	eSceneType sceneType{};
 	std::string name;
 	NiPoint3 unknown1;
@@ -204,6 +207,7 @@ struct Path {
 class Zone {
 public:
 	enum class FileFormatVersion : uint32_t { //Times are guessed.
+		Oldest = 20, // the client reads anything older as this
 		PrePreAlpha = 30,
 		PreAlpha = 32,
 		LatePreAlpha = 33,
@@ -231,6 +235,10 @@ public:
 	LUTriggers::Trigger* GetTrigger(uint32_t sceneID, uint32_t triggerID) const;
 	const Path* GetPath(std::string name) const;
 
+	// The CDClient SceneTable's sceneName for a scene of a file from before PrePreAlpha; the database's by default
+	using SceneTableLookup = std::function<std::optional<std::string>(uint32_t sceneTableID)>;
+	void SetSceneTableLookup(SceneTableLookup lookup) { m_SceneTableLookup = std::move(lookup); }
+
 	uint32_t GetWorldID() const { return m_WorldID; }
 	[[nodiscard]] std::string GetZoneName() const { return m_ZoneName; }
 	std::string GetZoneRawPath() const { return m_ZoneRawPath; }
@@ -276,7 +284,9 @@ private:
 
 	std::map<LWOSCENEID, uint32_t> m_MapRevisions; //rhs is the revision!
 	//private ("helper") functions:
-	void LoadScene(std::istream& file);
+	SceneTableLookup m_SceneTableLookup;
+	void LoadScene(std::istream& file, uint32_t index);
+	void KeepLoadedScenes();
 	void LoadLUTriggers(std::string triggerFile, SceneRef& scene);
 	void LoadSceneTransition(std::istream& file);
 	SceneTransitionInfo LoadSceneTransitionInfo(std::istream& file);