fix(dashboard): 3D views never draw a kept manifest with the game shaders

The game shader views looked up each shader's technique in the manifest's
"techniques". Property scenery manifests are cached by browsers for a day
(world ones for an hour), so after the update a browser drew the property
view from the manifest the older server had sent, which has no
techniques: every shader fell back to LEGO, whose decal texture alpha
laid the see-through tree, rock and water textures over white vertex
colors. Nimbus Isle came out with white trees, rocks and water, a yellow
build surface and a solid white build border.

- Manifest URLs carry the conversion format the views are written for
  (?format=5, scenery-core.js SCENERY_FORMAT), so a kept manifest from
  an older server is never used; SceneryCoreJs checks it matches
  Scenery.cpp FORMAT_VERSION.
- A manifest without techniques (an older server's) is drawn with the
  viewer's own lights and its textureAlpha table instead of every
  shader guessed as LEGO.
- A material whose NiAlphaController animates its alpha is drawn at its
  highest key. The AnimAlpha shaders now use the material alpha, and
  effects resting at 0 in the file (the Venture Explorer's lightning)
  had vanished. Conversion format 5.

Checked by rendering the world view of every zone with models and the
property view of every property template (headless, fixed cameras)
before and after, and the Nimbus Isle property with a manifest stripped
of its techniques, which reproduced the white look.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dCommon/NifFile.cpp

@@ -555,6 +555,56 @@ namespace {
 			}
 		}
 
+		// An NiFloatInterpolator's keys (its NiFloatData's) as time, value pairs; empty without data
+		std::vector<std::array<float, 2>> FloatKeys(int32_t interpolator) {
+			std::vector<std::array<float, 2>> out;
+			const auto* interpolatorType = TypeOf(interpolator);
+			if (!interpolatorType || *interpolatorType != "NiFloatInterpolator") return out;
+			m_Used.insert(interpolator);
+			auto value = BlockReader(interpolator);
+			value.Float();
+			const auto data = value.I32();
+			const auto* dataType = TypeOf(data);
+			if (!value.Ok() || !dataType || *dataType != "NiFloatData") return out;
+			m_Used.insert(data);
+			auto keys = BlockReader(data);
+			const auto count = keys.U32();
+			const auto keyType = count > 0 ? keys.U32() : 0;
+			// Linear keys are time and value; quadratic add two tangents; TBC three floats
+			const uint32_t floats = keyType == 1 ? 2 : keyType == 2 ? 4 : keyType == 3 ? 5 : 0;
+			if (count == 0 || floats == 0 || count > 100000) return out;
+			const auto values = keys.Array<float>(static_cast<uint64_t>(count) * floats);
+			if (!keys.Ok()) return out;
+			for (uint32_t key = 0; key < count; key++) out.push_back({ values[key * floats], values[key * floats + 1] });
+			return out;
+		}
+
+		// The highest alpha an NiAlphaController among the controllers from `first` on (an NiMaterialProperty's) gives
+		// the material: flickering and fading effects often rest at 0 in the file and only show while animated
+		std::optional<float> AnimatedAlpha(int32_t first) {
+			std::optional<float> highest;
+			std::set<int32_t> seen;
+			for (int32_t index = first; index >= 0 && !seen.contains(index);) {
+				seen.insert(index);
+				const auto* type = TypeOf(index);
+				if (!type) break;
+				auto reader = BlockReader(index);
+				const auto next = reader.I32();
+				if (*type == "NiAlphaController") {
+					m_Used.insert(index);
+					reader.Skip(2 + 16); // flags, frequency, phase, start, stop
+					reader.I32(); // target
+					const auto interpolator = reader.I32();
+					if (reader.Ok()) {
+						for (const auto& [time, value] : FloatKeys(interpolator)) highest = std::max(highest.value_or(value), value);
+					}
+				}
+				if (!reader.Ok()) break;
+				index = next;
+			}
+			return highest;
+		}
+
 		// Tiles a second the controllers from `first` on (an NiTexturingProperty's) move its base map in U and V: each
 		// NiTextureTransformController translating the base map, from its NiFloatInterpolator's NiFloatData's first key
 		// to its last, times its frequency
@@ -576,27 +626,11 @@ namespace {
 				const auto shaderMap = reader.U8();
 				const auto slot = reader.U32();
 				const auto operation = reader.U32();
-				const auto* interpolatorType = TypeOf(interpolator);
-				if (reader.Ok() && !shaderMap && slot == 0 && operation <= 1 && interpolatorType && *interpolatorType == "NiFloatInterpolator") {
-					m_Used.insert(interpolator);
-					auto value = BlockReader(interpolator);
-					value.Float();
-					const auto data = value.I32();
-					const auto* dataType = TypeOf(data);
-					if (value.Ok() && dataType && *dataType == "NiFloatData") {
-						m_Used.insert(data);
-						auto keys = BlockReader(data);
-						const auto count = keys.U32();
-						const auto keyType = count > 0 ? keys.U32() : 0;
-						// Linear keys are time and value; quadratic add two tangents; TBC three floats
-						const uint32_t floats = keyType == 1 ? 2 : keyType == 2 ? 4 : keyType == 3 ? 5 : 0;
-						if (count >= 2 && floats > 0 && count <= 100000) {
-							const auto values = keys.Array<float>(static_cast<uint64_t>(count) * floats);
-							if (keys.Ok()) {
-								const float duration = values[(count - 1) * floats] - values[0];
-								if (duration > 0.0f) scroll[operation] = (values[(count - 1) * floats + 1] - values[1]) / duration * frequency;
-							}
-						}
+				if (reader.Ok() && !shaderMap && slot == 0 && operation <= 1) {
+					const auto keys = FloatKeys(interpolator);
+					if (keys.size() >= 2) {
+						const float duration = keys.back()[0] - keys.front()[0];
+						if (duration > 0.0f) scroll[operation] = (keys.back()[1] - keys.front()[1]) / duration * frequency;
 					}
 				}
 				index = next;
@@ -608,7 +642,7 @@ namespace {
 			NifFile::Material material;
 			if (properties.material >= 0) {
 				auto reader = BlockReader(properties.material);
-				ReadNet(reader);
+				const auto controller = ReadNet(reader).controller;
 				reader.Skip(12); // ambient
 				std::array<float, 3> diffuse{}, emissive{};
 				for (auto& value : diffuse) value = reader.Float();
@@ -621,6 +655,8 @@ namespace {
 					material.emissive = emissive;
 					material.alpha = std::clamp(alpha, 0.0f, 1.0f);
 				}
+				// An animated alpha is drawn at its highest (the views don't play the controller)
+				if (const auto animated = AnimatedAlpha(controller)) material.alpha = std::clamp(*animated, 0.0f, 1.0f);
 			}
 			if (properties.alpha >= 0) {
 				auto reader = BlockReader(properties.alpha);

dDashboardServer/routes/Scenery.cpp

@@ -274,11 +274,13 @@ namespace {
 	/**
 	 * Bump when NifFile's output changes: converted models kept on disk are made again, and the manifests' "format"
 	 * goes into the viewers' model and texture URLs so browsers don't keep drawing the old ones (they're cached for
-	 * a week). 2: meshes carry their multishader tag; conversions without it drew glom parts with the LEGO shader.
-	 * 3: dark textures and the UV set each texture names. 4: the game's shaders draw the models (manifest
-	 * "techniques"), vertex colors go to them as stored.
+	 * a week). The viewers ask for manifests with the format they're written for (scenery-core.js SCENERY_FORMAT,
+	 * which must follow this; SceneryCoreJs checks), so a manifest a browser kept for a day from an older server
+	 * isn't drawn with newer code. 2: meshes carry their multishader tag; conversions without it drew glom parts with
+	 * the LEGO shader. 3: dark textures and the UV set each texture names. 4: the game's shaders draw the models
+	 * (manifest "techniques"), vertex colors go to them as stored. 5: an animated material alpha is its highest key.
 	 */
-	constexpr uint32_t FORMAT_VERSION = 4;
+	constexpr uint32_t FORMAT_VERSION = 5;
 
 	// A zone's lighting (WorldScene::Lighting) for the viewers' shaders
 	nlohmann::json LightingJson(const WorldScene::Lighting& lighting) {

dDashboardServer/static/js/scenery-core.js

@@ -107,6 +107,26 @@ export const TECHNIQUE = {
 	DIFFUSE_ONLY: 32768, ANIM_ALPHA: 65536, BASIC_EMISSIVE: 131072, NO_FOG: 262144, NOT_DRAWN: 524288, NO_BLEND: 1048576
 };
 
+/**
+ * The conversion format (Scenery.cpp FORMAT_VERSION) these views are written for. Manifest URLs carry it, so a
+ * browser never draws with a manifest it kept from an older server (they are cached for up to a day).
+ */
+export const SCENERY_FORMAT = 5;
+
+/** A manifest URL asking for SCENERY_FORMAT's manifest (the server ignores the parameter; browsers cache by it). */
+export function manifestUrl(url) {
+	return url + (url.includes('?') ? '&' : '?') + 'format=' + SCENERY_FORMAT;
+}
+
+/**
+ * Whether a manifest's models are drawn with the game's shaders: it has the zone's lighting and says which technique
+ * each shader is (format 4 on). Older manifests have neither all shaders' techniques nor the looks the shaders need,
+ * so they are drawn with the viewer's own lights rather than every shader guessed as LEGO.
+ */
+export function gameShaded(manifest) {
+	return !!(manifest && manifest.lighting && (manifest.techniques || manifest.technique));
+}
+
 // A technique the manifest doesn't name: the LEGO shader's, as the client falls back to it
 const LEGO_TECHNIQUE = { family: 'lego', look: 0, alpha: 'decal', flags: 0 };
 const FIXED_TECHNIQUE = { family: 'fixed', look: 0, alpha: 'opacity', flags: 0 };
@@ -129,6 +149,11 @@ export function techniqueOf(manifest, asset, mesh) {
  * alpha is; 'decal' the texture is laid over the vertex colors by its alpha (LEGO shaders); 'ignored' it does nothing.
  */
 export function textureAlphaMode(manifest, asset, mesh) {
+	if (!manifest || (!manifest.techniques && !manifest.technique)) {
+		// A manifest from before the techniques (format 3 and older) names only the shaders whose alpha isn't opacity
+		const shader = shaderOf(manifest, asset, mesh);
+		return (shader !== null && manifest && manifest.textureAlpha && manifest.textureAlpha[shader]) || 'opacity';
+	}
 	return techniqueOf(manifest, asset, mesh).alpha || 'opacity';
 }
 
@@ -141,10 +166,11 @@ export function textureAlphaMode(manifest, asset, mesh) {
  * whether it glows (LEGO-Emissive: the vertex alpha is then no opacity), whether its texture moves as the .nif's
  * texture transform says, its blending ('nif': as NiAlphaProperty says; 'blend': see-through without depth writes;
  * 'test': cut out; 'additive'; 'opaque') and whether the game draws it in the world at all (hidden: post-processing
- * and shadow shaders). Null without lighting in the manifest (older servers), for the viewer's own lights.
+ * and shadow shaders). Null without lighting or techniques in the manifest (older servers, gameShaded), for the
+ * viewer's own lights.
  */
 export function gameLook(manifest, asset, mesh) {
-	if (!manifest || !manifest.lighting) return null;
+	if (!gameShaded(manifest)) return null;
 	const technique = techniqueOf(manifest, asset, mesh);
 	const fixedFunction = technique.family === 'fixed';
 	const bits = technique.look || 0;

dDashboardServer/static/js/scenery.js

@@ -9,7 +9,7 @@
  * follows the camera, far away, behind everything.
  */
 import * as THREE from 'three';
-import { parseModel, mergeMeshes, parseDds, decodeDxt, completeChain, linearColors, groupObjects, cellsOf, textureAlphaMode, gameLook, blendingOf, TECHNIQUE, decodeSceneMap, sceneAt, loadedScenes } from '/js/scenery-core.js';
+import { parseModel, mergeMeshes, parseDds, decodeDxt, completeChain, linearColors, groupObjects, cellsOf, textureAlphaMode, gameLook, gameShaded, manifestUrl, blendingOf, TECHNIQUE, decodeSceneMap, sceneAt, loadedScenes } from '/js/scenery-core.js';
 import { createGameShading } from '/js/game-shaders.js';
 
 // Per detail level (the property view's 0 high, 1 medium, 2 low): the model LOD, how far objects are drawn, the
@@ -276,7 +276,7 @@ export function createScenery({ scene, camera, renderer, urls, focus, onProgress
 		// The sky's layers keep their order; everything else has its look-alike pieces joined
 		for (const mesh of forSky ? model.meshes : mergeMeshes(model.meshes)) {
 			if (!mesh.vertices || !mesh.indices.length) continue;
-			const look = forSky ? (manifest.lighting ? skyLook(mesh) : null) : gameLook(manifest, asset, mesh);
+			const look = forSky ? (gameShaded(manifest) ? skyLook(mesh) : null) : gameLook(manifest, asset, mesh);
 			// Post-processing and shadow shaders: the game draws nothing of these in the world
 			if (look && look.hidden) continue;
 			const textureOf = async (slot, clampU, clampV) => {
@@ -490,7 +490,7 @@ export function createScenery({ scene, camera, renderer, urls, focus, onProgress
 				manifest = null;
 				let loaded = null;
 				try {
-					const response = await fetch(url, { credentials: 'same-origin', signal: requests.signal });
+					const response = await fetch(manifestUrl(url), { credentials: 'same-origin', signal: requests.signal });
 					if (response.ok) loaded = await response.json();
 				} catch (error) {
 					// Aborted by a newer load or clear, or the network failed

docs/Dashboard.md

@@ -1118,6 +1118,12 @@ Powerups, Orb, TV Screen and Head Icon effects (drawn as LEGO or unlit), Flair's
 client's environment textures come from `/api/scenery/env/:name` (`reflection`, `polished`, `brushed`,
 `brushedNoise`). Placed player models are drawn by the viewer's own lighting.
 
+The views ask for a zone's manifest with the conversion format they're written for (`?format=5`,
+`scenery-core.js` `SCENERY_FORMAT`, which follows `Scenery.cpp` `FORMAT_VERSION`), since browsers keep manifests for up
+to a day. A manifest without `techniques` (an older server's) is drawn with the viewer's own lights and its
+`textureAlpha` table instead of guessing every shader as LEGO, which laid see-through textures over white vertex
+colors (white trees, rocks and water).
+
 Converting a model the caches don't have yet (a big "glom" file takes a moment) happens on a few worker threads, so
 the dashboard keeps answering everything else meanwhile: the route hands the request to a worker (`Web::Defer`) and
 the web thread sends the answer when it is ready. Flairs and small models (up to 256 KB) go first, and one of the

tests/dWebTests/CMakeLists.txt

@@ -87,5 +87,6 @@ target_compile_definitions(dWebTests PRIVATE DLU_SOURCE_DIR="${PROJECT_SOURCE_DI
 find_program(NODE_EXECUTABLE node)
 if(NODE_EXECUTABLE)
 	add_test(NAME SceneryCoreJs COMMAND ${NODE_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/scenery-core.test.mjs"
-		"${PROJECT_SOURCE_DIR}/dDashboardServer/static/js/scenery-core.js" "${PROJECT_SOURCE_DIR}/dCommon/NifFile.h")
+		"${PROJECT_SOURCE_DIR}/dDashboardServer/static/js/scenery-core.js" "${PROJECT_SOURCE_DIR}/dCommon/NifFile.h"
+		"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/Scenery.cpp")
 endif()

tests/dWebTests/NifFileTests.cpp

@@ -213,6 +213,36 @@ TEST(NifFileTests, PassesPropertiesDownTheTree) {
 	EXPECT_TRUE(m.doubleSided);
 }
 
+// A material whose NiAlphaController animates its alpha (flickering effects that rest at 0 in the file) is drawn at
+// its highest key; one without keeps its own alpha
+TEST(NifFileTests, DrawsAnAnimatedAlphaAtItsHighest) {
+	for (const bool animated : { true, false }) {
+		NifBuilder nif;
+		const auto root = nif.Add("NiNode", {});
+		const auto material = nif.Add("NiMaterialProperty", {});
+		const auto controller = nif.Add("NiAlphaController", {});
+		const auto interpolator = nif.Add("NiFloatInterpolator", {});
+		const auto keys = nif.Add("NiFloatData", Bytes().Put<uint32_t>(3).Put<uint32_t>(1).Floats({ 0.0f, 0.0f, 0.5f, 0.8f, 1.0f, 0.0f }));
+		// NiObjectNET with the controller, ambient, diffuse, specular, emissive, glossiness, alpha 0
+		Bytes body;
+		body.Put<uint32_t>(0xFFFFFFFF).Put<uint32_t>(0).Put<int32_t>(animated ? controller : -1);
+		body.Floats({ 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 10.0f, 0.0f });
+		nif.Set(material, body);
+		// NiTimeController: next, flags, frequency, phase, start, stop, target; the interpolator
+		nif.Set(controller, Bytes().Put<int32_t>(-1).Put<uint16_t>(8).Floats({ 1, 0, 0, 1 }).Put<int32_t>(material).Put<int32_t>(interpolator));
+		nif.Set(interpolator, Bytes().Put(0.0f).Put<int32_t>(keys));
+		const auto shape = nif.Add("NiTriShape", {});
+		const auto data = nif.Add("NiTriShapeData", TriShapeData());
+		nif.Set(root, Node(Av(0, { 0, 0, 0 }, IDENTITY, 1.0f, { material }), { shape }));
+		nif.Set(shape, Geometry(Av(0, { 0, 0, 0 }, IDENTITY, 1.0f, {}), data));
+		std::string error;
+		const auto model = NifFile::Parse(nif.Build({ root }), 0, error);
+		ASSERT_TRUE(model) << error;
+		ASSERT_EQ(model->meshes.size(), 1u);
+		EXPECT_FLOAT_EQ(model->meshes[0].material.alpha, animated ? 0.8f : 0.0f);
+	}
+}
+
 // Two layer shaders use NiTexturingProperty's dark texture too, each texture on the UV set its flags name
 TEST(NifFileTests, ReadsTheDarkTextureAndEachTexturesUvSet) {
 	NifBuilder nif;

tests/dWebTests/scenery-core.test.mjs

@@ -1,9 +1,9 @@
 // The 3D views' shader lookups (static/js/scenery-core.js): which technique draws a mesh and what it uses.
-// Run by ctest: node scenery-core.test.mjs <scenery-core.js> [NifFile.h]
+// Run by ctest: node scenery-core.test.mjs <scenery-core.js> [NifFile.h] [Scenery.cpp]
 import { pathToFileURL } from 'node:url';
 import { readFileSync } from 'node:fs';
 
-const [modulePath, nifHeader] = process.argv.slice(2);
+const [modulePath, nifHeader, sceneryRoutes] = process.argv.slice(2);
 const S = await import(pathToFileURL(modulePath).href);
 let failures = 0;
 const same = (actual, expected, what) => {
@@ -58,6 +58,19 @@ same(pick(S.gameLook(manifest, 2, { ...colored, vertexColors: 0 }), BASICS), { f
 // Without the zone's lighting the viewer lights scenery itself
 same(S.gameLook({ ...manifest, lighting: null }, 0, colored), null, 'no lighting');
 
+// A manifest from before the techniques (format 3, or one a browser kept): no shader is guessed as LEGO. The views
+// light it themselves and its textureAlpha table says what texture alpha does, so a Basic tree's see-through leaves
+// aren't laid over its white vertex colors
+const format3 = { ...manifest, format: 3, techniques: undefined, textureAlpha: { 5: 'decal' } };
+same(S.gameShaded(format3), false, 'format 3 manifest is not game shaded');
+same(S.gameShaded(manifest), true, 'manifest with techniques is game shaded');
+same(S.gameLook(format3, 0, colored), null, 'format 3 manifest: viewer lights');
+same(S.textureAlphaMode(format3, 0, {}), 'opacity', 'format 3 manifest: Basic VC texture alpha');
+same(S.textureAlphaMode(format3, 1, { shaderTag: 1 }), 'decal', 'format 3 manifest: LEGO texture alpha');
+// Manifest URLs name the format the views are written for, so an older manifest a browser kept isn't used
+same(S.manifestUrl('/api/world3d/1200/scenery'), '/api/world3d/1200/scenery?format=' + S.SCENERY_FORMAT, 'manifest URL');
+same(S.manifestUrl('/a?b=1'), '/a?b=1&format=' + S.SCENERY_FORMAT, 'manifest URL with a query');
+
 // What the flags turn into: moving textures, both sides, blending, not drawn
 const flagged = (flags, family = 'basic') => S.gameLook({ ...manifest, shaders: [1], techniques: { 1: { family, look: 0, alpha: 'opacity', flags } } }, 0, colored);
 same(flagged(T.UV_ANIM).uvAnim, true, 'UV animation');
@@ -137,6 +150,12 @@ if (nifHeader) {
 	same(bitsOf('eShaderLook'), L, 'SHADER_LOOK matches eShaderLook');
 }
 
+// The format the views are written for is the server's conversion format (Scenery.cpp FORMAT_VERSION)
+if (sceneryRoutes) {
+	const format = readFileSync(sceneryRoutes, 'utf8').match(/constexpr uint32_t FORMAT_VERSION = (\d+);/);
+	same(format && Number(format[1]), S.SCENERY_FORMAT, 'SCENERY_FORMAT matches FORMAT_VERSION');
+}
+
 // The near plane grows with the distance, within limits
 same([S.nearPlaneFor(10), S.nearPlaneFor(2000), S.nearPlaneFor(100000)], [0.5, 5, 20], 'near plane');