feat: normalize brick model positions (#1761)

* Add utilities for formats

* Normalize model positions when placing in the world

Have tested that placing a small and very large model both place and are located at the correct position.

* add migration

* Update Logger.cpp

* add some notes and remove some logs

* change arguments and add eof check

Revert "fix: buff station cycling and dying too soon"

This reverts commit 1c6cb2921e10eb2000ac40007d0c2636ba2ac151.

fix: buff station cycling and dying too soon

Tested that the buff station now only cycles after it has been built and has been alive for 25 seconds.

dCommon/BrickByBrickFix.cpp

@@ -8,6 +8,7 @@
 
 #include "Database.h"
 #include "Game.h"
+#include "Sd0.h"
 #include "ZCompression.h"
 #include "Logger.h"
 
@@ -44,10 +45,10 @@ uint32_t BrickByBrickFix::TruncateBrokenBrickByBrickXml() {
 				}
 
 				// Ignore the valgrind warning about uninitialized values.  These are discarded later when we know the actual uncompressed size.
-				std::unique_ptr<uint8_t[]> uncompressedChunk(new uint8_t[ZCompression::MAX_SD0_CHUNK_SIZE]);
+				std::unique_ptr<uint8_t[]> uncompressedChunk(new uint8_t[Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE]);
 				int32_t err{};
 				int32_t actualUncompressedSize = ZCompression::Decompress(
-					compressedChunk.get(), chunkSize, uncompressedChunk.get(), ZCompression::MAX_SD0_CHUNK_SIZE, err);
+					compressedChunk.get(), chunkSize, uncompressedChunk.get(), Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE, err);
 
 				if (actualUncompressedSize != -1) {
 					uint32_t previousSize = completeUncompressedModel.size();
@@ -117,7 +118,7 @@ uint32_t BrickByBrickFix::UpdateBrickByBrickModelsToSd0() {
 			}
 
 			std::string outputString(sd0ConvertedModel.get(), oldLxfmlSizeWithHeader);
-			std::istringstream outputStringStream(outputString);
+			std::stringstream outputStringStream(outputString);
 
 			try {
 				Database::Get()->UpdateUgcModelData(model.id, outputStringStream);

dCommon/CMakeLists.txt

@@ -16,6 +16,9 @@ set(DCOMMON_SOURCES
 		"BrickByBrickFix.cpp"
 		"BinaryPathFinder.cpp"
 		"FdbToSqlite.cpp"
+		"TinyXmlUtils.cpp"
+		"Sd0.cpp"
+		"Lxfml.cpp"
 )
 
 # Workaround for compiler bug where the optimized code could result in a memcpy of 0 bytes, even though that isnt possible.

dCommon/Logger.h

@@ -29,8 +29,8 @@ constexpr const char* GetFileNameFromAbsolutePath(const char* path) {
 // they will not be valid constexpr and will be evaluated at runtime instead of compile time!
 // The full string is still stored in the binary, however the offset of the filename in the absolute paths
 // is used in the instruction instead of the start of the absolute path.
-#define LOG(message, ...) do { auto str = FILENAME_AND_LINE; Game::logger->Log(str, message, ##__VA_ARGS__); } while(0)
-#define LOG_DEBUG(message, ...) do { auto str = FILENAME_AND_LINE; Game::logger->LogDebug(str, message, ##__VA_ARGS__); } while(0)
+#define LOG(message, ...) do { auto str_ = FILENAME_AND_LINE; Game::logger->Log(str_, message, ##__VA_ARGS__); } while(0)
+#define LOG_DEBUG(message, ...) do { auto str_ = FILENAME_AND_LINE; Game::logger->LogDebug(str_, message, ##__VA_ARGS__); } while(0)
 
 // Writer class for writing data to files.
 class Writer {

dCommon/Lxfml.cpp

@@ -0,0 +1,115 @@
+#include "Lxfml.h"
+
+#include "GeneralUtils.h"
+#include "StringifiedEnum.h"
+#include "TinyXmlUtils.h"
+
+#include <ranges>
+
+Lxfml::Result Lxfml::NormalizePosition(const std::string_view data) {
+	Result toReturn;
+	tinyxml2::XMLDocument doc;
+	const auto err = doc.Parse(data.data());
+	if (err != tinyxml2::XML_SUCCESS) {
+		LOG("Failed to parse xml %s.", StringifiedEnum::ToString(err).data());
+		return toReturn;
+	}
+
+	TinyXmlUtils::DocumentReader reader(doc);
+	std::map<std::string/* refID */, std::string> transformations;
+
+	auto lxfml = reader["LXFML"];
+	if (!lxfml) {
+		LOG("Failed to find LXFML element.");
+		return toReturn;
+	}
+
+	// First get all the positions of bricks
+	for (const auto& brick : lxfml["Bricks"]) {
+		const auto* part = brick.FirstChildElement("Part");
+		if (part) {
+			const auto* bone = part->FirstChildElement("Bone");
+			if (bone) {
+				auto* transformation = bone->Attribute("transformation");
+				if (transformation) {
+					auto* refID = bone->Attribute("refID");
+					if (refID) transformations[refID] = transformation;
+				}
+			}
+		}
+	}
+
+	// These points are well out of bounds for an actual player
+	NiPoint3 lowest{ 10'000.0f, 10'000.0f, 10'000.0f };
+	NiPoint3 highest{ -10'000.0f, -10'000.0f, -10'000.0f };
+
+	// Calculate the lowest and highest points on the entire model
+	for (const auto& transformation : transformations | std::views::values) {
+		auto split = GeneralUtils::SplitString(transformation, ',');
+		if (split.size() < 12) {
+			LOG("Not enough in the split?");
+			continue;
+		}
+
+		auto x = GeneralUtils::TryParse<float>(split[9]).value();
+		auto y = GeneralUtils::TryParse<float>(split[10]).value();
+		auto z = GeneralUtils::TryParse<float>(split[11]).value();
+		if (x < lowest.x) lowest.x = x;
+		if (y < lowest.y) lowest.y = y;
+		if (z < lowest.z) lowest.z = z;
+
+		if (highest.x < x) highest.x = x;
+		if (highest.y < y) highest.y = y;
+		if (highest.z < z) highest.z = z;
+	}
+
+	auto delta = (highest - lowest) / 2.0f;
+	auto newRootPos = lowest + delta;
+
+	// Clamp the Y to the lowest point on the model 
+	newRootPos.y = lowest.y;
+
+	// Adjust all positions to account for the new origin
+	for (auto& transformation : transformations | std::views::values) {
+		auto split = GeneralUtils::SplitString(transformation, ',');
+		if (split.size() < 12) {
+			LOG("Not enough in the split?");
+			continue;
+		}
+
+		auto x = GeneralUtils::TryParse<float>(split[9]).value() - newRootPos.x;
+		auto y = GeneralUtils::TryParse<float>(split[10]).value() - newRootPos.y;
+		auto z = GeneralUtils::TryParse<float>(split[11]).value() - newRootPos.z;
+		std::stringstream stream;
+		for (int i = 0; i < 9; i++) {
+			stream << split[i];
+			stream << ',';
+		}
+		stream << x << ',' << y << ',' << z;
+		transformation = stream.str();
+	}
+
+	// Finally write the new transformation back into the lxfml
+	for (auto& brick : lxfml["Bricks"]) {
+		auto* part = brick.FirstChildElement("Part");
+		if (part) {
+			auto* bone = part->FirstChildElement("Bone");
+			if (bone) {
+				auto* transformation = bone->Attribute("transformation");
+				if (transformation) {
+					auto* refID = bone->Attribute("refID");
+					if (refID) {
+						bone->SetAttribute("transformation", transformations[refID].c_str());
+					}
+				}
+			}
+		}
+	}
+
+	tinyxml2::XMLPrinter printer;
+	doc.Print(&printer);
+
+	toReturn.lxfml = printer.CStr();
+	toReturn.center = newRootPos;
+	return toReturn;
+}

dCommon/Lxfml.h

@@ -0,0 +1,23 @@
+// Darkflame Universe
+// Copyright 2025
+
+#ifndef LXFML_H
+#define LXFML_H
+
+#include <string>
+#include <string_view>
+
+#include "NiPoint3.h"
+
+namespace Lxfml {
+	struct Result {
+		std::string lxfml;
+		NiPoint3 center;
+	};
+
+	// Normalizes a LXFML model to be positioned relative to its local 0, 0, 0 rather than a game worlds 0, 0, 0.
+	// Returns a struct of its new center and the updated LXFML containing these edits.
+	[[nodiscard]] Result NormalizePosition(const std::string_view data);
+};
+
+#endif //!LXFML_H

dCommon/Sd0.cpp

@@ -0,0 +1,150 @@
+#include "Sd0.h"
+
+#include <array>
+#include <ranges>
+
+#include "BinaryIO.h"
+
+#include "Game.h"
+#include "Logger.h"
+
+#include "ZCompression.h"
+
+// Insert header if on first buffer
+void WriteHeader(Sd0::BinaryBuffer& chunk) {
+	chunk.push_back(Sd0::SD0_HEADER[0]);
+	chunk.push_back(Sd0::SD0_HEADER[1]);
+	chunk.push_back(Sd0::SD0_HEADER[2]);
+	chunk.push_back(Sd0::SD0_HEADER[3]);
+	chunk.push_back(Sd0::SD0_HEADER[4]);
+}
+
+// Write the size of the buffer to a chunk
+void WriteSize(Sd0::BinaryBuffer& chunk, uint32_t chunkSize) {
+	for (int i = 0; i < 4; i++) {
+		char toPush = chunkSize & 0xff;
+		chunkSize = chunkSize >> 8;
+		chunk.push_back(toPush);
+	}
+}
+
+int32_t GetDataOffset(bool firstBuffer) {
+	return firstBuffer ? 9 : 4;
+}
+
+Sd0::Sd0(std::istream& buffer) {
+	char header[5]{};
+
+	// Check if this is an sd0 buffer. It's possible we may be handed a zlib buffer directly due to old code so check for that too.
+	if (!BinaryIO::BinaryRead(buffer, header) || memcmp(header, SD0_HEADER, sizeof(header)) != 0) {
+		LOG("Failed to read SD0 header %i %i %i %i %i %i %i", buffer.good(), buffer.tellg(), header[0], header[1], header[2], header[3], header[4]);
+		LOG_DEBUG("This may be a zlib buffer directly? Trying again assuming its a zlib buffer.");
+		auto& firstChunk = m_Chunks.emplace_back();
+		WriteHeader(firstChunk);
+		buffer.seekg(0, std::ios::end);
+		uint32_t bufferSize = buffer.tellg();
+		buffer.seekg(0, std::ios::beg);
+		WriteSize(firstChunk, bufferSize);
+		firstChunk.resize(firstChunk.size() + bufferSize);
+		auto* dataStart = reinterpret_cast<char*>(firstChunk.data() + GetDataOffset(true));
+		if (!buffer.read(dataStart, bufferSize)) {
+			m_Chunks.pop_back();
+			LOG("Failed to read %u bytes from chunk %i", bufferSize, m_Chunks.size() - 1);
+		}
+		return;
+	}
+
+	while (buffer && buffer.peek() != std::istream::traits_type::eof()) {
+		uint32_t chunkSize{};
+		if (!BinaryIO::BinaryRead(buffer, chunkSize)) {
+			LOG("Failed to read chunk size from stream %lld %zu", buffer.tellg(), m_Chunks.size());
+			break;
+		}
+		auto& chunk = m_Chunks.emplace_back();
+		bool firstBuffer = m_Chunks.size() == 1;
+		auto dataOffset = GetDataOffset(firstBuffer);
+
+		// Insert header if on first buffer
+		if (firstBuffer) {
+			WriteHeader(chunk);
+		}
+
+		WriteSize(chunk, chunkSize);
+
+		chunk.resize(chunkSize + dataOffset);
+		auto* dataStart = reinterpret_cast<char*>(chunk.data() + dataOffset);
+		if (!buffer.read(dataStart, chunkSize)) {
+			m_Chunks.pop_back();
+			LOG("Failed to read %u bytes from chunk %i", chunkSize, m_Chunks.size() - 1);
+			break;
+		}
+	}
+}
+
+void Sd0::FromData(const uint8_t* data, size_t bufferSize) {
+	const auto originalBufferSize = bufferSize;
+	if (bufferSize == 0) return;
+
+	m_Chunks.clear();
+	while (bufferSize > 0) {
+		const auto numToCopy = std::min(MAX_UNCOMPRESSED_CHUNK_SIZE, bufferSize);
+		const auto* startOffset = data + originalBufferSize - bufferSize;
+		bufferSize -= numToCopy;
+		std::array<uint8_t, MAX_UNCOMPRESSED_CHUNK_SIZE> compressedChunk;
+		const auto compressedSize = ZCompression::Compress(
+			startOffset, numToCopy,
+			compressedChunk.data(), compressedChunk.size());
+
+		auto& chunk = m_Chunks.emplace_back();
+		bool firstBuffer = m_Chunks.size() == 1;
+		auto dataOffset = GetDataOffset(firstBuffer);
+
+		if (firstBuffer) {
+			WriteHeader(chunk);
+		}
+
+		WriteSize(chunk, compressedSize);
+
+		chunk.resize(compressedSize + dataOffset);
+		memcpy(chunk.data() + dataOffset, compressedChunk.data(), compressedSize);
+	}
+
+}
+
+std::string Sd0::GetAsStringUncompressed() const {
+	std::string toReturn;
+	bool first = true;
+	uint32_t totalSize{};
+	for (const auto& chunk : m_Chunks) {
+		auto dataOffset = GetDataOffset(first);
+		first = false;
+		const auto chunkSize = chunk.size();
+
+		auto oldSize = toReturn.size();
+		toReturn.resize(oldSize + MAX_UNCOMPRESSED_CHUNK_SIZE);
+		int32_t error{};
+		const auto uncompressedSize = ZCompression::Decompress(
+			chunk.data() + dataOffset, chunkSize - dataOffset,
+			reinterpret_cast<uint8_t*>(toReturn.data()) + oldSize, MAX_UNCOMPRESSED_CHUNK_SIZE,
+			error);
+
+		totalSize += uncompressedSize;
+	}
+
+	toReturn.resize(totalSize);
+	return toReturn;
+}
+
+std::stringstream Sd0::GetAsStream() const {
+	std::stringstream toReturn;
+
+	for (const auto& chunk : m_Chunks) {
+		toReturn.write(reinterpret_cast<const char*>(chunk.data()), chunk.size());
+	}
+
+	return toReturn;
+}
+
+const std::vector<Sd0::BinaryBuffer>& Sd0::GetAsVector() const {
+	return m_Chunks;
+}

dCommon/Sd0.h

@@ -0,0 +1,42 @@
+// Darkflame Universe
+// Copyright 2025
+
+#ifndef SD0_H
+#define SD0_H
+
+#include <fstream>
+#include <vector>
+
+// Sd0 is comprised of multiple zlib compressed buffers stored in a row.
+// The format starts with a SD0 header (see SD0_HEADER) followed by the size of a zlib buffer, and then the zlib buffer itself.
+// This repeats until end of file
+class Sd0 {
+public:
+	using BinaryBuffer = std::vector<uint8_t>;
+
+	static inline const char* SD0_HEADER = "sd0\x01\xff";
+
+	/**
+	 * @brief Max size of an inflated sd0 zlib chunk
+	 */
+	static constexpr inline size_t MAX_UNCOMPRESSED_CHUNK_SIZE = 1024 * 256;
+
+	// Read the input buffer into an internal chunk stream to be used later
+	Sd0(std::istream& buffer);
+
+	// Uncompresses the entire Sd0 buffer and returns it as a string
+	[[nodiscard]] std::string GetAsStringUncompressed() const;
+
+	// Gets the Sd0 buffer as a stream in its raw compressed form
+	[[nodiscard]] std::stringstream GetAsStream() const;
+
+	// Gets the Sd0 buffer as a vector in its raw compressed form
+	[[nodiscard]] const std::vector<BinaryBuffer>& GetAsVector() const;
+
+	// Compress data into a Sd0 buffer
+	void FromData(const uint8_t* data, size_t bufferSize);
+private:
+	std::vector<BinaryBuffer> m_Chunks{};
+};
+
+#endif //!SD0_H

dCommon/TinyXmlUtils.cpp

@@ -0,0 +1,37 @@
+#include "TinyXmlUtils.h"
+
+#include <tinyxml2.h>
+
+using namespace TinyXmlUtils;
+
+Element DocumentReader::operator[](const std::string_view elem) const {
+	return Element(m_Doc.FirstChildElement(elem.empty() ? nullptr : elem.data()), elem);
+}
+
+Element::Element(tinyxml2::XMLElement* xmlElem, const std::string_view elem) :
+	m_IteratedName{ elem },
+	m_Elem{ xmlElem } {
+}
+
+Element Element::operator[](const std::string_view elem) const {
+	const auto* usedElem = elem.empty() ? nullptr : elem.data();
+	auto* toReturn = m_Elem ? m_Elem->FirstChildElement(usedElem) : nullptr;
+	return Element(toReturn, m_IteratedName);
+}
+
+ElementIterator Element::begin() {
+	return ElementIterator(m_Elem ? m_Elem->FirstChildElement() : nullptr);
+}
+
+ElementIterator Element::end() {
+	return ElementIterator(nullptr);
+}
+
+ElementIterator::ElementIterator(tinyxml2::XMLElement* elem) :
+	m_CurElem{ elem } {
+}
+
+ElementIterator& ElementIterator::operator++() {
+	if (m_CurElem) m_CurElem = m_CurElem->NextSiblingElement();
+	return *this;
+}

dCommon/TinyXmlUtils.h

@@ -0,0 +1,66 @@
+// Darkflame Universe
+// Copyright 2025
+
+#ifndef TINYXMLUTILS_H
+#define TINYXMLUTILS_H
+
+#include <string>
+
+#include "DluAssert.h"
+
+#include <tinyxml2.h>
+
+namespace TinyXmlUtils {
+	// See cstdlib for iterator technicalities
+	struct ElementIterator {
+		ElementIterator(tinyxml2::XMLElement* elem);
+
+		ElementIterator& operator++();
+		[[nodiscard]] tinyxml2::XMLElement* operator->() { DluAssert(m_CurElem); return m_CurElem; }
+		[[nodiscard]] tinyxml2::XMLElement& operator*() { DluAssert(m_CurElem); return *m_CurElem; }
+
+		bool operator==(const ElementIterator& other) const { return other.m_CurElem == m_CurElem; }
+
+	private:
+		tinyxml2::XMLElement* m_CurElem{ nullptr };
+	};
+
+	// Wrapper class to act as an iterator over xml elements.
+	// All the normal rules that apply to Iterators in the std library apply here.
+	class Element {
+	public:
+		Element(tinyxml2::XMLElement* xmlElem, const std::string_view elem);
+
+		// The first child element of this element.
+		[[nodiscard]] ElementIterator begin();
+
+		// Always returns an ElementIterator which points to nullptr.
+		// TinyXml2 return NULL when you've reached the last child element so
+		// you can't do any funny one past end logic here.
+		[[nodiscard]] ElementIterator end();
+
+		// Get a child element
+		[[nodiscard]] Element operator[](const std::string_view elem) const;
+		[[nodiscard]] Element operator[](const char* elem) const { return operator[](std::string_view(elem)); };
+
+		// Whether or not data exists for this element
+		operator bool() const { return m_Elem != nullptr; }
+
+		[[nodiscard]] const tinyxml2::XMLElement* operator->() const { return m_Elem; }
+	private:
+		const char* GetElementName() const { return m_IteratedName.empty() ? nullptr : m_IteratedName.c_str(); }
+		const std::string m_IteratedName;
+		tinyxml2::XMLElement* m_Elem;
+	};
+
+	class DocumentReader {
+	public:
+		DocumentReader(tinyxml2::XMLDocument& doc) : m_Doc{ doc } {}
+
+		[[nodiscard]] Element operator[](const std::string_view elem) const;
+	private:
+		tinyxml2::XMLDocument& m_Doc;
+	};
+};
+
+#endif  //!TINYXMLUTILS_H

dCommon/ZCompression.h

@@ -8,11 +8,5 @@ namespace ZCompression {
 	int32_t Compress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst);
 
 	int32_t Decompress(const uint8_t* abSrc, int32_t nLenSrc, uint8_t* abDst, int32_t nLenDst, int32_t& nErr);
-
-	/**
-	 * @brief Max size of an inflated sd0 zlib chunk
-	 *
-	 */
-	constexpr uint32_t MAX_SD0_CHUNK_SIZE = 1024 * 256;
 }
 

dCommon/dClient/Pack.cpp

@@ -1,6 +1,7 @@
 #include "Pack.h"
 
 #include "BinaryIO.h"
+#include "Sd0.h"
 #include "ZCompression.h"
 
 Pack::Pack(const std::filesystem::path& filePath) {
@@ -106,7 +107,7 @@ bool Pack::ReadFileFromPack(const uint32_t crc, char** data, uint32_t* len) cons
 		pos += size; // Move pointer position the amount of bytes read to the right
 
 		int32_t err;
-		currentReadPos += ZCompression::Decompress(reinterpret_cast<uint8_t*>(chunk), size, reinterpret_cast<uint8_t*>(decompressedData + currentReadPos), ZCompression::MAX_SD0_CHUNK_SIZE, err);
+		currentReadPos += ZCompression::Decompress(reinterpret_cast<uint8_t*>(chunk), size, reinterpret_cast<uint8_t*>(decompressedData + currentReadPos), Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE, err);
 
 		free(chunk);
 	}