From 77c88575f94a76682cfe3aee783432b379e44af5 Mon Sep 17 00:00:00 2001 From: David Markowitz Date: Wed, 9 Apr 2025 23:35:55 -0700 Subject: [PATCH] Add utilities for formats --- dCommon/CMakeLists.txt | 3 + dCommon/Lxfml.cpp | 112 ++++++++++++++++++++++++++++++++ dCommon/Lxfml.h | 21 ++++++ dCommon/Sd0.cpp | 136 +++++++++++++++++++++++++++++++++++++++ dCommon/Sd0.h | 42 ++++++++++++ dCommon/TinyXmlUtils.cpp | 37 +++++++++++ dCommon/TinyXmlUtils.h | 66 +++++++++++++++++++ 7 files changed, 417 insertions(+) create mode 100644 dCommon/Lxfml.cpp create mode 100644 dCommon/Lxfml.h create mode 100644 dCommon/Sd0.cpp create mode 100644 dCommon/Sd0.h create mode 100644 dCommon/TinyXmlUtils.cpp create mode 100644 dCommon/TinyXmlUtils.h diff --git a/dCommon/CMakeLists.txt b/dCommon/CMakeLists.txt index 18fda0ed..74432e0f 100644 --- a/dCommon/CMakeLists.txt +++ b/dCommon/CMakeLists.txt @@ -16,6 +16,9 @@ set(DCOMMON_SOURCES "BrickByBrickFix.cpp" "BinaryPathFinder.cpp" "FdbToSqlite.cpp" + "TinyXmlUtils.cpp" + "Sd0.cpp" + "Lxfml.cpp" ) # Workaround for compiler bug where the optimized code could result in a memcpy of 0 bytes, even though that isnt possible. diff --git a/dCommon/Lxfml.cpp b/dCommon/Lxfml.cpp new file mode 100644 index 00000000..4bc074bb --- /dev/null +++ b/dCommon/Lxfml.cpp @@ -0,0 +1,112 @@ +#include "Lxfml.h" + +#include "GeneralUtils.h" +#include "StringifiedEnum.h" +#include "TinyXmlUtils.h" + +#include + +Lxfml::Result Lxfml::NormalizePosition(const std::string_view data) { + Result toReturn; + tinyxml2::XMLDocument doc; + const auto err = doc.Parse(data.data()); + if (err != tinyxml2::XML_SUCCESS) { + LOG("Failed to parse xml %s.", StringifiedEnum::ToString(err).data()); + return toReturn; + } + + TinyXmlUtils::DocumentReader reader(doc); + std::map transformations; + + auto lxfml = reader["LXFML"]; + if (!lxfml) return toReturn; + + // First get all the positions of bricks + for (const auto& brick : lxfml["Bricks"]) { + const auto* part = brick.FirstChildElement("Part"); + if (part) { + const auto* bone = part->FirstChildElement("Bone"); + if (bone) { + auto* transformation = bone->Attribute("transformation"); + if (transformation) { + auto* refID = bone->Attribute("refID"); + if (refID) transformations[refID] = transformation; + } + } + } + } + + // These points are well out of bounds for an actual player + NiPoint3 lowest{10'000, 10'000, 10'000}; + NiPoint3 highest{-10'000, -10'000, -10'000}; + + // Calculate the lowest and highest points on the entire model + for (const auto& transformation : transformations | std::views::values) { + auto split = GeneralUtils::SplitString(transformation, ','); + if (split.size() < 12) { + LOG("Not enough in the split?"); + continue; + } + + auto x = GeneralUtils::TryParse(split[9]).value(); + auto y = GeneralUtils::TryParse(split[10]).value(); + auto z = GeneralUtils::TryParse(split[11]).value(); + if (x < lowest.x) lowest.x = x; + if (y < lowest.y) lowest.y = y; + if (z < lowest.z) lowest.z = z; + + if (highest.x < x) highest.x = x; + if (highest.y < y) highest.y = y; + if (highest.z < z) highest.z = z; + } + + auto delta = (highest - lowest) / 2.0f; + auto newRootPos = lowest + delta; + + // Clamp the Y to the lowest point on the model + newRootPos.y = lowest.y; + + // Adjust all positions to account for the new origin + for (auto& transformation : transformations | std::views::values) { + auto split = GeneralUtils::SplitString(transformation, ','); + if (split.size() < 12) { + LOG("Not enough in the split?"); + continue; + } + + auto x = GeneralUtils::TryParse(split[9]).value() - newRootPos.x; + auto y = GeneralUtils::TryParse(split[10]).value() - newRootPos.y; + auto z = GeneralUtils::TryParse(split[11]).value() - newRootPos.z; + std::stringstream stream; + for (int i = 0; i < 9; i++) { + stream << split[i]; + stream << ','; + } + stream << x << ',' << y << ',' << z; + transformation = stream.str(); + } + + // Finally write the new transformation back into the lxfml + for (auto& brick : lxfml["Bricks"]) { + auto* part = brick.FirstChildElement("Part"); + if (part) { + auto* bone = part->FirstChildElement("Bone"); + if (bone) { + auto* transformation = bone->Attribute("transformation"); + if (transformation) { + auto* refID = bone->Attribute("refID"); + if (refID) { + bone->SetAttribute("transformation", transformations[refID].c_str()); + } + } + } + } + } + + tinyxml2::XMLPrinter printer; + doc.Print(&printer); + + toReturn.lxfml = printer.CStr(); + toReturn.center = newRootPos; + return toReturn; +} diff --git a/dCommon/Lxfml.h b/dCommon/Lxfml.h new file mode 100644 index 00000000..ab796473 --- /dev/null +++ b/dCommon/Lxfml.h @@ -0,0 +1,21 @@ +// Darkflame Universe +// Copyright 2025 + +#ifndef LXFML_H +#define LXFML_H + +#include +#include + +#include "NiPoint3.h" + +namespace Lxfml { + struct Result { + std::string lxfml; + NiPoint3 center; + }; + + [[nodiscard]] Result NormalizePosition(const std::string_view data); +}; + +#endif //!LXFML_H diff --git a/dCommon/Sd0.cpp b/dCommon/Sd0.cpp new file mode 100644 index 00000000..aa745380 --- /dev/null +++ b/dCommon/Sd0.cpp @@ -0,0 +1,136 @@ +#include "Sd0.h" + +#include +#include + +#include "BinaryIO.h" + +#include "Game.h" +#include "Logger.h" + +#include "ZCompression.h" + +// Insert header if on first buffer +void WriteHeader(Sd0::BinaryBuffer& chunk) { + chunk.push_back(Sd0::SD0_HEADER[0]); + chunk.push_back(Sd0::SD0_HEADER[1]); + chunk.push_back(Sd0::SD0_HEADER[2]); + chunk.push_back(Sd0::SD0_HEADER[3]); + chunk.push_back(Sd0::SD0_HEADER[4]); +} + +// Write the size of the buffer to a chunk +void WriteSize(Sd0::BinaryBuffer& chunk, uint32_t chunkSize) { + for (int i = 0; i < 4; i++) { + char toPush = chunkSize & 0xff; + chunkSize = chunkSize >> 8; + chunk.push_back(toPush); + } +} + +int32_t GetDataOffset(bool firstBuffer) { + return firstBuffer ? 9 : 4; +} + +Sd0::Sd0(std::istream& buffer) { + char header[5]{}; + if (!BinaryIO::BinaryRead(buffer, header) || memcmp(header, SD0_HEADER, sizeof(header)) != 0) { + LOG("Failed to read SD0 header %i %i %i %i %i %i %i", buffer.good(), buffer.tellg(), header[0], header[1], header[2], header[3], header[4]); + return; + } + + while (buffer) { + uint32_t chunkSize{}; + if (!BinaryIO::BinaryRead(buffer, chunkSize)) { + LOG("%i", m_Chunks.size()); + LOG("Failed to read chunk size from stream %i %i", buffer.tellg(), static_cast(m_Chunks.size())); + break; + } + auto& chunk = m_Chunks.emplace_back(); + bool firstBuffer = m_Chunks.size() == 1; + auto dataOffset = GetDataOffset(firstBuffer); + + // Insert header if on first buffer + if (firstBuffer) { + WriteHeader(chunk); + } + + WriteSize(chunk, chunkSize); + + chunk.resize(chunkSize + dataOffset); + auto* dataStart = reinterpret_cast(chunk.data() + dataOffset); + if (!buffer.read(dataStart, chunkSize)) { + m_Chunks.pop_back(); + LOG("Failed to read %u bytes from chunk %i", chunkSize, m_Chunks.size() - 1); + break; + } + } +} + +void Sd0::FromData(const uint8_t* data, size_t bufferSize) { + const auto originalBufferSize = bufferSize; + if (bufferSize == 0) return; + + m_Chunks.clear(); + while (bufferSize > 0) { + const auto numToCopy = std::min(MAX_UNCOMPRESSED_CHUNK_SIZE, bufferSize); + const auto* startOffset = data + originalBufferSize - bufferSize; + bufferSize -= numToCopy; + std::array compressedChunk; + const auto compressedSize = ZCompression::Compress( + startOffset, numToCopy, + compressedChunk.data(), compressedChunk.size()); + + auto& chunk = m_Chunks.emplace_back(); + bool firstBuffer = m_Chunks.size() == 1; + auto dataOffset = GetDataOffset(firstBuffer); + + if (firstBuffer) { + WriteHeader(chunk); + } + + WriteSize(chunk, compressedSize); + + chunk.resize(compressedSize + dataOffset); + memcpy(chunk.data() + dataOffset, compressedChunk.data(), compressedSize); + } + +} + +std::string Sd0::GetAsStringUncompressed() const { + std::string toReturn; + bool first = true; + uint32_t totalSize{}; + for (const auto& chunk : m_Chunks) { + auto dataOffset = GetDataOffset(first); + first = false; + const auto chunkSize = chunk.size(); + + auto oldSize = toReturn.size(); + toReturn.resize(oldSize + MAX_UNCOMPRESSED_CHUNK_SIZE); + int32_t error{}; + const auto uncompressedSize = ZCompression::Decompress( + chunk.data() + dataOffset, chunkSize - dataOffset, + reinterpret_cast(toReturn.data()) + oldSize, MAX_UNCOMPRESSED_CHUNK_SIZE, + error); + + totalSize += uncompressedSize; + } + + toReturn.resize(totalSize); + return toReturn; +} + +std::stringstream Sd0::GetAsStream() const { + std::stringstream toReturn; + + for (const auto& chunk : m_Chunks) { + toReturn.write(reinterpret_cast(chunk.data()), chunk.size()); + } + + return toReturn; +} + +const std::vector& Sd0::GetAsVector() const { + return m_Chunks; +} diff --git a/dCommon/Sd0.h b/dCommon/Sd0.h new file mode 100644 index 00000000..977c53da --- /dev/null +++ b/dCommon/Sd0.h @@ -0,0 +1,42 @@ +// Darkflame Universe +// Copyright 2025 + +#ifndef SD0_H +#define SD0_H + +#include +#include + +// Sd0 is comprised of multiple zlib compressed buffers stored in a row. +class Sd0 { +public: + using BinaryBuffer = std::vector; + + static inline const char* SD0_HEADER = "sd0\x01\xff"; + + /** + * @brief Max size of an inflated sd0 zlib chunk + */ + static constexpr inline size_t MAX_UNCOMPRESSED_CHUNK_SIZE = 1024 * 256; + + Sd0() {} + + // Read the input buffer into an internal chunk stream to be used later + Sd0(std::istream& buffer); + + // Uncompresses the entire Sd0 buffer and returns it as a string + std::string GetAsStringUncompressed() const; + + // Gets the Sd0 buffer as a stream in its raw compressed form + std::stringstream GetAsStream() const; + + // Gets the Sd0 buffer as a vector in its raw compressed form + const std::vector& GetAsVector() const; + + // Compress data into a Sd0 buffer + void FromData(const uint8_t* data, size_t bufferSize); +private: + std::vector m_Chunks{}; +}; + +#endif //!SD0_H diff --git a/dCommon/TinyXmlUtils.cpp b/dCommon/TinyXmlUtils.cpp new file mode 100644 index 00000000..9fe88eb7 --- /dev/null +++ b/dCommon/TinyXmlUtils.cpp @@ -0,0 +1,37 @@ +#include "TinyXmlUtils.h" + +#include + +using namespace TinyXmlUtils; + +Element DocumentReader::operator[](const std::string_view elem) const { + return Element(m_Doc.FirstChildElement(elem.empty() ? nullptr : elem.data()), elem); +} + +Element::Element(tinyxml2::XMLElement* xmlElem, const std::string_view elem) : + m_IteratedName{ elem }, + m_Elem{ xmlElem } { +} + +Element Element::operator[](const std::string_view elem) const { + const auto* usedElem = elem.empty() ? nullptr : elem.data(); + auto* toReturn = m_Elem ? m_Elem->FirstChildElement(usedElem) : nullptr; + return Element(toReturn, m_IteratedName); +} + +ElementIterator Element::begin() { + return ElementIterator(m_Elem ? m_Elem->FirstChildElement() : nullptr); +} + +ElementIterator Element::end() { + return ElementIterator(nullptr); +} + +ElementIterator::ElementIterator(tinyxml2::XMLElement* elem) : + m_CurElem{ elem } { +} + +ElementIterator& ElementIterator::operator++() { + if (m_CurElem) m_CurElem = m_CurElem->NextSiblingElement(); + return *this; +} diff --git a/dCommon/TinyXmlUtils.h b/dCommon/TinyXmlUtils.h new file mode 100644 index 00000000..1a7cf33e --- /dev/null +++ b/dCommon/TinyXmlUtils.h @@ -0,0 +1,66 @@ +// Darkflame Universe +// Copyright 2025 + +#ifndef TINYXMLUTILS_H +#define TINYXMLUTILS_H + +#include + +#include "DluAssert.h" + +#include + +namespace TinyXmlUtils { + // See cstdlib for iterator technicalities + struct ElementIterator { + ElementIterator(tinyxml2::XMLElement* elem); + + ElementIterator& operator++(); + tinyxml2::XMLElement* operator->() { DluAssert(m_CurElem); return m_CurElem; } + tinyxml2::XMLElement& operator*() { DluAssert(m_CurElem); return *m_CurElem; } + + bool operator==(const ElementIterator& other) const { return other.m_CurElem == m_CurElem; } + + private: + tinyxml2::XMLElement* m_CurElem{ nullptr }; + }; + + // Wrapper class to act as an iterator over xml elements. + // All the normal rules that apply to Iterators in the std library apply here. + class Element { + public: + Element(tinyxml2::XMLElement* xmlElem, const std::string_view elem); + + // The first child element of this element. + ElementIterator begin(); + + // Always returns an ElementIterator which points to nullptr. + // TinyXml2 return NULL when you've reached the last child element so + // you can't do any funny one past end logic here. + ElementIterator end(); + + // Get a child element + Element operator[](const std::string_view elem) const; + Element operator[](const char* elem) const { return operator[](std::string_view(elem)); }; + + // Whether or not data exists for this element + operator bool() const { return m_Elem != nullptr; } + + const tinyxml2::XMLElement* operator->() const { return m_Elem; } + private: + const char* GetElementName() const { return m_IteratedName.empty() ? nullptr : m_IteratedName.c_str(); } + const std::string m_IteratedName; + tinyxml2::XMLElement* m_Elem; + }; + + class DocumentReader { + public: + DocumentReader(tinyxml2::XMLDocument& doc) : m_Doc{ doc } {} + + Element operator[](const std::string_view elem) const; + private: + tinyxml2::XMLDocument& m_Doc; + }; +}; + +#endif //!TINYXMLUTILS_H