From 9f92f48a0f360a35924fa681bf437458b5912341 Mon Sep 17 00:00:00 2001 From: David Markowitz <39972741+EmosewaMC@users.noreply.github.com> Date: Mon, 23 Jun 2025 00:08:16 -0700 Subject: [PATCH] fix: models with multiple parts not being normalized properly (#1825) Tested that models are migrated to the new format a-ok Tested that the new logic works as expected. Old code needs to be kept so that models in both states can be brought to modern standards --- dCommon/CMakeLists.txt | 1 + dCommon/Lxfml.cpp | 6 +- dCommon/Lxfml.h | 4 + dCommon/LxfmlBugged.cpp | 210 ++++++++++++++++++ dDatabase/MigrationRunner.cpp | 9 +- dDatabase/ModelNormalizeMigration.cpp | 22 +- dDatabase/ModelNormalizeMigration.h | 1 + ...alize_model_positions_after_first_part.sql | 1 + ...alize_model_positions_after_first_part.sql | 1 + 9 files changed, 251 insertions(+), 4 deletions(-) create mode 100644 dCommon/LxfmlBugged.cpp create mode 100644 migrations/dlu/mysql/21_normalize_model_positions_after_first_part.sql create mode 100644 migrations/dlu/sqlite/4_normalize_model_positions_after_first_part.sql diff --git a/dCommon/CMakeLists.txt b/dCommon/CMakeLists.txt index 067e3c1c..3a073fc4 100644 --- a/dCommon/CMakeLists.txt +++ b/dCommon/CMakeLists.txt @@ -20,6 +20,7 @@ set(DCOMMON_SOURCES "TinyXmlUtils.cpp" "Sd0.cpp" "Lxfml.cpp" + "LxfmlBugged.cpp" ) # Workaround for compiler bug where the optimized code could result in a memcpy of 0 bytes, even though that isnt possible. diff --git a/dCommon/Lxfml.cpp b/dCommon/Lxfml.cpp index f713bcab..a244c3ae 100644 --- a/dCommon/Lxfml.cpp +++ b/dCommon/Lxfml.cpp @@ -27,7 +27,7 @@ Lxfml::Result Lxfml::NormalizePosition(const std::string_view data) { // First get all the positions of bricks for (const auto& brick : lxfml["Bricks"]) { const auto* part = brick.FirstChildElement("Part"); - if (part) { + while (part) { const auto* bone = part->FirstChildElement("Bone"); if (bone) { auto* transformation = bone->Attribute("transformation"); @@ -36,6 +36,7 @@ Lxfml::Result Lxfml::NormalizePosition(const std::string_view data) { if (refID) transformations[refID] = transformation; } } + part = part->NextSiblingElement("Part"); } } @@ -92,7 +93,7 @@ Lxfml::Result Lxfml::NormalizePosition(const std::string_view data) { // Finally write the new transformation back into the lxfml for (auto& brick : lxfml["Bricks"]) { auto* part = brick.FirstChildElement("Part"); - if (part) { + while (part) { auto* bone = part->FirstChildElement("Bone"); if (bone) { auto* transformation = bone->Attribute("transformation"); @@ -103,6 +104,7 @@ Lxfml::Result Lxfml::NormalizePosition(const std::string_view data) { } } } + part = part->NextSiblingElement("Part"); } } diff --git a/dCommon/Lxfml.h b/dCommon/Lxfml.h index 3f5f4d4a..5304d8ec 100644 --- a/dCommon/Lxfml.h +++ b/dCommon/Lxfml.h @@ -18,6 +18,10 @@ namespace Lxfml { // Normalizes a LXFML model to be positioned relative to its local 0, 0, 0 rather than a game worlds 0, 0, 0. // Returns a struct of its new center and the updated LXFML containing these edits. [[nodiscard]] Result NormalizePosition(const std::string_view data); + + // these are only for the migrations due to a bug in one of the implementations. + [[nodiscard]] Result NormalizePositionOnlyFirstPart(const std::string_view data); + [[nodiscard]] Result NormalizePositionAfterFirstPart(const std::string_view data, const NiPoint3& position); }; #endif //!LXFML_H diff --git a/dCommon/LxfmlBugged.cpp b/dCommon/LxfmlBugged.cpp new file mode 100644 index 00000000..f5eeebc8 --- /dev/null +++ b/dCommon/LxfmlBugged.cpp @@ -0,0 +1,210 @@ +#include "Lxfml.h" + +#include "GeneralUtils.h" +#include "StringifiedEnum.h" +#include "TinyXmlUtils.h" + +#include + +// this file should not be touched + +Lxfml::Result Lxfml::NormalizePositionOnlyFirstPart(const std::string_view data) { + Result toReturn; + tinyxml2::XMLDocument doc; + const auto err = doc.Parse(data.data()); + if (err != tinyxml2::XML_SUCCESS) { + LOG("Failed to parse xml %s.", StringifiedEnum::ToString(err).data()); + return toReturn; + } + + TinyXmlUtils::DocumentReader reader(doc); + std::map transformations; + + auto lxfml = reader["LXFML"]; + if (!lxfml) { + LOG("Failed to find LXFML element."); + return toReturn; + } + + // First get all the positions of bricks + for (const auto& brick : lxfml["Bricks"]) { + const auto* part = brick.FirstChildElement("Part"); + if (part) { + const auto* bone = part->FirstChildElement("Bone"); + if (bone) { + auto* transformation = bone->Attribute("transformation"); + if (transformation) { + auto* refID = bone->Attribute("refID"); + if (refID) transformations[refID] = transformation; + } + } + } + } + + // These points are well out of bounds for an actual player + NiPoint3 lowest{ 10'000.0f, 10'000.0f, 10'000.0f }; + NiPoint3 highest{ -10'000.0f, -10'000.0f, -10'000.0f }; + + // Calculate the lowest and highest points on the entire model + for (const auto& transformation : transformations | std::views::values) { + auto split = GeneralUtils::SplitString(transformation, ','); + if (split.size() < 12) { + LOG("Not enough in the split?"); + continue; + } + + auto x = GeneralUtils::TryParse(split[9]).value(); + auto y = GeneralUtils::TryParse(split[10]).value(); + auto z = GeneralUtils::TryParse(split[11]).value(); + if (x < lowest.x) lowest.x = x; + if (y < lowest.y) lowest.y = y; + if (z < lowest.z) lowest.z = z; + + if (highest.x < x) highest.x = x; + if (highest.y < y) highest.y = y; + if (highest.z < z) highest.z = z; + } + + auto delta = (highest - lowest) / 2.0f; + auto newRootPos = lowest + delta; + + // Clamp the Y to the lowest point on the model + newRootPos.y = lowest.y; + + // Adjust all positions to account for the new origin + for (auto& transformation : transformations | std::views::values) { + auto split = GeneralUtils::SplitString(transformation, ','); + if (split.size() < 12) { + LOG("Not enough in the split?"); + continue; + } + + auto x = GeneralUtils::TryParse(split[9]).value() - newRootPos.x; + auto y = GeneralUtils::TryParse(split[10]).value() - newRootPos.y; + auto z = GeneralUtils::TryParse(split[11]).value() - newRootPos.z; + std::stringstream stream; + for (int i = 0; i < 9; i++) { + stream << split[i]; + stream << ','; + } + stream << x << ',' << y << ',' << z; + transformation = stream.str(); + } + + // Finally write the new transformation back into the lxfml + for (auto& brick : lxfml["Bricks"]) { + auto* part = brick.FirstChildElement("Part"); + if (part) { + auto* bone = part->FirstChildElement("Bone"); + if (bone) { + auto* transformation = bone->Attribute("transformation"); + if (transformation) { + auto* refID = bone->Attribute("refID"); + if (refID) { + bone->SetAttribute("transformation", transformations[refID].c_str()); + } + } + } + } + } + + tinyxml2::XMLPrinter printer; + doc.Print(&printer); + + toReturn.lxfml = printer.CStr(); + toReturn.center = newRootPos; + return toReturn; +} + +Lxfml::Result Lxfml::NormalizePositionAfterFirstPart(const std::string_view data, const NiPoint3& position) { + Result toReturn; + tinyxml2::XMLDocument doc; + const auto err = doc.Parse(data.data()); + if (err != tinyxml2::XML_SUCCESS) { + LOG("Failed to parse xml %s.", StringifiedEnum::ToString(err).data()); + return toReturn; + } + + TinyXmlUtils::DocumentReader reader(doc); + std::map transformations; + + auto lxfml = reader["LXFML"]; + if (!lxfml) { + LOG("Failed to find LXFML element."); + return toReturn; + } + + // First get all the positions of bricks + for (const auto& brick : lxfml["Bricks"]) { + const auto* part = brick.FirstChildElement("Part"); + bool firstPart = true; + while (part) { + if (firstPart) { + firstPart = false; + } else { + LOG("Found extra bricks"); + const auto* bone = part->FirstChildElement("Bone"); + if (bone) { + auto* transformation = bone->Attribute("transformation"); + if (transformation) { + auto* refID = bone->Attribute("refID"); + if (refID) transformations[refID] = transformation; + } + } + } + part = part->NextSiblingElement("Part"); + } + } + + auto newRootPos = position; + + // Adjust all positions to account for the new origin + for (auto& transformation : transformations | std::views::values) { + auto split = GeneralUtils::SplitString(transformation, ','); + if (split.size() < 12) { + LOG("Not enough in the split?"); + continue; + } + + auto x = GeneralUtils::TryParse(split[9]).value() - newRootPos.x; + auto y = GeneralUtils::TryParse(split[10]).value() - newRootPos.y; + auto z = GeneralUtils::TryParse(split[11]).value() - newRootPos.z; + std::stringstream stream; + for (int i = 0; i < 9; i++) { + stream << split[i]; + stream << ','; + } + stream << x << ',' << y << ',' << z; + transformation = stream.str(); + } + + // Finally write the new transformation back into the lxfml + for (auto& brick : lxfml["Bricks"]) { + auto* part = brick.FirstChildElement("Part"); + bool firstPart = true; + while (part) { + if (firstPart) { + firstPart = false; + } else { + auto* bone = part->FirstChildElement("Bone"); + if (bone) { + auto* transformation = bone->Attribute("transformation"); + if (transformation) { + auto* refID = bone->Attribute("refID"); + if (refID) { + bone->SetAttribute("transformation", transformations[refID].c_str()); + } + } + } + } + part = part->NextSiblingElement("Part"); + } + } + + tinyxml2::XMLPrinter printer; + doc.Print(&printer); + + toReturn.lxfml = printer.CStr(); + toReturn.center = newRootPos; + return toReturn; +} diff --git a/dDatabase/MigrationRunner.cpp b/dDatabase/MigrationRunner.cpp index 8cdd17ae..ee767ba0 100644 --- a/dDatabase/MigrationRunner.cpp +++ b/dDatabase/MigrationRunner.cpp @@ -47,6 +47,7 @@ void MigrationRunner::RunMigrations() { std::string finalSQL = ""; bool runSd0Migrations = false; bool runNormalizeMigrations = false; + bool runNormalizeAfterFirstPartMigrations = false; for (const auto& entry : GeneralUtils::GetSqlFileNamesFromFolder((BinaryPathFinder::GetBinaryDir() / "./migrations/dlu/" / migrationFolder).string())) { auto migration = LoadMigration("dlu/" + migrationFolder + "/", entry); @@ -61,6 +62,8 @@ void MigrationRunner::RunMigrations() { runSd0Migrations = true; } else if (migration.name.ends_with("_normalize_model_positions.sql")) { runNormalizeMigrations = true; + } else if (migration.name.ends_with("_normalize_model_positions_after_first_part.sql")) { + runNormalizeAfterFirstPartMigrations = true; } else { finalSQL.append(migration.data.c_str()); } @@ -68,7 +71,7 @@ void MigrationRunner::RunMigrations() { Database::Get()->InsertMigration(migration.name); } - if (finalSQL.empty() && !runSd0Migrations && !runNormalizeMigrations) { + if (finalSQL.empty() && !runSd0Migrations && !runNormalizeMigrations && !runNormalizeAfterFirstPartMigrations) { LOG("Server database is up to date."); return; } @@ -96,6 +99,10 @@ void MigrationRunner::RunMigrations() { if (runNormalizeMigrations) { ModelNormalizeMigration::Run(); } + + if (runNormalizeAfterFirstPartMigrations) { + ModelNormalizeMigration::RunAfterFirstPart(); + } } void MigrationRunner::RunSQLiteMigrations() { diff --git a/dDatabase/ModelNormalizeMigration.cpp b/dDatabase/ModelNormalizeMigration.cpp index b8215733..e33e5e4f 100644 --- a/dDatabase/ModelNormalizeMigration.cpp +++ b/dDatabase/ModelNormalizeMigration.cpp @@ -14,7 +14,7 @@ void ModelNormalizeMigration::Run() { Sd0 sd0(lxfmlData); const auto asStr = sd0.GetAsStringUncompressed(); - const auto [newLxfml, newCenter] = Lxfml::NormalizePosition(asStr); + const auto [newLxfml, newCenter] = Lxfml::NormalizePositionOnlyFirstPart(asStr); if (newCenter == NiPoint3Constant::ZERO) { LOG("Failed to update model %llu due to failure reading xml."); continue; @@ -28,3 +28,23 @@ void ModelNormalizeMigration::Run() { } Database::Get()->SetAutoCommit(oldCommit); } + +void ModelNormalizeMigration::RunAfterFirstPart() { + const auto oldCommit = Database::Get()->GetAutoCommit(); + Database::Get()->SetAutoCommit(false); + for (auto& [lxfmlData, id, modelID] : Database::Get()->GetAllUgcModels()) { + const auto model = Database::Get()->GetModel(modelID); + // only BBB models (lot 14) need to have their position fixed from the above blunder + if (model.lot != 14) continue; + + Sd0 sd0(lxfmlData); + const auto asStr = sd0.GetAsStringUncompressed(); + const auto [newLxfml, newCenter] = Lxfml::NormalizePositionAfterFirstPart(asStr, model.position); + + sd0.FromData(reinterpret_cast(newLxfml.data()), newLxfml.size()); + auto asStream = sd0.GetAsStream(); + Database::Get()->UpdateModel(model.id, newCenter, model.rotation, model.behaviors); + Database::Get()->UpdateUgcModelData(id, asStream); + } + Database::Get()->SetAutoCommit(oldCommit); +} diff --git a/dDatabase/ModelNormalizeMigration.h b/dDatabase/ModelNormalizeMigration.h index 000781cd..686ca02f 100644 --- a/dDatabase/ModelNormalizeMigration.h +++ b/dDatabase/ModelNormalizeMigration.h @@ -6,6 +6,7 @@ namespace ModelNormalizeMigration { void Run(); + void RunAfterFirstPart(); }; #endif //!MODELNORMALIZEMIGRATION_H diff --git a/migrations/dlu/mysql/21_normalize_model_positions_after_first_part.sql b/migrations/dlu/mysql/21_normalize_model_positions_after_first_part.sql new file mode 100644 index 00000000..833fbe9b --- /dev/null +++ b/migrations/dlu/mysql/21_normalize_model_positions_after_first_part.sql @@ -0,0 +1 @@ +/* See ModelNormalizeMigration.cpp for details */ diff --git a/migrations/dlu/sqlite/4_normalize_model_positions_after_first_part.sql b/migrations/dlu/sqlite/4_normalize_model_positions_after_first_part.sql new file mode 100644 index 00000000..833fbe9b --- /dev/null +++ b/migrations/dlu/sqlite/4_normalize_model_positions_after_first_part.sql @@ -0,0 +1 @@ +/* See ModelNormalizeMigration.cpp for details */