From 7ea0453a6cf1006ea983f277942a2a53f2e01728 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Tue, 29 Sep 2026 06:05:32 -0500 Subject: [PATCH] feat(ugc): migration gives old cars and rockets a build id Cars and rockets built before builds were recorded were saved with subkey 0 and have no ugc_modular_build row, so the client has no blueprint id to ask for their icon with. The runtime fix only reaches characters that load again. Migrations mysql 98 and sqlite 81 (98/81 because the guilds work takes 97/80) run ModularBuildIdMigration after the SQL: every saved item with modules (x@ma) and no subkey gets what a new build gets, as the character-load fix does: a persistent id from object_id_tracker (with the character bit) as its subkey and a ugc_modular_build row with its modules and the character as owner. The XML is written with UpdateCharacterXml. Tried on a copy of a server's database with a couple of thousand such items: every one got an id and a build row, in seconds. Check: after the migration, old cars and rockets show their icons in the backpack (the UGC server makes them like any build) and still work (equip, race, launch). Co-Authored-By: Claude Opus 5.5 --- dDatabase/CMakeLists.txt | 2 +- dDatabase/MigrationRunner.cpp | 11 ++- dDatabase/ModularBuildIdMigration.cpp | 74 +++++++++++++++++++ dDatabase/ModularBuildIdMigration.h | 34 +++++++++ docs/UgcServer.md | 16 +++- migrations/dlu/mysql/98_modular_build_ids.sql | 1 + .../dlu/sqlite/81_modular_build_ids.sql | 1 + tests/dGameTests/CMakeLists.txt | 1 + .../ModularBuildIdMigrationTests.cpp | 45 +++++++++++ 9 files changed, 180 insertions(+), 5 deletions(-) create mode 100644 dDatabase/ModularBuildIdMigration.cpp create mode 100644 dDatabase/ModularBuildIdMigration.h create mode 100644 migrations/dlu/mysql/98_modular_build_ids.sql create mode 100644 migrations/dlu/sqlite/81_modular_build_ids.sql create mode 100644 tests/dGameTests/ModularBuildIdMigrationTests.cpp diff --git a/dDatabase/CMakeLists.txt b/dDatabase/CMakeLists.txt index e106df6c9..6d3a91a27 100644 --- a/dDatabase/CMakeLists.txt +++ b/dDatabase/CMakeLists.txt @@ -1,7 +1,7 @@ add_subdirectory(CDClientDatabase) add_subdirectory(GameDatabase) -add_library(dDatabase STATIC "MigrationRunner.cpp" "ModelNormalizeMigration.cpp" "ConfigSync.cpp") +add_library(dDatabase STATIC "MigrationRunner.cpp" "ModelNormalizeMigration.cpp" "ModularBuildIdMigration.cpp" "ConfigSync.cpp") add_custom_target(conncpp_dylib ${CMAKE_COMMAND} -E copy $ ${PROJECT_BINARY_DIR}) diff --git a/dDatabase/MigrationRunner.cpp b/dDatabase/MigrationRunner.cpp index b1fdc7308..9bf86fb67 100644 --- a/dDatabase/MigrationRunner.cpp +++ b/dDatabase/MigrationRunner.cpp @@ -8,6 +8,7 @@ #include "Logger.h" #include "BinaryPathFinder.h" #include "ModelNormalizeMigration.h" +#include "ModularBuildIdMigration.h" #include @@ -52,6 +53,7 @@ void MigrationRunner::RunMigrations() { bool runNormalizeMigrations = false; bool runNormalizeAfterFirstPartMigrations = false; bool runBrickBuildsNotOnGrid = false; + bool runModularBuildIds = false; for (const auto& entry : GeneralUtils::GetSqlFileNamesFromFolder((BinaryPathFinder::GetBinaryDir() / "./migrations/dlu/" / migrationFolder).string())) { auto migration = LoadMigration("dlu/" + migrationFolder + "/", entry); @@ -70,6 +72,8 @@ void MigrationRunner::RunMigrations() { runNormalizeAfterFirstPartMigrations = true; } else if (migration.name.ends_with("_brickbuilds_not_on_grid.sql")) { runBrickBuildsNotOnGrid = true; + } else if (migration.name.ends_with("_modular_build_ids.sql")) { + runModularBuildIds = true; } else { finalSQL.append(migration.data.c_str()); } @@ -77,7 +81,7 @@ void MigrationRunner::RunMigrations() { Database::Get()->InsertMigration(migration.name); } - if (finalSQL.empty() && !runSd0Migrations && !runNormalizeMigrations && !runNormalizeAfterFirstPartMigrations && !runBrickBuildsNotOnGrid) { + if (finalSQL.empty() && !runSd0Migrations && !runNormalizeMigrations && !runNormalizeAfterFirstPartMigrations && !runBrickBuildsNotOnGrid && !runModularBuildIds) { LOG("Server database is up to date."); return; } @@ -113,6 +117,11 @@ void MigrationRunner::RunMigrations() { if (runBrickBuildsNotOnGrid) { ModelNormalizeMigration::RunBrickBuildGrid(); } + + // After the SQL, which makes ugc_modular_build and object_id_tracker on a new database + if (runModularBuildIds) { + ModularBuildIdMigration::Run(); + } } void MigrationRunner::RunSQLiteMigrations() { diff --git a/dDatabase/ModularBuildIdMigration.cpp b/dDatabase/ModularBuildIdMigration.cpp new file mode 100644 index 000000000..884e411f5 --- /dev/null +++ b/dDatabase/ModularBuildIdMigration.cpp @@ -0,0 +1,74 @@ +#include "ModularBuildIdMigration.h" + +#include + +#include "tinyxml2.h" + +#include "Database.h" +#include "eObjectBits.h" +#include "GeneralUtils.h" +#include "Logger.h" + +namespace { + // A persistent id, as ObjectIDManager::GetPersistentID hands them out: from a range reserved in object_id_tracker, + // with the CHARACTER bit + class PersistentIds { + public: + LWOOBJID Next() { + if (!m_Range || m_Range->minID > m_Range->maxID) m_Range = Database::Get()->GetPersistentIdRange(); + LWOOBJID id = m_Range->minID++; + GeneralUtils::SetBit(id, eObjectBits::CHARACTER); + return id; + } + private: + std::optional m_Range; + }; +} + +std::vector ModularBuildIdMigration::AssignIds(tinyxml2::XMLDocument& document, const std::function& nextId) { + std::vector builds; + auto* obj = document.FirstChildElement("obj"); + auto* inv = obj ? obj->FirstChildElement("inv") : nullptr; + auto* items = inv ? inv->FirstChildElement("items") : nullptr; + for (auto* bag = items ? items->FirstChildElement("in") : nullptr; bag; bag = bag->NextSiblingElement("in")) { + for (auto* item = bag->FirstChildElement("i"); item; item = item->NextSiblingElement("i")) { + if (item->Int64Attribute("sk", 0) != 0) continue; + const auto* config = item->FirstChildElement("x"); + const char* saved = config ? config->Attribute("ma") : nullptr; + if (!saved) continue; + // Saved as the LDF type and value ("0:1:8129+1:8130"); the build row keeps the value + std::string modules(saved); + if (const auto colon = modules.find(':'); colon != std::string::npos && GeneralUtils::TryParse(modules.substr(0, colon))) modules = modules.substr(colon + 1); + if (modules.empty()) continue; + const auto id = nextId(); + item->SetAttribute("sk", id); + builds.push_back({ id, std::move(modules) }); + } + } + return builds; +} + +void ModularBuildIdMigration::Run() { + std::vector characters; + Database::Get()->ForEachCharacterXmlContaining("ma=\"", [&characters](LWOOBJID id, const std::string&) { characters.push_back(id); }); + + PersistentIds ids; + uint32_t builds = 0, changed = 0; + for (const auto characterId : characters) { + const auto xml = Database::Get()->GetCharacterXml(characterId); + tinyxml2::XMLDocument document; + if (xml.empty() || document.Parse(xml.c_str(), xml.size()) != tinyxml2::XML_SUCCESS) { + LOG("Character %llu: its saved XML can't be read, its cars and rockets keep no build id", characterId); + continue; + } + const auto made = AssignIds(document, [&ids] { return ids.Next(); }); + if (made.empty()) continue; + for (const auto& build : made) Database::Get()->InsertUgcBuild(build.modules, build.id, characterId); + tinyxml2::XMLPrinter printer(0, true, 0); + document.Print(&printer); + Database::Get()->UpdateCharacterXml(characterId, printer.CStr()); + builds += static_cast(made.size()); + changed++; + } + LOG("Gave %u cars and rockets of %u characters (made before builds were recorded) a build id", builds, changed); +} diff --git a/dDatabase/ModularBuildIdMigration.h b/dDatabase/ModularBuildIdMigration.h new file mode 100644 index 000000000..1051e5c09 --- /dev/null +++ b/dDatabase/ModularBuildIdMigration.h @@ -0,0 +1,34 @@ +#ifndef MODULARBUILDIDMIGRATION_H +#define MODULARBUILDIDMIGRATION_H + +#include +#include +#include +#include + +#include "dCommonVars.h" + +namespace tinyxml2 { + class XMLDocument; +} + +/** + * Cars and rockets built before builds were recorded: saved items with modules (x@ma, assemblyPartLOTs) and no subkey, + * so they have no ugc_modular_build row and the client has no blueprint id to ask for their icon with. The migration + * (dlu/mysql/98_modular_build_ids.sql, dlu/sqlite/81_modular_build_ids.sql) gives each what a new build gets + * (ModularBuildFinish) and what InventoryComponent gives one when its character loads: a persistent id as its subkey + * and a ugc_modular_build row with its modules and owner. Only modular builds have assemblyPartLOTs. + */ +namespace ModularBuildIdMigration { + struct NewBuild { + LWOOBJID id{}; + std::string modules; // "1:8129+1:8130+...", as ugc_modular_build.ldf_config + }; + + // Gives every item of a character's saved XML with modules and no subkey the next id as its subkey; the builds to insert + std::vector AssignIds(tinyxml2::XMLDocument& document, const std::function& nextId); + + void Run(); +}; + +#endif //!MODULARBUILDIDMIGRATION_H diff --git a/docs/UgcServer.md b/docs/UgcServer.md index cfff9b00d..b7a2497c1 100644 --- a/docs/UgcServer.md +++ b/docs/UgcServer.md @@ -48,7 +48,8 @@ it is and downloads it only when it has none. Structs: `WorldPackets::RequestUgc The blueprint id of an inventory item's icon (`LWOInventoryComponent_Client::LoadBlueprintIcon`) is the item's `blueprintid` config when it has one (Brick-by-Brick models, LOT 6662), else its subkey (cars and rockets: their `ugc_modular_build` id, which DLU gives them as subkey when they are built). Cars and rockets from before builds were -stored have subkey 0 and no build row, so the client never asks for their icons. +stored had subkey 0 and no build row, so the client never asked for their icons: see "Cars and rockets from before +builds were stored" below. A placed model (LOT 14) always loads its blueprint's NIF, HKX and LXFML through these requests; see "Models without 3D services" for how the worlds answer them. @@ -551,6 +552,17 @@ Migrations `dlu/mysql/92_ugc_triangles_before.sql` and `dlu/sqlite/75_ugc_triang Migrations `dlu/mysql/94_ugc_priority.sql` and `dlu/sqlite/77_ugc_priority.sql`: `ugc.priority`, 1 for models staff asked to be made again (made before any other, cleared once made). +### Cars and rockets from before builds were stored + +Migrations `dlu/mysql/98_modular_build_ids.sql` and `dlu/sqlite/81_modular_build_ids.sql` (run by +`ModularBuildIdMigration`, after the SQL migrations): every saved item with modules (`x@ma`, assemblyPartLOTs; only +modular builds have them) and no subkey gets what a new build gets (`ModularBuildFinish`): a persistent id (from +`object_id_tracker`, with the character bit, as `ObjectIDManager::GetPersistentID`) as its subkey (`sk`) and a +`ugc_modular_build` row with its modules and the character as owner. The character's XML is written with +`UpdateCharacterXml` (a world still holding an older copy can't save over it). The UGC server then makes their icons +like any other build's. Characters that load later with such an item (a database from before the migration, restored +XML) get the same when they load (`InventoryComponent`, `AssignModularBuildId`). Mail attachments are not changed. + ## Without 3D services (`UGCUSE3DSERVICES=7:0`) `UgcManifest` (dGame/dUtilities) answers `REQUEST_UGC_MANIFEST_INFO` when `ugc_manifest=1` (`sharedconfig.ini`, @@ -865,7 +877,5 @@ pages show the item's own icon. without collision behave. * HKX (physics) is not generated, so models downloaded from the UGC server have no collision for clients that never built them. -* Cars and rockets built before builds were stored (subkey 0, no `ugc_modular_build` row) never get an icon: the client - has no blueprint id to ask for. * A model's `.nif` made before `.sd0` files were written has none (and no checksum, so clients keep building that model from its LXFML) until the model is made again (Reprocess on the dashboard). diff --git a/migrations/dlu/mysql/98_modular_build_ids.sql b/migrations/dlu/mysql/98_modular_build_ids.sql new file mode 100644 index 000000000..4edbdf2e5 --- /dev/null +++ b/migrations/dlu/mysql/98_modular_build_ids.sql @@ -0,0 +1 @@ +/* Gives cars and rockets saved without a subkey a build id: see ModularBuildIdMigration.cpp */ diff --git a/migrations/dlu/sqlite/81_modular_build_ids.sql b/migrations/dlu/sqlite/81_modular_build_ids.sql new file mode 100644 index 000000000..4edbdf2e5 --- /dev/null +++ b/migrations/dlu/sqlite/81_modular_build_ids.sql @@ -0,0 +1 @@ +/* Gives cars and rockets saved without a subkey a build id: see ModularBuildIdMigration.cpp */ diff --git a/tests/dGameTests/CMakeLists.txt b/tests/dGameTests/CMakeLists.txt index 3b9822895..ac305dcc6 100644 --- a/tests/dGameTests/CMakeLists.txt +++ b/tests/dGameTests/CMakeLists.txt @@ -15,6 +15,7 @@ set(DGAMETEST_SOURCES "BrickByBrickTests.cpp" "PropertyBuildersTests.cpp" "ModularBuildTableTests.cpp" + "ModularBuildIdMigrationTests.cpp" "UgcManifestTests.cpp" "SceneGhostingTests.cpp" "HolidayEventsTests.cpp" diff --git a/tests/dGameTests/ModularBuildIdMigrationTests.cpp b/tests/dGameTests/ModularBuildIdMigrationTests.cpp new file mode 100644 index 000000000..4aff8a67e --- /dev/null +++ b/tests/dGameTests/ModularBuildIdMigrationTests.cpp @@ -0,0 +1,45 @@ +#include + +#include + +#include "tinyxml2.h" + +#include "ModularBuildIdMigration.h" + +// Cars and rockets saved without a subkey get the next id as their subkey and a build with their modules; everything +// else is left as it is +TEST(ModularBuildIdMigrationTests, GivesOldCarsAndRocketsAnId) { + const char* xml = + "" + "" + "" + "" + "" + "" + "" + "" + ""; + tinyxml2::XMLDocument document; + ASSERT_EQ(document.Parse(xml), tinyxml2::XML_SUCCESS); + LWOOBJID next = 1000; + const auto builds = ModularBuildIdMigration::AssignIds(document, [&next] { return next++; }); + + ASSERT_EQ(builds.size(), 2u); + EXPECT_EQ(builds[0].id, 1000); + EXPECT_EQ(builds[0].modules, "1:8129+1:8130+1:9332"); + EXPECT_EQ(builds[1].id, 1001); + EXPECT_EQ(builds[1].modules, "1:4713+1:4714+1:4715"); + + std::map subkeys; + for (auto* bag = document.FirstChildElement("obj")->FirstChildElement("inv")->FirstChildElement("items")->FirstChildElement("in"); bag; bag = bag->NextSiblingElement("in")) { + for (auto* item = bag->FirstChildElement("i"); item; item = item->NextSiblingElement("i")) subkeys[item->Int64Attribute("id")] = item->Int64Attribute("sk", 0); + } + EXPECT_EQ(subkeys[11], 1000); + EXPECT_EQ(subkeys[12], 777); + EXPECT_EQ(subkeys[13], 0); + EXPECT_EQ(subkeys[14], 0); + EXPECT_EQ(subkeys[15], 1001); + + // Run again: nothing left to give an id + EXPECT_TRUE(ModularBuildIdMigration::AssignIds(document, [&next] { return next++; }).empty()); +}