feat(ugc): migration gives old cars and rockets a build id

Cars and rockets built before builds were recorded were saved with
subkey 0 and have no ugc_modular_build row, so the client has no
blueprint id to ask for their icon with. The runtime fix only reaches
characters that load again.

Migrations mysql 98 and sqlite 81 (98/81 because the guilds work takes
97/80) run ModularBuildIdMigration after the SQL: every saved item with
modules (x@ma) and no subkey gets what a new build gets, as the
character-load fix does: a persistent id from object_id_tracker (with
the character bit) as its subkey and a ugc_modular_build row with its
modules and the character as owner. The XML is written with
UpdateCharacterXml. Tried on a copy of a server's database with a
couple of thousand such items: every one got an id and a build row, in
seconds.

Check: after the migration, old cars and rockets show their icons in the
backpack (the UGC server makes them like any build) and still work
(equip, race, launch).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dDatabase/CMakeLists.txt

@@ -1,7 +1,7 @@
 add_subdirectory(CDClientDatabase)
 add_subdirectory(GameDatabase)
 
-add_library(dDatabase STATIC "MigrationRunner.cpp" "ModelNormalizeMigration.cpp" "ConfigSync.cpp")
+add_library(dDatabase STATIC "MigrationRunner.cpp" "ModelNormalizeMigration.cpp" "ModularBuildIdMigration.cpp" "ConfigSync.cpp")
 
 add_custom_target(conncpp_dylib
 	${CMAKE_COMMAND} -E copy $<TARGET_FILE:MariaDB::ConnCpp> ${PROJECT_BINARY_DIR})

dDatabase/MigrationRunner.cpp

@@ -8,6 +8,7 @@
 #include "Logger.h"
 #include "BinaryPathFinder.h"
 #include "ModelNormalizeMigration.h"
+#include "ModularBuildIdMigration.h"
 
 #include <fstream>
 
@@ -52,6 +53,7 @@ void MigrationRunner::RunMigrations() {
 	bool runNormalizeMigrations = false;
 	bool runNormalizeAfterFirstPartMigrations = false;
 	bool runBrickBuildsNotOnGrid = false;
+	bool runModularBuildIds = false;
 	for (const auto& entry : GeneralUtils::GetSqlFileNamesFromFolder((BinaryPathFinder::GetBinaryDir() / "./migrations/dlu/" / migrationFolder).string())) {
 		auto migration = LoadMigration("dlu/" + migrationFolder + "/", entry);
 
@@ -70,6 +72,8 @@ void MigrationRunner::RunMigrations() {
 			runNormalizeAfterFirstPartMigrations = true;
 		} else if (migration.name.ends_with("_brickbuilds_not_on_grid.sql")) {
 			runBrickBuildsNotOnGrid = true;
+		} else if (migration.name.ends_with("_modular_build_ids.sql")) {
+			runModularBuildIds = true;
 		} else {
 			finalSQL.append(migration.data.c_str());
 		}
@@ -77,7 +81,7 @@ void MigrationRunner::RunMigrations() {
 		Database::Get()->InsertMigration(migration.name);
 	}
 
-	if (finalSQL.empty() && !runSd0Migrations && !runNormalizeMigrations && !runNormalizeAfterFirstPartMigrations && !runBrickBuildsNotOnGrid) {
+	if (finalSQL.empty() && !runSd0Migrations && !runNormalizeMigrations && !runNormalizeAfterFirstPartMigrations && !runBrickBuildsNotOnGrid && !runModularBuildIds) {
 		LOG("Server database is up to date.");
 		return;
 	}
@@ -113,6 +117,11 @@ void MigrationRunner::RunMigrations() {
 	if (runBrickBuildsNotOnGrid) {
 		ModelNormalizeMigration::RunBrickBuildGrid();
 	}
+
+	// After the SQL, which makes ugc_modular_build and object_id_tracker on a new database
+	if (runModularBuildIds) {
+		ModularBuildIdMigration::Run();
+	}
 }
 
 void MigrationRunner::RunSQLiteMigrations() {

dDatabase/ModularBuildIdMigration.cpp

@@ -0,0 +1,74 @@
+#include "ModularBuildIdMigration.h"
+
+#include <optional>
+
+#include "tinyxml2.h"
+
+#include "Database.h"
+#include "eObjectBits.h"
+#include "GeneralUtils.h"
+#include "Logger.h"
+
+namespace {
+	// A persistent id, as ObjectIDManager::GetPersistentID hands them out: from a range reserved in object_id_tracker,
+	// with the CHARACTER bit
+	class PersistentIds {
+	public:
+		LWOOBJID Next() {
+			if (!m_Range || m_Range->minID > m_Range->maxID) m_Range = Database::Get()->GetPersistentIdRange();
+			LWOOBJID id = m_Range->minID++;
+			GeneralUtils::SetBit(id, eObjectBits::CHARACTER);
+			return id;
+		}
+	private:
+		std::optional<IObjectIdTracker::Range> m_Range;
+	};
+}
+
+std::vector<ModularBuildIdMigration::NewBuild> ModularBuildIdMigration::AssignIds(tinyxml2::XMLDocument& document, const std::function<LWOOBJID()>& nextId) {
+	std::vector<NewBuild> builds;
+	auto* obj = document.FirstChildElement("obj");
+	auto* inv = obj ? obj->FirstChildElement("inv") : nullptr;
+	auto* items = inv ? inv->FirstChildElement("items") : nullptr;
+	for (auto* bag = items ? items->FirstChildElement("in") : nullptr; bag; bag = bag->NextSiblingElement("in")) {
+		for (auto* item = bag->FirstChildElement("i"); item; item = item->NextSiblingElement("i")) {
+			if (item->Int64Attribute("sk", 0) != 0) continue;
+			const auto* config = item->FirstChildElement("x");
+			const char* saved = config ? config->Attribute("ma") : nullptr;
+			if (!saved) continue;
+			// Saved as the LDF type and value ("0:1:8129+1:8130"); the build row keeps the value
+			std::string modules(saved);
+			if (const auto colon = modules.find(':'); colon != std::string::npos && GeneralUtils::TryParse<int32_t>(modules.substr(0, colon))) modules = modules.substr(colon + 1);
+			if (modules.empty()) continue;
+			const auto id = nextId();
+			item->SetAttribute("sk", id);
+			builds.push_back({ id, std::move(modules) });
+		}
+	}
+	return builds;
+}
+
+void ModularBuildIdMigration::Run() {
+	std::vector<LWOOBJID> characters;
+	Database::Get()->ForEachCharacterXmlContaining("ma=\"", [&characters](LWOOBJID id, const std::string&) { characters.push_back(id); });
+
+	PersistentIds ids;
+	uint32_t builds = 0, changed = 0;
+	for (const auto characterId : characters) {
+		const auto xml = Database::Get()->GetCharacterXml(characterId);
+		tinyxml2::XMLDocument document;
+		if (xml.empty() || document.Parse(xml.c_str(), xml.size()) != tinyxml2::XML_SUCCESS) {
+			LOG("Character %llu: its saved XML can't be read, its cars and rockets keep no build id", characterId);
+			continue;
+		}
+		const auto made = AssignIds(document, [&ids] { return ids.Next(); });
+		if (made.empty()) continue;
+		for (const auto& build : made) Database::Get()->InsertUgcBuild(build.modules, build.id, characterId);
+		tinyxml2::XMLPrinter printer(0, true, 0);
+		document.Print(&printer);
+		Database::Get()->UpdateCharacterXml(characterId, printer.CStr());
+		builds += static_cast<uint32_t>(made.size());
+		changed++;
+	}
+	LOG("Gave %u cars and rockets of %u characters (made before builds were recorded) a build id", builds, changed);
+}

dDatabase/ModularBuildIdMigration.h

@@ -0,0 +1,34 @@
+#ifndef MODULARBUILDIDMIGRATION_H
+#define MODULARBUILDIDMIGRATION_H
+
+#include <cstdint>
+#include <functional>
+#include <string>
+#include <vector>
+
+#include "dCommonVars.h"
+
+namespace tinyxml2 {
+	class XMLDocument;
+}
+
+/**
+ * Cars and rockets built before builds were recorded: saved items with modules (x@ma, assemblyPartLOTs) and no subkey,
+ * so they have no ugc_modular_build row and the client has no blueprint id to ask for their icon with. The migration
+ * (dlu/mysql/98_modular_build_ids.sql, dlu/sqlite/81_modular_build_ids.sql) gives each what a new build gets
+ * (ModularBuildFinish) and what InventoryComponent gives one when its character loads: a persistent id as its subkey
+ * and a ugc_modular_build row with its modules and owner. Only modular builds have assemblyPartLOTs.
+ */
+namespace ModularBuildIdMigration {
+	struct NewBuild {
+		LWOOBJID id{};
+		std::string modules; // "1:8129+1:8130+...", as ugc_modular_build.ldf_config
+	};
+
+	// Gives every item of a character's saved XML with modules and no subkey the next id as its subkey; the builds to insert
+	std::vector<NewBuild> AssignIds(tinyxml2::XMLDocument& document, const std::function<LWOOBJID()>& nextId);
+
+	void Run();
+};
+
+#endif //!MODULARBUILDIDMIGRATION_H

docs/UgcServer.md

@@ -48,7 +48,8 @@ it is and downloads it only when it has none. Structs: `WorldPackets::RequestUgc
 The blueprint id of an inventory item's icon (`LWOInventoryComponent_Client::LoadBlueprintIcon`) is the item's
 `blueprintid` config when it has one (Brick-by-Brick models, LOT 6662), else its subkey (cars and rockets: their
 `ugc_modular_build` id, which DLU gives them as subkey when they are built). Cars and rockets from before builds were
-stored have subkey 0 and no build row, so the client never asks for their icons.
+stored had subkey 0 and no build row, so the client never asked for their icons: see "Cars and rockets from before
+builds were stored" below.
 
 A placed model (LOT 14) always loads its blueprint's NIF, HKX and LXFML through these requests; see "Models without
 3D services" for how the worlds answer them.
@@ -551,6 +552,17 @@ Migrations `dlu/mysql/92_ugc_triangles_before.sql` and `dlu/sqlite/75_ugc_triang
 Migrations `dlu/mysql/94_ugc_priority.sql` and `dlu/sqlite/77_ugc_priority.sql`: `ugc.priority`, 1 for models staff
 asked to be made again (made before any other, cleared once made).
 
+### Cars and rockets from before builds were stored
+
+Migrations `dlu/mysql/98_modular_build_ids.sql` and `dlu/sqlite/81_modular_build_ids.sql` (run by
+`ModularBuildIdMigration`, after the SQL migrations): every saved item with modules (`x@ma`, assemblyPartLOTs; only
+modular builds have them) and no subkey gets what a new build gets (`ModularBuildFinish`): a persistent id (from
+`object_id_tracker`, with the character bit, as `ObjectIDManager::GetPersistentID`) as its subkey (`sk`) and a
+`ugc_modular_build` row with its modules and the character as owner. The character's XML is written with
+`UpdateCharacterXml` (a world still holding an older copy can't save over it). The UGC server then makes their icons
+like any other build's. Characters that load later with such an item (a database from before the migration, restored
+XML) get the same when they load (`InventoryComponent`, `AssignModularBuildId`). Mail attachments are not changed.
+
 ## Without 3D services (`UGCUSE3DSERVICES=7:0`)
 
 `UgcManifest` (dGame/dUtilities) answers `REQUEST_UGC_MANIFEST_INFO` when `ugc_manifest=1` (`sharedconfig.ini`,
@@ -865,7 +877,5 @@ pages show the item's own icon.
   without collision behave.
 * HKX (physics) is not generated, so models downloaded from the UGC server have no collision for clients that never
   built them.
-* Cars and rockets built before builds were stored (subkey 0, no `ugc_modular_build` row) never get an icon: the client
-  has no blueprint id to ask for.
 * A model's `.nif` made before `.sd0` files were written has none (and no checksum, so clients keep building that
   model from its LXFML) until the model is made again (Reprocess on the dashboard).

migrations/dlu/mysql/98_modular_build_ids.sql

@@ -0,0 +1 @@
+/* Gives cars and rockets saved without a subkey a build id: see ModularBuildIdMigration.cpp */

migrations/dlu/sqlite/81_modular_build_ids.sql

@@ -0,0 +1 @@
+/* Gives cars and rockets saved without a subkey a build id: see ModularBuildIdMigration.cpp */

tests/dGameTests/CMakeLists.txt

@@ -15,6 +15,7 @@ set(DGAMETEST_SOURCES
 	"BrickByBrickTests.cpp"
 	"PropertyBuildersTests.cpp"
 	"ModularBuildTableTests.cpp"
+	"ModularBuildIdMigrationTests.cpp"
 	"UgcManifestTests.cpp"
 	"SceneGhostingTests.cpp"
 	"HolidayEventsTests.cpp"

tests/dGameTests/ModularBuildIdMigrationTests.cpp

@@ -0,0 +1,45 @@
+#include <gtest/gtest.h>
+
+#include <map>
+
+#include "tinyxml2.h"
+
+#include "ModularBuildIdMigration.h"
+
+// Cars and rockets saved without a subkey get the next id as their subkey and a build with their modules; everything
+// else is left as it is
+TEST(ModularBuildIdMigrationTests, GivesOldCarsAndRocketsAnId) {
+	const char* xml =
+		"<obj v=\"1\"><inv><items>"
+		"<in t=\"5\">"
+		"<i l=\"8092\" id=\"11\" s=\"0\" c=\"1\" sk=\"0\"><x ma=\"0:1:8129+1:8130+1:9332\"/></i>"
+		"<i l=\"6416\" id=\"12\" s=\"1\" c=\"1\" sk=\"777\"><x ma=\"0:1:4713+1:4714+1:4715\"/></i>"
+		"<i l=\"6662\" id=\"13\" s=\"2\" c=\"1\"><x bp=\"9:555\"/></i>"
+		"<i l=\"4714\" id=\"14\" s=\"3\" c=\"1\"/>"
+		"</in>"
+		"<in t=\"14\"><i l=\"6416\" id=\"15\" s=\"0\" c=\"1\"><x ma=\"0:1:4713+1:4714+1:4715\"/></i></in>"
+		"</items></inv></obj>";
+	tinyxml2::XMLDocument document;
+	ASSERT_EQ(document.Parse(xml), tinyxml2::XML_SUCCESS);
+	LWOOBJID next = 1000;
+	const auto builds = ModularBuildIdMigration::AssignIds(document, [&next] { return next++; });
+
+	ASSERT_EQ(builds.size(), 2u);
+	EXPECT_EQ(builds[0].id, 1000);
+	EXPECT_EQ(builds[0].modules, "1:8129+1:8130+1:9332");
+	EXPECT_EQ(builds[1].id, 1001);
+	EXPECT_EQ(builds[1].modules, "1:4713+1:4714+1:4715");
+
+	std::map<int64_t, int64_t> subkeys;
+	for (auto* bag = document.FirstChildElement("obj")->FirstChildElement("inv")->FirstChildElement("items")->FirstChildElement("in"); bag; bag = bag->NextSiblingElement("in")) {
+		for (auto* item = bag->FirstChildElement("i"); item; item = item->NextSiblingElement("i")) subkeys[item->Int64Attribute("id")] = item->Int64Attribute("sk", 0);
+	}
+	EXPECT_EQ(subkeys[11], 1000);
+	EXPECT_EQ(subkeys[12], 777);
+	EXPECT_EQ(subkeys[13], 0);
+	EXPECT_EQ(subkeys[14], 0);
+	EXPECT_EQ(subkeys[15], 1001);
+
+	// Run again: nothing left to give an id
+	EXPECT_TRUE(ModularBuildIdMigration::AssignIds(document, [&next] { return next++; }).empty());
+}