diff --git a/dDatabase/CMakeLists.txt b/dDatabase/CMakeLists.txt index e106df6c9..6d3a91a27 100644 --- a/dDatabase/CMakeLists.txt +++ b/dDatabase/CMakeLists.txt @@ -1,7 +1,7 @@ add_subdirectory(CDClientDatabase) add_subdirectory(GameDatabase) -add_library(dDatabase STATIC "MigrationRunner.cpp" "ModelNormalizeMigration.cpp" "ConfigSync.cpp") +add_library(dDatabase STATIC "MigrationRunner.cpp" "ModelNormalizeMigration.cpp" "ModularBuildIdMigration.cpp" "ConfigSync.cpp") add_custom_target(conncpp_dylib ${CMAKE_COMMAND} -E copy $ ${PROJECT_BINARY_DIR}) diff --git a/dDatabase/MigrationRunner.cpp b/dDatabase/MigrationRunner.cpp index b1fdc7308..9bf86fb67 100644 --- a/dDatabase/MigrationRunner.cpp +++ b/dDatabase/MigrationRunner.cpp @@ -8,6 +8,7 @@ #include "Logger.h" #include "BinaryPathFinder.h" #include "ModelNormalizeMigration.h" +#include "ModularBuildIdMigration.h" #include @@ -52,6 +53,7 @@ void MigrationRunner::RunMigrations() { bool runNormalizeMigrations = false; bool runNormalizeAfterFirstPartMigrations = false; bool runBrickBuildsNotOnGrid = false; + bool runModularBuildIds = false; for (const auto& entry : GeneralUtils::GetSqlFileNamesFromFolder((BinaryPathFinder::GetBinaryDir() / "./migrations/dlu/" / migrationFolder).string())) { auto migration = LoadMigration("dlu/" + migrationFolder + "/", entry); @@ -70,6 +72,8 @@ void MigrationRunner::RunMigrations() { runNormalizeAfterFirstPartMigrations = true; } else if (migration.name.ends_with("_brickbuilds_not_on_grid.sql")) { runBrickBuildsNotOnGrid = true; + } else if (migration.name.ends_with("_modular_build_ids.sql")) { + runModularBuildIds = true; } else { finalSQL.append(migration.data.c_str()); } @@ -77,7 +81,7 @@ void MigrationRunner::RunMigrations() { Database::Get()->InsertMigration(migration.name); } - if (finalSQL.empty() && !runSd0Migrations && !runNormalizeMigrations && !runNormalizeAfterFirstPartMigrations && !runBrickBuildsNotOnGrid) { + if (finalSQL.empty() && !runSd0Migrations && !runNormalizeMigrations && !runNormalizeAfterFirstPartMigrations && !runBrickBuildsNotOnGrid && !runModularBuildIds) { LOG("Server database is up to date."); return; } @@ -113,6 +117,11 @@ void MigrationRunner::RunMigrations() { if (runBrickBuildsNotOnGrid) { ModelNormalizeMigration::RunBrickBuildGrid(); } + + // After the SQL, which makes ugc_modular_build and object_id_tracker on a new database + if (runModularBuildIds) { + ModularBuildIdMigration::Run(); + } } void MigrationRunner::RunSQLiteMigrations() { diff --git a/dDatabase/ModularBuildIdMigration.cpp b/dDatabase/ModularBuildIdMigration.cpp new file mode 100644 index 000000000..884e411f5 --- /dev/null +++ b/dDatabase/ModularBuildIdMigration.cpp @@ -0,0 +1,74 @@ +#include "ModularBuildIdMigration.h" + +#include + +#include "tinyxml2.h" + +#include "Database.h" +#include "eObjectBits.h" +#include "GeneralUtils.h" +#include "Logger.h" + +namespace { + // A persistent id, as ObjectIDManager::GetPersistentID hands them out: from a range reserved in object_id_tracker, + // with the CHARACTER bit + class PersistentIds { + public: + LWOOBJID Next() { + if (!m_Range || m_Range->minID > m_Range->maxID) m_Range = Database::Get()->GetPersistentIdRange(); + LWOOBJID id = m_Range->minID++; + GeneralUtils::SetBit(id, eObjectBits::CHARACTER); + return id; + } + private: + std::optional m_Range; + }; +} + +std::vector ModularBuildIdMigration::AssignIds(tinyxml2::XMLDocument& document, const std::function& nextId) { + std::vector builds; + auto* obj = document.FirstChildElement("obj"); + auto* inv = obj ? obj->FirstChildElement("inv") : nullptr; + auto* items = inv ? inv->FirstChildElement("items") : nullptr; + for (auto* bag = items ? items->FirstChildElement("in") : nullptr; bag; bag = bag->NextSiblingElement("in")) { + for (auto* item = bag->FirstChildElement("i"); item; item = item->NextSiblingElement("i")) { + if (item->Int64Attribute("sk", 0) != 0) continue; + const auto* config = item->FirstChildElement("x"); + const char* saved = config ? config->Attribute("ma") : nullptr; + if (!saved) continue; + // Saved as the LDF type and value ("0:1:8129+1:8130"); the build row keeps the value + std::string modules(saved); + if (const auto colon = modules.find(':'); colon != std::string::npos && GeneralUtils::TryParse(modules.substr(0, colon))) modules = modules.substr(colon + 1); + if (modules.empty()) continue; + const auto id = nextId(); + item->SetAttribute("sk", id); + builds.push_back({ id, std::move(modules) }); + } + } + return builds; +} + +void ModularBuildIdMigration::Run() { + std::vector characters; + Database::Get()->ForEachCharacterXmlContaining("ma=\"", [&characters](LWOOBJID id, const std::string&) { characters.push_back(id); }); + + PersistentIds ids; + uint32_t builds = 0, changed = 0; + for (const auto characterId : characters) { + const auto xml = Database::Get()->GetCharacterXml(characterId); + tinyxml2::XMLDocument document; + if (xml.empty() || document.Parse(xml.c_str(), xml.size()) != tinyxml2::XML_SUCCESS) { + LOG("Character %llu: its saved XML can't be read, its cars and rockets keep no build id", characterId); + continue; + } + const auto made = AssignIds(document, [&ids] { return ids.Next(); }); + if (made.empty()) continue; + for (const auto& build : made) Database::Get()->InsertUgcBuild(build.modules, build.id, characterId); + tinyxml2::XMLPrinter printer(0, true, 0); + document.Print(&printer); + Database::Get()->UpdateCharacterXml(characterId, printer.CStr()); + builds += static_cast(made.size()); + changed++; + } + LOG("Gave %u cars and rockets of %u characters (made before builds were recorded) a build id", builds, changed); +} diff --git a/dDatabase/ModularBuildIdMigration.h b/dDatabase/ModularBuildIdMigration.h new file mode 100644 index 000000000..1051e5c09 --- /dev/null +++ b/dDatabase/ModularBuildIdMigration.h @@ -0,0 +1,34 @@ +#ifndef MODULARBUILDIDMIGRATION_H +#define MODULARBUILDIDMIGRATION_H + +#include +#include +#include +#include + +#include "dCommonVars.h" + +namespace tinyxml2 { + class XMLDocument; +} + +/** + * Cars and rockets built before builds were recorded: saved items with modules (x@ma, assemblyPartLOTs) and no subkey, + * so they have no ugc_modular_build row and the client has no blueprint id to ask for their icon with. The migration + * (dlu/mysql/98_modular_build_ids.sql, dlu/sqlite/81_modular_build_ids.sql) gives each what a new build gets + * (ModularBuildFinish) and what InventoryComponent gives one when its character loads: a persistent id as its subkey + * and a ugc_modular_build row with its modules and owner. Only modular builds have assemblyPartLOTs. + */ +namespace ModularBuildIdMigration { + struct NewBuild { + LWOOBJID id{}; + std::string modules; // "1:8129+1:8130+...", as ugc_modular_build.ldf_config + }; + + // Gives every item of a character's saved XML with modules and no subkey the next id as its subkey; the builds to insert + std::vector AssignIds(tinyxml2::XMLDocument& document, const std::function& nextId); + + void Run(); +}; + +#endif //!MODULARBUILDIDMIGRATION_H diff --git a/docs/UgcServer.md b/docs/UgcServer.md index cfff9b00d..b7a2497c1 100644 --- a/docs/UgcServer.md +++ b/docs/UgcServer.md @@ -48,7 +48,8 @@ it is and downloads it only when it has none. Structs: `WorldPackets::RequestUgc The blueprint id of an inventory item's icon (`LWOInventoryComponent_Client::LoadBlueprintIcon`) is the item's `blueprintid` config when it has one (Brick-by-Brick models, LOT 6662), else its subkey (cars and rockets: their `ugc_modular_build` id, which DLU gives them as subkey when they are built). Cars and rockets from before builds were -stored have subkey 0 and no build row, so the client never asks for their icons. +stored had subkey 0 and no build row, so the client never asked for their icons: see "Cars and rockets from before +builds were stored" below. A placed model (LOT 14) always loads its blueprint's NIF, HKX and LXFML through these requests; see "Models without 3D services" for how the worlds answer them. @@ -551,6 +552,17 @@ Migrations `dlu/mysql/92_ugc_triangles_before.sql` and `dlu/sqlite/75_ugc_triang Migrations `dlu/mysql/94_ugc_priority.sql` and `dlu/sqlite/77_ugc_priority.sql`: `ugc.priority`, 1 for models staff asked to be made again (made before any other, cleared once made). +### Cars and rockets from before builds were stored + +Migrations `dlu/mysql/98_modular_build_ids.sql` and `dlu/sqlite/81_modular_build_ids.sql` (run by +`ModularBuildIdMigration`, after the SQL migrations): every saved item with modules (`x@ma`, assemblyPartLOTs; only +modular builds have them) and no subkey gets what a new build gets (`ModularBuildFinish`): a persistent id (from +`object_id_tracker`, with the character bit, as `ObjectIDManager::GetPersistentID`) as its subkey (`sk`) and a +`ugc_modular_build` row with its modules and the character as owner. The character's XML is written with +`UpdateCharacterXml` (a world still holding an older copy can't save over it). The UGC server then makes their icons +like any other build's. Characters that load later with such an item (a database from before the migration, restored +XML) get the same when they load (`InventoryComponent`, `AssignModularBuildId`). Mail attachments are not changed. + ## Without 3D services (`UGCUSE3DSERVICES=7:0`) `UgcManifest` (dGame/dUtilities) answers `REQUEST_UGC_MANIFEST_INFO` when `ugc_manifest=1` (`sharedconfig.ini`, @@ -865,7 +877,5 @@ pages show the item's own icon. without collision behave. * HKX (physics) is not generated, so models downloaded from the UGC server have no collision for clients that never built them. -* Cars and rockets built before builds were stored (subkey 0, no `ugc_modular_build` row) never get an icon: the client - has no blueprint id to ask for. * A model's `.nif` made before `.sd0` files were written has none (and no checksum, so clients keep building that model from its LXFML) until the model is made again (Reprocess on the dashboard). diff --git a/migrations/dlu/mysql/98_modular_build_ids.sql b/migrations/dlu/mysql/98_modular_build_ids.sql new file mode 100644 index 000000000..4edbdf2e5 --- /dev/null +++ b/migrations/dlu/mysql/98_modular_build_ids.sql @@ -0,0 +1 @@ +/* Gives cars and rockets saved without a subkey a build id: see ModularBuildIdMigration.cpp */ diff --git a/migrations/dlu/sqlite/81_modular_build_ids.sql b/migrations/dlu/sqlite/81_modular_build_ids.sql new file mode 100644 index 000000000..4edbdf2e5 --- /dev/null +++ b/migrations/dlu/sqlite/81_modular_build_ids.sql @@ -0,0 +1 @@ +/* Gives cars and rockets saved without a subkey a build id: see ModularBuildIdMigration.cpp */ diff --git a/tests/dGameTests/CMakeLists.txt b/tests/dGameTests/CMakeLists.txt index 3b9822895..ac305dcc6 100644 --- a/tests/dGameTests/CMakeLists.txt +++ b/tests/dGameTests/CMakeLists.txt @@ -15,6 +15,7 @@ set(DGAMETEST_SOURCES "BrickByBrickTests.cpp" "PropertyBuildersTests.cpp" "ModularBuildTableTests.cpp" + "ModularBuildIdMigrationTests.cpp" "UgcManifestTests.cpp" "SceneGhostingTests.cpp" "HolidayEventsTests.cpp" diff --git a/tests/dGameTests/ModularBuildIdMigrationTests.cpp b/tests/dGameTests/ModularBuildIdMigrationTests.cpp new file mode 100644 index 000000000..4aff8a67e --- /dev/null +++ b/tests/dGameTests/ModularBuildIdMigrationTests.cpp @@ -0,0 +1,45 @@ +#include + +#include + +#include "tinyxml2.h" + +#include "ModularBuildIdMigration.h" + +// Cars and rockets saved without a subkey get the next id as their subkey and a build with their modules; everything +// else is left as it is +TEST(ModularBuildIdMigrationTests, GivesOldCarsAndRocketsAnId) { + const char* xml = + "" + "" + "" + "" + "" + "" + "" + "" + ""; + tinyxml2::XMLDocument document; + ASSERT_EQ(document.Parse(xml), tinyxml2::XML_SUCCESS); + LWOOBJID next = 1000; + const auto builds = ModularBuildIdMigration::AssignIds(document, [&next] { return next++; }); + + ASSERT_EQ(builds.size(), 2u); + EXPECT_EQ(builds[0].id, 1000); + EXPECT_EQ(builds[0].modules, "1:8129+1:8130+1:9332"); + EXPECT_EQ(builds[1].id, 1001); + EXPECT_EQ(builds[1].modules, "1:4713+1:4714+1:4715"); + + std::map subkeys; + for (auto* bag = document.FirstChildElement("obj")->FirstChildElement("inv")->FirstChildElement("items")->FirstChildElement("in"); bag; bag = bag->NextSiblingElement("in")) { + for (auto* item = bag->FirstChildElement("i"); item; item = item->NextSiblingElement("i")) subkeys[item->Int64Attribute("id")] = item->Int64Attribute("sk", 0); + } + EXPECT_EQ(subkeys[11], 1000); + EXPECT_EQ(subkeys[12], 777); + EXPECT_EQ(subkeys[13], 0); + EXPECT_EQ(subkeys[14], 0); + EXPECT_EQ(subkeys[15], 1001); + + // Run again: nothing left to give an id + EXPECT_TRUE(ModularBuildIdMigration::AssignIds(document, [&next] { return next++; }).empty()); +}