add migration

This commit is contained in:
David Markowitz
2025-04-10 18:35:11 -07:00
parent fd27ffa9ae
commit 4b188b87ef
22 changed files with 170 additions and 47 deletions

View File

@@ -1,7 +1,7 @@
add_subdirectory(CDClientDatabase)
add_subdirectory(GameDatabase)
add_library(dDatabase STATIC "MigrationRunner.cpp")
add_library(dDatabase STATIC "MigrationRunner.cpp" "ModelNormalizeMigration.cpp")
add_custom_target(conncpp_dylib
${CMAKE_COMMAND} -E copy $<TARGET_FILE:MariaDB::ConnCpp> ${PROJECT_BINARY_DIR})

View File

@@ -34,9 +34,17 @@ public:
virtual void InsertNewPropertyModel(const LWOOBJID& propertyId, const IPropertyContents::Model& model, const std::string_view name) = 0;
// Update the model position and rotation for the given property id.
virtual void UpdateModel(const LWOOBJID& propertyId, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) = 0;
virtual void UpdateModel(const LWOOBJID& modelID, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) = 0;
virtual void UpdateModel(const LWOOBJID& modelID, const NiPoint3& position, const NiQuaternion& rotation, const std::array<int32_t, 5> behaviorIDs) {
std::array<std::pair<int32_t, std::string>, 5> behaviors;
for (int32_t i = 0; i < behaviors.size(); i++) behaviors[i].first = behaviorIDs[i];
UpdateModel(modelID, position, rotation, behaviors);
}
// Remove the model for the given property id.
virtual void RemoveModel(const LWOOBJID& modelId) = 0;
// Gets a model by ID
virtual Model GetModel(const LWOOBJID modelID) = 0;
};
#endif //!__IPROPERTIESCONTENTS__H__

View File

@@ -12,6 +12,7 @@ public:
struct Model {
std::stringstream lxfmlData;
LWOOBJID id{};
LWOOBJID modelID{};
};
// Gets all UGC models for the given property id.
@@ -27,6 +28,6 @@ public:
virtual void DeleteUgcModelData(const LWOOBJID& modelId) = 0;
// Inserts a new UGC model into the database.
virtual void UpdateUgcModelData(const LWOOBJID& modelId, std::istringstream& lxfml) = 0;
virtual void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) = 0;
};
#endif //!__IUGC__H__

View File

@@ -48,7 +48,7 @@ public:
void RemoveFriend(const uint32_t playerAccountId, const uint32_t friendAccountId) override;
void UpdateActivityLog(const uint32_t characterId, const eActivityType activityType, const LWOMAPID mapId) override;
void DeleteUgcModelData(const LWOOBJID& modelId) override;
void UpdateUgcModelData(const LWOOBJID& modelId, std::istringstream& lxfml) override;
void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) override;
std::vector<IUgc::Model> GetAllUgcModels() override;
void CreateMigrationHistoryTable() override;
bool IsMigrationRun(const std::string_view str) override;
@@ -74,7 +74,7 @@ public:
std::vector<IPropertyContents::Model> GetPropertyModels(const LWOOBJID& propertyId) override;
void RemoveUnreferencedUgcModels() override;
void InsertNewPropertyModel(const LWOOBJID& propertyId, const IPropertyContents::Model& model, const std::string_view name) override;
void UpdateModel(const LWOOBJID& propertyId, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) override;
void UpdateModel(const LWOOBJID& modelID, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) override;
void RemoveModel(const LWOOBJID& modelId) override;
void UpdatePerformanceCost(const LWOZONEID& zoneId, const float performanceCost) override;
void InsertNewBugReport(const IBugReports::Info& info) override;
@@ -126,6 +126,7 @@ public:
void DeleteUgcBuild(const LWOOBJID bigId) override;
uint32_t GetAccountCount() override;
bool IsNameInUse(const std::string_view name) override;
IPropertyContents::Model GetModel(const LWOOBJID modelID) override;
sql::PreparedStatement* CreatePreppedStmt(const std::string& query);
private:

View File

@@ -52,14 +52,39 @@ void MySQLDatabase::InsertNewPropertyModel(const LWOOBJID& propertyId, const IPr
}
}
void MySQLDatabase::UpdateModel(const LWOOBJID& propertyId, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) {
void MySQLDatabase::UpdateModel(const LWOOBJID& modelID, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) {
ExecuteUpdate(
"UPDATE properties_contents SET x = ?, y = ?, z = ?, rx = ?, ry = ?, rz = ?, rw = ?, "
"behavior_1 = ?, behavior_2 = ?, behavior_3 = ?, behavior_4 = ?, behavior_5 = ? WHERE id = ?;",
position.x, position.y, position.z, rotation.x, rotation.y, rotation.z, rotation.w,
behaviors[0].first, behaviors[1].first, behaviors[2].first, behaviors[3].first, behaviors[4].first, propertyId);
behaviors[0].first, behaviors[1].first, behaviors[2].first, behaviors[3].first, behaviors[4].first, modelID);
}
void MySQLDatabase::RemoveModel(const LWOOBJID& modelId) {
ExecuteDelete("DELETE FROM properties_contents WHERE id = ?;", modelId);
}
IPropertyContents::Model MySQLDatabase::GetModel(const LWOOBJID modelID) {
auto result = ExecuteSelect("SELECT * FROM properties_contents WHERE id = ?", modelID);
IPropertyContents::Model model{};
while (result->next()) {
model.id = result->getUInt64("id");
model.lot = static_cast<LOT>(result->getUInt("lot"));
model.position.x = result->getFloat("x");
model.position.y = result->getFloat("y");
model.position.z = result->getFloat("z");
model.rotation.w = result->getFloat("rw");
model.rotation.x = result->getFloat("rx");
model.rotation.y = result->getFloat("ry");
model.rotation.z = result->getFloat("rz");
model.ugcId = result->getUInt64("ugc_id");
model.behaviors[0] = result->getInt("behavior_1");
model.behaviors[1] = result->getInt("behavior_2");
model.behaviors[2] = result->getInt("behavior_3");
model.behaviors[3] = result->getInt("behavior_4");
model.behaviors[4] = result->getInt("behavior_5");
}
return model;
}

View File

@@ -2,7 +2,7 @@
std::vector<IUgc::Model> MySQLDatabase::GetUgcModels(const LWOOBJID& propertyId) {
auto result = ExecuteSelect(
"SELECT lxfml, u.id FROM ugc AS u JOIN properties_contents AS pc ON u.id = pc.ugc_id WHERE lot = 14 AND property_id = ? AND pc.ugc_id IS NOT NULL;",
"SELECT lxfml, u.id as ugcID, pc.id as modelID FROM ugc AS u JOIN properties_contents AS pc ON u.id = pc.ugc_id WHERE lot = 14 AND property_id = ? AND pc.ugc_id IS NOT NULL;",
propertyId);
std::vector<IUgc::Model> toReturn;
@@ -13,7 +13,8 @@ std::vector<IUgc::Model> MySQLDatabase::GetUgcModels(const LWOOBJID& propertyId)
// blob is owned by the query, so we need to do a deep copy :/
std::unique_ptr<std::istream> blob(result->getBlob("lxfml"));
model.lxfmlData << blob->rdbuf();
model.id = result->getUInt64("id");
model.id = result->getUInt64("ugcID");
model.modelID = result->getUInt64("modelID");
toReturn.push_back(std::move(model));
}
@@ -21,13 +22,14 @@ std::vector<IUgc::Model> MySQLDatabase::GetUgcModels(const LWOOBJID& propertyId)
}
std::vector<IUgc::Model> MySQLDatabase::GetAllUgcModels() {
auto result = ExecuteSelect("SELECT id, lxfml FROM ugc;");
auto result = ExecuteSelect("SELECT u.id AS ugcID, lxfml, pc.id AS modelID FROM ugc AS u JOIN properties_contents AS pc ON pc.ugc_id = u.id WHERE pc.lot = 14 AND pc.ugc_id IS NOT NULL;");
std::vector<IUgc::Model> models;
models.reserve(result->rowsCount());
while (result->next()) {
IUgc::Model model;
model.id = result->getInt64("id");
model.id = result->getInt64("ugcID");
model.modelID = result->getUInt64("modelID");
// blob is owned by the query, so we need to do a deep copy :/
std::unique_ptr<std::istream> blob(result->getBlob("lxfml"));
@@ -65,7 +67,7 @@ void MySQLDatabase::DeleteUgcModelData(const LWOOBJID& modelId) {
ExecuteDelete("DELETE FROM properties_contents WHERE ugc_id = ?;", modelId);
}
void MySQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::istringstream& lxfml) {
void MySQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) {
const std::istream stream(lxfml.rdbuf());
ExecuteUpdate("UPDATE ugc SET lxfml = ? WHERE id = ?;", &stream, modelId);
}

View File

@@ -46,7 +46,7 @@ public:
void RemoveFriend(const uint32_t playerAccountId, const uint32_t friendAccountId) override;
void UpdateActivityLog(const uint32_t characterId, const eActivityType activityType, const LWOMAPID mapId) override;
void DeleteUgcModelData(const LWOOBJID& modelId) override;
void UpdateUgcModelData(const LWOOBJID& modelId, std::istringstream& lxfml) override;
void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) override;
std::vector<IUgc::Model> GetAllUgcModels() override;
void CreateMigrationHistoryTable() override;
bool IsMigrationRun(const std::string_view str) override;
@@ -72,7 +72,7 @@ public:
std::vector<IPropertyContents::Model> GetPropertyModels(const LWOOBJID& propertyId) override;
void RemoveUnreferencedUgcModels() override;
void InsertNewPropertyModel(const LWOOBJID& propertyId, const IPropertyContents::Model& model, const std::string_view name) override;
void UpdateModel(const LWOOBJID& propertyId, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) override;
void UpdateModel(const LWOOBJID& modelID, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) override;
void RemoveModel(const LWOOBJID& modelId) override;
void UpdatePerformanceCost(const LWOZONEID& zoneId, const float performanceCost) override;
void InsertNewBugReport(const IBugReports::Info& info) override;
@@ -124,6 +124,7 @@ public:
void DeleteUgcBuild(const LWOOBJID bigId) override;
uint32_t GetAccountCount() override;
bool IsNameInUse(const std::string_view name) override;
IPropertyContents::Model GetModel(const LWOOBJID modelID) override;
private:
CppSQLite3Statement CreatePreppedStmt(const std::string& query);

View File

@@ -52,14 +52,41 @@ void SQLiteDatabase::InsertNewPropertyModel(const LWOOBJID& propertyId, const IP
}
}
void SQLiteDatabase::UpdateModel(const LWOOBJID& propertyId, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) {
void SQLiteDatabase::UpdateModel(const LWOOBJID& modelID, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) {
ExecuteUpdate(
"UPDATE properties_contents SET x = ?, y = ?, z = ?, rx = ?, ry = ?, rz = ?, rw = ?, "
"behavior_1 = ?, behavior_2 = ?, behavior_3 = ?, behavior_4 = ?, behavior_5 = ? WHERE id = ?;",
position.x, position.y, position.z, rotation.x, rotation.y, rotation.z, rotation.w,
behaviors[0].first, behaviors[1].first, behaviors[2].first, behaviors[3].first, behaviors[4].first, propertyId);
behaviors[0].first, behaviors[1].first, behaviors[2].first, behaviors[3].first, behaviors[4].first, modelID);
}
void SQLiteDatabase::RemoveModel(const LWOOBJID& modelId) {
ExecuteDelete("DELETE FROM properties_contents WHERE id = ?;", modelId);
}
IPropertyContents::Model SQLiteDatabase::GetModel(const LWOOBJID modelID) {
auto [_, result] = ExecuteSelect("SELECT * FROM properties_contents WHERE id = ?", modelID);
IPropertyContents::Model model{};
if (!result.eof()) {
do {
model.id = result.getInt64Field("id");
model.lot = static_cast<LOT>(result.getIntField("lot"));
model.position.x = result.getFloatField("x");
model.position.y = result.getFloatField("y");
model.position.z = result.getFloatField("z");
model.rotation.w = result.getFloatField("rw");
model.rotation.x = result.getFloatField("rx");
model.rotation.y = result.getFloatField("ry");
model.rotation.z = result.getFloatField("rz");
model.ugcId = result.getInt64Field("ugc_id");
model.behaviors[0] = result.getIntField("behavior_1");
model.behaviors[1] = result.getIntField("behavior_2");
model.behaviors[2] = result.getIntField("behavior_3");
model.behaviors[3] = result.getIntField("behavior_4");
model.behaviors[4] = result.getIntField("behavior_5");
} while (result.nextRow());
}
return model;
}

View File

@@ -2,7 +2,7 @@
std::vector<IUgc::Model> SQLiteDatabase::GetUgcModels(const LWOOBJID& propertyId) {
auto [_, result] = ExecuteSelect(
"SELECT lxfml, u.id FROM ugc AS u JOIN properties_contents AS pc ON u.id = pc.ugc_id WHERE lot = 14 AND property_id = ? AND pc.ugc_id IS NOT NULL;",
"SELECT lxfml, u.id AS ugcID, pc.id AS modelID FROM ugc AS u JOIN properties_contents AS pc ON u.id = pc.ugc_id WHERE lot = 14 AND property_id = ? AND pc.ugc_id IS NOT NULL;",
propertyId);
std::vector<IUgc::Model> toReturn;
@@ -13,7 +13,8 @@ std::vector<IUgc::Model> SQLiteDatabase::GetUgcModels(const LWOOBJID& propertyId
int blobSize{};
const auto* blob = result.getBlobField("lxfml", blobSize);
model.lxfmlData << std::string(reinterpret_cast<const char*>(blob), blobSize);
model.id = result.getInt64Field("id");
model.id = result.getInt64Field("ugcID");
model.modelID = result.getInt64Field("modelID");
toReturn.push_back(std::move(model));
result.nextRow();
}
@@ -22,12 +23,13 @@ std::vector<IUgc::Model> SQLiteDatabase::GetUgcModels(const LWOOBJID& propertyId
}
std::vector<IUgc::Model> SQLiteDatabase::GetAllUgcModels() {
auto [_, result] = ExecuteSelect("SELECT id, lxfml FROM ugc;");
auto [_, result] = ExecuteSelect("SELECT u.id AS ugcID, pc.id AS modelID, lxfml FROM ugc AS u JOIN properties_contents AS pc ON pc.id = u.id;");
std::vector<IUgc::Model> models;
while (!result.eof()) {
IUgc::Model model;
model.id = result.getInt64Field("id");
model.id = result.getInt64Field("ugcID");
model.modelID = result.getInt64Field("modelID");
int blobSize{};
const auto* blob = result.getBlobField("lxfml", blobSize);
@@ -66,7 +68,7 @@ void SQLiteDatabase::DeleteUgcModelData(const LWOOBJID& modelId) {
ExecuteDelete("DELETE FROM properties_contents WHERE ugc_id = ?;", modelId);
}
void SQLiteDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::istringstream& lxfml) {
void SQLiteDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) {
const std::istream stream(lxfml.rdbuf());
ExecuteUpdate("UPDATE ugc SET lxfml = ? WHERE id = ?;", &stream, modelId);
}

View File

@@ -60,7 +60,7 @@ void TestSQLDatabase::DeleteUgcModelData(const LWOOBJID& modelId) {
}
void TestSQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::istringstream& lxfml) {
void TestSQLDatabase::UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) {
}
@@ -164,7 +164,7 @@ void TestSQLDatabase::InsertNewPropertyModel(const LWOOBJID& propertyId, const I
}
void TestSQLDatabase::UpdateModel(const LWOOBJID& propertyId, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) {
void TestSQLDatabase::UpdateModel(const LWOOBJID& modelID, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) {
}

View File

@@ -25,7 +25,7 @@ class TestSQLDatabase : public GameDatabase {
void RemoveFriend(const uint32_t playerAccountId, const uint32_t friendAccountId) override;
void UpdateActivityLog(const uint32_t characterId, const eActivityType activityType, const LWOMAPID mapId) override;
void DeleteUgcModelData(const LWOOBJID& modelId) override;
void UpdateUgcModelData(const LWOOBJID& modelId, std::istringstream& lxfml) override;
void UpdateUgcModelData(const LWOOBJID& modelId, std::stringstream& lxfml) override;
std::vector<IUgc::Model> GetAllUgcModels() override;
void CreateMigrationHistoryTable() override;
bool IsMigrationRun(const std::string_view str) override;
@@ -51,7 +51,7 @@ class TestSQLDatabase : public GameDatabase {
std::vector<IPropertyContents::Model> GetPropertyModels(const LWOOBJID& propertyId) override;
void RemoveUnreferencedUgcModels() override;
void InsertNewPropertyModel(const LWOOBJID& propertyId, const IPropertyContents::Model& model, const std::string_view name) override;
void UpdateModel(const LWOOBJID& propertyId, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) override;
void UpdateModel(const LWOOBJID& modelID, const NiPoint3& position, const NiQuaternion& rotation, const std::array<std::pair<int32_t, std::string>, 5>& behaviors) override;
void RemoveModel(const LWOOBJID& modelId) override;
void UpdatePerformanceCost(const LWOZONEID& zoneId, const float performanceCost) override;
void InsertNewBugReport(const IBugReports::Info& info) override;
@@ -104,6 +104,7 @@ class TestSQLDatabase : public GameDatabase {
uint32_t GetAccountCount() override { return 0; };
bool IsNameInUse(const std::string_view name) override { return false; };
IPropertyContents::Model GetModel(const LWOOBJID modelID) override { return {}; }
};
#endif //!TESTSQLDATABASE_H

View File

@@ -7,6 +7,7 @@
#include "GeneralUtils.h"
#include "Logger.h"
#include "BinaryPathFinder.h"
#include "ModelNormalizeMigration.h"
#include <fstream>
@@ -35,7 +36,7 @@ void MigrationRunner::RunMigrations() {
Database::Get()->CreateMigrationHistoryTable();
// has to be here because when moving the files to the new folder, the migration_history table is not updated so it will run them all again.
const auto migrationFolder = Database::GetMigrationFolder();
if (!Database::Get()->IsMigrationRun("17_migration_for_migrations.sql") && migrationFolder == "mysql") {
LOG("Running migration: 17_migration_for_migrations.sql");
@@ -45,6 +46,7 @@ void MigrationRunner::RunMigrations() {
std::string finalSQL = "";
bool runSd0Migrations = false;
bool runNormalizeMigrations = false;
for (const auto& entry : GeneralUtils::GetSqlFileNamesFromFolder((BinaryPathFinder::GetBinaryDir() / "./migrations/dlu/" / migrationFolder).string())) {
auto migration = LoadMigration("dlu/" + migrationFolder + "/", entry);
@@ -57,6 +59,8 @@ void MigrationRunner::RunMigrations() {
LOG("Running migration: %s", migration.name.c_str());
if (migration.name == "5_brick_model_sd0.sql") {
runSd0Migrations = true;
} else if (migration.name.ends_with("_normalize_model_positions.sql")) {
runNormalizeMigrations = true;
} else {
finalSQL.append(migration.data.c_str());
}
@@ -64,7 +68,7 @@ void MigrationRunner::RunMigrations() {
Database::Get()->InsertMigration(migration.name);
}
if (finalSQL.empty() && !runSd0Migrations) {
if (finalSQL.empty() && !runSd0Migrations && !runNormalizeMigrations) {
LOG("Server database is up to date.");
return;
}
@@ -88,6 +92,10 @@ void MigrationRunner::RunMigrations() {
uint32_t numberOfTruncatedModels = BrickByBrickFix::TruncateBrokenBrickByBrickXml();
LOG("%i models were truncated from the database.", numberOfTruncatedModels);
}
if (runNormalizeMigrations) {
ModelNormalizeMigration::Run();
}
}
void MigrationRunner::RunSQLiteMigrations() {

View File

@@ -0,0 +1,30 @@
#include "ModelNormalizeMigration.h"
#include "Database.h"
#include "Lxfml.h"
#include "Sd0.h"
void ModelNormalizeMigration::Run() {
const auto oldCommit = Database::Get()->GetAutoCommit();
Database::Get()->SetAutoCommit(false);
for (auto& [lxfmlData, id, modelID] : Database::Get()->GetAllUgcModels()) {
const auto model = Database::Get()->GetModel(modelID);
// only BBB models (lot 14) and models with a position of NiPoint3::ZERO need to have their position fixed.
if (model.position != NiPoint3Constant::ZERO || model.lot != 14) continue;
Sd0 sd0(lxfmlData);
const auto asStr = sd0.GetAsStringUncompressed();
const auto [newLxfml, newCenter] = Lxfml::NormalizePosition(asStr);
if (newCenter == NiPoint3Constant::ZERO) {
LOG("Failed to update model %llu due to failure reading xml.");
continue;
}
LOG("Updated model %llu to have a center of %f %f %f", modelID, newCenter.x, newCenter.y, newCenter.z);
sd0.FromData(reinterpret_cast<const uint8_t*>(newLxfml.data()), newLxfml.size());
auto asStream = sd0.GetAsStream();
Database::Get()->UpdateModel(model.id, newCenter, model.rotation, model.behaviors);
Database::Get()->UpdateUgcModelData(id, asStream);
}
Database::Get()->SetAutoCommit(oldCommit);
}

View File

@@ -0,0 +1,11 @@
// Darkflame Universe
// Copyright 2025
#ifndef MODELNORMALIZEMIGRATION_H
#define MODELNORMALIZEMIGRATION_H
namespace ModelNormalizeMigration {
void Run();
};
#endif //!MODELNORMALIZEMIGRATION_H