From 6c414cec48dd871bc08e0a74b059b84d0f3ac6c5 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sun, 27 Sep 2026 08:21:53 -0500 Subject: [PATCH] feat(dashboard): check uploaded character XML before storing it The XML upload is the one place hand-written XML reaches the game, whose load path trusts the XML because the server writes it itself. Instead of making the game skip bad data at load, the upload is checked against what the load path assumes and refused (400, with the problems) when the game couldn't load it: required elements, attributes that must parse (flags read with std::stoul/stoull), required item and mission fields, known inventory types, mission states and character versions, items and missions that exist in the CDClient, unique item IDs and slots, and the acct attribute matching the owner. Suspicious but loadable content is returned as warnings that need confirm=true (409 otherwise): contraband (same matching as the world, now shared in ContrabandRules.h), stacks above the stack size, coins, level or u-score out of reach, a GM level above the account's. Contraband marked flag-and-remove is removed only if the uploader asks; once stored, findings are flagged (CONTRABAND) and audited. The XML editor shows the findings and offers "Save anyway". Related: issue 1332. Co-Authored-By: Claude Opus 5.5 --- dDashboardServer/routes/APIRoutes.cpp | 100 ++++- dDashboardServer/routes/CharacterXmlCheck.h | 380 ++++++++++++++++++ .../templates/character-view.jinja2 | 40 +- .../GameDatabase/ITables/ContrabandRules.h | 55 +++ dGame/dUtilities/Contraband.cpp | 14 - dGame/dUtilities/Contraband.h | 30 +- docs/Dashboard.md | 8 + tests/dWebTests/CMakeLists.txt | 1 + tests/dWebTests/CharacterXmlCheckTests.cpp | 174 ++++++++ 9 files changed, 743 insertions(+), 59 deletions(-) create mode 100644 dDashboardServer/routes/CharacterXmlCheck.h create mode 100644 dDatabase/GameDatabase/ITables/ContrabandRules.h create mode 100644 tests/dWebTests/CharacterXmlCheckTests.cpp diff --git a/dDashboardServer/routes/APIRoutes.cpp b/dDashboardServer/routes/APIRoutes.cpp index 3c44328f0..4aed3deaa 100644 --- a/dDashboardServer/routes/APIRoutes.cpp +++ b/dDashboardServer/routes/APIRoutes.cpp @@ -13,6 +13,7 @@ #include "CharacterTools.h" #include "ClientAssets.h" #include "CharacterXml.h" +#include "CharacterXmlCheck.h" #include "Scenery.h" #include "Workers.h" #include "LiveWorld.h" @@ -135,6 +136,45 @@ namespace { return info->accountId; } + // Checks an uploaded character XML (CharacterXmlCheck) with the CDClient, the contraband list and the owner's account + CharacterXmlCheck::Result CheckUploadedCharacterXml(const std::string& xml, uint32_t ownerAccountId) { + CharacterXmlCheck::Context context; + context.ownerAccountId = ownerAccountId; + const auto account = Database::Get()->GetAccountById(ownerAccountId); + context.accountGmLevel = account.is_object() && account.contains("gm_level") && account["gm_level"].is_number() ? account["gm_level"].get() : 0; + context.contrabandApplies = Contraband::Applies(static_cast(context.accountGmLevel), ConfigFlag("contraband_ignore_staff", true)); + for (const auto& item : Database::Get()->GetContrabandItems()) context.contraband[item.lot] = { item.reason, item.action }; + + auto& lookups = context.lookups; + lookups.isItem = [](LOT lot) { + auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT 1 FROM ComponentsRegistry WHERE id = ? AND component_type = 11 LIMIT 1;"); + stmt.bind(1, static_cast(lot)); + return !stmt.execQuery().eof(); + }; + lookups.stackSize = [](LOT lot) { + auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT ic.stackSize FROM ComponentsRegistry cr JOIN ItemComponent ic ON ic.id = cr.component_id " + "WHERE cr.id = ? AND cr.component_type = 11 LIMIT 1;"); + stmt.bind(1, static_cast(lot)); + auto result = stmt.execQuery(); + return result.eof() ? 0 : result.getIntField(0, 0); + }; + lookups.missionExists = [](int32_t id) { + auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT 1 FROM Missions WHERE id = ? LIMIT 1;"); + stmt.bind(1, id); + return !stmt.execQuery().eof(); + }; + lookups.levelUScore = [](uint32_t level) -> std::optional { + auto stmt = CDClientDatabase::CreatePreppedStmt("SELECT requiredUScore FROM LevelProgressionLookup WHERE id = ?;"); + stmt.bind(1, static_cast(level)); + auto result = stmt.execQuery(); + if (result.eof()) return std::nullopt; + return result.getInt64Field(0, 0); + }; + auto maxLevel = CDClientDatabase::ExecuteQuery("SELECT MAX(id) FROM LevelProgressionLookup;"); + if (!maxLevel.eof()) lookups.maxLevel = static_cast(maxLevel.getIntField(0, 0)); + return CharacterXmlCheck::Check(xml, context); + } + // Random play key in the XXXX-XXXX-XXXX-XXXX format used by NexusDashboard std::string GeneratePlayKey() { static constexpr std::string_view alphabet = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // no 0/O/1/I @@ -811,7 +851,9 @@ namespace { JsonSuccess(reply, { {"requestId", requestId}, {"message", "Rescue requested"} }); }); - Route(eHTTPMethod::POST, "/api/characters/:id/xml", Perm("characters_edit_xml"), "Replace a character's XML (enable_char_xml_upload=1). Disconnects the owner first. Body: {xml}", + Route(eHTTPMethod::POST, "/api/characters/:id/xml", Perm("characters_edit_xml"), "Replace a character's XML (enable_char_xml_upload=1). Disconnects the owner first. " + "Refused (400, errors) when the game couldn't load it; suspicious content (contraband, out of reach values) needs confirm=true (409, warnings). " + "Body: {xml, confirm, remove_contraband}", [](HTTPReply& reply, const HTTPContext& context) { if (!ConfigFlag("enable_char_xml_upload", false)) return JsonError(reply, eHTTPStatusCode::FORBIDDEN, "Character XML upload is disabled (enable_char_xml_upload)"); const auto charId = RequireId(context, 2, reply); @@ -821,21 +863,59 @@ namespace { const auto owner = CharacterOwner(*charId, reply); if (!owner || !AuthorizeAccountAction(context, *owner, reply, eAccountAction::ITEMS)) return; - const std::string xml = body->value("xml", ""); - tinyxml2::XMLDocument doc; - if (xml.empty() || doc.Parse(xml.c_str()) != tinyxml2::XML_SUCCESS) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "That is not valid XML"); - if (!doc.FirstChildElement("obj") || !doc.FirstChildElement("obj")->FirstChildElement("char")) { - return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Character XML must have an root with a element"); + const std::string xml = body->is_object() && (*body)["xml"].is_string() ? (*body)["xml"].get() : std::string(); + const auto check = CheckUploadedCharacterXml(xml, *owner); + if (!check.Ok()) { + return JsonReply(reply, eHTTPStatusCode::BAD_REQUEST, { {"success", false}, + {"error", "The XML was not stored: " + std::to_string(check.errors.size()) + (check.errors.size() >= CharacterXmlCheck::MAX_REPORTED ? "+" : "") + " problem(s) the game can't load"}, + {"errors", check.errors}, {"warnings", check.warnings} }); } + nlohmann::json contraband = nlohmann::json::array(); + for (const auto& found : check.contraband) { + contraband.push_back({ {"id", std::to_string(found.item.id)}, {"lot", found.item.lot}, {"name", ClientAssets::ItemName(found.item.lot)}, {"count", found.item.count}, + {"inventory", InventoryType::InventoryTypeToString(found.item.inventory)}, {"reason", found.entry.reason}, + {"action", found.entry.action == IContraband::eContrabandAction::REMOVE ? "remove" : "flag"} }); + } + if (!check.warnings.empty() && !body->value("confirm", false)) { + return JsonReply(reply, eHTTPStatusCode::CONFLICT, { {"success", false}, {"confirm_required", true}, + {"error", "Check the warnings and confirm to store this XML"}, {"warnings", check.warnings}, {"contraband", contraband} }); + } + + // Contraband marked "flag and remove" only goes when the uploader chose so + const bool removeContraband = body->value("remove_contraband", false); + std::set removeIds; + if (removeContraband) for (const auto& found : check.contraband) if (found.entry.action == IContraband::eContrabandAction::REMOVE) removeIds.insert(found.item.id); // Store compactly like the game does + tinyxml2::XMLDocument doc; + doc.Parse(xml.c_str()); tinyxml2::XMLPrinter printer(nullptr, true); doc.Print(&printer); - const std::string compact = printer.CStr(); + const std::string compact = removeIds.empty() ? std::string(printer.CStr()) : CharacterXmlCheck::RemoveItems(printer.CStr(), removeIds); - // Through the safe path: the owner is disconnected first and the current version is kept as a snapshot + // Through the safe path: the owner is disconnected first and the current version is kept as a snapshot. + // Once stored, what the checks found is flagged and audited like the world's own contraband check. + const auto target = *charId; + const auto ownerId = *owner; + const auto actor = context.authenticatedUser; + const auto actorId = context.accountId; + const auto findings = check.contraband; + const auto warnings = check.warnings; const auto requestId = WriteCharacterXml(*charId, *owner, context.authenticatedUser, context.accountId, "XML upload", - [compact](const std::string&, std::string&) { return std::optional(compact); }); - JsonSuccess(reply, { {"requestId", requestId}, {"message", "Uploading"} }); + [compact](const std::string&, std::string&) { return std::optional(compact); }, nullptr, + [target, ownerId, actor, actorId, findings, warnings, removeIds](const PlayerActions::Outcome& outcome) { + if (!outcome.success || warnings.empty()) return; + const auto today = static_cast(std::time(nullptr) / 86400); + for (const auto& found : findings) { + const bool removed = removeIds.contains(found.item.id); + Database::Get()->InsertEconomyFlag({ today, IDashboardAdmin::eFlagKind::CONTRABAND, target, found.item.lot, found.item.id, found.item.count, removed ? 1 : 0, + std::string(removed ? "Removed from" : "Kept in") + " an uploaded character XML (" + InventoryType::InventoryTypeToString(found.item.inventory) + ", by " + actor + "): " + found.entry.reason }); + } + std::string text = "Character " + std::to_string(target) + " XML uploaded with " + std::to_string(warnings.size()) + " warning(s)" + + (removeIds.empty() ? "" : ", " + std::to_string(removeIds.size()) + " contraband item stack(s) removed") + ":"; + for (const auto& warning : warnings) text += "\n- " + warning; + Database::Get()->InsertAuditLog(actorId, actor, "character_xml_warnings", text, ownerId, target); + }); + JsonSuccess(reply, { {"requestId", requestId}, {"message", "Uploading"}, {"warnings", check.warnings}, {"removed", removeIds.size()} }); }); Route(eHTTPMethod::GET, "/api/characters/:id/mail", 0, "A character's mailbox (up to 100 most recent). With characters_mail, or the character's owner", diff --git a/dDashboardServer/routes/CharacterXmlCheck.h b/dDashboardServer/routes/CharacterXmlCheck.h new file mode 100644 index 000000000..69ffe9cbf --- /dev/null +++ b/dDashboardServer/routes/CharacterXmlCheck.h @@ -0,0 +1,380 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "tinyxml2.h" +#include "ContrabandRules.h" +#include "GeneralUtils.h" +#include "dCommonVars.h" +#include "eCharacterVersion.h" +#include "eInventoryType.h" + +/** + * Checks a character XML uploaded on the dashboard before it is stored. That upload is the one place hand-written + * XML reaches the game, and the game's load path (Character::DoQuickXMLDataParse and the LoadFromXml/LoadXml of the + * Character, Inventory, Mission, Destroyable, Buff, LevelProgression, ControllablePhysics and Character components) + * trusts the XML because the server writes it itself. So instead of the game skipping bad data at load (and saving + * the loss back), bad uploads are refused here. + * + * - errors: things the load path would throw on, read uninitialized, stop loading at (dropping everything after it, + * which is then saved back), or silently drop (an unknown mission or item, a duplicate item id or slot). + * The upload is refused. + * - warnings: valid but suspicious content (contraband, stacks above the item's stack size, coins, level or u-score + * out of reach, GM level above the account's). Staff see them and must confirm to store the XML. + * + * Pure (the game data it needs comes in through Lookups) so it can be unit tested. + */ +namespace CharacterXmlCheck { + constexpr int64_t MAX_COINS = 2000000000; + constexpr int64_t MAX_COUNT = 999999; + constexpr size_t MAX_REPORTED = 50; + + // Game data from the CDClient; an unset lookup skips its check + struct Lookups { + std::function isItem; // the LOT has an item component + std::function stackSize; // the item's stack size, 0 if unknown + std::function missionExists; // the mission is in the Missions table + std::function(uint32_t)> levelUScore; // u-score needed to reach a level + uint32_t maxLevel = 0; // highest level (0: unknown) + }; + + struct Context { + uint32_t ownerAccountId = 0; // the account of the character being replaced + int32_t accountGmLevel = 0; // that account's GM level + bool contrabandApplies = true; // Contraband::Applies for that account + Contraband::List contraband; + Lookups lookups; + }; + + struct Result { + std::vector errors; + std::vector warnings; + std::vector contraband; + bool Ok() const { return errors.empty(); } + }; + + namespace Detail { + enum class eKind { INT, UINT, INT64, UINT64, FLOAT, BOOL }; + + inline const char* KindName(eKind kind) { + switch (kind) { + case eKind::INT: return "a whole number"; + case eKind::UINT: return "a non-negative whole number"; + case eKind::INT64: return "a whole number"; + case eKind::UINT64: return "a non-negative whole number"; + case eKind::FLOAT: return "a number"; + case eKind::BOOL: return "true/false or 1/0"; + } + return "a number"; + } + + // Parses like the game's tinyxml2 Query*Attribute calls + inline bool Parses(const tinyxml2::XMLElement& e, const char* name, eKind kind) { + int i; unsigned u; int64_t i64; uint64_t u64; float f; bool b; + switch (kind) { + case eKind::INT: return e.QueryIntAttribute(name, &i) == tinyxml2::XML_SUCCESS; + case eKind::UINT: return e.QueryUnsignedAttribute(name, &u) == tinyxml2::XML_SUCCESS && e.Attribute(name)[0] != '-'; + case eKind::INT64: return e.QueryInt64Attribute(name, &i64) == tinyxml2::XML_SUCCESS; + case eKind::UINT64: return e.QueryUnsigned64Attribute(name, &u64) == tinyxml2::XML_SUCCESS && e.Attribute(name)[0] != '-'; + case eKind::FLOAT: return e.QueryFloatAttribute(name, &f) == tinyxml2::XML_SUCCESS; + case eKind::BOOL: return e.QueryBoolAttribute(name, &b) == tinyxml2::XML_SUCCESS; + } + return false; + } + + struct Checker { + Result& result; + const Context& context; + + void Error(const std::string& text) { if (result.errors.size() < MAX_REPORTED) result.errors.push_back(text); } + void Warn(const std::string& text) { if (result.warnings.size() < MAX_REPORTED) result.warnings.push_back(text); } + + // Optional attributes: when present they must parse + void Numbers(const tinyxml2::XMLElement& e, const std::string& where, eKind kind, std::initializer_list names) { + for (const auto* name : names) { + const char* value = e.Attribute(name); + if (value && !Parses(e, name, kind)) Error(where + ": " + name + "=\"" + value + "\" must be " + KindName(kind)); + } + } + + // Required attributes: the game reads them into uninitialized variables + bool Required(const tinyxml2::XMLElement& e, const std::string& where, eKind kind, std::initializer_list names) { + bool ok = true; + for (const auto* name : names) { + if (!e.Attribute(name)) { Error(where + ": " + name + " is missing"); ok = false; } + } + Numbers(e, where, kind, names); + for (const auto* name : names) if (e.Attribute(name) && !Parses(e, name, kind)) ok = false; + return ok; + } + + static int64_t Int64(const tinyxml2::XMLElement& e, const char* name) { return e.Int64Attribute(name, 0); } + static std::string Str(int64_t value) { return std::to_string(value); } + + static bool KnownInventory(uint32_t type) { return type <= eInventoryType::ITEM_SETS; } + static bool KnownMissionState(int64_t state) { + switch (state) { + case -1: case 0: case 1: case 2: case 4: case 8: case 9: case 10: case 12: case 16: return true; + default: return false; + } + } + + void Mf(const tinyxml2::XMLElement& mf) { + Numbers(mf, "", eKind::UINT, { "hc", "hs", "hd", "t", "l", "hdc", "cd", "lh", "rh", "es", "ess", "ms" }); + } + + void Char(const tinyxml2::XMLElement& c) { + const std::string where = ""; + Numbers(c, where, eKind::INT64, { "cc", "rpt", "ls", "lrid", "gid", "llog" }); + Numbers(c, where, eKind::INT, { "gm", "ft" }); + Numbers(c, where, eKind::UINT, { "lwid", "lnzid", "acct" }); + Numbers(c, where, eKind::UINT64, { "lzid", "co", "co1", "co2", "co3", "time" }); + Numbers(c, where, eKind::FLOAT, { "lzx", "lzy", "lzz", "lzrx", "lzry", "lzrz", "lzrw" }); + + if (c.Attribute("acct") && Parses(c, "acct", eKind::UINT) && c.UnsignedAttribute("acct") != context.ownerAccountId) { + Error(where + ": acct=\"" + c.Attribute("acct") + "\" is not the account of this character (" + std::to_string(context.ownerAccountId) + ")"); + } + if (c.Attribute("gm") && Parses(c, "gm", eKind::INT)) { + const auto gm = c.IntAttribute("gm"); + if (gm < 0 || gm > 9) Error(where + ": gm=\"" + std::to_string(gm) + "\" is not a GM level (0 to 9)"); + else if (gm > context.accountGmLevel) Warn("GM level " + std::to_string(gm) + " in the XML is above the account's (" + std::to_string(context.accountGmLevel) + ")"); + } + if (c.Attribute("cc") && Parses(c, "cc", eKind::INT64)) { + const auto coins = Int64(c, "cc"); + if (coins < 0) Error(where + ": cc (coins) can't be negative"); + else if (coins > MAX_COINS) Warn("Coins " + Str(coins) + " are above " + Str(MAX_COINS)); + } + if (c.Attribute("ls") && Parses(c, "ls", eKind::INT64) && Int64(c, "ls") < 0) Error(where + ": ls (u-score) can't be negative"); + + if (const auto* ue = c.FirstChildElement("ue")) { + for (const auto* e = ue->FirstChildElement(); e; e = e->NextSiblingElement()) Required(*e, " emote", eKind::INT, { "id" }); + } + if (const auto* vl = c.FirstChildElement("vl")) { + for (const auto* l = vl->FirstChildElement("l"); l; l = l->NextSiblingElement("l")) Numbers(*l, "", eKind::INT, { "id", "cid" }); + } + if (const auto* zs = c.FirstChildElement("zs")) { + for (const auto* s = zs->FirstChildElement(); s; s = s->NextSiblingElement()) { + Required(*s, " zone statistics", eKind::UINT, { "map" }); + Numbers(*s, " zone statistics", eKind::UINT64, { "ac", "cc", "es", "qbc" }); + Numbers(*s, " zone statistics", eKind::INT64, { "bc" }); + } + } + } + + void Flags(const tinyxml2::XMLElement& flags) { + for (const auto* f = flags.FirstChildElement(); f; f = f->NextSiblingElement()) { + const char* id = f->Attribute("id"); + const char* v = f->Attribute("v"); + // Character::DoQuickXMLDataParse reads both with std::stoul/std::stoull, which throw + if (id && v) { + if (!GeneralUtils::TryParse(id)) Error(std::string(": id=\"") + id + "\" must be a non-negative whole number"); + if (!GeneralUtils::TryParse(v)) Error(std::string(": v=\"") + v + "\" must be a non-negative whole number"); + } + } + } + + void Dest(const tinyxml2::XMLElement& dest) { + Numbers(dest, "", eKind::INT, { "hc", "ic", "ac" }); + Numbers(dest, "", eKind::FLOAT, { "hm", "im", "am" }); + if (const auto* buff = dest.FirstChildElement("buff")) { + for (const auto* b = buff->FirstChildElement("b"); b; b = b->NextSiblingElement("b")) { + Numbers(*b, "", eKind::INT, { "id", "b", "refCount" }); + Numbers(*b, "", eKind::FLOAT, { "t", "tk", "tt", "s" }); + Numbers(*b, "", eKind::INT64, { "sr" }); + Numbers(*b, "", eKind::BOOL, { "cancelOnDamaged", "cancelOnDeath", "cancelOnLogout", "cancelOnRemoveBuff", "cancelOnUi", "cancelOnUnequip", "cancelOnZone", "applyOnTeammates" }); + } + } + } + + void Level(const tinyxml2::XMLElement& lvl, int64_t uscore) { + Numbers(lvl, "", eKind::UINT, { "l", "cv" }); + Numbers(lvl, "", eKind::FLOAT, { "sb" }); + if (lvl.Attribute("cv") && Parses(lvl, "cv", eKind::UINT) && lvl.UnsignedAttribute("cv") > GeneralUtils::ToUnderlying(eCharacterVersion::UP_TO_DATE)) { + Error(": cv=\"" + std::string(lvl.Attribute("cv")) + "\" is newer than this server's character version (" + std::to_string(GeneralUtils::ToUnderlying(eCharacterVersion::UP_TO_DATE)) + ")"); + } + if (!lvl.Attribute("l") || !Parses(lvl, "l", eKind::UINT)) return; + const auto level = lvl.UnsignedAttribute("l"); + const auto& lookups = context.lookups; + if (lookups.maxLevel > 0 && level > lookups.maxLevel) { + Warn("Level " + std::to_string(level) + " is above the highest level (" + std::to_string(lookups.maxLevel) + ")"); + return; + } + if (!lookups.levelUScore || uscore < 0) return; + const auto needed = lookups.levelUScore(level); + if (needed && uscore < *needed) Warn("Level " + std::to_string(level) + " needs " + Str(*needed) + " u-score but the character has " + Str(uscore)); + const auto next = level < lookups.maxLevel ? lookups.levelUScore(level + 1) : std::nullopt; + if (next && uscore >= *next) Warn("U-score " + Str(uscore) + " is enough for level " + std::to_string(level + 1) + " but the character is level " + std::to_string(level)); + } + + void Inventory(const tinyxml2::XMLElement& inv) { + Numbers(inv, "", eKind::INT, { "csl" }); + if (const auto* grps = inv.FirstChildElement("grps")) { + for (const auto* g = grps->FirstChildElement("grp"); g; g = g->NextSiblingElement("grp")) Numbers(*g, "", eKind::UINT, { "t" }); + } + const auto* bag = inv.FirstChildElement("bag"); + for (const auto* b = bag->FirstChildElement(); b; b = b->NextSiblingElement()) { + if (Required(*b, "", eKind::UINT, { "t", "m" }) && !KnownInventory(b->UnsignedAttribute("t"))) { + Error(": t=\"" + std::string(b->Attribute("t")) + "\" is not an inventory type"); + } + } + + std::set ids; + std::map> slots; + std::map stackSizes; + const auto& lookups = context.lookups; + for (const auto* in = inv.FirstChildElement("items")->FirstChildElement(); in; in = in->NextSiblingElement()) { + if (!Required(*in, "", eKind::UINT, { "t" })) continue; + const auto type = in->UnsignedAttribute("t"); + if (!KnownInventory(type)) { + Error(": t=\"" + std::to_string(type) + "\" is not an inventory type"); + continue; + } + const auto inventoryName = std::string(InventoryType::InventoryTypeToString(static_cast(type))); + // Items in the build inventories are moved (and given new slots) when the character loads + const bool reslotted = type == eInventoryType::MODELS_IN_BBB || type == eInventoryType::BRICKS_IN_BBB; + for (const auto* i = in->FirstChildElement(); i; i = i->NextSiblingElement()) { + const std::string where = "Item in " + inventoryName + (i->Attribute("l") ? std::string(" (LOT ") + i->Attribute("l") + ")" : std::string()); + bool ok = Required(*i, where, eKind::INT64, { "id" }); + ok = Required(*i, where, eKind::INT, { "l" }) && ok; + ok = Required(*i, where, eKind::UINT, { "s", "c" }) && ok; + ok = Required(*i, where, eKind::BOOL, { "eq", "b" }) && ok; + Numbers(*i, where, eKind::INT64, { "sk", "parent" }); + if (!ok) continue; + + const auto id = i->Int64Attribute("id"); + const LOT lot = i->IntAttribute("l"); + const auto slot = i->UnsignedAttribute("s"); + const auto count = i->UnsignedAttribute("c"); + if (id == LWOOBJID_EMPTY) Error(where + ": id can't be 0"); + else if (!ids.insert(id).second) Error(where + ": id " + std::to_string(id) + " is used by another item (the game would drop one)"); + if (!reslotted && !slots[type].insert(slot).second) Error(where + ": slot " + std::to_string(slot) + " is used by another item in " + inventoryName + " (the game would drop one)"); + if (lookups.isItem && !lookups.isItem(lot)) { + Error(where + ": LOT " + std::to_string(lot) + " is not an item"); + continue; + } + if (count == 0) Warn(where + ": count is 0"); + else if (count > MAX_COUNT) Warn(where + ": count " + std::to_string(count) + " is above " + std::to_string(MAX_COUNT)); + else if (lookups.stackSize) { + auto it = stackSizes.find(lot); + if (it == stackSizes.end()) it = stackSizes.emplace(lot, lookups.stackSize(lot)).first; + if (it->second > 0 && count > static_cast(it->second)) Warn(where + ": count " + std::to_string(count) + " is above the stack size (" + std::to_string(it->second) + ")"); + } + // Proxy items of item sets aren't what the player holds, like the world's check + if (i->Int64Attribute("parent", LWOOBJID_EMPTY) == LWOOBJID_EMPTY) held.push_back({ id, lot, count, static_cast(type) }); + } + } + } + + void Pets(const tinyxml2::XMLElement& pet) { + for (const auto* p = pet.FirstChildElement(); p; p = p->NextSiblingElement()) { + Numbers(*p, " pet", eKind::INT64, { "id" }); + Numbers(*p, " pet", eKind::INT, { "l", "m", "t" }); + } + } + + void Missions(const tinyxml2::XMLElement& mis) { + const auto& lookups = context.lookups; + const auto mission = [&](const tinyxml2::XMLElement& m, const char* where, bool done) { + if (!Required(m, where, eKind::INT, { "id" })) return; + const auto id = m.IntAttribute("id"); + const std::string name = std::string(where) + " " + std::to_string(id); + if (lookups.missionExists && !lookups.missionExists(id)) Error(name + ": no such mission (the game would lose it)"); + Numbers(m, name, eKind::INT, { "state" }); + Numbers(m, name, eKind::UINT, { "cct", "cts", "o" }); + if (m.Attribute("state") && Parses(m, "state", eKind::INT)) { + const auto state = m.IntAttribute("state"); + if (!KnownMissionState(state)) Error(name + ": state " + std::to_string(state) + " is not a mission state"); + else if (done && state < 8) Warn(name + " is in the completed list but its state (" + std::to_string(state) + ") is not complete"); + } + if (!done) for (const auto* t = m.FirstChildElement(); t; t = t->NextSiblingElement()) Numbers(*t, name + " task", eKind::UINT, { "v" }); + }; + if (const auto* done = mis.FirstChildElement("done")) for (const auto* m = done->FirstChildElement(); m; m = m->NextSiblingElement()) mission(*m, "Completed mission", true); + if (const auto* cur = mis.FirstChildElement("cur")) for (const auto* m = cur->FirstChildElement(); m; m = m->NextSiblingElement()) mission(*m, "Current mission", false); + } + + void Respawns(const tinyxml2::XMLElement& res) { + for (const auto* r = res.FirstChildElement("r"); r; r = r->NextSiblingElement("r")) { + Numbers(*r, "", eKind::INT, { "w" }); + Numbers(*r, "", eKind::FLOAT, { "x", "y", "z" }); + } + } + + std::vector held; + }; + } + + inline Result Check(const std::string& xml, const Context& context) { + Result result; + Detail::Checker check{ result, context }; + tinyxml2::XMLDocument doc; + if (xml.empty() || doc.Parse(xml.c_str()) != tinyxml2::XML_SUCCESS) { + check.Error(xml.empty() ? "The XML is empty" : std::string("That is not valid XML: ") + (doc.ErrorStr() ? doc.ErrorStr() : "parse error")); + return result; + } + const auto* obj = doc.FirstChildElement("obj"); + if (!obj) { + check.Error("The root element must be "); + return result; + } + + // Loading stops at the first of these that is missing; what would have been read after it is lost when the character is saved + const auto* character = obj->FirstChildElement("char"); + const auto* mf = obj->FirstChildElement("mf"); + const auto* inv = obj->FirstChildElement("inv"); + const auto* items = inv ? inv->FirstChildElement("items") : nullptr; + if (!character) check.Error(" must have a element"); + if (!mf) check.Error(" must have an (appearance) element"); + if (!inv) check.Error(" must have an element"); + if (inv && !inv->FirstChildElement("bag")) check.Error(" must have a element"); + if (inv && (!items || !items->FirstChildElement("in"))) check.Error(" must have an element with at least one "); + + if (mf) check.Mf(*mf); + int64_t uscore = -1; + if (character) { + check.Char(*character); + if (character->Attribute("ls") && Detail::Parses(*character, "ls", Detail::eKind::INT64)) uscore = character->Int64Attribute("ls"); + else uscore = 0; + } + if (const auto* flags = obj->FirstChildElement("flag")) check.Flags(*flags); + if (const auto* dest = obj->FirstChildElement("dest")) check.Dest(*dest); + if (const auto* lvl = obj->FirstChildElement("lvl")) check.Level(*lvl, uscore); + if (inv && inv->FirstChildElement("bag") && items) check.Inventory(*inv); + if (const auto* pet = obj->FirstChildElement("pet")) check.Pets(*pet); + if (const auto* mis = obj->FirstChildElement("mis")) check.Missions(*mis); + if (const auto* res = obj->FirstChildElement("res")) check.Respawns(*res); + + if (context.contrabandApplies) { + result.contraband = Contraband::Find(check.held, context.contraband); + for (const auto& found : result.contraband) { + const bool remove = found.entry.action == IContraband::eContrabandAction::REMOVE; + check.Warn("Contraband: " + std::to_string(found.item.count) + " of item " + std::to_string(found.item.lot) + " (id " + std::to_string(found.item.id) + ") in " + + InventoryType::InventoryTypeToString(found.item.inventory) + (remove ? " [flag and remove]" : " [flag]") + (found.entry.reason.empty() ? "" : ": " + found.entry.reason)); + } + } + return result; + } + + // The XML without the items with these ids (and without set proxy items whose parent is one of them), printed compactly + inline std::string RemoveItems(const std::string& xml, const std::set& ids) { + tinyxml2::XMLDocument doc; + if (doc.Parse(xml.c_str()) != tinyxml2::XML_SUCCESS) return xml; + auto* inv = doc.FirstChildElement("obj") ? doc.FirstChildElement("obj")->FirstChildElement("inv") : nullptr; + auto* items = inv ? inv->FirstChildElement("items") : nullptr; + for (auto* in = items ? items->FirstChildElement() : nullptr; in; in = in->NextSiblingElement()) { + for (auto* i = in->FirstChildElement(); i;) { + auto* next = i->NextSiblingElement(); + if (ids.contains(i->Int64Attribute("id")) || ids.contains(i->Int64Attribute("parent", LWOOBJID_EMPTY))) in->DeleteChild(i); + i = next; + } + } + tinyxml2::XMLPrinter printer(nullptr, true); + doc.Print(&printer); + return printer.CStr(); + } +} diff --git a/dDashboardServer/templates/character-view.jinja2 b/dDashboardServer/templates/character-view.jinja2 index f8073a583..e68b943f3 100644 --- a/dDashboardServer/templates/character-view.jinja2 +++ b/dDashboardServer/templates/character-view.jinja2 @@ -335,11 +335,13 @@
Saving replaces all of this character's data. If the player is online they are disconnected first. Invalid data can make the character unplayable.
+
- + +
@@ -575,16 +577,40 @@ function prettyXml(xml) { function openXmlEditor() { var editor = document.getElementById('xmlEditor'); editor.value = 'Loading…'; + showXmlFindings({}); + editor.oninput = function () { document.getElementById('xmlConfirm').classList.add('d-none'); }; // changed XML is checked again new bootstrap.Modal(document.getElementById('xmlModal')).show(); api.get('/api/characters/' + characterId + '/xml').then(function (r) { editor.value = prettyXml(r.text || ''); }); } -function saveXml() { - if (!confirm('Replace this character\'s data?')) return; - api.action('/api/characters/' + characterId + '/xml', { xml: document.getElementById('xmlEditor').value }).then(function (d) { - return d.result; - }).then(function (r) { - if (r && r.success) location.reload(); +// The server checks the XML first: problems the game can't load refuse it, suspicious content (contraband and values +// out of reach) needs "Save anyway" +function showXmlFindings(d) { + var box = document.getElementById('xmlFindings'); + var list = function (items) { return '
    ' + items.map(function (t) { return '
  • ' + esc(t) + '
    • '; }).join('') + '
'; }; + var html = ''; + if (d.errors && d.errors.length) html += '
' + esc(d.error || 'The XML was not stored') + '' + list(d.errors) + '
'; + if (d.warnings && d.warnings.length) html += '
Warnings' + list(d.warnings) + + ((d.contraband || []).some(function (c) { return c.action === 'remove'; }) + ? '
' + : '') + '
Contraband is flagged and the warnings are audited either way.
'; + box.innerHTML = html; + box.classList.toggle('d-none', !html); + document.getElementById('xmlConfirm').classList.toggle('d-none', !d.confirm_required); + box.scrollIntoView({ block: 'nearest' }); +} + +function saveXml(confirmed) { + if (!confirmed && !confirm('Replace this character\'s data?')) return; + var remove = document.getElementById('xmlRemoveContraband'); + api.post('/api/characters/' + characterId + '/xml', { xml: document.getElementById('xmlEditor').value, confirm: !!confirmed, remove_contraband: !!(remove && remove.checked) }).then(function (d) { + if (!d.success) { showXmlFindings(d); if (!d.errors && !d.confirm_required) toast(d.error || 'Request failed', 'danger'); return; } + showXmlFindings({}); + if (!d.requestId) return; + Live.waitForAction(d.requestId).then(function (r) { + if (r.message) toast(r.message, r.success ? 'info' : 'danger'); + if (r.success) location.reload(); + }); }).catch(function () {}); } diff --git a/dDatabase/GameDatabase/ITables/ContrabandRules.h b/dDatabase/GameDatabase/ITables/ContrabandRules.h new file mode 100644 index 000000000..7fdd2712d --- /dev/null +++ b/dDatabase/GameDatabase/ITables/ContrabandRules.h @@ -0,0 +1,55 @@ +#ifndef __CONTRABANDRULES__H__ +#define __CONTRABANDRULES__H__ + +#include +#include +#include +#include + +#include "dCommonVars.h" +#include "eGameMasterLevel.h" +#include "eInventoryType.h" +#include "IContraband.h" + +/** + * The pure contraband rules, shared by world servers (Contraband.h, checking characters as they load and receive + * items) and the dashboard (checking an uploaded character XML), so both find the same items. + */ +namespace Contraband { + struct Entry { + std::string reason; + IContraband::eContrabandAction action{}; + }; + + using List = std::map; + + struct HeldItem { + LWOOBJID id{}; + LOT lot{}; + uint32_t count{}; + eInventoryType inventory{}; + }; + + struct Finding { + HeldItem item; + Entry entry; + }; + + // The held items that are on the list, in the order given + inline std::vector Find(const std::vector& items, const List& list) { + std::vector found; + if (list.empty()) return found; + for (const auto& item : items) { + const auto it = list.find(item.lot); + if (it != list.end() && item.count > 0) found.push_back({ item, it->second }); + } + return found; + } + + // Whether an account at this GM level is checked + inline bool Applies(eGameMasterLevel accountLevel, bool ignoreStaff) { + return !ignoreStaff || accountLevel <= eGameMasterLevel::CIVILIAN; + } +} + +#endif //!__CONTRABANDRULES__H__ diff --git a/dGame/dUtilities/Contraband.cpp b/dGame/dUtilities/Contraband.cpp index d1cffb81d..b5ae06331 100644 --- a/dGame/dUtilities/Contraband.cpp +++ b/dGame/dUtilities/Contraband.cpp @@ -85,20 +85,6 @@ namespace { } namespace Contraband { - std::vector Find(const std::vector& items, const List& list) { - std::vector found; - if (list.empty()) return found; - for (const auto& item : items) { - const auto it = list.find(item.lot); - if (it != list.end() && item.count > 0) found.push_back({ item, it->second }); - } - return found; - } - - bool Applies(eGameMasterLevel accountLevel, bool ignoreStaff) { - return !ignoreStaff || accountLevel <= eGameMasterLevel::CIVILIAN; - } - bool CountsAsAdded(eLootSourceType source, eInventoryType sourceInventory) { return sourceInventory == eInventoryType::INVALID && source != eLootSourceType::RELOCATE && source != eLootSourceType::INVENTORY; } diff --git a/dGame/dUtilities/Contraband.h b/dGame/dUtilities/Contraband.h index ab00eb03e..0e74bf096 100644 --- a/dGame/dUtilities/Contraband.h +++ b/dGame/dUtilities/Contraband.h @@ -9,10 +9,9 @@ #include "dCommonVars.h" #include "eInventoryType.h" #include "eLootSourceType.h" -#include "IContraband.h" +#include "ContrabandRules.h" class Entity; -enum class eGameMasterLevel : uint8_t; /** * Contraband (issue #1563): items staff don't want players to have, listed on the dashboard's Contraband page @@ -29,32 +28,7 @@ enum class eGameMasterLevel : uint8_t; * Staff accounts (GM level above civilian) are not checked unless contraband_ignore_staff is 0. */ namespace Contraband { - struct Entry { - std::string reason; - IContraband::eContrabandAction action{}; - }; - - using List = std::map; - - struct HeldItem { - LWOOBJID id{}; - LOT lot{}; - uint32_t count{}; - eInventoryType inventory{}; - }; - - struct Finding { - HeldItem item; - Entry entry; - }; - - // ---- Pure rules, unit tested ---- - - // The held items that are on the list, in the order given - std::vector Find(const std::vector& items, const List& list); - - // Whether an account at this GM level is checked - bool Applies(eGameMasterLevel accountLevel, bool ignoreStaff); + // ---- Pure rules, unit tested (Entry, List, HeldItem, Finding, Find and Applies are in ContrabandRules.h) ---- // Whether an added item should be checked: moves between a player's own inventories are not new items bool CountsAsAdded(eLootSourceType source, eInventoryType sourceInventory); diff --git a/docs/Dashboard.md b/docs/Dashboard.md index 939ef0259..12e02461a 100644 --- a/docs/Dashboard.md +++ b/docs/Dashboard.md @@ -680,6 +680,14 @@ shows under the account's related data); that world saves nothing more for the c connected to it is disconnected (the client's save failure message) so they load the newer data. - **Edit** (GM 8+, `characters_edit`): coins, U-score, level, and adding or removing items. +- **Edit XML** (GM 8+, `characters_edit_xml`, needs `enable_char_xml_upload=1`): replace the whole character XML. The + upload is checked against everything the game assumes when it loads a character (required elements, numbers that + must parse, known inventory types and mission states, items and missions that exist, unique item IDs and slots, the + `acct` attribute matching the owner) and refused with the list of problems if the game couldn't load it. Suspicious + content (contraband, stacks above the item's stack size, coins, level or U-score out of reach, a GM level above the + account's) is shown as warnings and needs **Save anyway**; contraband marked *flag and remove* is taken out only if + you tick the box, and either way it is flagged (kind *Contraband*) and the warnings are audited + (`character_xml_warnings`). - **History** (GM 3+, `characters_history`): earlier versions of the character. Each one shows what differs from now, can be downloaded as XML, and (with `characters_edit`) restored. The `character_snapshots` task saves every character that changed once a day; old snapshots are removed after `snapshot_days` (90), but each character always diff --git a/tests/dWebTests/CMakeLists.txt b/tests/dWebTests/CMakeLists.txt index 40b13c1b6..3fcd361ba 100644 --- a/tests/dWebTests/CMakeLists.txt +++ b/tests/dWebTests/CMakeLists.txt @@ -17,6 +17,7 @@ set(DWEBTESTS_SOURCES "SettingsCatalogTests.cpp" "BehaviorXmlTests.cpp" "CharacterXmlTests.cpp" + "CharacterXmlCheckTests.cpp" "CharacterRestoreTests.cpp" "ZonePathsTests.cpp" "RawTerrainTests.cpp" diff --git a/tests/dWebTests/CharacterXmlCheckTests.cpp b/tests/dWebTests/CharacterXmlCheckTests.cpp new file mode 100644 index 000000000..b2d8e278d --- /dev/null +++ b/tests/dWebTests/CharacterXmlCheckTests.cpp @@ -0,0 +1,174 @@ +#include + +#include + +#include "CharacterXmlCheck.h" + +namespace { + // Shaped like what the server writes (UserManager's new character plus what the components save), made-up values + const std::string VALID = + "" + "" + "" + "" + "" + "" + "" + "" + "" + "" + "" + "

" + "" + ""; + + CharacterXmlCheck::Context MakeContext() { + CharacterXmlCheck::Context context; + context.ownerAccountId = 7; + auto& l = context.lookups; + l.isItem = [](LOT lot) { return lot == 1001 || lot == 1002 || lot == 2000 || lot == 6666; }; + l.stackSize = [](LOT lot) { return lot == 2000 ? 999 : 1; }; + l.missionExists = [](int32_t id) { return id == 1 || id == 2; }; + l.levelUScore = [](uint32_t level) -> std::optional { + if (level == 1) return 0; + if (level == 2) return 100; + if (level == 3) return 300; + return std::nullopt; + }; + l.maxLevel = 3; + return context; + } + + std::string Replace(std::string text, const std::string& from, const std::string& to) { + const auto at = text.find(from); + EXPECT_NE(at, std::string::npos) << from; + if (at != std::string::npos) text.replace(at, from.size(), to); + return text; + } + + bool Mentions(const std::vector& list, const std::string& part) { + return std::any_of(list.begin(), list.end(), [&](const std::string& s) { return s.find(part) != std::string::npos; }); + } + + CharacterXmlCheck::Result Check(const std::string& xml) { return CharacterXmlCheck::Check(xml, MakeContext()); } + + void ExpectError(const std::string& xml, const std::string& part) { + const auto result = Check(xml); + EXPECT_FALSE(result.Ok()) << part; + EXPECT_TRUE(Mentions(result.errors, part)) << "expected an error mentioning: " << part << (result.errors.empty() ? "" : " got: " + result.errors.front()); + } + + void ExpectWarning(const std::string& xml, const std::string& part) { + const auto result = Check(xml); + EXPECT_TRUE(result.Ok()) << (result.errors.empty() ? "" : result.errors.front()); + EXPECT_TRUE(Mentions(result.warnings, part)) << "expected a warning mentioning: " << part; + } +} + +TEST(CharacterXmlCheckTests, ValidCharacterPasses) { + const auto result = Check(VALID); + EXPECT_TRUE(result.errors.empty()) << (result.errors.empty() ? "" : result.errors.front()); + EXPECT_TRUE(result.warnings.empty()) << (result.warnings.empty() ? "" : result.warnings.front()); +} + +TEST(CharacterXmlCheckTests, Structure) { + ExpectError("", "empty"); + ExpectError("", "not valid XML"); + ExpectError("", ""); + ExpectError(Replace(Replace(VALID, "", ""), ""); + ExpectError(Replace(VALID, ""); + ExpectError(Replace(Replace(VALID, "", ""), "", ""), ""); + ExpectError(Replace(VALID, VALID.substr(VALID.find(""), VALID.find("") - VALID.find("")), ""), "at least one "); + ExpectError(Replace(Replace(VALID, "", ""), ""); +} + +TEST(CharacterXmlCheckTests, Numbers) { + ExpectError(Replace(VALID, "cc=\"150\"", "cc=\"lots\""), "cc=\"lots\""); + ExpectError(Replace(VALID, "cc=\"150\"", "cc=\"-5\""), "negative"); + ExpectError(Replace(VALID, "ls=\"150\"", "ls=\"-1\""), "negative"); + ExpectError(Replace(VALID, "lzx=\"-626.5\"", "lzx=\"west\""), "lzx"); + ExpectError(Replace(VALID, "hc=\"1\"", "hc=\"red\""), ""); + ExpectError(Replace(VALID, "hm=\"4\"", "hm=\"full\""), ""); + ExpectError(Replace(VALID, "refCount=\"1\"", "refCount=\"x\""), ""); + ExpectError(Replace(VALID, ""); + ExpectError(Replace(VALID, "", ""), "emote: id is missing"); + ExpectError(Replace(VALID, "map=\"1000\" ", ""), "map is missing"); + ExpectError(Replace(VALID, ""); + ExpectError(Replace(VALID, "l=\"2\" cv", "l=\"two\" cv"), ""); + ExpectError(Replace(VALID, "csl=\"-1\"", "csl=\"none\""), "csl"); + ExpectError(Replace(VALID, "

"); +} + +TEST(CharacterXmlCheckTests, Flags) { + // The game reads these with std::stoul/std::stoull, which throw + ExpectError(Replace(VALID, ": id"); + ExpectError(Replace(VALID, "v=\"18446744073709551615\"", "v=\"18446744073709551616\""), ": v"); + ExpectError(Replace(VALID, ": id"); +} + +TEST(CharacterXmlCheckTests, Enums) { + ExpectError(Replace(VALID, "gm=\"0\"", "gm=\"12\""), "not a GM level"); + ExpectError(Replace(VALID, "cv=\"1\"", "cv=\"99\""), "newer than"); + ExpectError(Replace(VALID, "", ""), "not an inventory type"); + ExpectError(Replace(VALID, "state=\"2\"", "state=\"3\""), "not a mission state"); +} + +TEST(CharacterXmlCheckTests, Items) { + ExpectError(Replace(VALID, " s=\"1\" c=\"1\"", " c=\"1\""), "s is missing"); + ExpectError(Replace(VALID, "c=\"500\" eq=\"0\"", "c=\"500\""), "eq is missing"); + ExpectError(Replace(VALID, "id=\"101\"", "id=\"abc\""), "id=\"abc\""); + ExpectError(Replace(VALID, "id=\"101\"", "id=\"0\""), "id can't be 0"); + ExpectError(Replace(VALID, "id=\"101\"", "id=\"100\""), "used by another item"); + ExpectError(Replace(VALID, "id=\"101\" s=\"1\"", "id=\"101\" s=\"0\""), "slot 0"); + ExpectError(Replace(VALID, "l=\"1002\"", "l=\"4242\""), "not an item"); + ExpectError(Replace(VALID, "c=\"500\"", "c=\"-3\""), "c=\"-3\""); + // Build inventories are re-slotted when loading, so shared slots there are fine + const auto bbb = Replace(VALID, "", ""); + EXPECT_TRUE(Check(bbb).Ok()); +} + +TEST(CharacterXmlCheckTests, MissionsAndOwnership) { + ExpectError(Replace(VALID, "", ""); + const auto removed = CharacterXmlCheck::RemoveItems(withProxy, { 102 }); + EXPECT_EQ(removed.find("id=\"102\""), std::string::npos); + EXPECT_EQ(removed.find("id=\"103\""), std::string::npos); + EXPECT_NE(removed.find("id=\"101\""), std::string::npos); + EXPECT_TRUE(Check(removed).Ok()); +}