From f34577b5a21a52166824d5ef4ead4c8689edc2db Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Tue, 10 Nov 2026 18:25:44 -0600 Subject: [PATCH] feat(zone): checking, keeping and locking a zone's edited files Three pieces that know nothing of how a set of files arrives: - ZonePublish reads every file of a set with the server's own readers and says, file by file, what is wrong; a set is good only when every file is. - ZoneOverrideStore keeps a good set in the override folder: built in a staging folder, flushed, swapped in with a rename, the set before it kept as a backup. Restoring a backup and clearing a zone's set are the same swap. - ZoneEditLock has the rules for a zone's editors and its one lock holder, and for holding a reload while players are in the zone. Co-Authored-By: Claude Fable 5.2 --- dCommon/CMakeLists.txt | 4 +- dCommon/ZoneEditLock.h | 91 ++++ dCommon/ZoneFile.cpp | 1 + dCommon/ZoneFile.h | 2 + dCommon/ZoneOverrideStore.cpp | 386 ++++++++++++++ dCommon/ZoneOverrideStore.h | 96 ++++ dCommon/ZonePublish.cpp | 394 ++++++++++++++ dCommon/ZonePublish.h | 108 ++++ tests/dCommonTests/CMakeLists.txt | 1 + tests/dCommonTests/ClientZoneFilesTests.cpp | 27 + tests/dCommonTests/ZonePublishTests.cpp | 548 ++++++++++++++++++++ 11 files changed, 1657 insertions(+), 1 deletion(-) create mode 100644 dCommon/ZoneEditLock.h create mode 100644 dCommon/ZoneOverrideStore.cpp create mode 100644 dCommon/ZoneOverrideStore.h create mode 100644 dCommon/ZonePublish.cpp create mode 100644 dCommon/ZonePublish.h create mode 100644 tests/dCommonTests/ZonePublishTests.cpp diff --git a/dCommon/CMakeLists.txt b/dCommon/CMakeLists.txt index 261aac37f..63a710426 100644 --- a/dCommon/CMakeLists.txt +++ b/dCommon/CMakeLists.txt @@ -37,6 +37,8 @@ set(DCOMMON_SOURCES "ZoneFile.cpp" "LevelFile.cpp" "ZoneScenes.cpp" + "ZonePublish.cpp" + "ZoneOverrideStore.cpp" "Process.cpp" "Permissions.cpp" "PermissionGrants.cpp" @@ -119,7 +121,7 @@ add_dependencies(dBuildInfo BuildInfoGenerate) target_link_libraries(dCommon PUBLIC glm::glm dBuildInfo - PRIVATE ZLIB::ZLIB bcrypt tinyxml2 + PRIVATE ZLIB::ZLIB bcrypt tinyxml2 OpenSSL::Crypto INTERFACE dDatabase) # Profiler.h's frames and scopes also go to Tracy (thirdparty/CMakeLists.txt, DLU_TRACY) diff --git a/dCommon/ZoneEditLock.h b/dCommon/ZoneEditLock.h new file mode 100644 index 000000000..256b4e3ba --- /dev/null +++ b/dCommon/ZoneEditLock.h @@ -0,0 +1,91 @@ +#pragma once + +#include +#include + +/** + * Who may edit a zone's files (docs/ZonePublish.md). A zone has up to MAX_EDITORS assigned editors and at most one + * lock holder at a time; a set can only be sent, restored or cleared by the account holding the zone's lock. A lock + * never runs out by itself: its holder gives it up, or an admin takes it away. + * + * Pure rules, the same whichever way a request arrives; the caller looks up the account's GM level (now, never from + * anything the requester says), the zone's holder and whether the account is assigned. + */ +namespace ZoneEditLock { + constexpr uint32_t MAX_EDITORS = 10; + constexpr uint8_t DEFAULT_PUBLISH_LEVEL = 9; + constexpr uint8_t DEFAULT_ADMIN_LEVEL = 9; + + // The numbers are part of what requesters are told; never change them + enum class eResult : uint8_t { + OK = 0, + LEVEL_TOO_LOW = 1, // the account's level is below the level editing needs + NOT_ASSIGNED = 2, // not one of the zone's editors (and not an admin) + LOCKED_BY_OTHER = 3, // another account holds the zone's lock + NO_CHANGE = 4, // nothing to do: already the holder, or nothing was locked + }; + + struct Levels { + uint8_t publish = DEFAULT_PUBLISH_LEVEL; // zone_publish_gm_level + uint8_t admin = DEFAULT_ADMIN_LEVEL; // zone_lock_admin_gm_level + }; + + struct Actor { + uint32_t accountId{}; + uint8_t gmLevel{}; + bool assigned{}; // one of the zone's editors + }; + + inline bool IsAdmin(const Actor& actor, const Levels& levels) { + return actor.gmLevel >= levels.admin; + } + + // Taking the zone's lock; holder: the account holding it now + inline eResult Take(const Actor& actor, std::optional holder, const Levels& levels) { + if (actor.accountId == 0 || actor.gmLevel < levels.publish) return eResult::LEVEL_TOO_LOW; + if (!actor.assigned && !IsAdmin(actor, levels)) return eResult::NOT_ASSIGNED; + if (holder && *holder == actor.accountId) return eResult::NO_CHANGE; + if (holder) return eResult::LOCKED_BY_OTHER; + return eResult::OK; + } + + // Giving the lock up: its holder, or an admin taking it from whoever holds it + inline eResult Release(const Actor& actor, std::optional holder, const Levels& levels) { + if (actor.accountId == 0 || actor.gmLevel < levels.publish) return eResult::LEVEL_TOO_LOW; + if (!holder) return eResult::NO_CHANGE; + if (*holder == actor.accountId || IsAdmin(actor, levels)) return eResult::OK; + return eResult::LOCKED_BY_OTHER; + } + + /** + * Changing the zone's files (a new set, a restore, a clear): only the lock's holder. OK, LEVEL_TOO_LOW, + * LOCKED_BY_OTHER, or NOT_ASSIGNED when nobody holds the lock (it has to be taken first). + */ + inline eResult Change(const Actor& actor, std::optional holder, const Levels& levels) { + if (actor.accountId == 0 || actor.gmLevel < levels.publish) return eResult::LEVEL_TOO_LOW; + if (!holder) return eResult::NOT_ASSIGNED; + if (*holder != actor.accountId) return eResult::LOCKED_BY_OTHER; + return eResult::OK; + } + + // What happens to the zone's running instances after its files changed + enum class eReload : uint8_t { + SKIPPED, // not asked for: the files are read when an instance next starts + STARTED, // the zone's instances are being replaced now + HELD, // players are there and they were not to be moved: waits until someone says to + }; + + inline eReload PlanReload(bool reload, uint32_t players, bool movePlayers) { + if (!reload) return eReload::SKIPPED; + return players > 0 && !movePlayers ? eReload::HELD : eReload::STARTED; + } + + inline const char* ReloadName(eReload reload) { + switch (reload) { + case eReload::SKIPPED: return "skipped"; + case eReload::STARTED: return "started"; + case eReload::HELD: return "held"; + } + return "skipped"; + } +} diff --git a/dCommon/ZoneFile.cpp b/dCommon/ZoneFile.cpp index dcebaed77..317eaddfd 100644 --- a/dCommon/ZoneFile.cpp +++ b/dCommon/ZoneFile.cpp @@ -72,6 +72,7 @@ void ZoneFile::Read(std::istream& file) { if (!ReadPaths(pathChunk)) { LOG("The zone file's paths are not ones the client reads; the zone has no paths, as in the client"); paths.clear(); + pathsRefused = true; } } } diff --git a/dCommon/ZoneFile.h b/dCommon/ZoneFile.h index ab57ed2cd..df015dc31 100644 --- a/dCommon/ZoneFile.h +++ b/dCommon/ZoneFile.h @@ -233,6 +233,8 @@ struct ZoneFile { uint32_t pathDataLength{}; uint32_t pathChunkVersion{}; std::vector paths; + // Set when the path chunk was not one the client reads (the zone then has no paths, as in the client) + bool pathsRefused = false; /** * Reads the whole file. Throws std::runtime_error (BinaryIO) if it ends early; the last value read may still come up diff --git a/dCommon/ZoneOverrideStore.cpp b/dCommon/ZoneOverrideStore.cpp new file mode 100644 index 000000000..5d81fafac --- /dev/null +++ b/dCommon/ZoneOverrideStore.cpp @@ -0,0 +1,386 @@ +#include "ZoneOverrideStore.h" + +#include +#include +#include +#include + +#include "BinaryPathFinder.h" +#include "ZonePublish.h" +#include "json.hpp" + +#ifdef _WIN32 +#include +#else +#include +#include +#endif +#ifdef __linux__ +#include +#include +#endif + +namespace fs = std::filesystem; +using json = nlohmann::json; + +namespace { + using namespace ZoneOverrideStore; + + [[noreturn]] void Fail(const std::string& what) { + throw StoreError(what); + } + + // Flush a file's (or, where the system lets a folder be opened, a folder's) contents to disk + void Sync(const fs::path& path) { +#ifdef _WIN32 + FILE* file = nullptr; + if (fopen_s(&file, path.string().c_str(), "rb+") != 0 || !file) return; // folders can't be opened this way + fflush(file); + _commit(_fileno(file)); + fclose(file); +#else + const int fd = ::open(path.c_str(), O_RDONLY); + if (fd < 0) Fail("Could not open " + path.string() + " to flush it"); + const int result = ::fsync(fd); + ::close(fd); + if (result != 0) Fail("Could not flush " + path.string()); +#endif + } + + void SyncTree(const fs::path& folder) { + std::error_code code; + for (fs::recursive_directory_iterator it(folder, code), end; !code && it != end; it.increment(code)) { + if (it->is_regular_file(code) || it->is_directory(code)) Sync(it->path()); + } + if (code) Fail("Could not read " + folder.string()); + Sync(folder); + } + + void MakeDirs(const fs::path& folder) { + std::error_code code; + fs::create_directories(folder, code); + if (code) Fail("Could not make " + folder.string()); + } + + // The same file under another name where the file system can (sets are never changed in place), else a copy + void LinkOrCopy(const fs::path& from, const fs::path& to) { + MakeDirs(to.parent_path()); + std::error_code code; + fs::create_hard_link(from, to, code); + if (!code) return; + fs::copy_file(from, to, fs::copy_options::overwrite_existing, code); + if (code) Fail("Could not copy " + from.string()); + } + + void MoveOrCopy(const fs::path& from, const fs::path& to) { + MakeDirs(to.parent_path()); + std::error_code code; + fs::rename(from, to, code); + if (!code) return; + fs::copy_file(from, to, fs::copy_options::overwrite_existing, code); + if (code) Fail("Could not copy " + from.string()); + } + + void WriteText(const fs::path& path, const std::string& text) { + MakeDirs(path.parent_path()); + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out << text; + out.flush(); + if (!out) Fail("Could not write " + path.string()); + } + + std::optional ReadJson(const fs::path& path) { + std::ifstream in(path, std::ios::binary); + if (!in) return std::nullopt; + auto parsed = json::parse(in, nullptr, false); + if (parsed.is_discarded() || !parsed.is_object()) return std::nullopt; + return parsed; + } + + json ToJson(const Entry& entry) { + return { {"path", entry.path}, {"bytes", entry.bytes}, {"sha256", entry.sha256}, {"version", entry.version}, {"who", entry.who}, {"when", entry.when} }; + } + + std::vector EntriesOf(const json& manifest) { + std::vector entries; + const auto files = manifest.find("files"); + if (files == manifest.end() || !files->is_array()) return entries; + for (const auto& file : *files) { + if (!file.is_object()) continue; + Entry entry; + entry.path = file.value("path", std::string()); + entry.bytes = file.value("bytes", uint64_t{}); + entry.sha256 = file.value("sha256", std::string()); + entry.version = file.value("version", uint32_t{}); + entry.who = file.value("who", std::string()); + entry.when = file.value("when", int64_t{}); + // A manifest is only ever written here, but it is read back from disk: its paths are checked again + if (ZonePublish::NameProblem(entry.path) || ZonePublish::Canonical(entry.path) != entry.path) continue; + entries.push_back(std::move(entry)); + } + return entries; + } + + std::string ManifestName(uint32_t zoneId) { + return ".zone-" + std::to_string(zoneId) + ".json"; + } + + struct Folders { + fs::path root; // resolved + fs::path maps; // /maps + fs::path live; // /maps/ + fs::path staging; // /staging/ + fs::path backups; // /backups/ + std::string folder; // the zone folder, no trailing '/' + }; + + Folders Resolve(const fs::path& root, uint32_t zoneId, const std::string& zoneFolder, const std::string& id) { + if (root.empty()) Fail("No override folder is set (zone_override_dir)"); + Folders folders; + std::error_code code; + fs::create_directories(root / "maps", code); + folders.root = fs::weakly_canonical(root, code); + if (code) Fail("The override folder can't be used: " + root.string()); + folders.maps = folders.root / "maps"; + folders.folder = zoneFolder; + while (!folders.folder.empty() && folders.folder.back() == '/') folders.folder.pop_back(); + if (folders.folder.empty()) Fail("The zone's files are not in a folder of their own"); + const auto live = ZonePublish::ResolveInside(folders.maps, folders.folder); + if (!live) Fail("The zone's folder is not inside the override folder"); + folders.live = *live; + if (!id.empty()) { + if (!ValidId(id)) Fail("Not a usable id"); + folders.staging = folders.root / "staging" / id; + } + folders.backups = folders.root / "backups" / std::to_string(zoneId); + return folders; + } + + // Swap two folders that both exist, in one step where the system has one + void SwapFolders(const fs::path& incoming, const fs::path& live, const fs::path& aside) { +#ifdef __linux__ + if (::syscall(SYS_renameat2, AT_FDCWD, incoming.c_str(), AT_FDCWD, live.c_str(), RENAME_EXCHANGE) == 0) return; +#endif + std::error_code code; + fs::rename(live, aside, code); + if (code) Fail("Could not move the zone's folder aside"); + fs::rename(incoming, live, code); + if (code) { + std::error_code undo; + fs::rename(aside, live, undo); + Fail("Could not move the new files into place"); + } + } + + std::vector CurrentOf(const Folders& folders, uint32_t zoneId, json* manifestOut = nullptr) { + const auto manifest = ReadJson(folders.live / ManifestName(zoneId)); + if (!manifest) return {}; + if (manifestOut) *manifestOut = *manifest; + auto entries = EntriesOf(*manifest); + const auto prefix = folders.folder + "/"; + std::erase_if(entries, [&](const Entry& entry) { return !entry.path.starts_with(prefix); }); + return entries; + } + + void PruneBackups(const fs::path& backups, uint32_t keep) { + std::vector ids; + std::error_code code; + for (fs::directory_iterator it(backups, code), end; !code && it != end; it.increment(code)) { + if (it->is_directory(code) && ValidId(it->path().filename().string())) ids.push_back(it->path().filename().string()); + } + // "-": newest first by time, then by name + const auto timeOf = [](const std::string& id) { return std::strtoll(id.c_str(), nullptr, 10); }; + std::ranges::sort(ids, [&](const std::string& a, const std::string& b) { + return timeOf(a) != timeOf(b) ? timeOf(a) > timeOf(b) : a > b; + }); + for (size_t i = keep; i < ids.size(); ++i) fs::remove_all(backups / ids[i], code); + } + + /** + * The one way a zone's set changes. `place` puts the new set's files under the folder it is given (laid out as + * under res/maps/) and returns their entries; everything else is the same for a new set, a restore and a clear. + */ + template + Applied Replace(const fs::path& root, const Change& change, Place place) { + const auto folders = Resolve(root, change.zoneId, change.zoneFolder, change.id); + const auto incomingMaps = folders.staging / "incoming"; + const auto incoming = incomingMaps / folders.folder; + std::error_code code; + fs::remove_all(incomingMaps, code); + MakeDirs(incoming); + + json oldManifest; + const auto previous = CurrentOf(folders, change.zoneId, &oldManifest); + std::set previousPaths; + for (const auto& entry : previous) previousPaths.insert(entry.path); + + // Everything in the folder that is not this zone's set stays (zones can share a folder) + const bool hadFolder = fs::is_directory(folders.live, code); + if (hadFolder) { + for (fs::recursive_directory_iterator it(folders.live, code), end; !code && it != end; it.increment(code)) { + if (!it->is_regular_file(code)) continue; + const auto relative = folders.folder + "/" + it->path().lexically_relative(folders.live).generic_string(); + if (previousPaths.contains(relative) || it->path().filename() == ManifestName(change.zoneId)) continue; + LinkOrCopy(it->path(), incomingMaps / relative); + } + if (code) Fail("Could not read the zone's folder"); + } + + auto entries = place(incomingMaps, folders); + const auto prefix = folders.folder + "/"; + for (auto& entry : entries) { + if (!entry.path.starts_with(prefix)) Fail("A file is not in the zone's folder: " + entry.path); + const auto target = ZonePublish::ResolveInside(incomingMaps, entry.path); + if (!target || !fs::is_regular_file(*target, code)) Fail("A file did not arrive where it belongs: " + entry.path); + } + if (!entries.empty()) { + json files = json::array(); + for (const auto& entry : entries) files.push_back(ToJson(entry)); + WriteText(incoming / ManifestName(change.zoneId), json{ {"zoneId", change.zoneId}, {"id", change.id}, {"who", change.who}, + {"note", change.note}, {"time", change.now}, {"files", files} }.dump(1)); + } + SyncTree(incomingMaps); + + // The set being replaced is kept first + Applied applied; + if (!previous.empty()) { + const auto backupId = std::to_string(change.now) + "-" + change.id; + const auto backup = folders.backups / backupId; + fs::remove_all(backup, code); + for (const auto& entry : previous) { + const auto from = ZonePublish::ResolveInside(folders.maps, entry.path); + if (!from || !fs::is_regular_file(*from, code)) continue; + LinkOrCopy(*from, backup / "files" / entry.path); + } + oldManifest["replacedAt"] = change.now; + WriteText(backup / "manifest.json", oldManifest.dump(1)); + SyncTree(backup); + Sync(folders.backups); + applied.backupId = backupId; + } + + // The swap + if (hadFolder) { + SwapFolders(incoming, folders.live, folders.staging / "replaced"); + } else { + MakeDirs(folders.live.parent_path()); + fs::rename(incoming, folders.live, code); + if (code) Fail("Could not move the new files into place"); + } + Sync(folders.live.parent_path()); + + fs::remove_all(folders.staging, code); + PruneBackups(folders.backups, change.keepBackups); + applied.files = std::move(entries); + return applied; + } +} + +namespace ZoneOverrideStore { + std::filesystem::path RootFromSetting(const std::string& value) { + if (value.empty()) return {}; + fs::path root(value); + if (root.is_relative()) root = BinaryPathFinder::GetBinaryDir() / root; + return root.lexically_normal(); + } + + bool ValidId(std::string_view text) { + if (text.empty() || text.size() > 64) return false; + return std::ranges::all_of(text, [](char c) { return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-'; }); + } + + std::filesystem::path MakeUploadDir(const fs::path& root, const std::string& id) { + if (root.empty() || !ValidId(id)) Fail("No override folder is set (zone_override_dir)"); + std::error_code code; + fs::create_directories(root, code); + const auto resolved = fs::weakly_canonical(root, code); + if (code) Fail("The override folder can't be used: " + root.string()); + const auto folder = resolved / "staging" / id / "upload"; + fs::remove_all(folder, code); + MakeDirs(folder); + return folder; + } + + void RemoveStaging(const fs::path& root, const std::string& id) { + if (root.empty() || !ValidId(id)) return; + std::error_code code; + fs::remove_all(root / "staging" / id, code); + } + + void RemoveAllStaging(const fs::path& root) { + if (root.empty()) return; + std::error_code code; + fs::remove_all(root / "staging", code); + } + + Applied Apply(const fs::path& root, const Change& change, const std::vector& files) { + if (files.empty()) Fail("A set has at least the zone's .luz"); + return Replace(root, change, [&](const fs::path& incomingMaps, const Folders&) { + std::vector entries; + std::set seen; + for (const auto& file : files) { + if (ZonePublish::Canonical(file.path) != file.path || !seen.insert(file.path).second) Fail("Not a usable file name: " + file.path); + const auto target = ZonePublish::ResolveInside(incomingMaps, file.path); + if (!target) Fail("Not a usable file name: " + file.path); + MoveOrCopy(file.source, *target); + entries.push_back({ file.path, file.bytes, file.sha256, file.version, change.who, change.now }); + } + return entries; + }); + } + + Applied Restore(const fs::path& root, const Change& change, const std::string& backupId) { + if (!ValidId(backupId)) Fail("No such backup"); + return Replace(root, change, [&](const fs::path& incomingMaps, const Folders& folders) { + const auto backup = folders.backups / backupId; + const auto manifest = ReadJson(backup / "manifest.json"); + if (!manifest) Fail("No such backup"); + auto entries = EntriesOf(*manifest); + if (entries.empty()) Fail("The backup holds no files"); + for (const auto& entry : entries) { + const auto from = ZonePublish::ResolveInside(backup / "files", entry.path); + const auto to = ZonePublish::ResolveInside(incomingMaps, entry.path); + std::error_code code; + if (!from || !to || !fs::is_regular_file(*from, code)) Fail("The backup is missing " + entry.path); + if (ZonePublish::Sha256File(*from) != std::optional(entry.sha256)) Fail("The backup's " + entry.path + " is not the file that was kept"); + LinkOrCopy(*from, *to); + } + return entries; + }); + } + + Applied Clear(const fs::path& root, const Change& change) { + return Replace(root, change, [](const fs::path&, const Folders&) { return std::vector{}; }); + } + + std::vector Current(const fs::path& root, uint32_t zoneId, const std::string& zoneFolder) { + if (root.empty()) return {}; + try { + return CurrentOf(Resolve(root, zoneId, zoneFolder, ""), zoneId); + } catch (const StoreError&) { + return {}; + } + } + + std::vector Backups(const fs::path& root, uint32_t zoneId) { + std::vector backups; + if (root.empty()) return backups; + const auto folder = root / "backups" / std::to_string(zoneId); + std::error_code code; + for (fs::directory_iterator it(folder, code), end; !code && it != end; it.increment(code)) { + const auto id = it->path().filename().string(); + if (!it->is_directory(code) || !ValidId(id)) continue; + const auto manifest = ReadJson(it->path() / "manifest.json"); + if (!manifest) continue; + Backup backup; + backup.id = id; + backup.time = manifest->value("replacedAt", int64_t{}); + backup.who = manifest->value("who", std::string()); + backup.note = manifest->value("note", std::string()); + backup.files = EntriesOf(*manifest); + backups.push_back(std::move(backup)); + } + std::ranges::sort(backups, [](const Backup& a, const Backup& b) { return a.time != b.time ? a.time > b.time : a.id > b.id; }); + return backups; + } +} diff --git a/dCommon/ZoneOverrideStore.h b/dCommon/ZoneOverrideStore.h new file mode 100644 index 000000000..5c62db6b4 --- /dev/null +++ b/dCommon/ZoneOverrideStore.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +/** + * The override folder (zone_override_dir, docs/ZonePublish.md): the edited zone files the servers read in place of + * the client's (AssetManager::SetOverrideDir), kept apart from the client's files, which are never written. + * + * /maps//... the files in use, where the asset manager looks + * /maps//.zone-.json which of them are zone 's set, who sent it and when + * /staging// a set while it is checked and built + * /backups//