feat(zone): checking, keeping and locking a zone's edited files

Three pieces that know nothing of how a set of files arrives:
- ZonePublish reads every file of a set with the server's own readers and says, file
  by file, what is wrong; a set is good only when every file is.
- ZoneOverrideStore keeps a good set in the override folder: built in a staging
  folder, flushed, swapped in with a rename, the set before it kept as a backup.
  Restoring a backup and clearing a zone's set are the same swap.
- ZoneEditLock has the rules for a zone's editors and its one lock holder, and for
  holding a reload while players are in the zone.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-11-10 18:25:44 -06:00
parent 0e57b95229
commit f34577b5a2
11 changed files with 1657 additions and 1 deletions

View File

@@ -37,6 +37,8 @@ set(DCOMMON_SOURCES
"ZoneFile.cpp"
"LevelFile.cpp"
"ZoneScenes.cpp"
"ZonePublish.cpp"
"ZoneOverrideStore.cpp"
"Process.cpp"
"Permissions.cpp"
"PermissionGrants.cpp"
@@ -119,7 +121,7 @@ add_dependencies(dBuildInfo BuildInfoGenerate)
target_link_libraries(dCommon
PUBLIC glm::glm dBuildInfo
PRIVATE ZLIB::ZLIB bcrypt tinyxml2
PRIVATE ZLIB::ZLIB bcrypt tinyxml2 OpenSSL::Crypto
INTERFACE dDatabase)
# Profiler.h's frames and scopes also go to Tracy (thirdparty/CMakeLists.txt, DLU_TRACY)

91
dCommon/ZoneEditLock.h Normal file
View File

@@ -0,0 +1,91 @@
#pragma once
#include <cstdint>
#include <optional>
/**
* Who may edit a zone's files (docs/ZonePublish.md). A zone has up to MAX_EDITORS assigned editors and at most one
* lock holder at a time; a set can only be sent, restored or cleared by the account holding the zone's lock. A lock
* never runs out by itself: its holder gives it up, or an admin takes it away.
*
* Pure rules, the same whichever way a request arrives; the caller looks up the account's GM level (now, never from
* anything the requester says), the zone's holder and whether the account is assigned.
*/
namespace ZoneEditLock {
constexpr uint32_t MAX_EDITORS = 10;
constexpr uint8_t DEFAULT_PUBLISH_LEVEL = 9;
constexpr uint8_t DEFAULT_ADMIN_LEVEL = 9;
// The numbers are part of what requesters are told; never change them
enum class eResult : uint8_t {
OK = 0,
LEVEL_TOO_LOW = 1, // the account's level is below the level editing needs
NOT_ASSIGNED = 2, // not one of the zone's editors (and not an admin)
LOCKED_BY_OTHER = 3, // another account holds the zone's lock
NO_CHANGE = 4, // nothing to do: already the holder, or nothing was locked
};
struct Levels {
uint8_t publish = DEFAULT_PUBLISH_LEVEL; // zone_publish_gm_level
uint8_t admin = DEFAULT_ADMIN_LEVEL; // zone_lock_admin_gm_level
};
struct Actor {
uint32_t accountId{};
uint8_t gmLevel{};
bool assigned{}; // one of the zone's editors
};
inline bool IsAdmin(const Actor& actor, const Levels& levels) {
return actor.gmLevel >= levels.admin;
}
// Taking the zone's lock; holder: the account holding it now
inline eResult Take(const Actor& actor, std::optional<uint32_t> holder, const Levels& levels) {
if (actor.accountId == 0 || actor.gmLevel < levels.publish) return eResult::LEVEL_TOO_LOW;
if (!actor.assigned && !IsAdmin(actor, levels)) return eResult::NOT_ASSIGNED;
if (holder && *holder == actor.accountId) return eResult::NO_CHANGE;
if (holder) return eResult::LOCKED_BY_OTHER;
return eResult::OK;
}
// Giving the lock up: its holder, or an admin taking it from whoever holds it
inline eResult Release(const Actor& actor, std::optional<uint32_t> holder, const Levels& levels) {
if (actor.accountId == 0 || actor.gmLevel < levels.publish) return eResult::LEVEL_TOO_LOW;
if (!holder) return eResult::NO_CHANGE;
if (*holder == actor.accountId || IsAdmin(actor, levels)) return eResult::OK;
return eResult::LOCKED_BY_OTHER;
}
/**
* Changing the zone's files (a new set, a restore, a clear): only the lock's holder. OK, LEVEL_TOO_LOW,
* LOCKED_BY_OTHER, or NOT_ASSIGNED when nobody holds the lock (it has to be taken first).
*/
inline eResult Change(const Actor& actor, std::optional<uint32_t> holder, const Levels& levels) {
if (actor.accountId == 0 || actor.gmLevel < levels.publish) return eResult::LEVEL_TOO_LOW;
if (!holder) return eResult::NOT_ASSIGNED;
if (*holder != actor.accountId) return eResult::LOCKED_BY_OTHER;
return eResult::OK;
}
// What happens to the zone's running instances after its files changed
enum class eReload : uint8_t {
SKIPPED, // not asked for: the files are read when an instance next starts
STARTED, // the zone's instances are being replaced now
HELD, // players are there and they were not to be moved: waits until someone says to
};
inline eReload PlanReload(bool reload, uint32_t players, bool movePlayers) {
if (!reload) return eReload::SKIPPED;
return players > 0 && !movePlayers ? eReload::HELD : eReload::STARTED;
}
inline const char* ReloadName(eReload reload) {
switch (reload) {
case eReload::SKIPPED: return "skipped";
case eReload::STARTED: return "started";
case eReload::HELD: return "held";
}
return "skipped";
}
}

View File

@@ -72,6 +72,7 @@ void ZoneFile::Read(std::istream& file) {
if (!ReadPaths(pathChunk)) {
LOG("The zone file's paths are not ones the client reads; the zone has no paths, as in the client");
paths.clear();
pathsRefused = true;
}
}
}

View File

@@ -233,6 +233,8 @@ struct ZoneFile {
uint32_t pathDataLength{};
uint32_t pathChunkVersion{};
std::vector<Path> paths;
// Set when the path chunk was not one the client reads (the zone then has no paths, as in the client)
bool pathsRefused = false;
/**
* Reads the whole file. Throws std::runtime_error (BinaryIO) if it ends early; the last value read may still come up

View File

@@ -0,0 +1,386 @@
#include "ZoneOverrideStore.h"
#include <algorithm>
#include <cstdio>
#include <fstream>
#include <set>
#include "BinaryPathFinder.h"
#include "ZonePublish.h"
#include "json.hpp"
#ifdef _WIN32
#include <io.h>
#else
#include <fcntl.h>
#include <unistd.h>
#endif
#ifdef __linux__
#include <sys/syscall.h>
#include <linux/fs.h>
#endif
namespace fs = std::filesystem;
using json = nlohmann::json;
namespace {
using namespace ZoneOverrideStore;
[[noreturn]] void Fail(const std::string& what) {
throw StoreError(what);
}
// Flush a file's (or, where the system lets a folder be opened, a folder's) contents to disk
void Sync(const fs::path& path) {
#ifdef _WIN32
FILE* file = nullptr;
if (fopen_s(&file, path.string().c_str(), "rb+") != 0 || !file) return; // folders can't be opened this way
fflush(file);
_commit(_fileno(file));
fclose(file);
#else
const int fd = ::open(path.c_str(), O_RDONLY);
if (fd < 0) Fail("Could not open " + path.string() + " to flush it");
const int result = ::fsync(fd);
::close(fd);
if (result != 0) Fail("Could not flush " + path.string());
#endif
}
void SyncTree(const fs::path& folder) {
std::error_code code;
for (fs::recursive_directory_iterator it(folder, code), end; !code && it != end; it.increment(code)) {
if (it->is_regular_file(code) || it->is_directory(code)) Sync(it->path());
}
if (code) Fail("Could not read " + folder.string());
Sync(folder);
}
void MakeDirs(const fs::path& folder) {
std::error_code code;
fs::create_directories(folder, code);
if (code) Fail("Could not make " + folder.string());
}
// The same file under another name where the file system can (sets are never changed in place), else a copy
void LinkOrCopy(const fs::path& from, const fs::path& to) {
MakeDirs(to.parent_path());
std::error_code code;
fs::create_hard_link(from, to, code);
if (!code) return;
fs::copy_file(from, to, fs::copy_options::overwrite_existing, code);
if (code) Fail("Could not copy " + from.string());
}
void MoveOrCopy(const fs::path& from, const fs::path& to) {
MakeDirs(to.parent_path());
std::error_code code;
fs::rename(from, to, code);
if (!code) return;
fs::copy_file(from, to, fs::copy_options::overwrite_existing, code);
if (code) Fail("Could not copy " + from.string());
}
void WriteText(const fs::path& path, const std::string& text) {
MakeDirs(path.parent_path());
std::ofstream out(path, std::ios::binary | std::ios::trunc);
out << text;
out.flush();
if (!out) Fail("Could not write " + path.string());
}
std::optional<json> ReadJson(const fs::path& path) {
std::ifstream in(path, std::ios::binary);
if (!in) return std::nullopt;
auto parsed = json::parse(in, nullptr, false);
if (parsed.is_discarded() || !parsed.is_object()) return std::nullopt;
return parsed;
}
json ToJson(const Entry& entry) {
return { {"path", entry.path}, {"bytes", entry.bytes}, {"sha256", entry.sha256}, {"version", entry.version}, {"who", entry.who}, {"when", entry.when} };
}
std::vector<Entry> EntriesOf(const json& manifest) {
std::vector<Entry> entries;
const auto files = manifest.find("files");
if (files == manifest.end() || !files->is_array()) return entries;
for (const auto& file : *files) {
if (!file.is_object()) continue;
Entry entry;
entry.path = file.value("path", std::string());
entry.bytes = file.value("bytes", uint64_t{});
entry.sha256 = file.value("sha256", std::string());
entry.version = file.value("version", uint32_t{});
entry.who = file.value("who", std::string());
entry.when = file.value("when", int64_t{});
// A manifest is only ever written here, but it is read back from disk: its paths are checked again
if (ZonePublish::NameProblem(entry.path) || ZonePublish::Canonical(entry.path) != entry.path) continue;
entries.push_back(std::move(entry));
}
return entries;
}
std::string ManifestName(uint32_t zoneId) {
return ".zone-" + std::to_string(zoneId) + ".json";
}
struct Folders {
fs::path root; // resolved
fs::path maps; // <root>/maps
fs::path live; // <root>/maps/<zone folder>
fs::path staging; // <root>/staging/<id>
fs::path backups; // <root>/backups/<zone id>
std::string folder; // the zone folder, no trailing '/'
};
Folders Resolve(const fs::path& root, uint32_t zoneId, const std::string& zoneFolder, const std::string& id) {
if (root.empty()) Fail("No override folder is set (zone_override_dir)");
Folders folders;
std::error_code code;
fs::create_directories(root / "maps", code);
folders.root = fs::weakly_canonical(root, code);
if (code) Fail("The override folder can't be used: " + root.string());
folders.maps = folders.root / "maps";
folders.folder = zoneFolder;
while (!folders.folder.empty() && folders.folder.back() == '/') folders.folder.pop_back();
if (folders.folder.empty()) Fail("The zone's files are not in a folder of their own");
const auto live = ZonePublish::ResolveInside(folders.maps, folders.folder);
if (!live) Fail("The zone's folder is not inside the override folder");
folders.live = *live;
if (!id.empty()) {
if (!ValidId(id)) Fail("Not a usable id");
folders.staging = folders.root / "staging" / id;
}
folders.backups = folders.root / "backups" / std::to_string(zoneId);
return folders;
}
// Swap two folders that both exist, in one step where the system has one
void SwapFolders(const fs::path& incoming, const fs::path& live, const fs::path& aside) {
#ifdef __linux__
if (::syscall(SYS_renameat2, AT_FDCWD, incoming.c_str(), AT_FDCWD, live.c_str(), RENAME_EXCHANGE) == 0) return;
#endif
std::error_code code;
fs::rename(live, aside, code);
if (code) Fail("Could not move the zone's folder aside");
fs::rename(incoming, live, code);
if (code) {
std::error_code undo;
fs::rename(aside, live, undo);
Fail("Could not move the new files into place");
}
}
std::vector<Entry> CurrentOf(const Folders& folders, uint32_t zoneId, json* manifestOut = nullptr) {
const auto manifest = ReadJson(folders.live / ManifestName(zoneId));
if (!manifest) return {};
if (manifestOut) *manifestOut = *manifest;
auto entries = EntriesOf(*manifest);
const auto prefix = folders.folder + "/";
std::erase_if(entries, [&](const Entry& entry) { return !entry.path.starts_with(prefix); });
return entries;
}
void PruneBackups(const fs::path& backups, uint32_t keep) {
std::vector<std::string> ids;
std::error_code code;
for (fs::directory_iterator it(backups, code), end; !code && it != end; it.increment(code)) {
if (it->is_directory(code) && ValidId(it->path().filename().string())) ids.push_back(it->path().filename().string());
}
// "<unix time>-<id>": newest first by time, then by name
const auto timeOf = [](const std::string& id) { return std::strtoll(id.c_str(), nullptr, 10); };
std::ranges::sort(ids, [&](const std::string& a, const std::string& b) {
return timeOf(a) != timeOf(b) ? timeOf(a) > timeOf(b) : a > b;
});
for (size_t i = keep; i < ids.size(); ++i) fs::remove_all(backups / ids[i], code);
}
/**
* The one way a zone's set changes. `place` puts the new set's files under the folder it is given (laid out as
* under res/maps/) and returns their entries; everything else is the same for a new set, a restore and a clear.
*/
template<typename Place>
Applied Replace(const fs::path& root, const Change& change, Place place) {
const auto folders = Resolve(root, change.zoneId, change.zoneFolder, change.id);
const auto incomingMaps = folders.staging / "incoming";
const auto incoming = incomingMaps / folders.folder;
std::error_code code;
fs::remove_all(incomingMaps, code);
MakeDirs(incoming);
json oldManifest;
const auto previous = CurrentOf(folders, change.zoneId, &oldManifest);
std::set<std::string> previousPaths;
for (const auto& entry : previous) previousPaths.insert(entry.path);
// Everything in the folder that is not this zone's set stays (zones can share a folder)
const bool hadFolder = fs::is_directory(folders.live, code);
if (hadFolder) {
for (fs::recursive_directory_iterator it(folders.live, code), end; !code && it != end; it.increment(code)) {
if (!it->is_regular_file(code)) continue;
const auto relative = folders.folder + "/" + it->path().lexically_relative(folders.live).generic_string();
if (previousPaths.contains(relative) || it->path().filename() == ManifestName(change.zoneId)) continue;
LinkOrCopy(it->path(), incomingMaps / relative);
}
if (code) Fail("Could not read the zone's folder");
}
auto entries = place(incomingMaps, folders);
const auto prefix = folders.folder + "/";
for (auto& entry : entries) {
if (!entry.path.starts_with(prefix)) Fail("A file is not in the zone's folder: " + entry.path);
const auto target = ZonePublish::ResolveInside(incomingMaps, entry.path);
if (!target || !fs::is_regular_file(*target, code)) Fail("A file did not arrive where it belongs: " + entry.path);
}
if (!entries.empty()) {
json files = json::array();
for (const auto& entry : entries) files.push_back(ToJson(entry));
WriteText(incoming / ManifestName(change.zoneId), json{ {"zoneId", change.zoneId}, {"id", change.id}, {"who", change.who},
{"note", change.note}, {"time", change.now}, {"files", files} }.dump(1));
}
SyncTree(incomingMaps);
// The set being replaced is kept first
Applied applied;
if (!previous.empty()) {
const auto backupId = std::to_string(change.now) + "-" + change.id;
const auto backup = folders.backups / backupId;
fs::remove_all(backup, code);
for (const auto& entry : previous) {
const auto from = ZonePublish::ResolveInside(folders.maps, entry.path);
if (!from || !fs::is_regular_file(*from, code)) continue;
LinkOrCopy(*from, backup / "files" / entry.path);
}
oldManifest["replacedAt"] = change.now;
WriteText(backup / "manifest.json", oldManifest.dump(1));
SyncTree(backup);
Sync(folders.backups);
applied.backupId = backupId;
}
// The swap
if (hadFolder) {
SwapFolders(incoming, folders.live, folders.staging / "replaced");
} else {
MakeDirs(folders.live.parent_path());
fs::rename(incoming, folders.live, code);
if (code) Fail("Could not move the new files into place");
}
Sync(folders.live.parent_path());
fs::remove_all(folders.staging, code);
PruneBackups(folders.backups, change.keepBackups);
applied.files = std::move(entries);
return applied;
}
}
namespace ZoneOverrideStore {
std::filesystem::path RootFromSetting(const std::string& value) {
if (value.empty()) return {};
fs::path root(value);
if (root.is_relative()) root = BinaryPathFinder::GetBinaryDir() / root;
return root.lexically_normal();
}
bool ValidId(std::string_view text) {
if (text.empty() || text.size() > 64) return false;
return std::ranges::all_of(text, [](char c) { return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-'; });
}
std::filesystem::path MakeUploadDir(const fs::path& root, const std::string& id) {
if (root.empty() || !ValidId(id)) Fail("No override folder is set (zone_override_dir)");
std::error_code code;
fs::create_directories(root, code);
const auto resolved = fs::weakly_canonical(root, code);
if (code) Fail("The override folder can't be used: " + root.string());
const auto folder = resolved / "staging" / id / "upload";
fs::remove_all(folder, code);
MakeDirs(folder);
return folder;
}
void RemoveStaging(const fs::path& root, const std::string& id) {
if (root.empty() || !ValidId(id)) return;
std::error_code code;
fs::remove_all(root / "staging" / id, code);
}
void RemoveAllStaging(const fs::path& root) {
if (root.empty()) return;
std::error_code code;
fs::remove_all(root / "staging", code);
}
Applied Apply(const fs::path& root, const Change& change, const std::vector<NewFile>& files) {
if (files.empty()) Fail("A set has at least the zone's .luz");
return Replace(root, change, [&](const fs::path& incomingMaps, const Folders&) {
std::vector<Entry> entries;
std::set<std::string> seen;
for (const auto& file : files) {
if (ZonePublish::Canonical(file.path) != file.path || !seen.insert(file.path).second) Fail("Not a usable file name: " + file.path);
const auto target = ZonePublish::ResolveInside(incomingMaps, file.path);
if (!target) Fail("Not a usable file name: " + file.path);
MoveOrCopy(file.source, *target);
entries.push_back({ file.path, file.bytes, file.sha256, file.version, change.who, change.now });
}
return entries;
});
}
Applied Restore(const fs::path& root, const Change& change, const std::string& backupId) {
if (!ValidId(backupId)) Fail("No such backup");
return Replace(root, change, [&](const fs::path& incomingMaps, const Folders& folders) {
const auto backup = folders.backups / backupId;
const auto manifest = ReadJson(backup / "manifest.json");
if (!manifest) Fail("No such backup");
auto entries = EntriesOf(*manifest);
if (entries.empty()) Fail("The backup holds no files");
for (const auto& entry : entries) {
const auto from = ZonePublish::ResolveInside(backup / "files", entry.path);
const auto to = ZonePublish::ResolveInside(incomingMaps, entry.path);
std::error_code code;
if (!from || !to || !fs::is_regular_file(*from, code)) Fail("The backup is missing " + entry.path);
if (ZonePublish::Sha256File(*from) != std::optional<std::string>(entry.sha256)) Fail("The backup's " + entry.path + " is not the file that was kept");
LinkOrCopy(*from, *to);
}
return entries;
});
}
Applied Clear(const fs::path& root, const Change& change) {
return Replace(root, change, [](const fs::path&, const Folders&) { return std::vector<Entry>{}; });
}
std::vector<Entry> Current(const fs::path& root, uint32_t zoneId, const std::string& zoneFolder) {
if (root.empty()) return {};
try {
return CurrentOf(Resolve(root, zoneId, zoneFolder, ""), zoneId);
} catch (const StoreError&) {
return {};
}
}
std::vector<Backup> Backups(const fs::path& root, uint32_t zoneId) {
std::vector<Backup> backups;
if (root.empty()) return backups;
const auto folder = root / "backups" / std::to_string(zoneId);
std::error_code code;
for (fs::directory_iterator it(folder, code), end; !code && it != end; it.increment(code)) {
const auto id = it->path().filename().string();
if (!it->is_directory(code) || !ValidId(id)) continue;
const auto manifest = ReadJson(it->path() / "manifest.json");
if (!manifest) continue;
Backup backup;
backup.id = id;
backup.time = manifest->value("replacedAt", int64_t{});
backup.who = manifest->value("who", std::string());
backup.note = manifest->value("note", std::string());
backup.files = EntriesOf(*manifest);
backups.push_back(std::move(backup));
}
std::ranges::sort(backups, [](const Backup& a, const Backup& b) { return a.time != b.time ? a.time > b.time : a.id > b.id; });
return backups;
}
}

View File

@@ -0,0 +1,96 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <optional>
#include <stdexcept>
#include <string>
#include <vector>
/**
* The override folder (zone_override_dir, docs/ZonePublish.md): the edited zone files the servers read in place of
* the client's (AssetManager::SetOverrideDir), kept apart from the client's files, which are never written.
*
* <root>/maps/<zone folder>/... the files in use, where the asset manager looks
* <root>/maps/<zone folder>/.zone-<id>.json which of them are zone <id>'s set, who sent it and when
* <root>/staging/<id>/ a set while it is checked and built
* <root>/backups/<zone id>/<time>-<id>/ the sets a zone had before, newest kept
*
* A zone's set is replaced as a whole: the new contents of its folder are built in staging, flushed to disk, and
* swapped in with a rename; the set it replaces goes to backups first. Restoring a backup and clearing a zone's set
* are the same swap. Knows nothing of who may do this (ZoneEditLock) or how files arrive.
*
* Only reads and writes files under root, and takes no lock of its own: callers apply one set at a time. No
* settings, database or logging, so it can run on a worker thread.
*/
namespace ZoneOverrideStore {
struct StoreError : std::runtime_error {
using std::runtime_error::runtime_error;
};
struct Entry {
std::string path; // relative to res/maps/
uint64_t bytes{};
std::string sha256;
uint32_t version{};
std::string who;
int64_t when{};
};
struct Backup {
std::string id; // "<unix time>-<set id>"
int64_t time{}; // when the set was replaced
std::string who; // who sent the set it holds
std::string note;
std::vector<Entry> files;
};
// A file of a new set: where its bytes are now (moved or copied into the store)
struct NewFile {
std::string path;
std::filesystem::path source;
uint64_t bytes{};
std::string sha256;
uint32_t version{};
};
struct Applied {
std::optional<std::string> backupId; // the set that was replaced, when there was one
std::vector<Entry> files; // the zone's set now
};
struct Change {
uint32_t zoneId{};
std::string zoneFolder; // the zone's .luz folder relative to res/maps/, with its trailing '/'
std::string id; // unique per change: letters, digits and '-' only
std::string who;
std::string note;
int64_t now{}; // unix seconds
uint32_t keepBackups = 20;
};
// The folder a zone_override_dir value names: as written when absolute, else under the server's own folder.
// Empty for an empty value (the feature is off).
std::filesystem::path RootFromSetting(const std::string& value);
// Whether text can be a set or backup id: 1 to 64 of a-z, 0-9 and '-'
bool ValidId(std::string_view text);
// Where a transport may put a set's files while it is checked: <root>/staging/<id>/upload (made empty)
std::filesystem::path MakeUploadDir(const std::filesystem::path& root, const std::string& id);
// Removes everything of <root>/staging/<id>
void RemoveStaging(const std::filesystem::path& root, const std::string& id);
// Removes every staging folder (left behind when a server stopped mid-way); call before any set is taken
void RemoveAllStaging(const std::filesystem::path& root);
// Replace the zone's set with these files. Throws StoreError with nothing changed when it can't.
Applied Apply(const std::filesystem::path& root, const Change& change, const std::vector<NewFile>& files);
// Replace the zone's set with a backup's. Throws StoreError (also for an unknown backup).
Applied Restore(const std::filesystem::path& root, const Change& change, const std::string& backupId);
// Take the zone's set away, so the client's files are read again
Applied Clear(const std::filesystem::path& root, const Change& change);
std::vector<Entry> Current(const std::filesystem::path& root, uint32_t zoneId, const std::string& zoneFolder);
// Newest first
std::vector<Backup> Backups(const std::filesystem::path& root, uint32_t zoneId);
}

394
dCommon/ZonePublish.cpp Normal file
View File

@@ -0,0 +1,394 @@
#include "ZonePublish.h"
#include <algorithm>
#include <array>
#include <fstream>
#include <map>
#include <sstream>
#include <openssl/evp.h>
#include "LevelFile.h"
#include "Raw.h"
#include "ZoneFile.h"
#include "eTriggerCommandType.h"
#include "eTriggerEventType.h"
#include "tinyxml2.h"
namespace {
using namespace ZonePublish;
std::string Hex(const unsigned char* bytes, size_t size) {
static constexpr char digits[] = "0123456789abcdef";
std::string out;
out.reserve(size * 2);
for (size_t i = 0; i < size; ++i) {
out += digits[bytes[i] >> 4];
out += digits[bytes[i] & 15];
}
return out;
}
void Add(File& file, std::string code, std::string message, std::optional<uint64_t> offset = std::nullopt) {
file.ok = false;
if (file.errors.size() >= MAX_ERRORS_PER_FILE) return;
file.errors.push_back({ std::move(code), std::move(message), offset });
}
std::optional<std::string> ReadWhole(const std::filesystem::path& path, uint64_t maxBytes) {
std::ifstream in(path, std::ios::binary);
if (!in) return std::nullopt;
in.seekg(0, std::ios::end);
const auto size = in.tellg();
if (size < 0 || static_cast<uint64_t>(size) > maxBytes) return std::nullopt;
in.seekg(0);
std::string data(static_cast<size_t>(size), '\0');
in.read(data.data(), static_cast<std::streamsize>(data.size()));
if (in.gcount() != static_cast<std::streamsize>(data.size())) return std::nullopt;
return data;
}
// Where a reader stopped, for the error
std::optional<uint64_t> Position(std::istream& stream) {
stream.clear();
const auto position = stream.tellg();
if (position < 0) return std::nullopt;
return static_cast<uint64_t>(position);
}
std::string Stem(const std::string& path) {
const auto dot = path.find_last_of('.');
const auto slash = path.find_last_of('/');
if (dot == std::string::npos || (slash != std::string::npos && dot < slash)) return path;
return path.substr(0, dot);
}
// The .luz by itself: read to its end, in a version the server takes. false when its names can't be looked at.
bool ReadZoneFile(File& file, const std::string& data, ZoneFile& luz) {
std::istringstream stream(data);
try {
luz.Read(stream);
} catch (const std::exception&) {
Add(file, "parse_failed", "The zone file ends before everything in it was read", Position(stream));
return false;
}
file.version = static_cast<uint32_t>(luz.fileFormatVersion);
if (stream.fail()) {
Add(file, "parse_failed", "The zone file ends before everything in it was read", Position(stream));
return false;
}
if (luz.fileFormatVersion < ZoneFile::FileFormatVersion::PrePreAlpha || luz.fileFormatVersion > ZoneFile::FileFormatVersion::Latest) {
Add(file, "unsupported_version", "Zone file version " + std::to_string(file.version) + " is not one the server takes (30 to 41)");
return false;
}
if (luz.pathsRefused) Add(file, "parse_failed", "The zone file's paths can't be read");
if (stream.peek() != std::char_traits<char>::eof()) {
Add(file, "parse_failed", "The zone file has more in it after its last part", Position(stream));
}
return true;
}
/**
* The .luz: read to its end, a version the server reads, every scene one the zone has in the client files, and a
* terrain file in the zone's folder. Fills the names the rest of the set may have.
*/
void CheckZoneFile(File& file, const std::string& data, const Zone& zone, const std::string& folder,
std::set<std::string>& sceneFiles, std::string& terrain) {
ZoneFile luz;
if (!ReadZoneFile(file, data, luz)) return;
if (luz.scenes.empty()) Add(file, "scene_mismatch", "The zone file lists no scenes");
for (const auto& scene : luz.scenes) {
if (NameProblem(scene.filename) || !Canonical(scene.filename).ends_with(".lvl")) {
Add(file, "scene_mismatch", "Scene " + std::to_string(scene.id) + " names a file that is not a scene file in the zone's folder");
continue;
}
const auto path = folder + Canonical(scene.filename);
if (!zone.clientScenes.contains(path)) {
Add(file, "scene_mismatch", "Scene file " + path + " is not one of this zone's scenes");
continue;
}
sceneFiles.insert(path);
}
if (NameProblem(luz.zoneRawPath) || !Canonical(luz.zoneRawPath).ends_with(".raw")) {
Add(file, "terrain_mismatch", "The zone file names a terrain file that is not a .raw in the zone's folder");
} else {
terrain = folder + Canonical(luz.zoneRawPath);
}
}
void CheckLevelFile(File& file, const std::string& data, uint32_t& objects, uint32_t maxObjects) {
std::istringstream stream(data);
LevelFile level;
try {
level.Read(stream);
} catch (const std::exception&) {
file.version = level.version;
return Add(file, "parse_failed", "The scene file ends before everything in it was read", Position(stream));
}
file.version = level.version;
if (stream.fail()) return Add(file, "parse_failed", "The scene file ends before everything in it was read", Position(stream));
if (level.version > 48) return Add(file, "unsupported_version", "Scene file version " + std::to_string(level.version) + " is newer than the server reads");
for (const auto& object : level.objects) {
for (const auto& [key, value] : object.settings) {
if (!key.starts_with(u"FAILED_TO_PARSE_")) continue;
Add(file, "bad_object_config", "Object " + std::to_string(object.id) + " (LOT " + std::to_string(object.lot) + ") has a setting that can't be read");
break;
}
}
objects += static_cast<uint32_t>(std::min<size_t>(level.objects.size(), UINT32_MAX - objects));
if (objects > maxObjects) {
Add(file, "too_many_objects", "The set has more than " + std::to_string(maxObjects) + " objects in its scenes");
}
}
void CheckTerrainFile(File& file, const std::string& data) {
std::istringstream stream(data);
Raw::Raw raw;
const bool read = Raw::ReadRaw(stream, raw);
file.version = raw.version;
if (!read || stream.fail()) return Add(file, "parse_failed", "The terrain file can't be read to its end", Position(stream));
// A terrain file the game uses has its chunks in it, and nothing after them
if (raw.dev != 0 || raw.chunks.empty()) return Add(file, "parse_failed", "The terrain file has no terrain in it");
if (stream.peek() != std::char_traits<char>::eof()) Add(file, "parse_failed", "The terrain file has more in it after its last chunk", Position(stream));
}
}
namespace ZonePublish {
std::string FolderOf(std::string_view luzPath) {
const auto slash = luzPath.find_last_of('/');
return slash == std::string_view::npos ? std::string() : std::string(luzPath.substr(0, slash + 1));
}
std::string Canonical(std::string_view name) {
std::string out(name);
for (auto& c : out) {
if (c >= 'A' && c <= 'Z') c = static_cast<char>(c - 'A' + 'a');
}
return out;
}
std::optional<std::string> NameProblem(std::string_view name) {
if (name.empty()) return "The name is empty";
if (name.size() > MAX_NAME_LENGTH) return "The name is longer than " + std::to_string(MAX_NAME_LENGTH) + " characters";
if (name.find('\0') != std::string_view::npos) return "The name has a NUL in it";
if (name.find('\\') != std::string_view::npos) return "The name has a backslash in it; use forward slashes";
if (name.front() == '/') return "The name is an absolute path";
for (const char c : name) {
const auto byte = static_cast<unsigned char>(c);
if (byte < 0x20 || byte >= 0x7F) return "The name has a character in it that is not printable ASCII";
if (std::string_view(":*?\"<>|").find(c) != std::string_view::npos) return "The name has a character in it that file names can't have";
}
size_t start = 0;
while (start <= name.size()) {
const auto end = std::min(name.find('/', start), name.size());
const auto part = name.substr(start, end - start);
if (part.empty()) return "The name has an empty part";
if (part == "." || part == "..") return "The name has a \".\" or \"..\" part";
if (part.front() == ' ' || part.back() == ' ' || part.back() == '.') return "A part of the name starts or ends with a space, or ends with a dot";
start = end + 1;
}
return std::nullopt;
}
std::optional<std::filesystem::path> ResolveInside(const std::filesystem::path& root, const std::string& relative) {
if (root.empty() || NameProblem(relative)) return std::nullopt;
std::error_code code;
const auto base = std::filesystem::weakly_canonical(root, code);
if (code) return std::nullopt;
const auto target = std::filesystem::weakly_canonical(base / relative, code);
if (code) return std::nullopt;
const auto inside = target.lexically_relative(base);
if (inside.empty() || inside.native().starts_with("..") || inside.is_absolute()) return std::nullopt;
return target;
}
std::string Sha256(std::string_view bytes) {
std::array<unsigned char, EVP_MAX_MD_SIZE> digest{};
unsigned int size = 0;
EVP_Digest(bytes.data(), bytes.size(), digest.data(), &size, EVP_sha256(), nullptr);
return Hex(digest.data(), size);
}
std::optional<std::string> Sha256File(const std::filesystem::path& file) {
std::ifstream in(file, std::ios::binary);
if (!in) return std::nullopt;
auto* context = EVP_MD_CTX_new();
if (!context) return std::nullopt;
EVP_DigestInit_ex(context, EVP_sha256(), nullptr);
std::array<char, 65536> buffer{};
while (in) {
in.read(buffer.data(), static_cast<std::streamsize>(buffer.size()));
if (in.gcount() > 0) EVP_DigestUpdate(context, buffer.data(), static_cast<size_t>(in.gcount()));
}
std::array<unsigned char, EVP_MAX_MD_SIZE> digest{};
unsigned int size = 0;
EVP_DigestFinal_ex(context, digest.data(), &size);
EVP_MD_CTX_free(context);
if (in.bad()) return std::nullopt;
return Hex(digest.data(), size);
}
std::vector<Error> CheckTriggers(std::string_view xml) {
std::vector<Error> errors;
const auto add = [&errors](std::string code, std::string message) {
if (errors.size() < MAX_ERRORS_PER_FILE) errors.push_back({ std::move(code), std::move(message), std::nullopt });
};
// An empty file has no triggers (the world reads it as none)
if (xml.empty()) return errors;
tinyxml2::XMLDocument doc;
if (doc.Parse(xml.data(), xml.size()) != tinyxml2::XML_SUCCESS) {
add("parse_failed", "The triggers file is not well formed XML (line " + std::to_string(doc.ErrorLineNum()) + ")");
return errors;
}
const auto* triggers = doc.FirstChildElement("triggers");
if (!triggers) {
add("parse_failed", "The triggers file has no <triggers> element");
return errors;
}
for (const auto* trigger = triggers->FirstChildElement("trigger"); trigger; trigger = trigger->NextSiblingElement("trigger")) {
uint32_t id = 0;
if (trigger->QueryUnsignedAttribute("id", &id) != tinyxml2::XML_SUCCESS) add("parse_failed", "A trigger has no numeric id");
const auto where = "Trigger " + std::to_string(id);
for (const auto* event = trigger->FirstChildElement("event"); event; event = event->NextSiblingElement("event")) {
const char* eventId = event->Attribute("id");
if (!eventId || TriggerEventType::StringToTriggerEventType(eventId) == eTriggerEventType::INVALID) {
add("unknown_trigger_event", where + " has an event the server doesn't know: " + std::string(eventId ? eventId : "(no id)"));
}
for (const auto* command = event->FirstChildElement("command"); command; command = command->NextSiblingElement("command")) {
const char* commandId = command->Attribute("id");
if (!commandId || TriggerCommandType::StringToTriggerCommandType(commandId) == eTriggerCommandType::INVALID) {
add("unknown_trigger_command", where + " has a command the server doesn't know: " + std::string(commandId ? commandId : "(no id)"));
}
const char* target = command->Attribute("target");
if (!target || !*target) {
add("trigger_missing_target", where + " has a " + std::string(commandId ? commandId : "command") + " command without a target");
}
}
}
}
return errors;
}
std::vector<Error> CheckContents(std::string_view name, const std::string& data, uint32_t* version) {
File file;
uint32_t objects = 0;
const auto lower = Canonical(name);
if (lower.ends_with(".luz")) {
ZoneFile luz;
ReadZoneFile(file, data, luz);
} else if (lower.ends_with(".lvl")) CheckLevelFile(file, data, objects, UINT32_MAX);
else if (lower.ends_with(".raw")) CheckTerrainFile(file, data);
else if (lower.ends_with(".lutriggers")) file.errors = CheckTriggers(data);
if (version) *version = file.version;
return file.errors;
}
Result Validate(const Zone& zone, const std::vector<Upload>& uploads, const Limits& limits) {
Result result;
const auto luzPath = Canonical(zone.luzPath);
const auto folder = FolderOf(luzPath);
result.files.reserve(uploads.size() + 1);
// Names first: what each file is called, twice the same, and whether the .luz is there
std::map<std::string, size_t> byPath;
std::optional<size_t> luzIndex;
uint64_t total = 0;
for (size_t i = 0; i < uploads.size(); ++i) {
const auto& upload = uploads[i];
File file;
file.path = upload.name;
file.bytes = upload.bytes;
file.source = upload.file;
total += upload.bytes;
if (i >= limits.maxFiles) {
Add(file, "too_large", "The set has more than " + std::to_string(limits.maxFiles) + " files");
} else if (const auto problem = NameProblem(upload.name)) {
Add(file, "name_not_allowed", *problem);
} else {
file.path = Canonical(upload.name);
if (!file.path.starts_with(folder) || file.path.size() == folder.size()) {
Add(file, "name_not_allowed", "The file is not in the zone's folder (" + folder + ")");
} else if (!byPath.try_emplace(file.path, i).second) {
Add(file, "duplicate", "The set has this file more than once");
} else if (file.path == luzPath) {
luzIndex = i;
}
if (upload.bytes > limits.maxFileBytes) {
Add(file, "too_large", "The file is larger than " + std::to_string(limits.maxFileBytes) + " bytes");
} else if (file.ok && total > limits.maxTotalBytes) {
Add(file, "too_large", "The set is larger than " + std::to_string(limits.maxTotalBytes) + " bytes");
}
}
result.files.push_back(std::move(file));
}
// The .luz says which other names the set may have
std::set<std::string> sceneFiles;
std::string terrain;
bool luzRead = false;
if (!luzIndex) {
File missing;
missing.path = luzPath;
Add(missing, "missing_luz", "The set has no " + luzPath + "; every set has the zone's .luz");
result.files.push_back(std::move(missing));
} else if (auto& file = result.files[*luzIndex]; file.ok) {
const auto data = ReadWhole(file.source, limits.maxFileBytes);
if (!data) {
Add(file, "parse_failed", "The file could not be read");
} else {
file.sha256 = Sha256(*data);
CheckZoneFile(file, *data, zone, folder, sceneFiles, terrain);
luzRead = file.ok;
}
}
std::set<std::string> allowed{ luzPath };
if (!terrain.empty()) allowed.insert(terrain);
for (const auto& scene : sceneFiles) {
allowed.insert(scene);
allowed.insert(Stem(scene) + ".lutriggers");
}
for (size_t i = 0; i < result.files.size(); ++i) {
auto& file = result.files[i];
if (!file.ok || (luzIndex && i == *luzIndex) || file.source.empty()) continue;
if (!luzRead) {
// Without a good .luz nothing says which files belong; only the plainly wrong ones are named
const bool known = file.path.ends_with(".lvl") || file.path.ends_with(".lutriggers") || file.path.ends_with(".raw");
if (!known) Add(file, "name_not_allowed", "Only the zone's .luz, its scenes' .lvl and .lutriggers files and its .raw can be sent");
continue;
}
if (!allowed.contains(file.path)) {
if (file.path.ends_with(".raw")) Add(file, "terrain_mismatch", "The zone file names another terrain file (" + terrain + ")");
else Add(file, "name_not_allowed", "The zone file doesn't list this file: only its scenes' .lvl and .lutriggers files and its .raw can be sent");
continue;
}
const auto data = ReadWhole(file.source, limits.maxFileBytes);
if (!data) {
Add(file, "parse_failed", "The file could not be read");
continue;
}
file.sha256 = Sha256(*data);
if (file.path.ends_with(".lvl")) CheckLevelFile(file, *data, result.objects, limits.maxObjects);
else if (file.path.ends_with(".lutriggers")) {
for (auto& error : CheckTriggers(*data)) Add(file, std::move(error.code), std::move(error.message), error.offset);
} else CheckTerrainFile(file, *data);
}
// What the .luz names and the set leaves out has to be there already
if (luzRead) {
auto& luz = result.files[*luzIndex];
const auto there = [&](const std::string& path) { return byPath.contains(path) || (zone.hasFile && zone.hasFile(path)); };
for (const auto& scene : sceneFiles) {
if (!there(scene)) Add(luz, "scene_mismatch", "Scene file " + scene + " is not in the set and the zone doesn't have it");
}
if (!terrain.empty() && !there(terrain)) Add(luz, "terrain_mismatch", "Terrain file " + terrain + " is not in the set and the zone doesn't have it");
}
for (const auto& file : result.files) result.ok = result.ok && file.ok;
return result;
}
}

108
dCommon/ZonePublish.h Normal file
View File

@@ -0,0 +1,108 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <functional>
#include <optional>
#include <set>
#include <string>
#include <string_view>
#include <vector>
/**
* Publishing edited zone files (docs/ZonePublish.md): checking a set of files an external level editor sent for one
* zone before any of it is kept. Nothing here knows how the files arrived (the dashboard's API, or any other way in)
* or who sent them; it reads the files it is given and says, file by file, what is wrong with them. Applying a set
* that passed is ZoneOverrideStore's, who may send one is ZoneEditLock's.
*
* Reads files and nothing else (no settings, no database, no logging of its own), so it can run on a worker thread.
*/
namespace ZonePublish {
constexpr size_t MAX_NAME_LENGTH = 200;
// Errors listed per file; a file with more is cut off there
constexpr size_t MAX_ERRORS_PER_FILE = 20;
struct Limits {
uint64_t maxFileBytes = 64ULL * 1024 * 1024;
uint64_t maxTotalBytes = 256ULL * 1024 * 1024;
uint32_t maxFiles = 128;
uint32_t maxObjects = 100000; // in every scene of the set together
};
// The zone a set is for, as the server knows it
struct Zone {
uint32_t zoneId{};
// The zone's .luz from the ZoneTable, relative to res/maps/, lowercase with forward slashes
std::string luzPath;
// The scene files the zone has in the server's client files, as luzPath is written; a set can't add scenes
std::set<std::string> clientScenes;
// Whether a file the set leaves out is there for the zone already (an earlier set's, or the client's)
std::function<bool(const std::string& path)> hasFile;
};
// One file of a set: its name as sent, and where its bytes are
struct Upload {
std::string name;
std::filesystem::path file;
uint64_t bytes{};
};
struct Error {
std::string code;
std::string message;
std::optional<uint64_t> offset;
};
struct File {
std::string path; // as sent when it was refused by name, else relative to res/maps/ as it is kept
bool ok = true;
std::vector<Error> errors;
uint64_t bytes{};
std::string sha256;
uint32_t version{}; // the file's own format version (0 for triggers)
std::filesystem::path source; // the upload it came from
};
struct Result {
bool ok = true;
std::vector<File> files; // every file sent, in the order sent; then a missing .luz
uint32_t objects{};
};
// A zone's folder: its .luz path up to and including the last '/'
std::string FolderOf(std::string_view luzPath);
// Lowercase with forward slashes kept: how a path is compared and kept. ASCII only (other bytes never pass NameProblem).
std::string Canonical(std::string_view name);
/**
* What is wrong with a file name as sent, by its spelling alone; nullopt when nothing is. A name is a path relative
* to res/maps/: no empty, "." or ".." part, no leading '/', no backslash, drive or NUL, printable ASCII only, and
* no part that starts or ends with a space or ends with a dot.
*/
std::optional<std::string> NameProblem(std::string_view name);
/**
* The path a file of this zone is written to under root, when that path is inside root once every link and
* ".." on the way is resolved (std::filesystem::weakly_canonical). nullopt otherwise.
*/
std::optional<std::filesystem::path> ResolveInside(const std::filesystem::path& root, const std::string& relative);
/**
* Checks a whole set. Every file is read with the server's own readers; a set is good only when every file is
* (Result::ok), and nothing is written here either way.
*/
Result Validate(const Zone& zone, const std::vector<Upload>& uploads, const Limits& limits);
// What a .lutriggers file has wrong with it (parse_failed, unknown_trigger_event, unknown_trigger_command,
// trigger_missing_target)
std::vector<Error> CheckTriggers(std::string_view xml);
// One file's own checks, picked by what its name ends in (.luz, .lvl, .lutriggers, .raw): whether it reads to its
// end in a version the server takes. What a .luz names is Validate's to check.
std::vector<Error> CheckContents(std::string_view name, const std::string& data, uint32_t* version = nullptr);
// SHA-256 of a file as lowercase hex; nullopt when it can't be read
std::optional<std::string> Sha256File(const std::filesystem::path& file);
std::string Sha256(std::string_view bytes);
}