feat(zone): zone files as the game client's level editor packs them

ZoneFilePack reads and writes the editor's packed files: named entries whose data is
sd0, one entry for a single file or the zone, terrain, scenes and triggers of a whole
map. Every length is checked, data that is not sd0 or has a chunk over 256 KiB is
refused, and only a name's base name is used. ZoneFilePush makes a set of what was
sent (one file replaces that file in the zone's set), checks it and applies it like
any other set.

The override store links a set's files instead of moving them, so a file the zone
already has can be carried into its next set, and one change at a time is made in an
override folder whichever server makes it. The asset manager can read the client's
own file past an override.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-11-10 19:11:29 -06:00
parent dbe0722095
commit e50a2255f8
13 changed files with 880 additions and 17 deletions

View File

@@ -39,6 +39,8 @@ set(DCOMMON_SOURCES
"ZoneScenes.cpp"
"ZonePublish.cpp"
"ZoneOverrideStore.cpp"
"ZoneFilePack.cpp"
"ZoneFilePush.cpp"
"Process.cpp"
"Permissions.cpp"
"PermissionGrants.cpp"

View File

@@ -68,6 +68,27 @@ namespace ZoneEditLock {
return eResult::OK;
}
/**
* Reading the zone's files as they are on the server: an account at the level that is one of the zone's
* editors, or an admin. OK, LEVEL_TOO_LOW or NOT_ASSIGNED; the lock is not needed to look.
*/
inline eResult Read(const Actor& actor, const Levels& levels) {
if (actor.accountId == 0 || actor.gmLevel < levels.publish) return eResult::LEVEL_TOO_LOW;
if (!actor.assigned && !IsAdmin(actor, levels)) return eResult::NOT_ASSIGNED;
return eResult::OK;
}
/**
* Change, for a requester that is only ever told 0 to 3: an account that could take the lock and hasn't is
* told LOCKED_BY_OTHER (by nobody: it has to lock the zone first), any other account without the lock what
* keeps it from taking it.
*/
inline eResult ChangeOrWhyNot(const Actor& actor, std::optional<uint32_t> holder, const Levels& levels) {
const auto result = Change(actor, holder, levels);
if (result != eResult::NOT_ASSIGNED) return result;
return Read(actor, levels) == eResult::OK ? eResult::LOCKED_BY_OTHER : eResult::NOT_ASSIGNED;
}
// What happens to the zone's running instances after its files changed
enum class eReload : uint8_t {
SKIPPED, // not asked for: the files are read when an instance next starts

183
dCommon/ZoneFilePack.cpp Normal file
View File

@@ -0,0 +1,183 @@
#include "ZoneFilePack.h"
#include <cstring>
#include "Sd0.h"
#include "ZCompression.h"
namespace {
using namespace ZoneFilePack;
constexpr std::string_view SD0_HEADER("sd0\x01\xff", 5);
// A chunk's compressed bytes are never much more than what it inflates to
constexpr uint32_t MAX_COMPRESSED_CHUNK = Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE + 1024;
struct Reader {
std::string_view bytes;
size_t at = 0;
size_t Left() const { return bytes.size() - at; }
template<typename T>
bool Read(T& value) {
if (Left() < sizeof(T)) return false;
std::memcpy(&value, bytes.data() + at, sizeof(T));
at += sizeof(T);
return true;
}
bool Take(size_t count, std::string_view& out) {
if (Left() < count) return false;
out = bytes.substr(at, count);
at += count;
return true;
}
};
eError ReadEntry(Reader& reader, const Limits& limits, uint64_t& total, Entry& entry) {
uint32_t nameBytes = 0, dataBytes = 0;
if (!reader.Read(nameBytes) || !reader.Read(dataBytes)) return eError::TRUNCATED;
std::string_view name, data;
if (!reader.Take(nameBytes, name) || !reader.Take(dataBytes, data)) return eError::TRUNCATED;
// UTF-16LE with its null: at least the null, an even length
if (nameBytes < 2 || nameBytes % 2 != 0 || nameBytes > 2 * 1024) return eError::BAD_NAME;
std::string text;
for (size_t i = 0; i + 1 < name.size(); i += 2) {
const auto low = static_cast<unsigned char>(name[i]);
const auto high = static_cast<unsigned char>(name[i + 1]);
const bool last = i + 2 == name.size();
if (last) {
if (low != 0 || high != 0) return eError::BAD_NAME;
break;
}
if (high != 0 || low == 0) return eError::BAD_NAME;
text += static_cast<char>(low);
}
const auto base = BaseName(text);
if (!base) return eError::BAD_NAME;
entry.name = *base;
const auto room = limits.maxTotalBytes > total ? limits.maxTotalBytes - total : 0;
const auto error = Inflate(data, std::min(limits.maxFileBytes, room), entry.data);
total += entry.data.size();
return error;
}
void WriteEntry(std::string& out, const Entry& entry) {
const auto data = Sd0::Compress(entry.data);
const auto nameBytes = static_cast<uint32_t>((entry.name.size() + 1) * 2);
const auto dataBytes = static_cast<uint32_t>(data.size());
out.append(reinterpret_cast<const char*>(&nameBytes), sizeof(nameBytes));
out.append(reinterpret_cast<const char*>(&dataBytes), sizeof(dataBytes));
for (const char c : entry.name) {
out += c;
out += '\0';
}
out.append(2, '\0');
out += data;
}
}
namespace ZoneFilePack {
const char* Describe(eError error) {
switch (error) {
case eError::NONE: return "nothing";
case eError::TRUNCATED: return "a length in it runs past its end, or bytes are left over";
case eError::BAD_NAME: return "a file name in it can't be used";
case eError::NOT_SD0: return "a file in it is not sd0";
case eError::BAD_CHUNK: return "a file in it has a chunk that can't be inflated or is too large";
case eError::TOO_LARGE: return "it is larger than this server takes";
case eError::INCOMPLETE: return "a whole map needs its zone, its terrain and every scene";
}
return "unknown";
}
std::optional<std::string> BaseName(std::string_view name) {
const auto slash = name.find_last_of("/\\");
const auto base = slash == std::string_view::npos ? name : name.substr(slash + 1);
if (base.empty() || base == "." || base == ".." || base.size() > 128) return std::nullopt;
for (const char c : base) {
const auto byte = static_cast<unsigned char>(c);
if (byte < 0x20 || byte >= 0x7F || std::string_view(":*?\"<>|").find(c) != std::string_view::npos) return std::nullopt;
}
if (base.front() == ' ' || base.back() == ' ' || base.back() == '.') return std::nullopt;
return std::string(base);
}
eError Inflate(std::string_view sd0, uint64_t maxBytes, std::string& out) {
out.clear();
if (!sd0.starts_with(SD0_HEADER)) return eError::NOT_SD0;
Reader reader{ sd0, SD0_HEADER.size() };
std::string chunk(Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE, '\0');
while (reader.Left() > 0) {
uint32_t size = 0;
std::string_view compressed;
if (!reader.Read(size)) return eError::TRUNCATED;
if (size > MAX_COMPRESSED_CHUNK) return eError::BAD_CHUNK;
if (!reader.Take(size, compressed)) return eError::TRUNCATED;
int32_t zlibError = 0;
// A chunk that inflates to more than the room it is given fails here
const auto inflated = ZCompression::Decompress(reinterpret_cast<const uint8_t*>(compressed.data()), size,
reinterpret_cast<uint8_t*>(chunk.data()), static_cast<uint32_t>(chunk.size()), zlibError);
if (inflated < 0 || zlibError < 0) return eError::BAD_CHUNK;
if (out.size() + static_cast<uint64_t>(inflated) > maxBytes) return eError::TOO_LARGE;
out.append(chunk.data(), static_cast<size_t>(inflated));
}
return eError::NONE;
}
eError Unpack(eFileType type, std::string_view body, const Limits& limits, Files& out) {
out = {};
Reader reader{ body };
uint64_t total = 0;
const auto entry = [&](Entry& target) { return ReadEntry(reader, limits, total, target); };
if (type != eFileType::WHOLE_MAP) {
Entry single;
if (const auto error = entry(single); error != eError::NONE) return error;
if (reader.Left() != 0) return eError::TRUNCATED;
switch (type) {
case eFileType::SCENE: out.scenes.push_back(std::move(single)); break;
case eFileType::TERRAIN: out.terrain = std::move(single); break;
case eFileType::ZONE: out.zone = std::move(single); break;
case eFileType::TRIGGERS: out.triggers.push_back(std::move(single)); break;
default: return eError::TRUNCATED;
}
return eError::NONE;
}
out.zone.emplace();
if (const auto error = entry(*out.zone); error != eError::NONE) return error;
out.terrain.emplace();
if (const auto error = entry(*out.terrain); error != eError::NONE) return error;
for (auto* list : { &out.scenes, &out.triggers }) {
uint16_t count = 0;
if (!reader.Read(count)) return eError::TRUNCATED;
if (out.Count() + count > limits.maxFiles) return eError::TOO_LARGE;
for (uint16_t i = 0; i < count; ++i) {
if (const auto error = entry(list->emplace_back()); error != eError::NONE) return error;
}
}
if (reader.Left() != 0) return eError::TRUNCATED;
// As the client takes a whole map: only with a zone, a terrain and every scene having data
if (out.zone->data.empty() || out.terrain->data.empty()) return eError::INCOMPLETE;
for (const auto& scene : out.scenes) if (scene.data.empty()) return eError::INCOMPLETE;
return eError::NONE;
}
std::string Pack(eFileType type, const Files& files) {
std::string out;
switch (type) {
case eFileType::SCENE: if (!files.scenes.empty()) WriteEntry(out, files.scenes.front()); return out;
case eFileType::TERRAIN: if (files.terrain) WriteEntry(out, *files.terrain); return out;
case eFileType::ZONE: if (files.zone) WriteEntry(out, *files.zone); return out;
case eFileType::TRIGGERS: if (!files.triggers.empty()) WriteEntry(out, files.triggers.front()); return out;
case eFileType::WHOLE_MAP: break;
}
if (!files.zone || !files.terrain) return out;
WriteEntry(out, *files.zone);
WriteEntry(out, *files.terrain);
for (const auto* list : { &files.scenes, &files.triggers }) {
const auto count = static_cast<uint16_t>(list->size());
out.append(reinterpret_cast<const char*>(&count), sizeof(count));
for (const auto& entry : *list) WriteEntry(out, entry);
}
return out;
}
}

81
dCommon/ZoneFilePack.h Normal file
View File

@@ -0,0 +1,81 @@
#pragma once
#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
/**
* A zone's files as the game client's level editor sends and takes them in one packet (docs/ZonePublish.md): a
* list of named entries whose data is sd0 (zlib chunks of at most 256 KiB each).
*
* entry: u32 nameBytes (UTF-16LE with its null, so (characters + 1) * 2), u32 dataBytes (as compressed),
* the name, the data. No padding.
* whole map: the zone (.luz) entry, the terrain (.raw) entry, u16 N and N scene (.lvl) entries, u16 M and M
* trigger (.lutriggers) entries
* one file: exactly one entry (a scene, the terrain, the zone, or a scene's triggers)
*
* There is no checksum and no total size. A name is only ever a base name here: the folder part of what was sent is
* dropped, as the client does with what it is sent, and a name is never used as a path. Pure: bytes in, bytes out,
* with every length checked against what is left; it runs on a worker thread.
*/
namespace ZoneFilePack {
enum class eFileType : uint8_t {
WHOLE_MAP = 0,
SCENE = 1,
TERRAIN = 2,
ZONE = 3,
TRIGGERS = 4,
};
inline bool ValidType(uint8_t type) { return type <= static_cast<uint8_t>(eFileType::TRIGGERS); }
struct Entry {
std::string name; // the base name, as sent (ASCII)
std::string data; // uncompressed
};
struct Files {
std::optional<Entry> zone;
std::optional<Entry> terrain;
std::vector<Entry> scenes;
std::vector<Entry> triggers;
size_t Count() const { return (zone ? 1 : 0) + (terrain ? 1 : 0) + scenes.size() + triggers.size(); }
};
struct Limits {
uint64_t maxFileBytes = 64ULL * 1024 * 1024; // one file, uncompressed
uint64_t maxTotalBytes = 256ULL * 1024 * 1024; // all files, uncompressed
uint32_t maxFiles = 128;
};
enum class eError : uint8_t {
NONE,
TRUNCATED, // a length runs past the end, or bytes are left over
BAD_NAME, // not UTF-16 with its null, not ASCII, or no usable base name (empty, ".", "..")
NOT_SD0, // an entry's data does not start with the sd0 header
BAD_CHUNK, // a chunk that is not a zlib stream, or one that inflates to more than 256 KiB
TOO_LARGE, // over a limit
INCOMPLETE, // a whole map without its zone, its terrain or a scene's data
};
const char* Describe(eError error);
// The base name of a sent name: what follows its last '/' or '\'. nullopt for one that is empty, "." or "..",
// or has a character that is not printable ASCII or that file names can't have.
std::optional<std::string> BaseName(std::string_view name);
// sd0 to bytes, strictly: the header, then chunks that each inflate to at most 256 KiB, at most maxBytes in all
eError Inflate(std::string_view sd0, uint64_t maxBytes, std::string& out);
// What follows the file type and level file ID in a push (or a pull response)
eError Unpack(eFileType type, std::string_view body, const Limits& limits, Files& out);
/**
* The same bytes from files, their data compressed to sd0. For one file the entry is the one of its kind (the
* first scene or trigger entry); empty when it is missing.
*/
std::string Pack(eFileType type, const Files& files);
}

86
dCommon/ZoneFilePush.cpp Normal file
View File

@@ -0,0 +1,86 @@
#include "ZoneFilePush.h"
#include <fstream>
#include <vector>
#include "ZoneOverrideStore.h"
namespace {
std::string BaseOf(const std::string& path) {
const auto slash = path.find_last_of('/');
return slash == std::string::npos ? path : path.substr(slash + 1);
}
}
ZoneFilePush::Result ZoneFilePush::Run(const Job& job) {
Result result;
const auto zoneFolder = ZonePublish::FolderOf(job.luzPath);
try {
ZoneFilePack::Files files;
const ZoneFilePack::Limits packLimits{ job.limits.maxFileBytes, job.limits.maxTotalBytes, job.limits.maxFiles };
if (const auto error = ZoneFilePack::Unpack(job.fileType, job.packedFiles, packLimits, files); error != ZoneFilePack::eError::NONE) {
result.problem = std::string("the files could not be unpacked: ") + ZoneFilePack::Describe(error);
return result;
}
const auto folder = ZoneOverrideStore::MakeUploadDir(job.root, job.id);
std::vector<ZonePublish::Upload> uploads;
std::set<std::string> sent;
uint32_t index = 0;
// A name is a base name here; it becomes a file of this zone's folder, and the checks decide whether the
// zone has such a file. Its bytes are written under a name of our own.
const auto add = [&](const ZoneFilePack::Entry& entry) {
const auto path = zoneFolder + ZonePublish::Canonical(entry.name);
const auto file = folder / ("file-" + std::to_string(index++));
std::ofstream out(file, std::ios::binary | std::ios::trunc);
out.write(entry.data.data(), static_cast<std::streamsize>(entry.data.size()));
out.flush();
if (!out) throw ZoneOverrideStore::StoreError("A file could not be written");
uploads.push_back({ path, file, entry.data.size() });
sent.insert(path);
};
if (files.zone) add(*files.zone);
if (files.terrain) add(*files.terrain);
for (const auto& scene : files.scenes) add(scene);
for (const auto& triggers : files.triggers) add(triggers);
// One file replaces that file in the zone's set: the rest of the set stays as it is
if (job.fileType != ZoneFilePack::eFileType::WHOLE_MAP) {
for (const auto& current : ZoneOverrideStore::Current(job.root, job.zoneId, zoneFolder)) {
if (sent.contains(current.path)) continue;
const auto file = ZonePublish::ResolveInside(job.root / "maps", current.path);
if (!file) continue;
uploads.push_back({ current.path, *file, current.bytes });
sent.insert(current.path);
}
}
// Every set has the zone's .luz: the client's own when none was sent or kept
if (!sent.contains(job.luzPath) && job.clientLuz) add({ BaseOf(job.luzPath), *job.clientLuz });
ZonePublish::Zone zone;
zone.zoneId = job.zoneId;
zone.luzPath = job.luzPath;
zone.clientScenes = job.clientScenes;
zone.hasFile = [&job](const std::string& path) { return job.clientFiles.contains(path); };
const auto validation = ZonePublish::Validate(zone, uploads, job.limits);
if (!validation.ok) {
for (const auto& file : validation.files) {
if (file.ok || file.errors.empty()) continue;
result.problem = BaseOf(file.path) + ": " + file.errors.front().message + " (" + file.errors.front().code + ")";
break;
}
ZoneOverrideStore::RemoveStaging(job.root, job.id);
return result;
}
std::vector<ZoneOverrideStore::NewFile> newFiles;
for (const auto& file : validation.files) newFiles.push_back({ file.path, file.source, file.bytes, file.sha256, file.version });
const auto applied = ZoneOverrideStore::Apply(job.root, { job.zoneId, zoneFolder, job.id, job.who, "sent from the game client", job.now, job.keepBackups }, newFiles);
result.applied = true;
result.files = applied.files.size();
result.backupId = applied.backupId;
} catch (const std::exception& ex) {
result.problem = ex.what();
ZoneOverrideStore::RemoveStaging(job.root, job.id);
}
return result;
}

46
dCommon/ZoneFilePush.h Normal file
View File

@@ -0,0 +1,46 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <optional>
#include <set>
#include <string>
#include "ZoneFilePack.h"
#include "ZonePublish.h"
/**
* Taking a zone's files as the game client's level editor sends them (ZoneFilePack) into the zone's set: unpacked,
* made into a whole set, checked like any other set (ZonePublish) and applied (ZoneOverrideStore), or refused with
* nothing written. A whole map is the zone's new set. One file replaces that file in the set the zone has and keeps
* the rest; a set always has the zone's .luz, the client's own when none was sent or kept.
*
* Only reads the bytes it is given and files under the override folder: no connection, database or setting, so it
* runs on a worker thread.
*/
namespace ZoneFilePush {
struct Job {
uint32_t zoneId{};
std::string luzPath; // from the ZoneTable, relative to res/maps/, lowercase
ZoneFilePack::eFileType fileType{};
std::string packedFiles;
ZonePublish::Limits limits;
std::filesystem::path root; // the override folder
std::string id; // of this change (ZoneOverrideStore::ValidId)
std::string who;
int64_t now{};
uint32_t keepBackups = 20;
std::set<std::string> clientScenes; // the scenes the zone has in the client's files
std::set<std::string> clientFiles; // which of the files a set can leave out the client has
std::optional<std::string> clientLuz; // the client's own zone file
};
struct Result {
bool applied{};
std::string problem; // why not, for the sender
size_t files{}; // in the zone's set now
std::optional<std::string> backupId;
};
Result Run(const Job& job);
}

View File

@@ -13,6 +13,7 @@
#include <io.h>
#else
#include <fcntl.h>
#include <sys/file.h>
#include <unistd.h>
#endif
#ifdef __linux__
@@ -72,15 +73,6 @@ namespace {
if (code) Fail("Could not copy " + from.string());
}
void MoveOrCopy(const fs::path& from, const fs::path& to) {
MakeDirs(to.parent_path());
std::error_code code;
fs::rename(from, to, code);
if (!code) return;
fs::copy_file(from, to, fs::copy_options::overwrite_existing, code);
if (code) Fail("Could not copy " + from.string());
}
void WriteText(const fs::path& path, const std::string& text) {
MakeDirs(path.parent_path());
std::ofstream out(path, std::ios::binary | std::ios::trunc);
@@ -196,6 +188,27 @@ namespace {
for (size_t i = keep; i < ids.size(); ++i) fs::remove_all(backups / ids[i], code);
}
// One change at a time in an override folder, whichever server makes it (the lock goes when its holder does)
struct FolderLock {
explicit FolderLock(const fs::path& root) {
#ifndef _WIN32
m_File = ::open((root / ".lock").c_str(), O_CREAT | O_RDWR, 0644);
if (m_File < 0 || ::flock(m_File, LOCK_EX) != 0) Fail("The override folder can't be locked");
#else
(void)root;
#endif
}
~FolderLock() {
#ifndef _WIN32
if (m_File >= 0) ::close(m_File);
#endif
}
FolderLock(const FolderLock&) = delete;
FolderLock& operator=(const FolderLock&) = delete;
private:
int m_File = -1;
};
/**
* The one way a zone's set changes. `place` puts the new set's files under the folder it is given (laid out as
* under res/maps/) and returns their entries; everything else is the same for a new set, a restore and a clear.
@@ -203,6 +216,7 @@ namespace {
template<typename Place>
Applied Replace(const fs::path& root, const Change& change, Place place) {
const auto folders = Resolve(root, change.zoneId, change.zoneFolder, change.id);
const FolderLock lock(folders.root);
const auto incomingMaps = folders.staging / "incoming";
const auto incoming = incomingMaps / folders.folder;
std::error_code code;
@@ -322,7 +336,8 @@ namespace ZoneOverrideStore {
if (ZonePublish::Canonical(file.path) != file.path || !seen.insert(file.path).second) Fail("Not a usable file name: " + file.path);
const auto target = ZonePublish::ResolveInside(incomingMaps, file.path);
if (!target) Fail("Not a usable file name: " + file.path);
MoveOrCopy(file.source, *target);
// Linked, never moved: a file may be one the zone has in use already, carried over into the new set
LinkOrCopy(file.source, *target);
entries.push_back({ file.path, file.bytes, file.sha256, file.version, change.who, change.now });
}
return entries;

View File

@@ -20,8 +20,9 @@
* swapped in with a rename; the set it replaces goes to backups first. Restoring a backup and clearing a zone's set
* are the same swap. Knows nothing of who may do this (ZoneEditLock) or how files arrive.
*
* Only reads and writes files under root, and takes no lock of its own: callers apply one set at a time. No
* settings, database or logging, so it can run on a worker thread.
* Only reads and writes files under root. Changes in one override folder happen one at a time, whichever server
* makes them (a file lock, where the system has one). No settings, database or logging, so it can run on a worker
* thread.
*/
namespace ZoneOverrideStore {
struct StoreError : std::runtime_error {
@@ -45,7 +46,7 @@ namespace ZoneOverrideStore {
std::vector<Entry> files;
};
// A file of a new set: where its bytes are now (moved or copied into the store)
// A file of a new set: where its bytes are now (linked or copied into the store; the source stays)
struct NewFile {
std::string path;
std::filesystem::path source;

View File

@@ -126,8 +126,27 @@ std::optional<std::filesystem::path> AssetManager::GetOverridePath(std::string n
return path;
}
bool AssetManager::HasFile(std::string fixedName) const {
if (GetOverridePath(fixedName)) return true;
bool AssetManager::HasFile(std::string name) const {
return HasFile(std::move(name), true);
}
bool AssetManager::HasClientFile(const std::string& name) const {
return HasFile(name, false);
}
bool AssetManager::GetFile(std::string name, char** data, uint32_t* len) const {
return GetFile(std::move(name), data, len, true);
}
AssetStream AssetManager::GetClientFile(const std::string& name) const {
char* buf = nullptr;
uint32_t len = 0;
const bool success = GetFile(name, &buf, &len, false);
return AssetStream(buf, len, success);
}
bool AssetManager::HasFile(std::string fixedName, const bool useOverride) const {
if (useOverride && GetOverridePath(fixedName)) return true;
std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); });
// Special case for unpacked client have BrickModels in upper case
@@ -152,8 +171,10 @@ bool AssetManager::HasFile(std::string fixedName) const {
return false;
}
bool AssetManager::GetFile(std::string fixedName, char** data, uint32_t* len) const {
if (const auto overridden = GetOverridePath(fixedName)) return ReadLooseFile(*overridden, data, len);
bool AssetManager::GetFile(std::string fixedName, char** data, uint32_t* len, const bool useOverride) const {
if (useOverride) {
if (const auto overridden = GetOverridePath(fixedName)) return ReadLooseFile(*overridden, data, len);
}
std::transform(fixedName.begin(), fixedName.end(), fixedName.begin(), [](uint8_t c) { return std::tolower(c); });
std::replace(fixedName.begin(), fixedName.end(), '\\', '/'); // On the off chance someone has the wrong slashes, force forward slashes

View File

@@ -121,11 +121,20 @@ public:
[[nodiscard]]
AssetStream GetFile(const char* name) const;
// The client's own file, whatever the override folder holds for the name
[[nodiscard]]
AssetStream GetClientFile(const std::string& name) const;
[[nodiscard]]
bool HasClientFile(const std::string& name) const;
[[nodiscard]]
AssetStream GetFile(const std::string& name) const { return GetFile(name.c_str()); };
private:
void LoadPackIndex();
bool HasFile(std::string name, bool useOverride) const;
bool GetFile(std::string name, char** data, uint32_t* len, bool useOverride) const;
// Modified crc algorithm (mpeg2)
// Reference: https://stackoverflow.com/questions/54339800/how-to-modify-crc-32-to-crc-32-mpeg-2

View File

@@ -43,6 +43,7 @@ set(DCOMMONTEST_SOURCES
"ChatFilterCoreTests.cpp"
"BrickBoundsTests.cpp"
"ZonePublishTests.cpp"
"ZoneFilePackTests.cpp"
)
add_subdirectory(dEnumsTests)

View File

@@ -0,0 +1,265 @@
#include <gtest/gtest.h>
#include <cstring>
#include "Sd0.h"
#include "ZCompression.h"
#include "ZoneFilePack.h"
using namespace ZoneFilePack;
namespace {
template<typename T>
void Put(std::string& out, T value) { out.append(reinterpret_cast<const char*>(&value), sizeof(T)); }
std::string Wide(const std::string& text) {
std::string out;
for (const char c : text) { out += c; out += '\0'; }
out.append(2, '\0');
return out;
}
// "hello" as a client sends it: the sd0 header, one chunk's length, the zlib stream (level 9)
const std::string HELLO_SD0("sd0\x01\xff" "\x0d\x00\x00\x00" "\x78\xda\xcb\x48\xcd\xc9\xc9\x07\x00\x06\x2c\x02\x15", 22);
std::string EntryBytes(const std::string& name, const std::string& sd0) {
std::string out;
Put<uint32_t>(out, static_cast<uint32_t>((name.size() + 1) * 2));
Put<uint32_t>(out, static_cast<uint32_t>(sd0.size()));
return out + Wide(name) + sd0;
}
// One chunk of sd0 holding `data`, whatever its size
std::string OneChunk(const std::string& data) {
std::string compressed(ZCompression::GetMaxCompressedLength(static_cast<uint32_t>(data.size())), '\0');
const auto size = ZCompression::Compress(reinterpret_cast<const uint8_t*>(data.data()), static_cast<uint32_t>(data.size()),
reinterpret_cast<uint8_t*>(compressed.data()), static_cast<uint32_t>(compressed.size()));
std::string out("sd0\x01\xff", 5);
Put<uint32_t>(out, static_cast<uint32_t>(size));
return out + compressed.substr(0, static_cast<size_t>(size));
}
}
// What a client sends for one scene, byte for byte
TEST(ZoneFilePackTests, UnpacksOneFile) {
std::string body;
Put<uint32_t>(body, 12); // "a.lvl" and its null, in UTF-16
Put<uint32_t>(body, 22); // the sd0 data
body += std::string("a\0.\0l\0v\0l\0\0\0", 12);
body += HELLO_SD0;
ASSERT_EQ(body.size(), 8u + 12u + 22u);
Files files;
ASSERT_EQ(Unpack(eFileType::SCENE, body, {}, files), eError::NONE);
ASSERT_EQ(files.scenes.size(), 1u);
EXPECT_EQ(files.scenes[0].name, "a.lvl");
EXPECT_EQ(files.scenes[0].data, "hello");
EXPECT_FALSE(files.zone || files.terrain);
// The other single kinds land in their own place
ASSERT_EQ(Unpack(eFileType::TERRAIN, body, {}, files), eError::NONE);
EXPECT_EQ(files.terrain->data, "hello");
ASSERT_EQ(Unpack(eFileType::ZONE, body, {}, files), eError::NONE);
EXPECT_EQ(files.zone->data, "hello");
ASSERT_EQ(Unpack(eFileType::TRIGGERS, body, {}, files), eError::NONE);
EXPECT_EQ(files.triggers[0].data, "hello");
}
// A whole map: the zone, the terrain, a u16 count of scenes, a u16 count of triggers
TEST(ZoneFilePackTests, UnpacksAWholeMap) {
std::string body = EntryBytes("z.luz", HELLO_SD0) + EntryBytes("z.raw", HELLO_SD0);
Put<uint16_t>(body, 2);
body += EntryBytes("a.lvl", HELLO_SD0) + EntryBytes("b.lvl", HELLO_SD0);
Put<uint16_t>(body, 1);
body += EntryBytes("a.lutriggers", HELLO_SD0);
Files files;
ASSERT_EQ(Unpack(eFileType::WHOLE_MAP, body, {}, files), eError::NONE);
EXPECT_EQ(files.zone->name, "z.luz");
EXPECT_EQ(files.terrain->name, "z.raw");
ASSERT_EQ(files.scenes.size(), 2u);
EXPECT_EQ(files.scenes[1].name, "b.lvl");
ASSERT_EQ(files.triggers.size(), 1u);
EXPECT_EQ(files.triggers[0].name, "a.lutriggers");
EXPECT_EQ(files.Count(), 5u);
// No triggers is a count of 0, not a missing count
std::string none = EntryBytes("z.luz", HELLO_SD0) + EntryBytes("z.raw", HELLO_SD0);
Put<uint16_t>(none, 0);
EXPECT_EQ(Unpack(eFileType::WHOLE_MAP, none, {}, files), eError::TRUNCATED);
Put<uint16_t>(none, 0);
EXPECT_EQ(Unpack(eFileType::WHOLE_MAP, none, {}, files), eError::NONE);
// A zone or a scene without data is not a whole map
std::string empty = EntryBytes("z.luz", std::string("sd0\x01\xff", 5)) + EntryBytes("z.raw", HELLO_SD0);
Put<uint16_t>(empty, 0);
Put<uint16_t>(empty, 0);
EXPECT_EQ(Unpack(eFileType::WHOLE_MAP, empty, {}, files), eError::INCOMPLETE);
}
// What the server sends back, byte for byte: the same framing, the data as sd0
TEST(ZoneFilePackTests, PacksOneFileAndAWholeMap) {
Files files;
files.zone = Entry{ "z.luz", "zone bytes" };
files.terrain = Entry{ "z.raw", "terrain bytes" };
files.scenes = { { "a.lvl", "hello" } };
files.triggers = { { "a.lutriggers", "<triggers/>" } };
const auto sd0 = [](const std::string& data) { return Sd0::Compress(data); };
EXPECT_EQ(Pack(eFileType::SCENE, files), EntryBytes("a.lvl", sd0("hello")));
EXPECT_EQ(Pack(eFileType::TERRAIN, files), EntryBytes("z.raw", sd0("terrain bytes")));
EXPECT_EQ(Pack(eFileType::ZONE, files), EntryBytes("z.luz", sd0("zone bytes")));
EXPECT_EQ(Pack(eFileType::TRIGGERS, files), EntryBytes("a.lutriggers", sd0("<triggers/>")));
std::string whole = EntryBytes("z.luz", sd0("zone bytes")) + EntryBytes("z.raw", sd0("terrain bytes"));
Put<uint16_t>(whole, 1);
whole += EntryBytes("a.lvl", sd0("hello"));
Put<uint16_t>(whole, 1);
whole += EntryBytes("a.lutriggers", sd0("<triggers/>"));
EXPECT_EQ(Pack(eFileType::WHOLE_MAP, files), whole);
// The data is sd0: its header, a chunk length, a zlib stream
const auto packed = Pack(eFileType::SCENE, files);
EXPECT_EQ(packed.substr(8 + 12, 5), std::string("sd0\x01\xff", 5));
EXPECT_EQ(static_cast<unsigned char>(packed[8 + 12 + 9]), 0x78);
// And it comes back out
Files read;
ASSERT_EQ(Unpack(eFileType::WHOLE_MAP, whole, {}, read), eError::NONE);
EXPECT_EQ(read.zone->data, "zone bytes");
EXPECT_EQ(read.triggers[0].data, "<triggers/>");
// A kind that is not there packs to nothing
EXPECT_TRUE(Pack(eFileType::SCENE, Files{}).empty());
EXPECT_TRUE(Pack(eFileType::WHOLE_MAP, Files{}).empty());
}
// A file larger than one chunk is several chunks, each at most 256 KiB
TEST(ZoneFilePackTests, LargeFilesAreChunked) {
std::string big(700 * 1024, '\0');
for (size_t i = 0; i < big.size(); ++i) big[i] = static_cast<char>((i * 7 + i / 513) & 0xFF);
Files files;
files.scenes = { { "big.lvl", big } };
Files read;
ASSERT_EQ(Unpack(eFileType::SCENE, Pack(eFileType::SCENE, files), {}, read), eError::NONE);
EXPECT_TRUE(read.scenes[0].data == big);
}
TEST(ZoneFilePackTests, TruncatedEntries) {
const auto whole = EntryBytes("a.lvl", HELLO_SD0);
Files files;
for (const size_t length : { size_t{ 0 }, size_t{ 3 }, size_t{ 8 }, size_t{ 15 }, size_t{ 25 }, whole.size() - 1 }) {
EXPECT_EQ(Unpack(eFileType::SCENE, whole.substr(0, length), {}, files), eError::TRUNCATED) << length;
}
// Bytes after the one entry
EXPECT_EQ(Unpack(eFileType::SCENE, whole + "x", {}, files), eError::TRUNCATED);
// A data length far past the end
std::string lying;
Put<uint32_t>(lying, 12);
Put<uint32_t>(lying, 0xFFFFFFF0);
lying += Wide("a.lvl") + HELLO_SD0;
EXPECT_EQ(Unpack(eFileType::SCENE, lying, {}, files), eError::TRUNCATED);
// A chunk length past the end of the entry's data
auto cut = HELLO_SD0;
cut.resize(cut.size() - 4);
EXPECT_EQ(Unpack(eFileType::SCENE, EntryBytes("a.lvl", cut), {}, files), eError::TRUNCATED);
// A whole map that stops after its terrain
EXPECT_EQ(Unpack(eFileType::WHOLE_MAP, EntryBytes("z.luz", HELLO_SD0) + EntryBytes("z.raw", HELLO_SD0), {}, files), eError::TRUNCATED);
}
TEST(ZoneFilePackTests, DataWithoutTheSd0HeaderIsRefused) {
Files files;
// The bare zlib stream, and plain bytes
EXPECT_EQ(Unpack(eFileType::SCENE, EntryBytes("a.lvl", HELLO_SD0.substr(9)), {}, files), eError::NOT_SD0);
EXPECT_EQ(Unpack(eFileType::SCENE, EntryBytes("a.lvl", "hello"), {}, files), eError::NOT_SD0);
EXPECT_EQ(Unpack(eFileType::SCENE, EntryBytes("a.lvl", ""), {}, files), eError::NOT_SD0);
// The header and a chunk that is not zlib
EXPECT_EQ(Unpack(eFileType::SCENE, EntryBytes("a.lvl", std::string("sd0\x01\xff\x04\x00\x00\x00junk", 13)), {}, files), eError::BAD_CHUNK);
}
TEST(ZoneFilePackTests, OversizedChunksAndFilesAreRefused) {
Files files;
// One chunk that inflates to a byte more than 256 KiB
EXPECT_EQ(Unpack(eFileType::SCENE, EntryBytes("a.lvl", OneChunk(std::string(Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE, 'x'))), {}, files), eError::NONE);
EXPECT_EQ(Unpack(eFileType::SCENE, EntryBytes("a.lvl", OneChunk(std::string(Sd0::MAX_UNCOMPRESSED_CHUNK_SIZE + 1, 'x'))), {}, files), eError::BAD_CHUNK);
// A chunk whose compressed length is itself too large
std::string huge("sd0\x01\xff", 5);
Put<uint32_t>(huge, 64 * 1024 * 1024);
EXPECT_EQ(Unpack(eFileType::SCENE, EntryBytes("a.lvl", huge), {}, files), eError::BAD_CHUNK);
// Many small chunks that add up to more than a file may be
Limits limits;
limits.maxFileBytes = 12;
std::string three = HELLO_SD0 + HELLO_SD0.substr(5) + HELLO_SD0.substr(5); // 15 bytes in 3 chunks
EXPECT_EQ(Unpack(eFileType::SCENE, EntryBytes("a.lvl", three), limits, files), eError::TOO_LARGE);
limits.maxFileBytes = 15;
EXPECT_EQ(Unpack(eFileType::SCENE, EntryBytes("a.lvl", three), limits, files), eError::NONE);
EXPECT_EQ(files.scenes[0].data, "hellohellohello");
// All the files together, and how many there are
std::string whole = EntryBytes("z.luz", HELLO_SD0) + EntryBytes("z.raw", HELLO_SD0);
Put<uint16_t>(whole, 1);
whole += EntryBytes("a.lvl", HELLO_SD0);
Put<uint16_t>(whole, 0);
limits = {};
limits.maxTotalBytes = 14;
EXPECT_EQ(Unpack(eFileType::WHOLE_MAP, whole, limits, files), eError::TOO_LARGE);
limits = {};
limits.maxFiles = 2;
EXPECT_EQ(Unpack(eFileType::WHOLE_MAP, whole, limits, files), eError::TOO_LARGE);
// A count that promises more entries than there are bytes
std::string many = EntryBytes("z.luz", HELLO_SD0) + EntryBytes("z.raw", HELLO_SD0);
Put<uint16_t>(many, 100);
EXPECT_EQ(Unpack(eFileType::WHOLE_MAP, many, {}, files), eError::TRUNCATED);
}
// A name's folder part is dropped, as the client drops it: only the base name is ever looked at
TEST(ZoneFilePackTests, NamesAreBaseNamesOnly) {
EXPECT_EQ(BaseName("a.lvl"), "a.lvl");
EXPECT_EQ(BaseName("maps/zone/a.lvl"), "a.lvl");
EXPECT_EQ(BaseName("..\\..\\windows\\a.lvl"), "a.lvl");
EXPECT_EQ(BaseName("../../../etc/passwd"), "passwd");
EXPECT_EQ(BaseName("C:\\maps\\a.lvl"), "a.lvl");
EXPECT_FALSE(BaseName(""));
EXPECT_FALSE(BaseName(".."));
EXPECT_FALSE(BaseName("maps/.."));
EXPECT_FALSE(BaseName("maps/"));
EXPECT_FALSE(BaseName("maps\\."));
EXPECT_FALSE(BaseName("C:a.lvl"));
EXPECT_FALSE(BaseName("a.lvl."));
EXPECT_FALSE(BaseName(std::string("a\0.lvl", 6)));
EXPECT_FALSE(BaseName("a\n.lvl"));
EXPECT_FALSE(BaseName("\xC3\xA9.lvl"));
Files files;
ASSERT_EQ(Unpack(eFileType::SCENE, EntryBytes("..\\other_zone/a.lvl", HELLO_SD0), {}, files), eError::NONE);
EXPECT_EQ(files.scenes[0].name, "a.lvl");
EXPECT_EQ(Unpack(eFileType::SCENE, EntryBytes("maps/..", HELLO_SD0), {}, files), eError::BAD_NAME);
EXPECT_EQ(Unpack(eFileType::SCENE, EntryBytes("", HELLO_SD0), {}, files), eError::BAD_NAME);
// A name that is not UTF-16 with its null, or not ASCII
std::string odd;
Put<uint32_t>(odd, 5);
Put<uint32_t>(odd, 22);
odd += std::string("a\0b\0c", 5) + HELLO_SD0;
EXPECT_EQ(Unpack(eFileType::SCENE, odd, {}, files), eError::BAD_NAME);
std::string unterminated;
Put<uint32_t>(unterminated, 4);
Put<uint32_t>(unterminated, 22);
unterminated += std::string("a\0b\0", 4) + HELLO_SD0;
EXPECT_EQ(Unpack(eFileType::SCENE, unterminated, {}, files), eError::BAD_NAME);
std::string wide;
Put<uint32_t>(wide, 4);
Put<uint32_t>(wide, 22);
wide += std::string("\x42\x04\0\0", 4) + HELLO_SD0;
EXPECT_EQ(Unpack(eFileType::SCENE, wide, {}, files), eError::BAD_NAME);
std::string embedded;
Put<uint32_t>(embedded, 6);
Put<uint32_t>(embedded, 22);
embedded += std::string("a\0\0\0\0\0", 6) + HELLO_SD0;
EXPECT_EQ(Unpack(eFileType::SCENE, embedded, {}, files), eError::BAD_NAME);
EXPECT_TRUE(ValidType(0));
EXPECT_TRUE(ValidType(4));
EXPECT_FALSE(ValidType(5));
}

View File

@@ -546,3 +546,135 @@ TEST(ZoneEditLockTests, ReloadHoldsWhilePlayersAreThere) {
EXPECT_EQ(PlanReload(true, 3, false), eReload::HELD);
EXPECT_EQ(PlanReload(true, 3, true), eReload::STARTED);
}
// ---- Files sent by the game client's level editor (ZoneFilePush): the same checks, store and backups ----
#include "ZoneFilePush.h"
namespace {
struct PushFixture : Fixture {
uint32_t pushes = 0;
ZoneFilePush::Job Job(ZoneFilePack::eFileType type, const ZoneFilePack::Files& files) {
ZoneFilePush::Job job;
job.zoneId = zone.zoneId;
job.luzPath = LUZ;
job.fileType = type;
job.packedFiles = ZoneFilePack::Pack(type, files);
job.root = root;
job.id = "w" + std::to_string(++pushes);
job.who = "editor";
job.now = 1000 + pushes;
job.clientScenes = zone.clientScenes;
job.clientFiles = { FOLDER + "a.lvl", FOLDER + "b.lvl", FOLDER + "test_zone.raw" };
job.clientLuz = ZoneBytes({ "a.lvl", "b.lvl" });
return job;
}
};
using ZoneFilePushTests = PushFixture;
}
// One scene sent: the set is that scene and the zone's .luz (the client's own, as none was sent)
TEST_F(ZoneFilePushTests, OneFileBecomesASetWithTheZoneFile) {
ZoneFilePack::Files files;
files.scenes = { { "A.lvl", LevelBytes({ "pushed=0:yes" }) } };
const auto result = ZoneFilePush::Run(Job(ZoneFilePack::eFileType::SCENE, files));
ASSERT_TRUE(result.applied) << result.problem;
EXPECT_EQ(result.files, 2u);
EXPECT_FALSE(result.backupId);
EXPECT_EQ(Slurp(root / "maps" / FOLDER / "a.lvl"), LevelBytes({ "pushed=0:yes" }));
EXPECT_EQ(Slurp(root / "maps" / LUZ), ZoneBytes({ "a.lvl", "b.lvl" }));
EXPECT_FALSE(fs::exists(root / "staging" / "w1"));
}
// A second single file keeps the rest of the set, and the set before is a backup
TEST_F(ZoneFilePushTests, OneFileKeepsTheRestOfTheSet) {
ZoneFilePack::Files first;
first.scenes = { { "a.lvl", LevelBytes({ "first=0:a" }) } };
ASSERT_TRUE(ZoneFilePush::Run(Job(ZoneFilePack::eFileType::SCENE, first)).applied);
ZoneFilePack::Files second;
second.triggers = { { "a.lutriggers", GOOD_TRIGGERS } };
const auto result = ZoneFilePush::Run(Job(ZoneFilePack::eFileType::TRIGGERS, second));
ASSERT_TRUE(result.applied) << result.problem;
EXPECT_EQ(result.files, 3u);
ASSERT_TRUE(result.backupId);
EXPECT_EQ(Slurp(root / "maps" / FOLDER / "a.lvl"), LevelBytes({ "first=0:a" }));
EXPECT_EQ(Slurp(root / "maps" / FOLDER / "a.lutriggers"), GOOD_TRIGGERS);
// The backup still holds the first set, whole
const auto backups = ZoneOverrideStore::Backups(root, zone.zoneId);
ASSERT_EQ(backups.size(), 1u);
EXPECT_EQ(backups[0].files.size(), 2u);
EXPECT_EQ(Slurp(root / "backups" / "1900" / backups[0].id / "files" / FOLDER / "a.lvl"), LevelBytes({ "first=0:a" }));
}
// A whole map is checked whole: one bad file in it and the set the zone has stays as it is
TEST_F(ZoneFilePushTests, ARefusedWholeMapLeavesTheSetAlone) {
ZoneFilePack::Files first;
first.triggers = { { "a.lutriggers", GOOD_TRIGGERS } };
ASSERT_TRUE(ZoneFilePush::Run(Job(ZoneFilePack::eFileType::TRIGGERS, first)).applied);
ZoneFilePack::Files whole;
whole.zone = ZoneFilePack::Entry{ "test_zone.luz", ZoneBytes({ "a.lvl" }) };
whole.terrain = ZoneFilePack::Entry{ "test_zone.raw", "not a terrain file" };
whole.scenes = { { "a.lvl", LevelBytes({ "whole=0:map" }) } };
auto result = ZoneFilePush::Run(Job(ZoneFilePack::eFileType::WHOLE_MAP, whole));
EXPECT_FALSE(result.applied);
EXPECT_NE(result.problem.find("test_zone.raw"), std::string::npos);
EXPECT_NE(result.problem.find("parse_failed"), std::string::npos);
// Refused whole: the set before is untouched
EXPECT_EQ(Slurp(root / "maps" / FOLDER / "a.lutriggers"), GOOD_TRIGGERS);
EXPECT_FALSE(fs::exists(root / "maps" / FOLDER / "a.lvl"));
EXPECT_FALSE(fs::exists(root / "staging" / "w2"));
}
// Whatever folder a sent name has, the file can only become one of this zone's own
TEST_F(ZoneFilePushTests, NamesNeverLeaveTheZoneFolder) {
ZoneFilePack::Files files;
files.scenes = { { "x", LevelBytes({}) } };
// (Pack writes the name as given; a client could send any)
files.scenes[0].name = "../../../../etc/a.lvl";
auto result = ZoneFilePush::Run(Job(ZoneFilePack::eFileType::SCENE, files));
ASSERT_TRUE(result.applied) << result.problem;
EXPECT_TRUE(fs::exists(root / "maps" / FOLDER / "a.lvl")); // its base name, in the zone's folder
EXPECT_FALSE(fs::exists(dir / "etc"));
// A base name the zone has no file for is refused, and so is a file of another kind
files.scenes[0].name = "..\\other_zone\\c.lvl";
result = ZoneFilePush::Run(Job(ZoneFilePack::eFileType::SCENE, files));
EXPECT_FALSE(result.applied);
EXPECT_NE(result.problem.find("name_not_allowed"), std::string::npos);
files.scenes[0].name = "passwd";
result = ZoneFilePush::Run(Job(ZoneFilePack::eFileType::SCENE, files));
EXPECT_FALSE(result.applied);
files.scenes[0].name = "..";
result = ZoneFilePush::Run(Job(ZoneFilePack::eFileType::SCENE, files));
EXPECT_FALSE(result.applied);
EXPECT_NE(result.problem.find("unpacked"), std::string::npos);
}
TEST_F(ZoneFilePushTests, DamagedPacksAndFilesAreRefusedWithNothingWritten) {
ZoneFilePack::Files files;
files.scenes = { { "a.lvl", LevelBytes({ "a=0:b", "c=0:d" }) } };
auto job = Job(ZoneFilePack::eFileType::SCENE, files);
job.packedFiles.resize(job.packedFiles.size() - 3);
auto result = ZoneFilePush::Run(job);
EXPECT_FALSE(result.applied);
EXPECT_NE(result.problem.find("unpacked"), std::string::npos);
// A scene cut short inside good packing
auto level = LevelBytes({ "a=0:b", "c=0:d" });
files.scenes[0].data = level.substr(0, level.size() - 2);
result = ZoneFilePush::Run(Job(ZoneFilePack::eFileType::SCENE, files));
EXPECT_FALSE(result.applied);
EXPECT_NE(result.problem.find("parse_failed"), std::string::npos);
// Larger than a file may be
files.scenes[0].data = level;
job = Job(ZoneFilePack::eFileType::SCENE, files);
job.limits.maxFileBytes = 16;
result = ZoneFilePush::Run(job);
EXPECT_FALSE(result.applied);
EXPECT_TRUE(ZoneOverrideStore::Current(root, zone.zoneId, FOLDER).empty());
EXPECT_FALSE(fs::exists(root / "maps" / FOLDER / "a.lvl"));
}