Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-02 02:43:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(dashboard): log bundles, zipped log files picked by date, server and world

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-27 10:05:10 -05:00
parent dd083ffc21
commit 709cfab140
7 changed files with 965 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
dDashboardServer/routes/CMakeLists.txt
View File

@@ -28,6 +28,7 @@ set(DASHBOARDROUTES_SOURCES
"MissionTools.cpp"
"CharacterProgress.cpp"
"ServerRoutes.cpp"
"LogBundle.cpp"
"Traffic.cpp"
"TrafficHistory.cpp"
"LeaderboardRoutes.cpp"

484
dDashboardServer/routes/LogBundle.cpp Normal file
View File

@@ -0,0 +1,484 @@
#include "LogBundle.h"
#include <algorithm>
#include <chrono>
#include <cstdio>
#include <ctime>
#include <fstream>
#include "GeneralUtils.h"
#include "ZCompression.h"
namespace fs = std::filesystem;
namespace {
constexpr std::string_view SERVERS[][2] = {
{ "master", "MasterServer" }, { "auth", "AuthServer" }, { "chat", "ChatServer" },
{ "dashboard", "DashboardServer" }, { "ugc", "UgcServer" }, { "world", "WorldServer" },
};
std::optional<uint32_t> Number(std::string_view text) {
if (text.empty() || text.size() > 10 || !std::all_of(text.begin(), text.end(), [](char c) { return c >= '0' && c <= '9'; })) return std::nullopt;
const auto value = std::stoull(std::string(text));
if (value > UINT32_MAX) return std::nullopt;
return static_cast<uint32_t>(value);
}
std::vector<std::string_view> Split(std::string_view text, char separator) {
std::vector<std::string_view> parts;
size_t start = 0;
while (true) {
const auto end = text.find(separator, start);
parts.push_back(text.substr(start, end == std::string_view::npos ? std::string_view::npos : end - start));
if (end == std::string_view::npos) return parts;
start = end + 1;
}
}
bool KnownServer(std::string_view name) {
return std::any_of(std::begin(SERVERS), std::end(SERVERS), [&](const auto& s) { return s[1] == name; });
}
std::tm LocalTime(int64_t time) {
const auto t = static_cast<std::time_t>(time);
std::tm out{};
#ifdef _WIN32
localtime_s(&out, &t);
#else
localtime_r(&t, &out);
#endif
return out;
}
std::string TimeText(int64_t time) {
if (time <= 0) return "-";
const auto tm = LocalTime(time);
char text[32];
std::strftime(text, sizeof(text), "%Y-%m-%d %H:%M:%S", &tm);
return text;
}
int64_t WriteTime(const fs::path& path) {
std::error_code ec;
const auto time = fs::last_write_time(path, ec);
if (ec) return 0;
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::clock_cast<std::chrono::system_clock>(time).time_since_epoch()).count();
}
bool Matches(const LogBundle::LogName& name, const LogBundle::Filter& filter) {
if (!filter.servers.empty() && !filter.servers.contains(name.server)) return false;
if (name.server != "WorldServer") return true;
if (!filter.zones.empty() && (!name.zone || !filter.zones.contains(*name.zone))) return false;
if (filter.clone && name.clone != filter.clone) return false;
if (filter.instance && name.instance != filter.instance) return false;
return true;
}
bool Overlaps(int64_t started, int64_t written, const LogBundle::Filter& filter) {
return (filter.to == 0 || started <= filter.to) && (filter.from == 0 || written >= filter.from);
}
bool IsHex(char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); }
bool IsDigit(char c) { return c >= '0' && c <= '9'; }
bool IsWordChar(char c) { return std::isalnum(static_cast<unsigned char>(c)) || c == '_'; }
// Writes a zip file a piece at a time: each entry's sizes and CRC follow its data (flag bit 3), so nothing needs
// to be known before it's compressed. Without zip64, so everything must stay under 4 GB (the size limit sees to it).
class ZipWriter {
public:
explicit ZipWriter(const fs::path& path) : m_File(path, std::ios::binary | std::ios::trunc) {}
bool Ok() const { return m_File.good(); }
uint64_t Size() const { return m_Offset; }
bool Begin(const std::string& name, int64_t time) {
m_Entry = { name, static_cast<uint32_t>(m_Offset), DosTime(time) };
std::string header;
Put32(header, 0x04034b50);
Put16(header, 20); // version needed: deflate
Put16(header, 0x0808); // sizes after the data, UTF-8 names
Put16(header, 8); // deflate
Put32(header, m_Entry.dosTime);
Put32(header, 0); Put32(header, 0); Put32(header, 0);
Put16(header, static_cast<uint16_t>(name.size()));
Put16(header, 0);
header += name;
m_Crc = 0;
m_Deflater = std::make_unique<ZCompression::RawDeflater>([this](std::string_view data) { return Raw(data); });
return Raw(header);
}
bool Write(std::string_view data) {
m_Crc = ZCompression::Crc32(m_Crc, data);
return m_Deflater->Write(data);
}
bool End() {
if (!m_Deflater->Finish()) return false;
m_Entry.crc = m_Crc;
m_Entry.compressed = static_cast<uint32_t>(m_Deflater->BytesOut());
m_Entry.size = static_cast<uint32_t>(m_Deflater->BytesIn());
m_Deflater.reset();
std::string descriptor;
Put32(descriptor, 0x08074b50);
Put32(descriptor, m_Entry.crc);
Put32(descriptor, m_Entry.compressed);
Put32(descriptor, m_Entry.size);
m_Entries.push_back(m_Entry);
return Raw(descriptor);
}
bool Close() {
const auto start = m_Offset;
std::string directory;
for (const auto& entry : m_Entries) {
Put32(directory, 0x02014b50);
Put16(directory, 0x0314); // made by: Unix, 2.0 (so the permissions below count)
Put16(directory, 20);
Put16(directory, 0x0808);
Put16(directory, 8);
Put32(directory, entry.dosTime);
Put32(directory, entry.crc);
Put32(directory, entry.compressed);
Put32(directory, entry.size);
Put16(directory, static_cast<uint16_t>(entry.name.size()));
Put16(directory, 0); Put16(directory, 0); Put16(directory, 0); Put16(directory, 0);
Put32(directory, 0100644u << 16); // a plain file, rw-r--r--
Put32(directory, entry.offset);
directory += entry.name;
}
const auto directorySize = directory.size();
Put32(directory, 0x06054b50);
Put16(directory, 0); Put16(directory, 0);
Put16(directory, static_cast<uint16_t>(m_Entries.size()));
Put16(directory, static_cast<uint16_t>(m_Entries.size()));
Put32(directory, static_cast<uint32_t>(directorySize));
Put32(directory, static_cast<uint32_t>(start));
Put16(directory, 0);
if (!Raw(directory)) return false;
m_File.close();
return !m_File.fail();
}
private:
struct Entry {
std::string name;
uint32_t offset{};
uint32_t dosTime{};
uint32_t crc{}, compressed{}, size{};
};
static void Put16(std::string& out, uint16_t value) { out += static_cast<char>(value & 0xFF); out += static_cast<char>(value >> 8); }
static void Put32(std::string& out, uint32_t value) { Put16(out, value & 0xFFFF); Put16(out, value >> 16); }
static uint32_t DosTime(int64_t time) {
const auto tm = LocalTime(std::max<int64_t>(time, 315532800)); // zip times start in 1980
const uint32_t date = ((tm.tm_year - 80) << 9) | ((tm.tm_mon + 1) << 5) | tm.tm_mday;
const uint32_t clock = (tm.tm_hour << 11) | (tm.tm_min << 5) | (tm.tm_sec / 2);
return (date << 16) | clock;
}
bool Raw(std::string_view data) {
m_File.write(data.data(), static_cast<std::streamsize>(data.size()));
m_Offset += data.size();
return m_File.good();
}
std::ofstream m_File;
uint64_t m_Offset{};
Entry m_Entry;
uint32_t m_Crc{};
std::unique_ptr<ZCompression::RawDeflater> m_Deflater;
std::vector<Entry> m_Entries;
};
}
namespace LogBundle {
std::optional<LogName> ParseLogName(std::string_view fileName) {
if (!fileName.ends_with(".log")) return std::nullopt;
fileName.remove_suffix(4);
const auto parts = Split(fileName, '_');
if (parts.empty() || !KnownServer(parts[0])) return std::nullopt;
LogName name{ std::string(parts[0]) };
std::vector<uint32_t> numbers;
for (size_t i = 1; i < parts.size(); i++) {
const auto number = Number(parts[i]);
if (!number) return std::nullopt;
numbers.push_back(*number);
}
if (name.server == "WorldServer") {
// WorldServer_<zone>_<clone>_<instance>_<time>, or just WorldServer_<time> from before the zone was known
if (numbers.size() == 4) {
name.zone = numbers[0];
name.clone = numbers[1];
name.instance = numbers[2];
name.started = numbers[3];
} else if (numbers.size() == 1) {
name.started = numbers[0];
} else if (!numbers.empty()) return std::nullopt;
} else {
if (numbers.size() > 1) return std::nullopt;
if (numbers.size() == 1) name.started = numbers[0];
}
return name;
}
std::optional<LogName> ParseCrashName(std::string_view fileName) {
if (!fileName.starts_with("Crash_") || !fileName.ends_with(".log")) return std::nullopt;
fileName.remove_prefix(6);
fileName.remove_suffix(4);
const auto pid = fileName.rfind('_'); // the process ID
if (pid == std::string_view::npos || !Number(fileName.substr(pid + 1))) return std::nullopt;
return ParseLogName(std::string(fileName.substr(0, pid)) + ".log");
}
std::optional<std::string> ServerFromShortName(std::string_view name) {
for (const auto& server : SERVERS) {
if (server[0] == name || server[1] == name) return std::string(server[1]);
}
return std::nullopt;
}
std::optional<Filter> Filter::FromQuery(const std::function<std::string(const std::string&)>& value, std::string& error) {
Filter filter;
const auto time = [&](const std::string& key, int64_t& out) {
const auto text = value(key);
if (text.empty()) return true;
const auto parsed = GeneralUtils::TryParse<int64_t>(text);
if (!parsed || *parsed < 0) { error = key + " must be a Unix time in seconds"; return false; }
out = *parsed;
return true;
};
if (!time("from", filter.from) || !time("to", filter.to)) return std::nullopt;
if (filter.from && filter.to && filter.from > filter.to) { error = "from is after to"; return std::nullopt; }
if (const auto servers = value("servers"); !servers.empty()) {
for (const auto part : Split(servers, ',')) {
const auto server = ServerFromShortName(part);
if (!server) { error = "Unknown server " + std::string(part) + " (master, auth, chat, dashboard, ugc or world)"; return std::nullopt; }
filter.servers.insert(*server);
}
}
if (const auto zones = value("zones"); !zones.empty()) {
for (const auto part : Split(zones, ',')) {
const auto zone = Number(part);
if (!zone) { error = "zones must be zone IDs separated by commas"; return std::nullopt; }
filter.zones.insert(*zone);
}
}
for (const auto& [key, out] : { std::pair{ "clone", &filter.clone }, std::pair{ "instance", &filter.instance } }) {
const auto text = value(key);
if (text.empty()) continue;
*out = Number(text);
if (!*out) { error = std::string(key) + " must be a number"; return std::nullopt; }
}
filter.crashDumps = value("crash") == "1";
filter.trim = value("trim") == "1";
filter.redactIps = value("redact") == "1";
filter.text = value("text");
if (filter.text.size() > 200) { error = "text is longer than 200 characters"; return std::nullopt; }
std::transform(filter.text.begin(), filter.text.end(), filter.text.begin(), [](unsigned char c) { return std::tolower(c); });
return filter;
}
nlohmann::json Filter::ToJson() const {
nlohmann::json json{ {"from", from}, {"to", to}, {"servers", servers}, {"zones", zones}, {"crash_dumps", crashDumps},
{"trim", trim}, {"text", text}, {"redact_ips", redactIps} };
json["clone"] = clone ? nlohmann::json(*clone) : nlohmann::json(nullptr);
json["instance"] = instance ? nlohmann::json(*instance) : nlohmann::json(nullptr);
return json;
}
std::vector<File> Select(const fs::path& logFolder, const fs::path& dumpFolder, const Filter& filter) {
std::vector<File> files;
std::error_code ec;
for (auto it = fs::recursive_directory_iterator(logFolder, ec); !ec && it != fs::recursive_directory_iterator(); it.increment(ec)) {
if (!it->is_regular_file(ec)) continue;
const auto name = ParseLogName(it->path().filename().string());
if (!name || !Matches(*name, filter)) continue;
File file{ it->path(), "logs/" + fs::relative(it->path(), logFolder, ec).generic_string(), *name };
file.written = WriteTime(it->path());
file.started = name->started ? name->started : file.written;
file.size = it->file_size(ec);
if (Overlaps(file.started, file.written, filter)) files.push_back(std::move(file));
}
if (filter.crashDumps && !dumpFolder.empty()) {
for (const auto& entry : fs::directory_iterator(dumpFolder, ec)) {
if (!entry.is_regular_file(ec)) continue;
if (!entry.path().filename().string().starts_with("Crash_")) continue;
// A dump written by a process this can't name still goes in when every server is wanted
auto name = ParseCrashName(entry.path().filename().string());
if (name ? !Matches(*name, filter) : !filter.servers.empty() || !filter.zones.empty()) continue;
File file{ entry.path(), "crash_dumps/" + entry.path().filename().string(), name.value_or(LogName{}) };
file.crashDump = true;
file.written = WriteTime(entry.path());
file.started = file.name.started ? file.name.started : file.written;
file.size = entry.file_size(ec);
if (Overlaps(file.started, file.written, filter)) files.push_back(std::move(file));
}
}
std::sort(files.begin(), files.end(), [](const File& a, const File& b) {
return std::tie(a.started, a.archiveName) < std::tie(b.started, b.archiveName);
});
return files;
}
std::optional<int64_t> LineTime(std::string_view line) {
// [dd-mm-yy HH:MM:SS
if (line.size() < 19 || line[0] != '[' || line[3] != '-' || line[6] != '-' || line[9] != ' ' || line[12] != ':' || line[15] != ':') return std::nullopt;
for (const size_t at : { 1, 2, 4, 5, 7, 8, 10, 11, 13, 14, 16, 17 }) {
if (!IsDigit(line[at])) return std::nullopt;
}
const auto two = [&](size_t at) { return (line[at] - '0') * 10 + (line[at + 1] - '0'); };
std::tm tm{};
tm.tm_mday = two(1);
tm.tm_mon = two(4) - 1;
tm.tm_year = 100 + two(7);
tm.tm_hour = two(10);
tm.tm_min = two(13);
tm.tm_sec = two(16);
tm.tm_isdst = -1;
const auto time = std::mktime(&tm);
if (time == -1) return std::nullopt;
return static_cast<int64_t>(time);
}
std::string RedactIps(std::string_view line) {
std::string out;
out.reserve(line.size());
size_t i = 0;
while (i < line.size()) {
const bool boundary = i == 0 || (!IsWordChar(line[i - 1]) && line[i - 1] != '.' && line[i - 1] != ':');
if (boundary && (IsHex(line[i]) || line[i] == ':')) {
// The longest run of hex digits, dots and colons from here
size_t end = i;
while (end < line.size() && (IsHex(line[end]) || line[end] == '.' || line[end] == ':')) end++;
// A run that ends in a word ("Foo::Bar" is not an address)
if (end < line.size() && IsWordChar(line[end])) { out += line.substr(i, end - i); i = end; continue; }
auto run = line.substr(i, end - i);
// Trailing punctuation isn't part of an address ("from 1.2.3.4." or "at ::1:")
while (!run.empty() && (run.back() == '.' || (run.back() == ':' && !run.ends_with("::")))) run.remove_suffix(1);
const auto colons = std::count(run.begin(), run.end(), ':');
const auto dots = std::count(run.begin(), run.end(), '.');
bool address = false;
if (colons == 0 && dots == 3) {
// IPv4: four numbers up to 255
const auto parts = Split(run, '.');
address = std::all_of(parts.begin(), parts.end(), [](std::string_view p) {
return !p.empty() && p.size() <= 3 && std::all_of(p.begin(), p.end(), IsDigit) && std::stoi(std::string(p)) <= 255;
});
} else if (colons >= 2 && (dots == 0 || dots == 3)) {
// IPv6: eight groups, or fewer around a "::"; times like 09:49:54 have neither
const bool compressed = run.find("::") != std::string_view::npos;
const auto groups = Split(run.substr(0, dots ? run.rfind(':') : run.size()), ':');
const bool groupsOk = std::all_of(groups.begin(), groups.end(), [](std::string_view g) { return g.size() <= 4 && std::all_of(g.begin(), g.end(), IsHex); });
address = groupsOk && (compressed || colons == 7) && std::any_of(run.begin(), run.end(), IsHex);
} else if (colons == 1 && dots == 3) {
// IPv4 with a port: redact the address, keep the port
const auto colon = run.find(':');
const auto ip = RedactIps(run.substr(0, colon));
if (ip != run.substr(0, colon)) { out += ip; out += run.substr(colon); i += run.size(); continue; }
}
if (address) out += "[ip]";
else out += run;
i += run.size();
continue;
}
out += line[i++];
}
return out;
}
std::string SizeText(uint64_t bytes) {
char text[32];
if (bytes >= 1024ull * 1024 * 1024) std::snprintf(text, sizeof(text), "%.1f GB", bytes / 1073741824.0);
else if (bytes >= 1024 * 1024) std::snprintf(text, sizeof(text), "%.1f MB", bytes / 1048576.0);
else if (bytes >= 1024) std::snprintf(text, sizeof(text), "%.1f kB", bytes / 1024.0);
else std::snprintf(text, sizeof(text), "%llu bytes", static_cast<unsigned long long>(bytes));
return text;
}
Result WriteZip(const fs::path& out, const std::vector<File>& files, const Filter& filter, const std::string& manifestHeader,
uint64_t maxBytes, const std::function<bool()>& cancelled) {
Result result;
std::string manifest = manifestHeader;
manifest += "\nFiles (archive name, server, first and last time, source size, size in this bundle):\n";
const auto fail = [&](std::string error) {
std::error_code ec;
fs::remove(out, ec);
result.ok = false;
result.error = std::move(error);
return result;
};
if (files.size() > 60000) return fail("More than 60,000 files; pick a shorter date range or fewer servers");
ZipWriter zip(out);
if (!zip.Ok()) return fail("Can't write " + out.string());
for (const auto& file : files) {
if (cancelled && cancelled()) return fail("Cancelled");
std::ifstream in(file.path, std::ios::binary);
if (!in) {
manifest += file.archiveName + " (couldn't be read)\n";
continue;
}
// Begun with the first bytes, so files with no lines left stay out
bool begun = false;
uint64_t written = 0;
const auto put = [&](std::string_view data) {
if (!begun && !(begun = zip.Begin(file.archiveName, file.written))) return false;
written += data.size();
result.bytesIn += data.size();
if (maxBytes && result.bytesIn > maxBytes) {
result.overLimit = true;
return false;
}
return zip.Write(data);
};
bool ok = true;
const bool trim = filter.trim && !file.crashDump && (filter.from || filter.to);
const bool text = !filter.text.empty() && !file.crashDump;
if (!trim && !text && !filter.redactIps) {
// Whole files: copied a piece at a time
char buffer[64 * 1024];
while (ok && in) {
in.read(buffer, sizeof(buffer));
if (in.gcount() > 0) ok = put(std::string_view(buffer, static_cast<size_t>(in.gcount())));
}
} else {
// A line without a time of its own (a continued message) belongs to the time of the line before
int64_t lineTime = file.started;
std::string line, lower, pending;
while (ok && std::getline(in, line)) {
if (const auto time = LineTime(line)) lineTime = *time;
if (trim && ((filter.from && lineTime < filter.from) || (filter.to && lineTime > filter.to))) continue;
if (text) {
lower = line;
std::transform(lower.begin(), lower.end(), lower.begin(), [](unsigned char c) { return std::tolower(c); });
if (lower.find(filter.text) == std::string::npos) continue;
}
pending += filter.redactIps ? RedactIps(line) : line;
pending += '\n';
if (pending.size() >= 64 * 1024) {
ok = put(pending);
pending.clear();
}
}
if (ok && !pending.empty()) ok = put(pending);
}
if (!ok) {
if (result.overLimit) return fail("The bundle would hold more than " + SizeText(maxBytes) + " (log_bundle_max_mb); pick a shorter date range, fewer servers, or only lines with some text");
return fail("Couldn't write the bundle (is the disk full?)");
}
if (begun && !zip.End()) return fail("Couldn't write the bundle (is the disk full?)");
if (begun) result.files++;
manifest += file.archiveName + " " + file.name.server + (file.name.zone ? " zone " + std::to_string(*file.name.zone) : "") +
" " + TimeText(file.started) + " to " + TimeText(file.written) + " " + SizeText(file.size) + " " +
(written == 0 ? "(no lines left)" : SizeText(written)) + "\n";
}
if (files.empty()) manifest += "(none)\n";
if (!zip.Begin("manifest.txt", std::time(nullptr)) || !zip.Write(manifest) || !zip.End() || !zip.Close()) {
return fail("Couldn't write the bundle (is the disk full?)");
}
result.ok = true;
result.archiveSize = zip.Size();
return result;
}
}

90
dDashboardServer/routes/LogBundle.h Normal file
View File

@@ -0,0 +1,90 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <functional>
#include <optional>
#include <set>
#include <string>
#include <string_view>
#include <vector>
#include "json.hpp"
/**
* Log bundles: the servers' log files (and crash dumps) picked by date range, server and world, put in one zip file to
* download from the System Log page or GET /api/logs/bundle. Only reads files, so it runs on the dashboard's worker
* threads; the routes (ServerRoutes.cpp) read the settings on the web thread and pass the folders in.
*/
namespace LogBundle {
// What a log file's name says. Log files are <Server>_<start time>.log, world servers'
// WorldServer_<zone>_<clone>_<instance>_<start time>.log; crash dumps are Crash_<process name>_<pid>.log.
struct LogName {
std::string server; // MasterServer, AuthServer, ChatServer, DashboardServer, UgcServer, WorldServer
std::optional<uint32_t> zone, clone, instance;
int64_t started{}; // Unix seconds, 0 when the name has none
};
std::optional<LogName> ParseLogName(std::string_view fileName);
std::optional<LogName> ParseCrashName(std::string_view fileName);
// Short names used in queries and on the page (master, auth, chat, dashboard, ugc, world) to server names, and back
std::optional<std::string> ServerFromShortName(std::string_view name);
struct Filter {
int64_t from{}, to{}; // Unix seconds; 0 leaves that end open
std::set<std::string> servers; // server names; empty: all of them
std::set<uint32_t> zones; // world servers of these zones; empty: all
std::optional<uint32_t> clone, instance;
bool crashDumps{};
bool trim{}; // only the lines written in the date range, rather than whole files
std::string text; // only lines with this in them (case doesn't matter)
bool redactIps{};
// Query string values: from, to (Unix seconds), servers (comma separated short names), zones, clone, instance,
// crash, trim, redact (1 for on), text. Sets `error` and returns nullopt when a value is wrong.
static std::optional<Filter> FromQuery(const std::function<std::string(const std::string&)>& value, std::string& error);
nlohmann::json ToJson() const;
// Whether only some lines of a file go in
bool FiltersLines() const { return trim || !text.empty() || redactIps; }
};
struct File {
std::filesystem::path path;
std::string archiveName; // logs/<path under logs>, or crash_dumps/<name>
LogName name;
int64_t started{}; // from the name, or when it was last written if the name has none
int64_t written{}; // last written, Unix seconds
uintmax_t size{};
bool crashDump{};
};
// The files matching `filter` (a file matches a date range when the time from its start to its last write
// overlaps it), oldest first. `dumpFolder` may be empty (no crash dumps).
std::vector<File> Select(const std::filesystem::path& logFolder, const std::filesystem::path& dumpFolder, const Filter& filter);
// The time at the start of a log line ("[27-09-26 09:49:54 ..." in the server's local time), if it has one
std::optional<int64_t> LineTime(std::string_view line);
// `line` with IPv4 and IPv6 addresses replaced by [ip]
std::string RedactIps(std::string_view line);
struct Result {
bool ok{};
std::string error;
bool overLimit{};
size_t files{}; // files put in (files with no lines left are left out)
uint64_t bytesIn{}; // uncompressed, what the files put in hold after filtering
uint64_t archiveSize{};
};
/**
* Write a zip file of `files` (filtered as `filter` says) and a manifest.txt that starts with `manifestHeader` to
* `out`. Gives up once more than `maxBytes` (uncompressed) would go in, or when `cancelled` says so. Reads and
* compresses a piece at a time. On failure `out` is removed.
*/
Result WriteZip(const std::filesystem::path& out, const std::vector<File>& files, const Filter& filter,
const std::string& manifestHeader, uint64_t maxBytes, const std::function<bool()>& cancelled = {});
// "1.5 MB" for manifests and errors
std::string SizeText(uint64_t bytes);
}

119
dDashboardServer/routes/ServerRoutes.cpp
View File

@@ -24,6 +24,9 @@
#include "dConfig.h"
#include "GeneralUtils.h"
#include "eHTTPMethod.h"
#include "DashboardRoutes.h"
#include "LogBundle.h"
#include "Workers.h"
#ifdef __linux__
#include <unistd.h>
@@ -125,6 +128,34 @@ namespace {
return folder.is_absolute() ? folder : BinaryPathFinder::GetBinaryDir() / folder;
}
// Log bundles are built here and deleted once sent (or, where that fails, an hour later)
fs::path BundleFolder() {
std::error_code ec;
auto folder = fs::temp_directory_path(ec);
if (ec) folder = BinaryPathFinder::GetBinaryDir();
return folder / "darkflame-log-bundles";
}
void ClearOldBundles(const fs::path& folder) {
std::error_code ec;
const auto cutoff = fs::file_time_type::clock::now() - std::chrono::hours(1);
for (const auto& entry : fs::directory_iterator(folder, ec)) {
if (entry.is_regular_file(ec) && entry.last_write_time(ec) < cutoff) fs::remove(entry.path(), ec);
}
}
std::optional<LogBundle::Filter> BundleFilter(HTTPReply& reply, const HTTPContext& context) {
std::string error;
auto filter = LogBundle::Filter::FromQuery([&](const std::string& key) { return QueryValue(context.queryString, key); }, error);
if (!filter) JsonError(reply, eHTTPStatusCode::BAD_REQUEST, error);
return filter;
}
uint64_t BundleMaxBytes() {
const auto mb = GeneralUtils::TryParse<uint64_t>(Game::config->GetValue("log_bundle_max_mb")).value_or(512);
return std::clamp<uint64_t>(mb, 1, 4000) * 1024 * 1024;
}
bool PlainFileName(const std::string& name) {
static const std::regex pattern("^[A-Za-z0-9._-]{1,200}$");
return std::regex_match(name, pattern) && name.find("..") == std::string::npos;
@@ -321,6 +352,94 @@ void RegisterServerRoutes() {
reply.headers.push_back("Content-Disposition: attachment; filename=\"" + name + "\"");
});
static const std::string bundleQuery = "Query: from, to (Unix seconds; a file matches when the time from its start to its last write overlaps them), "
"servers (comma separated: master, auth, chat, dashboard, ugc, world; default all), zones (world zone IDs, comma separated), clone, instance, "
"crash=1 (crash dumps too), trim=1 (only lines written between from and to), text= (only lines with it, any case), redact=1 (IP addresses become [ip])";
Route(eHTTPMethod::GET, "/api/logs/bundle/preview", Perm("logs_system"),
"The log files a bundle would hold. " + bundleQuery + ". Returns {files: [{name, server, zone, zone_name, clone, instance, size, started, written, "
"crash_dump}] (the first 2000), count, total_size, max_bytes, over_limit (whole files over log_bundle_max_mb), filters_lines}",
[](HTTPReply& reply, const HTTPContext& context) {
const auto filter = BundleFilter(reply, context);
if (!filter) return;
const auto files = LogBundle::Select(LogFolder(), DumpFolder(), *filter);
const auto& zoneNames = ZoneNames();
nlohmann::json list = nlohmann::json::array();
uint64_t total = 0;
for (const auto& file : files) {
total += file.size;
if (list.size() >= 2000) continue;
nlohmann::json item{ {"name", file.archiveName}, {"server", file.name.server}, {"size", file.size}, {"started", file.started},
{"written", file.written}, {"crash_dump", file.crashDump}, {"zone", nullptr}, {"zone_name", nullptr}, {"clone", nullptr}, {"instance", nullptr} };
if (file.name.zone) {
item["zone"] = *file.name.zone;
item["zone_name"] = zoneNames.value(std::to_string(*file.name.zone), "");
item["clone"] = *file.name.clone;
item["instance"] = *file.name.instance;
}
list.push_back(std::move(item));
}
const auto maxBytes = BundleMaxBytes();
JsonSuccess(reply, { {"files", list}, {"count", files.size()}, {"total_size", total}, {"max_bytes", maxBytes},
{"over_limit", !filter->FiltersLines() && total > maxBytes}, {"filters_lines", filter->FiltersLines()} });
});
Route(eHTTPMethod::GET, "/api/logs/bundle", Perm("logs_system"),
"Download log files as one zip file, with a manifest.txt of the filters and files. " + bundleQuery + ". Built in the background and "
"streamed; 413 when it would hold more than log_bundle_max_mb (before compression)",
[](HTTPReply& reply, const HTTPContext& context) {
const auto filter = BundleFilter(reply, context);
if (!filter) return;
// Everything the worker needs is read here: it must not touch the settings or the database
auto files = LogBundle::Select(LogFolder(), DumpFolder(), *filter);
const auto maxBytes = BundleMaxBytes();
uint64_t total = 0;
for (const auto& file : files) total += file.size;
if (files.empty()) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No log files match");
if (!filter->FiltersLines() && total > maxBytes) {
return JsonError(reply, eHTTPStatusCode::PAYLOAD_TOO_LARGE, "These files hold " + LogBundle::SizeText(total) + ", more than the " +
LogBundle::SizeText(maxBytes) + " a bundle may (log_bundle_max_mb); pick a shorter date range or fewer servers");
}
const auto folder = BundleFolder();
std::error_code ec;
fs::create_directories(folder, ec);
ClearOldBundles(folder);
static uint64_t counter = 0;
const auto now = std::time(nullptr);
char stamp[32], made[32];
const auto local = *std::localtime(&now);
std::strftime(stamp, sizeof(stamp), "%Y%m%d-%H%M%S", &local);
std::strftime(made, sizeof(made), "%Y-%m-%d %H:%M:%S", &local);
const std::string downloadName = std::string("logs_") + stamp + ".zip";
const auto out = folder / ("bundle_" + std::to_string(context.accountId) + "_" + std::to_string(now) + "_" + std::to_string(++counter) + ".zip");
std::string header = "DarkflameServer log bundle\nServer version: " PROJECT_VERSION "\nMade: " + std::string(made) + " (server time) by " +
context.authenticatedUser + "\nFilters: " + filter->ToJson().dump() + "\nSize limit: " + LogBundle::SizeText(maxBytes) + " before compression\n";
Audit(context, "download_logs", "Downloaded a log bundle: " + std::to_string(files.size()) + " file(s), " + LogBundle::SizeText(total) +
" of logs; filters " + filter->ToJson().dump());
Workers::Reply(reply, context, false, [out, downloadName, files = std::move(files), filter = *filter, header, maxBytes](HTTPReply& reply) {
const auto result = LogBundle::WriteZip(out, files, filter, header, maxBytes);
if (!result.ok) {
LOG("Log bundle failed: %s", result.error.c_str());
return JsonError(reply, result.overLimit ? eHTTPStatusCode::PAYLOAD_TOO_LARGE : eHTTPStatusCode::INTERNAL_SERVER_ERROR, result.error);
}
if (result.files == 0) {
std::error_code ec;
fs::remove(out, ec);
return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "No lines match these filters");
}
LOG("Log bundle %s: %zu file(s), %s of logs, %s zipped", downloadName.c_str(), result.files, LogBundle::SizeText(result.bytesIn).c_str(),
LogBundle::SizeText(result.archiveSize).c_str());
reply.file = out.string();
reply.removeFile = true;
reply.message.clear();
reply.status = eHTTPStatusCode::OK;
reply.contentType = eContentType::APPLICATION_OCTET_STREAM;
reply.headers.push_back("Content-Disposition: attachment; filename=\"" + downloadName + "\"");
}, WorkerPool::ePriority::LARGE);
});
Route(eHTTPMethod::GET, "/api/logs/search", Perm("logs_system"),
"Search the servers' log files. Query: ?q= (text, case-insensitive), &server= (e.g. WorldServer), &files= (newest per server, default 3, max 20). "
"Runs in the background: returns {requestId}; the result's data has {matches: [{file, line, text}], truncated}",

3
dDashboardServer/routes/SettingsCatalog.cpp
View File

@@ -390,6 +390,9 @@ namespace {
c.Add(Days("health_days", "Server health history", "Minute-by-minute player counts and uptime.", "30"));
c.Add(Days("traffic_days", "Server traffic history", "Minute-by-minute packets, bytes and HTTP requests of every server (Diagnostics). The last hour at one second is only kept in memory.", "30"));
c.AddSection("Log bundles", "Server log files downloaded together from the System Log page (Download logs).");
c.Add(Unit(Int(DASHBOARD, "log_bundle_max_mb", "At most", "How much log text (before compression) one download may hold. Bundles are built in the system's temporary folder and deleted once sent.", "512", 1, 4000), "MB"));
c.AddSection("Player movement", "Where players went, for replays on the 3D world view (staff with players_history only).");
c.Add(Bool(DASHBOARD, "position_history", "Record player movement", "Keeps a player's position every few seconds while they move, and every 30 seconds while they stand still.", true));
c.Add(Unit(Int(DASHBOARD, "position_history_seconds", "Record every", "While a player moves. Less often keeps the table smaller; replays then move in straighter lines.", "5", 1, 600), "seconds"));

2
tests/dWebTests/CMakeLists.txt
View File

@@ -41,6 +41,8 @@ set(DWEBTESTS_SOURCES
"OnceCacheTests.cpp"
"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/WorkerPool.cpp"
"BackupFilesTests.cpp"
"LogBundleTests.cpp"
"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/LogBundle.cpp"
"SecurityFixesTests.cpp"
"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/BackupFiles.cpp"
"MetricsFormatTests.cpp"

266
tests/dWebTests/LogBundleTests.cpp Normal file
View File

@@ -0,0 +1,266 @@
#include <gtest/gtest.h>
#include <chrono>
#include <cstring>
#include <ctime>
#include <fstream>
#include <map>
#include "LogBundle.h"
#include "ZCompression.h"
namespace fs = std::filesystem;
using namespace LogBundle;
namespace {
// "[dd-mm-yy HH:MM:SS " for a Unix time, in local time as the servers write it
std::string Stamp(int64_t time) {
const auto t = static_cast<std::time_t>(time);
std::tm tm{};
#ifdef _WIN32
localtime_s(&tm, &t);
#else
localtime_r(&t, &tm);
#endif
char text[32];
std::strftime(text, sizeof(text), "[%d-%m-%y %H:%M:%S ", &tm);
return text;
}
void SetWritten(const fs::path& path, int64_t time) {
const auto sys = std::chrono::system_clock::time_point(std::chrono::seconds(time));
fs::last_write_time(path, std::chrono::clock_cast<fs::file_time_type::clock>(sys));
}
// The entries of a zip file, inflated
std::map<std::string, std::string> ReadZip(const fs::path& path) {
std::ifstream in(path, std::ios::binary);
const std::string data((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
const auto u16 = [&](size_t at) { return static_cast<uint32_t>(static_cast<uint8_t>(data[at]) | static_cast<uint8_t>(data[at + 1]) << 8); };
const auto u32 = [&](size_t at) { return u16(at) | u16(at + 2) << 16; };
std::map<std::string, std::string> entries;
const auto end = data.size() - 22;
EXPECT_EQ(u32(end), 0x06054b50u);
size_t at = u32(end + 16);
for (uint32_t i = 0; i < u16(end + 10); i++) {
EXPECT_EQ(u32(at), 0x02014b50u);
const auto crc = u32(at + 16), compressed = u32(at + 20), size = u32(at + 24), nameLength = u16(at + 28), offset = u32(at + 42);
const auto name = data.substr(at + 46, nameLength);
const auto body = offset + 30 + u16(offset + 26) + u16(offset + 28);
auto content = ZCompression::InflateRaw(std::string_view(data).substr(body, compressed), size);
EXPECT_TRUE(content.has_value()) << name;
if (content) {
EXPECT_EQ(ZCompression::Crc32(0, *content), crc) << name;
entries[name] = *content;
}
at += 46 + nameLength + u16(at + 30) + u16(at + 32);
}
return entries;
}
class LogBundleTest : public ::testing::Test {
protected:
void SetUp() override {
m_Root = fs::temp_directory_path() / ("log_bundle_test_" + std::to_string(::testing::UnitTest::GetInstance()->random_seed()) + "_" +
::testing::UnitTest::GetInstance()->current_test_info()->name());
fs::remove_all(m_Root);
fs::create_directories(m_Root / "logs");
fs::create_directories(m_Root / "dumps");
}
void TearDown() override { fs::remove_all(m_Root); }
fs::path Make(const fs::path& relative, const std::string& content, int64_t written) {
const auto path = m_Root / relative;
fs::create_directories(path.parent_path());
std::ofstream(path, std::ios::binary) << content;
SetWritten(path, written);
return path;
}
std::vector<std::string> Names(const Filter& filter) {
std::vector<std::string> names;
for (const auto& file : Select(m_Root / "logs", m_Root / "dumps", filter)) names.push_back(file.archiveName);
return names;
}
fs::path m_Root;
};
constexpr int64_t T = 1790000000; // a start time
}
TEST(LogBundleNames, ParsesServerAndWorldNames) {
auto name = ParseLogName("MasterServer_1790520594.log");
ASSERT_TRUE(name);
EXPECT_EQ(name->server, "MasterServer");
EXPECT_EQ(name->started, 1790520594);
EXPECT_FALSE(name->zone);
name = ParseLogName("WorldServer_1000_2_3_1790520596.log");
ASSERT_TRUE(name);
EXPECT_EQ(name->server, "WorldServer");
EXPECT_EQ(name->zone, 1000u);
EXPECT_EQ(name->clone, 2u);
EXPECT_EQ(name->instance, 3u);
EXPECT_EQ(name->started, 1790520596);
EXPECT_FALSE(ParseLogName("MasterServer_1790520594.txt"));
EXPECT_FALSE(ParseLogName("SomethingElse_1790520594.log"));
EXPECT_FALSE(ParseLogName("WorldServer_1000_2_1790520596.log"));
EXPECT_FALSE(ParseLogName("AuthServer_abc.log"));
}
TEST(LogBundleNames, ParsesCrashDumps) {
auto name = ParseCrashName("Crash_WorldServer_1200_0_4_1790520596_31337.log");
ASSERT_TRUE(name);
EXPECT_EQ(name->server, "WorldServer");
EXPECT_EQ(name->zone, 1200u);
name = ParseCrashName("Crash_UgcServer_1790520596_42.log");
ASSERT_TRUE(name);
EXPECT_EQ(name->server, "UgcServer");
EXPECT_EQ(name->started, 1790520596);
name = ParseCrashName("Crash_WorldServer_42.log"); // crashed before it knew its zone
ASSERT_TRUE(name);
EXPECT_FALSE(name->zone);
EXPECT_FALSE(ParseCrashName("WorldServer_42.log"));
}
TEST(LogBundleFilter, ReadsTheQuery) {
std::map<std::string, std::string> query{ {"from", "100"}, {"to", "200"}, {"servers", "world,auth"}, {"zones", "1000,1200"},
{"instance", "3"}, {"crash", "1"}, {"trim", "1"}, {"text", "Error"}, {"redact", "1"} };
const auto value = [&](const std::string& key) { return query.contains(key) ? query[key] : std::string(); };
std::string error;
const auto filter = Filter::FromQuery(value, error);
ASSERT_TRUE(filter) << error;
EXPECT_EQ(filter->from, 100);
EXPECT_EQ(filter->to, 200);
EXPECT_EQ(filter->servers, (std::set<std::string>{ "WorldServer", "AuthServer" }));
EXPECT_EQ(filter->zones, (std::set<uint32_t>{ 1000, 1200 }));
EXPECT_FALSE(filter->clone);
EXPECT_EQ(filter->instance, 3u);
EXPECT_TRUE(filter->crashDumps && filter->trim && filter->redactIps && filter->FiltersLines());
EXPECT_EQ(filter->text, "error");
for (const auto& [key, bad] : std::map<std::string, std::string>{ {"from", "yesterday"}, {"servers", "world,moon"}, {"zones", "1000,x"}, {"clone", "-1"} }) {
query = { {key, bad} };
error.clear();
EXPECT_FALSE(Filter::FromQuery(value, error)) << key;
EXPECT_FALSE(error.empty()) << key;
}
query = { {"from", "300"}, {"to", "200"} };
EXPECT_FALSE(Filter::FromQuery(value, error));
}
TEST_F(LogBundleTest, SelectsByDateServerAndWorld) {
Make("logs/MasterServer/MasterServer_" + std::to_string(T) + ".log", "m\n", T + 3600);
Make("logs/AuthServer/AuthServer_" + std::to_string(T + 7200) + ".log", "a\n", T + 9000);
Make("logs/WorldServer/1000/0/WorldServer_1000_0_1_" + std::to_string(T) + ".log", "w\n", T + 600);
Make("logs/WorldServer/1200/1/WorldServer_1200_1_2_" + std::to_string(T) + ".log", "w\n", T + 600);
Make("logs/ChatServer_" + std::to_string(T - 86400) + ".log", "old layout\n", T - 80000);
Make("logs/notes.txt", "not a log\n", T);
Make("dumps/Crash_WorldServer_1200_1_2_" + std::to_string(T) + "_99.log", "backtrace\n", T + 700);
Filter all;
EXPECT_EQ(Names(all).size(), 5u); // no crash dumps unless asked, and not notes.txt
all.crashDumps = true;
EXPECT_EQ(Names(all).size(), 6u);
// Overlap: the master log (T to T+3600) matches a range inside it; the auth log starts after it
Filter range;
range.from = T + 1000;
range.to = T + 2000;
EXPECT_EQ(Names(range), std::vector<std::string>{ "logs/MasterServer/MasterServer_" + std::to_string(T) + ".log" });
Filter worlds;
worlds.servers = { "WorldServer" };
worlds.zones = { 1200 };
worlds.crashDumps = true;
EXPECT_EQ(Names(worlds), (std::vector<std::string>{ "crash_dumps/Crash_WorldServer_1200_1_2_" + std::to_string(T) + "_99.log",
"logs/WorldServer/1200/1/WorldServer_1200_1_2_" + std::to_string(T) + ".log" }));
worlds.clone = 0;
EXPECT_TRUE(Names(worlds).empty());
// Old files at the top of logs/ count too
Filter chat;
chat.servers = { "ChatServer" };
EXPECT_EQ(Names(chat), std::vector<std::string>{ "logs/ChatServer_" + std::to_string(T - 86400) + ".log" });
}
TEST(LogBundleLines, ReadsLineTimes) {
EXPECT_EQ(LineTime(Stamp(T) + "dConfig.cpp:97] hi"), T);
EXPECT_FALSE(LineTime(" continued line"));
EXPECT_FALSE(LineTime("[ab-cd-ef 00:00:00 x"));
}
TEST(LogBundleLines, RedactsAddresses) {
EXPECT_EQ(RedactIps("login from 192.168.1.20 ok"), "login from [ip] ok");
EXPECT_EQ(RedactIps("peer 10.0.0.5:2001 connected."), "peer [ip]:2001 connected.");
EXPECT_EQ(RedactIps("from 2001:db8::1 and ::1."), "from [ip] and [ip].");
EXPECT_EQ(RedactIps("full fe80:0:0:0:202:b3ff:fe1e:8329 x"), "full [ip] x");
// Not addresses
EXPECT_EQ(RedactIps("[27-09-26 09:49:54 dConfig.cpp:97] v1.2.3 999.1.1.1"), "[27-09-26 09:49:54 dConfig.cpp:97] v1.2.3 999.1.1.1");
EXPECT_EQ(RedactIps("Foo::Bar called at 12:30:45"), "Foo::Bar called at 12:30:45");
}
TEST_F(LogBundleTest, ZipsWholeAndFilteredFiles) {
const std::string log = Stamp(T) + "a.cpp:1] start from 1.2.3.4\n" + Stamp(T + 100) + "a.cpp:2] ERROR one\n detail of one\n" +
Stamp(T + 200) + "a.cpp:3] fine\n" + Stamp(T + 300) + "a.cpp:4] error two\n";
Make("logs/AuthServer/AuthServer_" + std::to_string(T) + ".log", log, T + 300);
Make("logs/ChatServer/ChatServer_" + std::to_string(T) + ".log", Stamp(T) + "c.cpp:1] nothing\n", T + 300);
const auto out = m_Root / "bundle.zip";
Filter whole;
auto files = Select(m_Root / "logs", m_Root / "dumps", whole);
auto result = WriteZip(out, files, whole, "header\n", 1024 * 1024);
ASSERT_TRUE(result.ok) << result.error;
EXPECT_EQ(result.files, 2u);
auto entries = ReadZip(out);
EXPECT_EQ(entries["logs/AuthServer/AuthServer_" + std::to_string(T) + ".log"], log);
EXPECT_TRUE(entries["manifest.txt"].starts_with("header\n"));
EXPECT_NE(entries["manifest.txt"].find("logs/ChatServer/ChatServer_"), std::string::npos);
EXPECT_EQ(result.archiveSize, fs::file_size(out));
// Lines from T+50 to T+250 with "error", addresses hidden: the continued line keeps the time of the one before
Filter lines;
lines.from = T + 50;
lines.to = T + 250;
lines.trim = true;
lines.text = "error";
lines.redactIps = true;
result = WriteZip(out, files, lines, "header\n", 1024 * 1024);
ASSERT_TRUE(result.ok) << result.error;
EXPECT_EQ(result.files, 1u); // nothing of the chat log is left, so it stays out
entries = ReadZip(out);
EXPECT_EQ(entries["logs/AuthServer/AuthServer_" + std::to_string(T) + ".log"], Stamp(T + 100) + "a.cpp:2] ERROR one\n");
EXPECT_FALSE(entries.contains("logs/ChatServer/ChatServer_" + std::to_string(T) + ".log"));
lines = {};
lines.redactIps = true;
result = WriteZip(out, files, lines, "", 1024 * 1024);
ASSERT_TRUE(result.ok);
EXPECT_NE(ReadZip(out)["logs/AuthServer/AuthServer_" + std::to_string(T) + ".log"].find("start from [ip]"), std::string::npos);
}
TEST_F(LogBundleTest, StopsAtTheSizeLimit) {
Make("logs/MasterServer/MasterServer_" + std::to_string(T) + ".log", std::string(200 * 1024, 'x'), T);
const auto out = m_Root / "bundle.zip";
Filter filter;
const auto result = WriteZip(out, Select(m_Root / "logs", {}, filter), filter, "", 100 * 1024);
EXPECT_FALSE(result.ok);
EXPECT_TRUE(result.overLimit);
EXPECT_NE(result.error.find("log_bundle_max_mb"), std::string::npos);
EXPECT_FALSE(fs::exists(out));
}
TEST(LogBundleCompression, DeflatesInPieces) {
std::string compressed;
ZCompression::RawDeflater deflater([&](std::string_view data) { compressed += data; return true; });
std::string input;
for (int i = 0; i < 5000; i++) input += "line " + std::to_string(i) + " of the log\n";
for (size_t at = 0; at < input.size(); at += 1000) ASSERT_TRUE(deflater.Write(std::string_view(input).substr(at, 1000)));
ASSERT_TRUE(deflater.Finish());
EXPECT_EQ(deflater.BytesIn(), input.size());
EXPECT_EQ(deflater.BytesOut(), compressed.size());
EXPECT_LT(compressed.size(), input.size() / 3);
EXPECT_EQ(ZCompression::InflateRaw(compressed, input.size()), input);
}