From 709cfab1406ed6569afca7838372166b263273a9 Mon Sep 17 00:00:00 2001 From: Aaron Kimbrell Date: Sun, 27 Sep 2026 10:05:10 -0500 Subject: [PATCH] feat(dashboard): log bundles, zipped log files picked by date, server and world Co-Authored-By: Claude Opus 5.5 --- dDashboardServer/routes/CMakeLists.txt | 1 + dDashboardServer/routes/LogBundle.cpp | 484 ++++++++++++++++++++ dDashboardServer/routes/LogBundle.h | 90 ++++ dDashboardServer/routes/ServerRoutes.cpp | 119 +++++ dDashboardServer/routes/SettingsCatalog.cpp | 3 + tests/dWebTests/CMakeLists.txt | 2 + tests/dWebTests/LogBundleTests.cpp | 266 +++++++++++ 7 files changed, 965 insertions(+) create mode 100644 dDashboardServer/routes/LogBundle.cpp create mode 100644 dDashboardServer/routes/LogBundle.h create mode 100644 tests/dWebTests/LogBundleTests.cpp diff --git a/dDashboardServer/routes/CMakeLists.txt b/dDashboardServer/routes/CMakeLists.txt index fccbfdea6..77e810a4d 100644 --- a/dDashboardServer/routes/CMakeLists.txt +++ b/dDashboardServer/routes/CMakeLists.txt @@ -28,6 +28,7 @@ set(DASHBOARDROUTES_SOURCES "MissionTools.cpp" "CharacterProgress.cpp" "ServerRoutes.cpp" + "LogBundle.cpp" "Traffic.cpp" "TrafficHistory.cpp" "LeaderboardRoutes.cpp" diff --git a/dDashboardServer/routes/LogBundle.cpp b/dDashboardServer/routes/LogBundle.cpp new file mode 100644 index 000000000..22785abf4 --- /dev/null +++ b/dDashboardServer/routes/LogBundle.cpp @@ -0,0 +1,484 @@ +#include "LogBundle.h" + +#include +#include +#include +#include +#include + +#include "GeneralUtils.h" +#include "ZCompression.h" + +namespace fs = std::filesystem; + +namespace { + constexpr std::string_view SERVERS[][2] = { + { "master", "MasterServer" }, { "auth", "AuthServer" }, { "chat", "ChatServer" }, + { "dashboard", "DashboardServer" }, { "ugc", "UgcServer" }, { "world", "WorldServer" }, + }; + + std::optional Number(std::string_view text) { + if (text.empty() || text.size() > 10 || !std::all_of(text.begin(), text.end(), [](char c) { return c >= '0' && c <= '9'; })) return std::nullopt; + const auto value = std::stoull(std::string(text)); + if (value > UINT32_MAX) return std::nullopt; + return static_cast(value); + } + + std::vector Split(std::string_view text, char separator) { + std::vector parts; + size_t start = 0; + while (true) { + const auto end = text.find(separator, start); + parts.push_back(text.substr(start, end == std::string_view::npos ? std::string_view::npos : end - start)); + if (end == std::string_view::npos) return parts; + start = end + 1; + } + } + + bool KnownServer(std::string_view name) { + return std::any_of(std::begin(SERVERS), std::end(SERVERS), [&](const auto& s) { return s[1] == name; }); + } + + std::tm LocalTime(int64_t time) { + const auto t = static_cast(time); + std::tm out{}; +#ifdef _WIN32 + localtime_s(&out, &t); +#else + localtime_r(&t, &out); +#endif + return out; + } + + std::string TimeText(int64_t time) { + if (time <= 0) return "-"; + const auto tm = LocalTime(time); + char text[32]; + std::strftime(text, sizeof(text), "%Y-%m-%d %H:%M:%S", &tm); + return text; + } + + int64_t WriteTime(const fs::path& path) { + std::error_code ec; + const auto time = fs::last_write_time(path, ec); + if (ec) return 0; + return std::chrono::duration_cast(std::chrono::clock_cast(time).time_since_epoch()).count(); + } + + bool Matches(const LogBundle::LogName& name, const LogBundle::Filter& filter) { + if (!filter.servers.empty() && !filter.servers.contains(name.server)) return false; + if (name.server != "WorldServer") return true; + if (!filter.zones.empty() && (!name.zone || !filter.zones.contains(*name.zone))) return false; + if (filter.clone && name.clone != filter.clone) return false; + if (filter.instance && name.instance != filter.instance) return false; + return true; + } + + bool Overlaps(int64_t started, int64_t written, const LogBundle::Filter& filter) { + return (filter.to == 0 || started <= filter.to) && (filter.from == 0 || written >= filter.from); + } + + bool IsHex(char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } + bool IsDigit(char c) { return c >= '0' && c <= '9'; } + bool IsWordChar(char c) { return std::isalnum(static_cast(c)) || c == '_'; } + + // Writes a zip file a piece at a time: each entry's sizes and CRC follow its data (flag bit 3), so nothing needs + // to be known before it's compressed. Without zip64, so everything must stay under 4 GB (the size limit sees to it). + class ZipWriter { + public: + explicit ZipWriter(const fs::path& path) : m_File(path, std::ios::binary | std::ios::trunc) {} + bool Ok() const { return m_File.good(); } + uint64_t Size() const { return m_Offset; } + + bool Begin(const std::string& name, int64_t time) { + m_Entry = { name, static_cast(m_Offset), DosTime(time) }; + std::string header; + Put32(header, 0x04034b50); + Put16(header, 20); // version needed: deflate + Put16(header, 0x0808); // sizes after the data, UTF-8 names + Put16(header, 8); // deflate + Put32(header, m_Entry.dosTime); + Put32(header, 0); Put32(header, 0); Put32(header, 0); + Put16(header, static_cast(name.size())); + Put16(header, 0); + header += name; + m_Crc = 0; + m_Deflater = std::make_unique([this](std::string_view data) { return Raw(data); }); + return Raw(header); + } + + bool Write(std::string_view data) { + m_Crc = ZCompression::Crc32(m_Crc, data); + return m_Deflater->Write(data); + } + + bool End() { + if (!m_Deflater->Finish()) return false; + m_Entry.crc = m_Crc; + m_Entry.compressed = static_cast(m_Deflater->BytesOut()); + m_Entry.size = static_cast(m_Deflater->BytesIn()); + m_Deflater.reset(); + std::string descriptor; + Put32(descriptor, 0x08074b50); + Put32(descriptor, m_Entry.crc); + Put32(descriptor, m_Entry.compressed); + Put32(descriptor, m_Entry.size); + m_Entries.push_back(m_Entry); + return Raw(descriptor); + } + + bool Close() { + const auto start = m_Offset; + std::string directory; + for (const auto& entry : m_Entries) { + Put32(directory, 0x02014b50); + Put16(directory, 0x0314); // made by: Unix, 2.0 (so the permissions below count) + Put16(directory, 20); + Put16(directory, 0x0808); + Put16(directory, 8); + Put32(directory, entry.dosTime); + Put32(directory, entry.crc); + Put32(directory, entry.compressed); + Put32(directory, entry.size); + Put16(directory, static_cast(entry.name.size())); + Put16(directory, 0); Put16(directory, 0); Put16(directory, 0); Put16(directory, 0); + Put32(directory, 0100644u << 16); // a plain file, rw-r--r-- + Put32(directory, entry.offset); + directory += entry.name; + } + const auto directorySize = directory.size(); + Put32(directory, 0x06054b50); + Put16(directory, 0); Put16(directory, 0); + Put16(directory, static_cast(m_Entries.size())); + Put16(directory, static_cast(m_Entries.size())); + Put32(directory, static_cast(directorySize)); + Put32(directory, static_cast(start)); + Put16(directory, 0); + if (!Raw(directory)) return false; + m_File.close(); + return !m_File.fail(); + } + + private: + struct Entry { + std::string name; + uint32_t offset{}; + uint32_t dosTime{}; + uint32_t crc{}, compressed{}, size{}; + }; + + static void Put16(std::string& out, uint16_t value) { out += static_cast(value & 0xFF); out += static_cast(value >> 8); } + static void Put32(std::string& out, uint32_t value) { Put16(out, value & 0xFFFF); Put16(out, value >> 16); } + + static uint32_t DosTime(int64_t time) { + const auto tm = LocalTime(std::max(time, 315532800)); // zip times start in 1980 + const uint32_t date = ((tm.tm_year - 80) << 9) | ((tm.tm_mon + 1) << 5) | tm.tm_mday; + const uint32_t clock = (tm.tm_hour << 11) | (tm.tm_min << 5) | (tm.tm_sec / 2); + return (date << 16) | clock; + } + + bool Raw(std::string_view data) { + m_File.write(data.data(), static_cast(data.size())); + m_Offset += data.size(); + return m_File.good(); + } + + std::ofstream m_File; + uint64_t m_Offset{}; + Entry m_Entry; + uint32_t m_Crc{}; + std::unique_ptr m_Deflater; + std::vector m_Entries; + }; +} + +namespace LogBundle { + std::optional ParseLogName(std::string_view fileName) { + if (!fileName.ends_with(".log")) return std::nullopt; + fileName.remove_suffix(4); + const auto parts = Split(fileName, '_'); + if (parts.empty() || !KnownServer(parts[0])) return std::nullopt; + LogName name{ std::string(parts[0]) }; + std::vector numbers; + for (size_t i = 1; i < parts.size(); i++) { + const auto number = Number(parts[i]); + if (!number) return std::nullopt; + numbers.push_back(*number); + } + if (name.server == "WorldServer") { + // WorldServer____