diff --git a/dDashboardServer/routes/DatabaseUsageReport.cpp b/dDashboardServer/routes/DatabaseUsageReport.cpp new file mode 100644 index 000000000..03c67642d --- /dev/null +++ b/dDashboardServer/routes/DatabaseUsageReport.cpp @@ -0,0 +1,92 @@ +#include "DatabaseUsageReport.h" + +#include +#include + +namespace DatabaseUsageReport { + bool IsLocalHost(const std::string& mysqlHost) { + std::string host = mysqlHost; + std::ranges::transform(host, host.begin(), ::tolower); + if (host.starts_with("unix://") || host.starts_with("pipe://")) return true; + if (host.starts_with("tcp://")) host = host.substr(6); + if (host.starts_with("[")) host = host.substr(1, host.find(']') == std::string::npos ? std::string::npos : host.find(']') - 1); + else if (std::ranges::count(host, ':') == 1) host = host.substr(0, host.find(':')); // host:port (an IPv6 address has more colons) + return host.empty() || host == "localhost" || host == "::1" || host.starts_with("127."); + } + + Disk DiskState(uint64_t free, uint64_t total, double warnPercent) { + Disk disk; + if (total == 0) return disk; + disk.known = true; + disk.free = std::min(free, total); + disk.total = total; + disk.freePercent = 100.0 * static_cast(disk.free) / static_cast(total); + disk.low = warnPercent > 0 && disk.freePercent < warnPercent; + return disk; + } + + nlohmann::json TotalsJson(const IDatabaseUsage::UsageTotals& totals, double warnPercent) { + const auto disk = DiskState(totals.diskFree, totals.diskTotal, warnPercent); + return { + {"time", totals.time}, {"total_bytes", totals.totalBytes}, {"free_bytes", totals.freeBytes}, {"data_bytes", totals.dataBytes}, + {"index_bytes", totals.indexBytes}, {"rows", totals.rows}, + {"disk", { {"known", disk.known}, {"free", disk.free}, {"total", disk.total}, {"free_percent", disk.freePercent}, {"low", disk.low} }}, + }; + } + + nlohmann::json Series(const std::vector& samples, size_t maxPoints) { + nlohmann::json out = { {"times", nlohmann::json::array()}, {"total", nlohmann::json::array()}, {"data", nlohmann::json::array()}, + {"index", nlohmann::json::array()}, {"free", nlohmann::json::array()}, {"rows", nlohmann::json::array()}, {"disk_free", nlohmann::json::array()} }; + if (samples.empty() || maxPoints == 0) return out; + const auto add = [&out](const IDatabaseUsage::UsageTotals& s) { + out["times"].push_back(s.time); + out["total"].push_back(s.totalBytes); + out["data"].push_back(s.dataBytes); + out["index"].push_back(s.indexBytes); + out["free"].push_back(s.freeBytes); + out["rows"].push_back(s.rows); + out["disk_free"].push_back(s.diskTotal ? nlohmann::json(s.diskFree) : nlohmann::json(nullptr)); + }; + if (samples.size() <= maxPoints) { + for (const auto& s : samples) add(s); + return out; + } + const auto from = samples.front().time, to = samples.back().time; + const auto span = std::max(to - from, 1); + for (size_t i = 0; i < samples.size(); i++) { + // The last sample of each slice (always the newest one) + const auto here = std::min(static_cast((samples[i].time - from) * static_cast(maxPoints) / (span + 1)), maxPoints - 1); + const bool last = i + 1 == samples.size(); + const auto next = last ? maxPoints : std::min(static_cast((samples[i + 1].time - from) * static_cast(maxPoints) / (span + 1)), maxPoints - 1); + if (last || next != here) add(samples[i]); + } + return out; + } + + nlohmann::json Tables(const std::vector& now, const std::vector& before, int64_t seconds) { + std::map then; + for (const auto& table : before) then[table.name] = &table; + const double days = seconds > 0 ? static_cast(seconds) / 86400.0 : 0.0; + auto out = nlohmann::json::array(); + for (const auto& table : now) { + const auto it = then.find(table.name); + const auto total = table.dataBytes + table.indexBytes; + const auto oldTotal = it == then.end() ? 0 : it->second->dataBytes + it->second->indexBytes; + const auto oldRows = it == then.end() ? 0 : it->second->rows; + const auto growth = static_cast(total) - static_cast(oldTotal); + out.push_back({ {"name", table.name}, {"data_bytes", table.dataBytes}, {"index_bytes", table.indexBytes}, {"total_bytes", total}, + {"rows", table.rows}, {"growth_bytes", growth}, {"growth_rows", static_cast(table.rows) - static_cast(oldRows)}, + {"growth_per_day", days > 0 ? static_cast(growth) / days : 0.0} }); + } + std::stable_sort(out.begin(), out.end(), [](const auto& a, const auto& b) { return a["total_bytes"].template get() > b["total_bytes"].template get(); }); + return out; + } + + nlohmann::json Fastest(const nlohmann::json& tables, size_t limit) { + auto out = nlohmann::json::array(); + for (const auto& table : tables) if (table.value("growth_bytes", int64_t{}) > 0) out.push_back(table); + std::stable_sort(out.begin(), out.end(), [](const auto& a, const auto& b) { return a["growth_bytes"].template get() > b["growth_bytes"].template get(); }); + if (out.size() > limit) out.erase(out.begin() + static_cast(limit), out.end()); + return out; + } +} diff --git a/dDashboardServer/routes/DatabaseUsageReport.h b/dDashboardServer/routes/DatabaseUsageReport.h new file mode 100644 index 000000000..71dde8ec6 --- /dev/null +++ b/dDashboardServer/routes/DatabaseUsageReport.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + +#include "IDatabaseUsage.h" +#include "json.hpp" + +/** + * The Database usage page's numbers, worked out from the samples (IDatabaseUsage): the growth chart's series, each + * table's size and growth over the range, the fastest-growing tables and the disk space warning. Pure (no database, + * config or clock), so it is unit tested; DatabaseUsage.cpp feeds it. + */ +namespace DatabaseUsageReport { + // Whether mysql_host is this machine, so the database's data directory can be looked at (localhost, 127.0.0.1, ::1, + // unix:// or pipe://, with or without tcp:// and a port) + bool IsLocalHost(const std::string& mysqlHost); + + struct Disk { + bool known{}; + uint64_t free{}; + uint64_t total{}; + double freePercent{}; + bool low{}; // free below warnPercent of the volume (warnPercent 0: never) + }; + Disk DiskState(uint64_t free, uint64_t total, double warnPercent); + + // One sample: {time, total_bytes, free_bytes, data_bytes, index_bytes, rows, disk: {known, free, total, free_percent, low}} + nlohmann::json TotalsJson(const IDatabaseUsage::UsageTotals& totals, double warnPercent); + + // The samples for the chart, at most maxPoints (the last sample of each equal slice of the range): + // {times, total, data, index, free, rows, disk_free} + nlohmann::json Series(const std::vector& samples, size_t maxPoints); + + // Every table now with how much it grew since `before` (a table missing then grew by all it has): + // [{name, data_bytes, index_bytes, total_bytes, rows, growth_bytes, growth_rows, growth_per_day}], biggest first + nlohmann::json Tables(const std::vector& now, const std::vector& before, int64_t seconds); + + // From Tables(): the `limit` tables that grew the most bytes (only those that grew) + nlohmann::json Fastest(const nlohmann::json& tables, size_t limit); +} diff --git a/tests/dWebTests/CMakeLists.txt b/tests/dWebTests/CMakeLists.txt index 1cd3528e9..695448118 100644 --- a/tests/dWebTests/CMakeLists.txt +++ b/tests/dWebTests/CMakeLists.txt @@ -56,6 +56,8 @@ set(DWEBTESTS_SOURCES "${PROJECT_SOURCE_DIR}/dDashboardServer/routes/BackupFiles.cpp" "MetricsFormatTests.cpp" "TrafficHistoryTests.cpp" + "DatabaseUsageReportTests.cpp" + "${PROJECT_SOURCE_DIR}/dDashboardServer/routes/DatabaseUsageReport.cpp" "${PROJECT_SOURCE_DIR}/dDashboardServer/routes/TrafficHistory.cpp" "NetworkViewTests.cpp" "${PROJECT_SOURCE_DIR}/dDashboardServer/routes/NetworkView.cpp" diff --git a/tests/dWebTests/DatabaseUsageReportTests.cpp b/tests/dWebTests/DatabaseUsageReportTests.cpp new file mode 100644 index 000000000..e4875cd5f --- /dev/null +++ b/tests/dWebTests/DatabaseUsageReportTests.cpp @@ -0,0 +1,67 @@ +#include + +#include "DatabaseUsageReport.h" + +using namespace DatabaseUsageReport; + +TEST(DatabaseUsageReport, LocalHosts) { + EXPECT_TRUE(IsLocalHost("")); + EXPECT_TRUE(IsLocalHost("localhost")); + EXPECT_TRUE(IsLocalHost("LOCALHOST:3306")); + EXPECT_TRUE(IsLocalHost("127.0.0.1")); + EXPECT_TRUE(IsLocalHost("tcp://127.0.0.1:3307")); + EXPECT_TRUE(IsLocalHost("[::1]:3306")); + EXPECT_TRUE(IsLocalHost("::1")); + EXPECT_TRUE(IsLocalHost("unix:///run/mysqld/mysqld.sock")); + EXPECT_TRUE(IsLocalHost("pipe://MySQL")); + EXPECT_FALSE(IsLocalHost("db.example.com")); + EXPECT_FALSE(IsLocalHost("tcp://10.0.0.5:3306")); + EXPECT_FALSE(IsLocalHost("192.168.1.2")); +} + +TEST(DatabaseUsageReport, DiskWarning) { + EXPECT_FALSE(DiskState(0, 0, 10).known); + const auto low = DiskState(5, 100, 10); + EXPECT_TRUE(low.known); + EXPECT_TRUE(low.low); + EXPECT_DOUBLE_EQ(low.freePercent, 5.0); + EXPECT_FALSE(DiskState(10, 100, 10).low); + EXPECT_FALSE(DiskState(5, 100, 0).low); // 0: never warn + EXPECT_EQ(DiskState(200, 100, 10).free, 100u); + const auto json = TotalsJson({ 7, 1000, 0, 600, 400, 3, 5, 100 }, 10); + EXPECT_TRUE(json["disk"]["low"].get()); + EXPECT_EQ(json["total_bytes"].get(), 1000u); +} + +TEST(DatabaseUsageReport, SeriesKeepsTheNewestSampleOfEachSlice) { + std::vector samples; + for (int64_t i = 0; i < 100; i++) samples.push_back({ i * 3600, static_cast(i), 0, 0, 0, 0, 0, i % 2 ? 0u : 50u }); + const auto all = Series(samples, 200); + EXPECT_EQ(all["times"].size(), 100u); + EXPECT_TRUE(all["disk_free"][1].is_null()); // disk not known then + const auto few = Series(samples, 10); + EXPECT_LE(few["times"].size(), 10u); + EXPECT_GE(few["times"].size(), 9u); + EXPECT_EQ(few["times"].back().get(), 99 * 3600); + EXPECT_EQ(few["total"].back().get(), 99u); + for (size_t i = 1; i < few["times"].size(); i++) EXPECT_LT(few["times"][i - 1].get(), few["times"][i].get()); + EXPECT_TRUE(Series({}, 10)["times"].empty()); +} + +TEST(DatabaseUsageReport, GrowthPerTable) { + const std::vector before = { { "chat_log", 1000, 100, 10 }, { "accounts", 500, 500, 2 }, { "gone", 9, 0, 1 } }; + const std::vector now = { { "chat_log", 5000, 300, 50 }, { "accounts", 400, 500, 2 }, { "new_table", 200, 0, 4 } }; + const auto tables = Tables(now, before, 2 * 86400); + ASSERT_EQ(tables.size(), 3u); + EXPECT_EQ(tables[0]["name"], "chat_log"); // biggest first + EXPECT_EQ(tables[0]["growth_bytes"].get(), 4200); + EXPECT_EQ(tables[0]["growth_rows"].get(), 40); + EXPECT_DOUBLE_EQ(tables[0]["growth_per_day"].get(), 2100.0); + EXPECT_EQ(tables[1]["growth_bytes"].get(), -100); // accounts shrank + EXPECT_EQ(tables[2]["growth_bytes"].get(), 200); // new: all of it + + const auto fastest = Fastest(tables, 1); + ASSERT_EQ(fastest.size(), 1u); + EXPECT_EQ(fastest[0]["name"], "chat_log"); + EXPECT_EQ(Fastest(tables, 10).size(), 2u); // the shrinking table is left out +}