feat(dashboard): database usage report: growth, fastest tables, disk warning

Pure helpers for the Database usage page, unit tested.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dDashboardServer/routes/DatabaseUsageReport.cpp

@@ -0,0 +1,92 @@
+#include "DatabaseUsageReport.h"
+
+#include <algorithm>
+#include <map>
+
+namespace DatabaseUsageReport {
+	bool IsLocalHost(const std::string& mysqlHost) {
+		std::string host = mysqlHost;
+		std::ranges::transform(host, host.begin(), ::tolower);
+		if (host.starts_with("unix://") || host.starts_with("pipe://")) return true;
+		if (host.starts_with("tcp://")) host = host.substr(6);
+		if (host.starts_with("[")) host = host.substr(1, host.find(']') == std::string::npos ? std::string::npos : host.find(']') - 1);
+		else if (std::ranges::count(host, ':') == 1) host = host.substr(0, host.find(':')); // host:port (an IPv6 address has more colons)
+		return host.empty() || host == "localhost" || host == "::1" || host.starts_with("127.");
+	}
+
+	Disk DiskState(uint64_t free, uint64_t total, double warnPercent) {
+		Disk disk;
+		if (total == 0) return disk;
+		disk.known = true;
+		disk.free = std::min(free, total);
+		disk.total = total;
+		disk.freePercent = 100.0 * static_cast<double>(disk.free) / static_cast<double>(total);
+		disk.low = warnPercent > 0 && disk.freePercent < warnPercent;
+		return disk;
+	}
+
+	nlohmann::json TotalsJson(const IDatabaseUsage::UsageTotals& totals, double warnPercent) {
+		const auto disk = DiskState(totals.diskFree, totals.diskTotal, warnPercent);
+		return {
+			{"time", totals.time}, {"total_bytes", totals.totalBytes}, {"free_bytes", totals.freeBytes}, {"data_bytes", totals.dataBytes},
+			{"index_bytes", totals.indexBytes}, {"rows", totals.rows},
+			{"disk", { {"known", disk.known}, {"free", disk.free}, {"total", disk.total}, {"free_percent", disk.freePercent}, {"low", disk.low} }},
+		};
+	}
+
+	nlohmann::json Series(const std::vector<IDatabaseUsage::UsageTotals>& samples, size_t maxPoints) {
+		nlohmann::json out = { {"times", nlohmann::json::array()}, {"total", nlohmann::json::array()}, {"data", nlohmann::json::array()},
+			{"index", nlohmann::json::array()}, {"free", nlohmann::json::array()}, {"rows", nlohmann::json::array()}, {"disk_free", nlohmann::json::array()} };
+		if (samples.empty() || maxPoints == 0) return out;
+		const auto add = [&out](const IDatabaseUsage::UsageTotals& s) {
+			out["times"].push_back(s.time);
+			out["total"].push_back(s.totalBytes);
+			out["data"].push_back(s.dataBytes);
+			out["index"].push_back(s.indexBytes);
+			out["free"].push_back(s.freeBytes);
+			out["rows"].push_back(s.rows);
+			out["disk_free"].push_back(s.diskTotal ? nlohmann::json(s.diskFree) : nlohmann::json(nullptr));
+		};
+		if (samples.size() <= maxPoints) {
+			for (const auto& s : samples) add(s);
+			return out;
+		}
+		const auto from = samples.front().time, to = samples.back().time;
+		const auto span = std::max<int64_t>(to - from, 1);
+		for (size_t i = 0; i < samples.size(); i++) {
+			// The last sample of each slice (always the newest one)
+			const auto here = std::min<size_t>(static_cast<size_t>((samples[i].time - from) * static_cast<int64_t>(maxPoints) / (span + 1)), maxPoints - 1);
+			const bool last = i + 1 == samples.size();
+			const auto next = last ? maxPoints : std::min<size_t>(static_cast<size_t>((samples[i + 1].time - from) * static_cast<int64_t>(maxPoints) / (span + 1)), maxPoints - 1);
+			if (last || next != here) add(samples[i]);
+		}
+		return out;
+	}
+
+	nlohmann::json Tables(const std::vector<IDatabaseUsage::TableUsage>& now, const std::vector<IDatabaseUsage::TableUsage>& before, int64_t seconds) {
+		std::map<std::string, const IDatabaseUsage::TableUsage*> then;
+		for (const auto& table : before) then[table.name] = &table;
+		const double days = seconds > 0 ? static_cast<double>(seconds) / 86400.0 : 0.0;
+		auto out = nlohmann::json::array();
+		for (const auto& table : now) {
+			const auto it = then.find(table.name);
+			const auto total = table.dataBytes + table.indexBytes;
+			const auto oldTotal = it == then.end() ? 0 : it->second->dataBytes + it->second->indexBytes;
+			const auto oldRows = it == then.end() ? 0 : it->second->rows;
+			const auto growth = static_cast<int64_t>(total) - static_cast<int64_t>(oldTotal);
+			out.push_back({ {"name", table.name}, {"data_bytes", table.dataBytes}, {"index_bytes", table.indexBytes}, {"total_bytes", total},
+				{"rows", table.rows}, {"growth_bytes", growth}, {"growth_rows", static_cast<int64_t>(table.rows) - static_cast<int64_t>(oldRows)},
+				{"growth_per_day", days > 0 ? static_cast<double>(growth) / days : 0.0} });
+		}
+		std::stable_sort(out.begin(), out.end(), [](const auto& a, const auto& b) { return a["total_bytes"].template get<uint64_t>() > b["total_bytes"].template get<uint64_t>(); });
+		return out;
+	}
+
+	nlohmann::json Fastest(const nlohmann::json& tables, size_t limit) {
+		auto out = nlohmann::json::array();
+		for (const auto& table : tables) if (table.value("growth_bytes", int64_t{}) > 0) out.push_back(table);
+		std::stable_sort(out.begin(), out.end(), [](const auto& a, const auto& b) { return a["growth_bytes"].template get<int64_t>() > b["growth_bytes"].template get<int64_t>(); });
+		if (out.size() > limit) out.erase(out.begin() + static_cast<std::ptrdiff_t>(limit), out.end());
+		return out;
+	}
+}

dDashboardServer/routes/DatabaseUsageReport.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "IDatabaseUsage.h"
+#include "json.hpp"
+
+/**
+ * The Database usage page's numbers, worked out from the samples (IDatabaseUsage): the growth chart's series, each
+ * table's size and growth over the range, the fastest-growing tables and the disk space warning. Pure (no database,
+ * config or clock), so it is unit tested; DatabaseUsage.cpp feeds it.
+ */
+namespace DatabaseUsageReport {
+	// Whether mysql_host is this machine, so the database's data directory can be looked at (localhost, 127.0.0.1, ::1,
+	// unix:// or pipe://, with or without tcp:// and a port)
+	bool IsLocalHost(const std::string& mysqlHost);
+
+	struct Disk {
+		bool known{};
+		uint64_t free{};
+		uint64_t total{};
+		double freePercent{};
+		bool low{}; // free below warnPercent of the volume (warnPercent 0: never)
+	};
+	Disk DiskState(uint64_t free, uint64_t total, double warnPercent);
+
+	// One sample: {time, total_bytes, free_bytes, data_bytes, index_bytes, rows, disk: {known, free, total, free_percent, low}}
+	nlohmann::json TotalsJson(const IDatabaseUsage::UsageTotals& totals, double warnPercent);
+
+	// The samples for the chart, at most maxPoints (the last sample of each equal slice of the range):
+	// {times, total, data, index, free, rows, disk_free}
+	nlohmann::json Series(const std::vector<IDatabaseUsage::UsageTotals>& samples, size_t maxPoints);
+
+	// Every table now with how much it grew since `before` (a table missing then grew by all it has):
+	// [{name, data_bytes, index_bytes, total_bytes, rows, growth_bytes, growth_rows, growth_per_day}], biggest first
+	nlohmann::json Tables(const std::vector<IDatabaseUsage::TableUsage>& now, const std::vector<IDatabaseUsage::TableUsage>& before, int64_t seconds);
+
+	// From Tables(): the `limit` tables that grew the most bytes (only those that grew)
+	nlohmann::json Fastest(const nlohmann::json& tables, size_t limit);
+}

tests/dWebTests/CMakeLists.txt

@@ -56,6 +56,8 @@ set(DWEBTESTS_SOURCES
 	"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/BackupFiles.cpp"
 	"MetricsFormatTests.cpp"
 	"TrafficHistoryTests.cpp"
+	"DatabaseUsageReportTests.cpp"
+	"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/DatabaseUsageReport.cpp"
 	"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/TrafficHistory.cpp"
 	"NetworkViewTests.cpp"
 	"${PROJECT_SOURCE_DIR}/dDashboardServer/routes/NetworkView.cpp"

tests/dWebTests/DatabaseUsageReportTests.cpp

@@ -0,0 +1,67 @@
+#include <gtest/gtest.h>
+
+#include "DatabaseUsageReport.h"
+
+using namespace DatabaseUsageReport;
+
+TEST(DatabaseUsageReport, LocalHosts) {
+	EXPECT_TRUE(IsLocalHost(""));
+	EXPECT_TRUE(IsLocalHost("localhost"));
+	EXPECT_TRUE(IsLocalHost("LOCALHOST:3306"));
+	EXPECT_TRUE(IsLocalHost("127.0.0.1"));
+	EXPECT_TRUE(IsLocalHost("tcp://127.0.0.1:3307"));
+	EXPECT_TRUE(IsLocalHost("[::1]:3306"));
+	EXPECT_TRUE(IsLocalHost("::1"));
+	EXPECT_TRUE(IsLocalHost("unix:///run/mysqld/mysqld.sock"));
+	EXPECT_TRUE(IsLocalHost("pipe://MySQL"));
+	EXPECT_FALSE(IsLocalHost("db.example.com"));
+	EXPECT_FALSE(IsLocalHost("tcp://10.0.0.5:3306"));
+	EXPECT_FALSE(IsLocalHost("192.168.1.2"));
+}
+
+TEST(DatabaseUsageReport, DiskWarning) {
+	EXPECT_FALSE(DiskState(0, 0, 10).known);
+	const auto low = DiskState(5, 100, 10);
+	EXPECT_TRUE(low.known);
+	EXPECT_TRUE(low.low);
+	EXPECT_DOUBLE_EQ(low.freePercent, 5.0);
+	EXPECT_FALSE(DiskState(10, 100, 10).low);
+	EXPECT_FALSE(DiskState(5, 100, 0).low); // 0: never warn
+	EXPECT_EQ(DiskState(200, 100, 10).free, 100u);
+	const auto json = TotalsJson({ 7, 1000, 0, 600, 400, 3, 5, 100 }, 10);
+	EXPECT_TRUE(json["disk"]["low"].get<bool>());
+	EXPECT_EQ(json["total_bytes"].get<uint64_t>(), 1000u);
+}
+
+TEST(DatabaseUsageReport, SeriesKeepsTheNewestSampleOfEachSlice) {
+	std::vector<IDatabaseUsage::UsageTotals> samples;
+	for (int64_t i = 0; i < 100; i++) samples.push_back({ i * 3600, static_cast<uint64_t>(i), 0, 0, 0, 0, 0, i % 2 ? 0u : 50u });
+	const auto all = Series(samples, 200);
+	EXPECT_EQ(all["times"].size(), 100u);
+	EXPECT_TRUE(all["disk_free"][1].is_null()); // disk not known then
+	const auto few = Series(samples, 10);
+	EXPECT_LE(few["times"].size(), 10u);
+	EXPECT_GE(few["times"].size(), 9u);
+	EXPECT_EQ(few["times"].back().get<int64_t>(), 99 * 3600);
+	EXPECT_EQ(few["total"].back().get<uint64_t>(), 99u);
+	for (size_t i = 1; i < few["times"].size(); i++) EXPECT_LT(few["times"][i - 1].get<int64_t>(), few["times"][i].get<int64_t>());
+	EXPECT_TRUE(Series({}, 10)["times"].empty());
+}
+
+TEST(DatabaseUsageReport, GrowthPerTable) {
+	const std::vector<IDatabaseUsage::TableUsage> before = { { "chat_log", 1000, 100, 10 }, { "accounts", 500, 500, 2 }, { "gone", 9, 0, 1 } };
+	const std::vector<IDatabaseUsage::TableUsage> now = { { "chat_log", 5000, 300, 50 }, { "accounts", 400, 500, 2 }, { "new_table", 200, 0, 4 } };
+	const auto tables = Tables(now, before, 2 * 86400);
+	ASSERT_EQ(tables.size(), 3u);
+	EXPECT_EQ(tables[0]["name"], "chat_log"); // biggest first
+	EXPECT_EQ(tables[0]["growth_bytes"].get<int64_t>(), 4200);
+	EXPECT_EQ(tables[0]["growth_rows"].get<int64_t>(), 40);
+	EXPECT_DOUBLE_EQ(tables[0]["growth_per_day"].get<double>(), 2100.0);
+	EXPECT_EQ(tables[1]["growth_bytes"].get<int64_t>(), -100); // accounts shrank
+	EXPECT_EQ(tables[2]["growth_bytes"].get<int64_t>(), 200); // new: all of it
+
+	const auto fastest = Fastest(tables, 1);
+	ASSERT_EQ(fastest.size(), 1u);
+	EXPECT_EQ(fastest[0]["name"], "chat_log");
+	EXPECT_EQ(Fastest(tables, 10).size(), 2u); // the shrinking table is left out
+}