feat(db): chat log guild, team and filter result; chat flags

chat_log gets guild_id, team_id and filtered (migrations mysql 100, sqlite 83),
and the chat log queries take a time range, a guild, a team, a conversation
between two characters and paging back by id. Whispers are their own visibility
switch (includeWhispers), apart from team and guild chat (includePrivate).
Whisper partners and teams that talked are listed with counts.

chat_flags, chat_flag_messages and chat_flag_events (mysql 101, sqlite 84) hold
flagged chat with a copy of the messages around it, its status, note, linked
player report and history. Open flags are in the dashboard snapshot.

Tests on SQLite alone (dDatabaseSqliteTests) and in the MySQL parity tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dDatabase/GameDatabase/GameDatabase.h

@@ -40,6 +40,7 @@
 #include "IAccountStrikes.h"
 #include "IServerHealth.h"
 #include "IChatLog.h"
+#include "IChatFlags.h"
 #include "IRelatedData.h"
 #include "IServerOperations.h"
 #include "IPlayerPositions.h"
@@ -69,7 +70,7 @@ class GameDatabase :
 	public IPropertyContents, public IProperty, public IPetNames, public ICharXml,
 	public IMigrationHistory, public IUgc, public IFriends, public ICharInfo,
 	public IAccounts, public IActivityLog, public IAccountsRewardCodes, public IIgnoreList,
-	public IBehaviors, public IUgcModularBuild, public IAuditLog, public IDashboardStats, public IAccountEmails, public IDashboardMaintenance, public IEconomyLedger, public IDashboardAdmin, public IServerConfig, public IScheduledTasks, public ICharacterSnapshots, public IAccountNotes, public IServerHealth, public IRelatedData, public IChatLog, public IAccountStrikes, public ISlashCommands, public IModeration, public IServerOperations, public IPlayerPositions, public IAiSuggestions, public ILiveOps, public IFeaturedProperties, public IMessageCaptures, public IContraband, public IPropertyRent, public IPropertyReputation, public IBbbAutosave, public IServerTraffic, public IApiKeys, public IUgcLookup, public IPermissionGrants, public IGuilds {
+	public IBehaviors, public IUgcModularBuild, public IAuditLog, public IDashboardStats, public IAccountEmails, public IDashboardMaintenance, public IEconomyLedger, public IDashboardAdmin, public IServerConfig, public IScheduledTasks, public ICharacterSnapshots, public IAccountNotes, public IServerHealth, public IRelatedData, public IChatLog, public IChatFlags, public IAccountStrikes, public ISlashCommands, public IModeration, public IServerOperations, public IPlayerPositions, public IAiSuggestions, public ILiveOps, public IFeaturedProperties, public IMessageCaptures, public IContraband, public IPropertyRent, public IPropertyReputation, public IBbbAutosave, public IServerTraffic, public IApiKeys, public IUgcLookup, public IPermissionGrants, public IGuilds {
 public:
 	virtual ~GameDatabase() = default;
 	// TODO: These should be made private.

dDatabase/GameDatabase/ITables/IChatFlags.h

@@ -0,0 +1,75 @@
+#ifndef __ICHATFLAGS__H__
+#define __ICHATFLAGS__H__
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "dCommonVars.h"
+
+/**
+ * Chat messages staff flagged for review (the Chat Flags page). A flag covers one message or several from one
+ * conversation, keeps a copy of them and the chat around them (chat_log is pruned after log_chat_days; the flag keeps
+ * its evidence), and has a history of who did what with it.
+ */
+class IChatFlags {
+public:
+	struct ChatFlag {
+		uint64_t id{};
+		int64_t createdAt{};
+		uint32_t createdById{};     // the staff account
+		std::string createdBy;
+		std::string status{ "open" }; // open, actioned, dismissed
+		std::string channel;        // of the flagged messages
+		LWOOBJID characterId{};     // who the flag is about
+		std::string characterName;
+		uint32_t accountId{};
+		uint64_t firstMessageId{};
+		uint64_t lastMessageId{};
+		int64_t firstTime{};
+		int64_t lastTime{};
+		std::string excerpt;        // the flagged text, shortened, for the queue
+		std::string note;           // why it was flagged, and what was done
+		std::string messages;       // JSON array: the flagged messages and the chat around them, as they were
+		uint64_t playerReportId{};  // a player report this goes with (0: none)
+		int64_t updatedAt{};
+		std::string updatedBy;
+	};
+
+	struct ChatFlagEvent {
+		uint64_t id{};
+		uint64_t flagId{};
+		int64_t time{};
+		uint32_t accountId{};
+		std::string actor;
+		std::string action;         // created, actioned, dismissed, reopened, note, report
+		std::string detail;
+	};
+
+	struct ChatFlagQuery {
+		std::string status;         // empty: any
+		LWOOBJID characterId{};     // 0: anyone
+		uint32_t accountId{};       // 0: any
+		bool includePrivate{};      // flags on team and guild chat
+		bool includeWhispers{};     // flags on whispers
+		uint32_t offset{};
+		uint32_t limit{ 50 };
+	};
+
+	// Adds the flag and remembers which messages it covers. Returns its id.
+	virtual uint64_t InsertChatFlag(const ChatFlag& flag, const std::vector<uint64_t>& messageIds) = 0;
+	virtual std::optional<ChatFlag> GetChatFlag(uint64_t id) = 0;
+	// Newest first
+	virtual std::vector<ChatFlag> GetChatFlags(const ChatFlagQuery& query) = 0;
+	virtual uint64_t CountChatFlags(const ChatFlagQuery& query) = 0;
+	// Status, note and linked report; returns false when there is no such flag
+	virtual bool UpdateChatFlag(uint64_t id, const std::string& status, const std::string& note, uint64_t playerReportId, int64_t time, const std::string& updatedBy) = 0;
+	virtual void InsertChatFlagEvent(const ChatFlagEvent& event) = 0;
+	// Oldest first
+	virtual std::vector<ChatFlagEvent> GetChatFlagEvents(uint64_t flagId) = 0;
+	// Of these messages, the ones in a flag, with the newest flag covering each
+	virtual std::vector<std::pair<uint64_t, uint64_t>> GetFlaggedMessages(const std::vector<uint64_t>& messageIds) = 0;
+};
+
+#endif  //!__ICHATFLAGS__H__

dDatabase/GameDatabase/ITables/IChatLog.h

@@ -28,18 +28,27 @@ public:
 		uint32_t cloneId{};
 		std::string message;
 		bool blocked{};            // stopped by the chat filter: nobody saw it
+		int64_t guildId{};         // guild chat: the guild
+		LWOOBJID teamId{};         // team chat: the chat server's team
+		bool filtered{};           // the chat filter found words it doesn't allow (delivered anyway unless blocked)
 	};
 
 	struct ChatQuery {
 		uint64_t afterId{};        // only messages newer than this (for bots catching up)
 		uint32_t limit{ 100 };
 		std::string channel;       // empty: any
-		bool includePrivate{};     // whispers and team chat
+		bool includePrivate{};     // team and guild chat
+		bool includeWhispers{};    // whispers
 		LWOOBJID characterId{};    // sent or received by this character (0: anyone)
+		LWOOBJID otherCharacterId{}; // with characterId: only whispers between the two
 		uint32_t accountId{};      // sent by this account (0: anyone)
 		uint32_t zoneId{};         // 0: any
 		int64_t instanceId{ -1 };  // -1: any (instance 0 is character select)
 		int64_t since{};           // only messages at or after this time (0: any)
+		int64_t until{};           // only messages before this time (0: any)
+		uint64_t beforeId{};       // only messages older than this (0: any; for paging back through a conversation)
+		int64_t guildId{};         // 0: any
+		LWOOBJID teamId{};         // 0: any
 		std::string search;        // text in the message or a name
 		bool blockedOnly{};
 		bool newestFirst{};
@@ -49,6 +58,28 @@ public:
 	virtual uint64_t InsertChatMessage(const ChatMessage& message) = 0;
 	virtual std::vector<ChatMessage> GetChatMessages(const ChatQuery& query) = 0;
 	virtual uint64_t CountChatMessages(const ChatQuery& query) = 0;
+
+	// The characters a character whispered with, most recent conversation first
+	struct WhisperPartner {
+		LWOOBJID characterId{};
+		std::string name;          // the name in their newest message
+		uint64_t messages{};
+		int64_t firstTime{};
+		int64_t lastTime{};
+	};
+	virtual std::vector<WhisperPartner> GetWhisperPartners(LWOOBJID characterId, uint32_t offset, uint32_t limit) = 0;
+	virtual uint64_t CountWhisperPartners(LWOOBJID characterId) = 0;
+
+	// Teams that talked in team chat, most recent first; characterId (0: any) only teams that character talked in
+	struct ChatTeam {
+		LWOOBJID teamId{};
+		uint64_t messages{};
+		int64_t firstTime{};
+		int64_t lastTime{};
+		std::string senders;       // the names that talked, comma separated
+	};
+	virtual std::vector<ChatTeam> GetChatTeams(LWOOBJID characterId, uint32_t offset, uint32_t limit) = 0;
+	virtual uint64_t CountChatTeams(LWOOBJID characterId) = 0;
 };
 
 #endif  //!__ICHATLOG__H__

dDatabase/GameDatabase/ITables/IDashboardStats.h

@@ -27,6 +27,9 @@ public:
 		uint64_t mailMaxId{};
 		uint64_t openEconomyFlags{};
 		uint64_t economyFlagsMaxId{};
+		uint64_t openChatFlags{};
+		uint64_t chatFlagsMaxId{};
+		uint64_t chatFlagsUpdatedAt{};
 
 		bool operator==(const Snapshot&) const = default;
 	};

dDatabase/GameDatabase/MySQL/MySQLDatabase.h

@@ -335,6 +335,18 @@ public:
 	uint64_t InsertChatMessage(const ChatMessage& message) override;
 	std::vector<ChatMessage> GetChatMessages(const ChatQuery& query) override;
 	uint64_t CountChatMessages(const ChatQuery& query) override;
+	std::vector<WhisperPartner> GetWhisperPartners(LWOOBJID characterId, uint32_t offset, uint32_t limit) override;
+	uint64_t CountWhisperPartners(LWOOBJID characterId) override;
+	std::vector<ChatTeam> GetChatTeams(LWOOBJID characterId, uint32_t offset, uint32_t limit) override;
+	uint64_t CountChatTeams(LWOOBJID characterId) override;
+	uint64_t InsertChatFlag(const ChatFlag& flag, const std::vector<uint64_t>& messageIds) override;
+	std::optional<ChatFlag> GetChatFlag(uint64_t id) override;
+	std::vector<ChatFlag> GetChatFlags(const ChatFlagQuery& query) override;
+	uint64_t CountChatFlags(const ChatFlagQuery& query) override;
+	bool UpdateChatFlag(uint64_t id, const std::string& status, const std::string& note, uint64_t playerReportId, int64_t time, const std::string& updatedBy) override;
+	void InsertChatFlagEvent(const ChatFlagEvent& event) override;
+	std::vector<ChatFlagEvent> GetChatFlagEvents(uint64_t flagId) override;
+	std::vector<std::pair<uint64_t, uint64_t>> GetFlaggedMessages(const std::vector<uint64_t>& messageIds) override;
 	void InsertModerationDecision(const std::string& kind, int64_t subjectId, const std::string& subject, bool approved, const std::string& reason, int64_t time) override;
 	nlohmann::json GetModerationDecisions(const std::string& kind, int64_t subjectId, uint32_t limit) override;
 	Totp GetTotp(uint32_t accountId) override;

dDatabase/GameDatabase/MySQL/Tables/CMakeLists.txt

@@ -28,6 +28,7 @@ set(DDATABASES_DATABASES_MYSQL_TABLES_SOURCES
 	"ServerTraffic.cpp"
 	"MessageCaptures.cpp"
 	"ChatLog.cpp"
+	"ChatFlags.cpp"
 	"RelatedData.cpp"
 	"ActivityLog.cpp"
 	"Behaviors.cpp"

dDatabase/GameDatabase/MySQL/Tables/ChatFlags.cpp

@@ -0,0 +1,107 @@
+#include "MySQLDatabase.h"
+
+namespace {
+	// Every column but messages (the copy of the chat, only read for one flag)
+	constexpr const char* LIST_COLUMNS = "id, created_at, created_by_id, created_by, status, channel, character_id, character_name, account_id, "
+		"first_message_id, last_message_id, first_time, last_time, excerpt, note, player_report_id, updated_at, updated_by";
+
+	// Each filter binds the same parameters whether it is on or off (see ChatLog.cpp)
+	std::string Where(const IChatFlags::ChatFlagQuery& q) {
+		std::string where = " WHERE (channel IN ('zone', 'web') OR (? = 1 AND channel IN ('team', 'guild')) OR (? = 1 AND channel = 'whisper'))";
+		where += q.status.empty() ? " AND (? = '' AND ? = '')" : " AND (? <> '' AND status = ?)";
+		where += q.characterId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND character_id = ?)";
+		where += q.accountId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND account_id = ?)";
+		return where;
+	}
+
+	IChatFlags::ChatFlag Row(sql::ResultSet& r, bool withMessages) {
+		IChatFlags::ChatFlag f;
+		f.id = r.getUInt64("id");
+		f.createdAt = r.getInt64("created_at");
+		f.createdById = r.getUInt("created_by_id");
+		f.createdBy = r.getString("created_by").c_str();
+		f.status = r.getString("status").c_str();
+		f.channel = r.getString("channel").c_str();
+		f.characterId = r.getInt64("character_id");
+		f.characterName = r.getString("character_name").c_str();
+		f.accountId = r.getUInt("account_id");
+		f.firstMessageId = r.getUInt64("first_message_id");
+		f.lastMessageId = r.getUInt64("last_message_id");
+		f.firstTime = r.getInt64("first_time");
+		f.lastTime = r.getInt64("last_time");
+		f.excerpt = r.getString("excerpt").c_str();
+		f.note = r.getString("note").c_str();
+		if (withMessages) f.messages = r.getString("messages").c_str();
+		f.playerReportId = r.getUInt64("player_report_id");
+		f.updatedAt = r.getInt64("updated_at");
+		f.updatedBy = r.getString("updated_by").c_str();
+		return f;
+	}
+}
+
+#define CHAT_FLAG_QUERY_PARAMS(q) (q).includePrivate, (q).includeWhispers, (q).status, (q).status, (q).characterId, (q).characterId, (q).accountId, (q).accountId
+
+uint64_t MySQLDatabase::InsertChatFlag(const ChatFlag& f, const std::vector<uint64_t>& messageIds) {
+	ExecuteInsert("INSERT INTO chat_flags (created_at, created_by_id, created_by, status, channel, character_id, character_name, account_id, first_message_id, "
+		"last_message_id, first_time, last_time, excerpt, note, messages, player_report_id, updated_at, updated_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
+		f.createdAt, f.createdById, f.createdBy, f.status, f.channel, f.characterId, f.characterName, f.accountId, f.firstMessageId, f.lastMessageId,
+		f.firstTime, f.lastTime, f.excerpt, f.note, f.messages, f.playerReportId, f.updatedAt, f.updatedBy);
+	auto last = ExecuteSelect("SELECT LAST_INSERT_ID() AS id;");
+	const uint64_t id = last->next() ? last->getUInt64("id") : 0;
+	for (const auto messageId : messageIds) {
+		ExecuteInsert("INSERT IGNORE INTO chat_flag_messages (flag_id, message_id) VALUES (?, ?);", id, messageId);
+	}
+	return id;
+}
+
+std::optional<IChatFlags::ChatFlag> MySQLDatabase::GetChatFlag(const uint64_t id) {
+	auto result = ExecuteSelect("SELECT * FROM chat_flags WHERE id = ?;", id);
+	if (!result->next()) return std::nullopt;
+	return Row(*result.m_resultSet, true);
+}
+
+std::vector<IChatFlags::ChatFlag> MySQLDatabase::GetChatFlags(const ChatFlagQuery& q) {
+	std::vector<ChatFlag> flags;
+	auto result = ExecuteSelect(std::string("SELECT ") + LIST_COLUMNS + " FROM chat_flags" + Where(q) + " ORDER BY id DESC LIMIT ? OFFSET ?;",
+		CHAT_FLAG_QUERY_PARAMS(q), q.limit, q.offset);
+	while (result->next()) flags.push_back(Row(*result.m_resultSet, false));
+	return flags;
+}
+
+uint64_t MySQLDatabase::CountChatFlags(const ChatFlagQuery& q) {
+	auto result = ExecuteSelect("SELECT COUNT(*) AS count FROM chat_flags" + Where(q) + ";", CHAT_FLAG_QUERY_PARAMS(q));
+	return result->next() ? result->getUInt64("count") : 0;
+}
+
+bool MySQLDatabase::UpdateChatFlag(const uint64_t id, const std::string& status, const std::string& note, const uint64_t playerReportId, const int64_t time, const std::string& updatedBy) {
+	auto exists = ExecuteSelect("SELECT id FROM chat_flags WHERE id = ?;", id);
+	if (!exists->next()) return false;
+	ExecuteUpdate("UPDATE chat_flags SET status = ?, note = ?, player_report_id = ?, updated_at = ?, updated_by = ? WHERE id = ?;", status, note, playerReportId, time, updatedBy, id);
+	return true;
+}
+
+void MySQLDatabase::InsertChatFlagEvent(const ChatFlagEvent& e) {
+	ExecuteInsert("INSERT INTO chat_flag_events (flag_id, time, account_id, actor, action, detail) VALUES (?, ?, ?, ?, ?, ?);",
+		e.flagId, e.time, e.accountId, e.actor, e.action, e.detail);
+}
+
+std::vector<IChatFlags::ChatFlagEvent> MySQLDatabase::GetChatFlagEvents(const uint64_t flagId) {
+	std::vector<ChatFlagEvent> events;
+	auto result = ExecuteSelect("SELECT * FROM chat_flag_events WHERE flag_id = ? ORDER BY id ASC;", flagId);
+	while (result->next()) {
+		events.push_back({ result->getUInt64("id"), result->getUInt64("flag_id"), result->getInt64("time"), result->getUInt("account_id"),
+			std::string(result->getString("actor").c_str()), std::string(result->getString("action").c_str()), std::string(result->getString("detail").c_str()) });
+	}
+	return events;
+}
+
+std::vector<std::pair<uint64_t, uint64_t>> MySQLDatabase::GetFlaggedMessages(const std::vector<uint64_t>& messageIds) {
+	std::vector<std::pair<uint64_t, uint64_t>> flagged;
+	if (messageIds.empty()) return flagged;
+	// Numbers only, written by us: safe to put in the query
+	std::string list;
+	for (const auto id : messageIds) list += (list.empty() ? "" : ",") + std::to_string(id);
+	auto result = ExecuteSelect("SELECT message_id, MAX(flag_id) AS flag_id FROM chat_flag_messages WHERE message_id IN (" + list + ") GROUP BY message_id ORDER BY message_id;");
+	while (result->next()) flagged.emplace_back(result->getUInt64("message_id"), result->getUInt64("flag_id"));
+	return flagged;
+}

dDatabase/GameDatabase/MySQL/Tables/ChatLog.cpp

@@ -1,48 +1,123 @@
 #include "MySQLDatabase.h"
 #include "GeneralUtils.h"
 
+#include <algorithm>
+
 // The filters are all optional. Each binds the same parameters whether it is on or off, so both queries below take one
 // argument list. A filter that is off only compares its own parameters (worked out once, not per row); one that is on is
-// a plain comparison, so the character and account filters are answered from chat_log's indexes instead of a full scan.
+// a plain comparison, so the character, account, guild and team filters are answered from chat_log's indexes instead of
+// a full scan.
 namespace {
 	std::string Where(const IChatLog::ChatQuery& q) {
 		std::string where = " WHERE id > ?";
+		where += q.beforeId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND id < ?)";
 		where += q.channel.empty() ? " AND (? = '' AND ? = '')" : " AND (? <> '' AND channel = ?)";
-		where += q.includePrivate ? " AND (? = 1)" : " AND (? = 1 OR channel IN ('zone', 'web'))";
+		// Zone and web chat for everyone who may read chat; team and guild chat, and whispers, each behind their own permission
+		where += " AND (channel IN ('zone', 'web') OR (? = 1 AND channel IN ('team', 'guild')) OR (? = 1 AND channel = 'whisper'))";
 		where += q.characterId == 0 ? " AND (? = 0 AND ? = 0 AND ? = 0)" : " AND (? <> 0 AND (sender_id = ? OR recipient_id = ?))";
+		where += q.otherCharacterId == 0 ? " AND (? = 0 AND ? = 0 AND ? = 0)" : " AND (? <> 0 AND (sender_id = ? OR recipient_id = ?))";
 		where += q.accountId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND account_id = ?)";
 		where += q.zoneId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND zone_id = ?)";
 		where += q.instanceId < 0 ? " AND (? < 0 AND ? < 0)" : " AND (? >= 0 AND instance_id = ?)";
 		where += q.since == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND time >= ?)";
+		where += q.until == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND time < ?)";
+		where += q.guildId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND guild_id = ?)";
+		where += q.teamId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND team_id = ?)";
 		where += q.search.empty() ? " AND (? = '' AND ? = '' AND ? = '' AND ? = '')"
 			: " AND (? <> '' AND (message LIKE CONCAT('%', ?, '%') ESCAPE '!' OR sender_name LIKE CONCAT('%', ?, '%') ESCAPE '!' OR recipient_name LIKE CONCAT('%', ?, '%') ESCAPE '!'))";
 		where += q.blockedOnly ? " AND (? <> 0 AND blocked = 1)" : " AND (? = 0)";
 		return where;
 	}
+
+	IChatLog::ChatMessage Row(sql::ResultSet& result) {
+		IChatLog::ChatMessage m{ result.getUInt64("id"), result.getInt64("time"), std::string(result.getString("channel").c_str()),
+			result.getInt64("sender_id"), std::string(result.getString("sender_name").c_str()), result.getUInt("account_id"),
+			result.getInt64("recipient_id"), std::string(result.getString("recipient_name").c_str()), result.getUInt("zone_id"),
+			result.getUInt("instance_id"), result.getUInt("clone_id"), std::string(result.getString("message").c_str()),
+			result.getInt("blocked") != 0 };
+		m.guildId = result.getInt64("guild_id");
+		m.teamId = result.getInt64("team_id");
+		m.filtered = result.getInt("filtered") != 0;
+		return m;
+	}
+
+	// GROUP_CONCAT lists names in whatever order the database likes; sorted, so both databases agree
+	std::string SortedNames(const std::string& concatenated) {
+		auto names = GeneralUtils::SplitString(concatenated, ',');
+		std::ranges::sort(names);
+		std::string out;
+		for (const auto& name : names) {
+			if (name.empty()) continue;
+			out += (out.empty() ? "" : ",") + name;
+		}
+		return out;
+	}
 }
 
+// Every query ending in Where(q) takes the filter's parameters in this order, then its own
+#define CHAT_QUERY_PARAMS(q, search) static_cast<int64_t>((q).afterId), static_cast<int64_t>((q).beforeId), static_cast<int64_t>((q).beforeId), (q).channel, (q).channel, \
+	(q).includePrivate, (q).includeWhispers, (q).characterId, (q).characterId, (q).characterId, (q).otherCharacterId, (q).otherCharacterId, (q).otherCharacterId, \
+	(q).accountId, (q).accountId, (q).zoneId, (q).zoneId, (q).instanceId, (q).instanceId, (q).since, (q).since, (q).until, (q).until, (q).guildId, (q).guildId, \
+	(q).teamId, (q).teamId, (q).search, search, search, search, (q).blockedOnly
+
 uint64_t MySQLDatabase::InsertChatMessage(const ChatMessage& m) {
-	ExecuteInsert("INSERT INTO chat_log (time, channel, sender_id, sender_name, account_id, recipient_id, recipient_name, zone_id, instance_id, clone_id, message, blocked) "
+	ExecuteInsert("INSERT INTO chat_log (time, channel, sender_id, sender_name, account_id, recipient_id, recipient_name, zone_id, instance_id, clone_id, message, blocked, guild_id, team_id, filtered) "
-		"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
+		"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
-		m.time, m.channel, m.senderId, m.senderName, m.accountId, m.recipientId, m.recipientName, m.zoneId, m.instanceId, m.cloneId, m.message, m.blocked);
+		m.time, m.channel, m.senderId, m.senderName, m.accountId, m.recipientId, m.recipientName, m.zoneId, m.instanceId, m.cloneId, m.message, m.blocked,
+		m.guildId, m.teamId, m.filtered || m.blocked);
 	auto last = ExecuteSelect("SELECT LAST_INSERT_ID() AS id;"); // this connection's insert, not another server's
 	return last->next() ? last->getUInt64("id") : 0;
 }
 
 std::vector<IChatLog::ChatMessage> MySQLDatabase::GetChatMessages(const ChatQuery& q) {
 	std::vector<ChatMessage> messages;
-	auto result = ExecuteSelect("SELECT * FROM chat_log" + Where(q) + " ORDER BY id " + std::string(q.newestFirst ? "DESC" : "ASC") + " LIMIT ? OFFSET ?;", static_cast<int64_t>(q.afterId), q.channel, q.channel, q.includePrivate, q.characterId, q.characterId, q.characterId, q.accountId, q.accountId, q.zoneId, q.zoneId, q.instanceId, q.instanceId, q.since, q.since, q.search, GeneralUtils::LikeEscape(q.search, '!'), GeneralUtils::LikeEscape(q.search, '!'), GeneralUtils::LikeEscape(q.search, '!'), q.blockedOnly, q.limit, q.offset);
+	const auto search = GeneralUtils::LikeEscape(q.search, '!');
-	while (result->next()) {
+	auto result = ExecuteSelect("SELECT * FROM chat_log" + Where(q) + " ORDER BY id " + std::string(q.newestFirst ? "DESC" : "ASC") + " LIMIT ? OFFSET ?;",
-		messages.push_back({ result->getUInt64("id"), result->getInt64("time"), std::string(result->getString("channel").c_str()),
+		CHAT_QUERY_PARAMS(q, search), q.limit, q.offset);
-			result->getInt64("sender_id"), std::string(result->getString("sender_name").c_str()), result->getUInt("account_id"),
+	while (result->next()) messages.push_back(Row(*result.m_resultSet));
-			result->getInt64("recipient_id"), std::string(result->getString("recipient_name").c_str()), result->getUInt("zone_id"),
-			result->getUInt("instance_id"), result->getUInt("clone_id"), std::string(result->getString("message").c_str()),
-			result->getInt("blocked") != 0 });
-	}
 	return messages;
 }
 
 uint64_t MySQLDatabase::CountChatMessages(const ChatQuery& q) {
-	auto result = ExecuteSelect("SELECT COUNT(*) AS count FROM chat_log" + Where(q) + ";", static_cast<int64_t>(q.afterId), q.channel, q.channel, q.includePrivate, q.characterId, q.characterId, q.characterId, q.accountId, q.accountId, q.zoneId, q.zoneId, q.instanceId, q.instanceId, q.since, q.since, q.search, GeneralUtils::LikeEscape(q.search, '!'), GeneralUtils::LikeEscape(q.search, '!'), GeneralUtils::LikeEscape(q.search, '!'), q.blockedOnly);
+	const auto search = GeneralUtils::LikeEscape(q.search, '!');
+	auto result = ExecuteSelect("SELECT COUNT(*) AS count FROM chat_log" + Where(q) + ";", CHAT_QUERY_PARAMS(q, search));
+	return result->next() ? result->getUInt64("count") : 0;
+}
+
+std::vector<IChatLog::WhisperPartner> MySQLDatabase::GetWhisperPartners(const LWOOBJID characterId, const uint32_t offset, const uint32_t limit) {
+	std::vector<WhisperPartner> partners;
+	auto result = ExecuteSelect(
+		"SELECT g.partner, g.messages, g.first_time, g.last_time, CASE WHEN l.sender_id = g.partner THEN l.sender_name ELSE l.recipient_name END AS name FROM "
+		"(SELECT CASE WHEN sender_id = ? THEN recipient_id ELSE sender_id END AS partner, COUNT(*) AS messages, MIN(time) AS first_time, MAX(time) AS last_time, MAX(id) AS last_id "
+		"FROM chat_log WHERE channel = 'whisper' AND (sender_id = ? OR recipient_id = ?) GROUP BY partner) g "
+		"JOIN chat_log l ON l.id = g.last_id ORDER BY g.last_id DESC LIMIT ? OFFSET ?;", characterId, characterId, characterId, limit, offset);
+	while (result->next()) {
+		partners.push_back({ result->getInt64("partner"), std::string(result->getString("name").c_str()), result->getUInt64("messages"),
+			result->getInt64("first_time"), result->getInt64("last_time") });
+	}
+	return partners;
+}
+
+uint64_t MySQLDatabase::CountWhisperPartners(const LWOOBJID characterId) {
+	auto result = ExecuteSelect("SELECT COUNT(DISTINCT CASE WHEN sender_id = ? THEN recipient_id ELSE sender_id END) AS count FROM chat_log "
+		"WHERE channel = 'whisper' AND (sender_id = ? OR recipient_id = ?);", characterId, characterId, characterId);
+	return result->next() ? result->getUInt64("count") : 0;
+}
+
+std::vector<IChatLog::ChatTeam> MySQLDatabase::GetChatTeams(const LWOOBJID characterId, const uint32_t offset, const uint32_t limit) {
+	std::vector<ChatTeam> teams;
+	auto result = ExecuteSelect("SELECT team_id, COUNT(*) AS messages, MIN(time) AS first_time, MAX(time) AS last_time, MAX(id) AS last_id, GROUP_CONCAT(DISTINCT sender_name) AS senders "
+		"FROM chat_log WHERE channel = 'team' AND team_id <> 0 AND (? = 0 OR team_id IN (SELECT team_id FROM chat_log WHERE channel = 'team' AND sender_id = ?)) "
+		"GROUP BY team_id ORDER BY last_id DESC LIMIT ? OFFSET ?;", characterId, characterId, limit, offset);
+	while (result->next()) {
+		teams.push_back({ result->getInt64("team_id"), result->getUInt64("messages"), result->getInt64("first_time"), result->getInt64("last_time"),
+			SortedNames(std::string(result->getString("senders").c_str())) });
+	}
+	return teams;
+}
+
+uint64_t MySQLDatabase::CountChatTeams(const LWOOBJID characterId) {
+	auto result = ExecuteSelect("SELECT COUNT(DISTINCT team_id) AS count FROM chat_log WHERE channel = 'team' AND team_id <> 0 "
+		"AND (? = 0 OR team_id IN (SELECT team_id FROM chat_log WHERE channel = 'team' AND sender_id = ?));", characterId, characterId);
 	return result->next() ? result->getUInt64("count") : 0;
 }

dDatabase/GameDatabase/MySQL/Tables/Dashboard.cpp

@@ -26,7 +26,10 @@ namespace {
 		"(SELECT COALESCE(MAX(id), 0) FROM audit_log) AS audit_log_max_id, "
 		"(SELECT COALESCE(MAX(id), 0) FROM mail) AS mail_max_id, "
 		"(SELECT COUNT(*) FROM economy_flags WHERE status = 0) AS open_economy_flags, "
-		"(SELECT COALESCE(MAX(id), 0) FROM economy_flags) AS economy_flags_max_id;";
+		"(SELECT COALESCE(MAX(id), 0) FROM economy_flags) AS economy_flags_max_id, "
+		"(SELECT COUNT(*) FROM chat_flags WHERE status = 'open') AS open_chat_flags, "
+		"(SELECT COALESCE(MAX(id), 0) FROM chat_flags) AS chat_flags_max_id, "
+		"(SELECT COALESCE(MAX(updated_at), 0) FROM chat_flags) AS chat_flags_updated_at;";
 }
 
 namespace {
@@ -264,6 +267,9 @@ IDashboardStats::Snapshot MySQLDatabase::GetDashboardSnapshot() {
 	snapshot.mailMaxId = result->getUInt64("mail_max_id");
 	snapshot.openEconomyFlags = result->getUInt64("open_economy_flags");
 	snapshot.economyFlagsMaxId = result->getUInt64("economy_flags_max_id");
+	snapshot.openChatFlags = result->getUInt64("open_chat_flags");
+	snapshot.chatFlagsMaxId = result->getUInt64("chat_flags_max_id");
+	snapshot.chatFlagsUpdatedAt = result->getUInt64("chat_flags_updated_at");
 	return snapshot;
 }
 

dDatabase/GameDatabase/SQLite/SQLiteDatabase.h

@@ -319,6 +319,18 @@ public:
 	uint64_t InsertChatMessage(const ChatMessage& message) override;
 	std::vector<ChatMessage> GetChatMessages(const ChatQuery& query) override;
 	uint64_t CountChatMessages(const ChatQuery& query) override;
+	std::vector<WhisperPartner> GetWhisperPartners(LWOOBJID characterId, uint32_t offset, uint32_t limit) override;
+	uint64_t CountWhisperPartners(LWOOBJID characterId) override;
+	std::vector<ChatTeam> GetChatTeams(LWOOBJID characterId, uint32_t offset, uint32_t limit) override;
+	uint64_t CountChatTeams(LWOOBJID characterId) override;
+	uint64_t InsertChatFlag(const ChatFlag& flag, const std::vector<uint64_t>& messageIds) override;
+	std::optional<ChatFlag> GetChatFlag(uint64_t id) override;
+	std::vector<ChatFlag> GetChatFlags(const ChatFlagQuery& query) override;
+	uint64_t CountChatFlags(const ChatFlagQuery& query) override;
+	bool UpdateChatFlag(uint64_t id, const std::string& status, const std::string& note, uint64_t playerReportId, int64_t time, const std::string& updatedBy) override;
+	void InsertChatFlagEvent(const ChatFlagEvent& event) override;
+	std::vector<ChatFlagEvent> GetChatFlagEvents(uint64_t flagId) override;
+	std::vector<std::pair<uint64_t, uint64_t>> GetFlaggedMessages(const std::vector<uint64_t>& messageIds) override;
 	void InsertModerationDecision(const std::string& kind, int64_t subjectId, const std::string& subject, bool approved, const std::string& reason, int64_t time) override;
 	nlohmann::json GetModerationDecisions(const std::string& kind, int64_t subjectId, uint32_t limit) override;
 	Totp GetTotp(uint32_t accountId) override;

dDatabase/GameDatabase/SQLite/Tables/CMakeLists.txt

@@ -28,6 +28,7 @@ set(DDATABASES_DATABASES_SQLITE_TABLES_SOURCES
 	"ServerTraffic.cpp"
 	"MessageCaptures.cpp"
 	"ChatLog.cpp"
+	"ChatFlags.cpp"
 	"RelatedData.cpp"
 	"ActivityLog.cpp"
 	"Behaviors.cpp"

dDatabase/GameDatabase/SQLite/Tables/ChatFlags.cpp

@@ -0,0 +1,104 @@
+#include "SQLiteDatabase.h"
+
+namespace {
+	// Every column but messages (the copy of the chat, only read for one flag)
+	constexpr const char* LIST_COLUMNS = "id, created_at, created_by_id, created_by, status, channel, character_id, character_name, account_id, "
+		"first_message_id, last_message_id, first_time, last_time, excerpt, note, player_report_id, updated_at, updated_by";
+
+	// Each filter binds the same parameters whether it is on or off (see ChatLog.cpp)
+	std::string Where(const IChatFlags::ChatFlagQuery& q) {
+		std::string where = " WHERE (channel IN ('zone', 'web') OR (? = 1 AND channel IN ('team', 'guild')) OR (? = 1 AND channel = 'whisper'))";
+		where += q.status.empty() ? " AND (? = '' AND ? = '')" : " AND (? <> '' AND status = ?)";
+		where += q.characterId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND character_id = ?)";
+		where += q.accountId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND account_id = ?)";
+		return where;
+	}
+
+	IChatFlags::ChatFlag Row(CppSQLite3Query& r, bool withMessages) {
+		IChatFlags::ChatFlag f;
+		f.id = static_cast<uint64_t>(r.getInt64Field("id"));
+		f.createdAt = r.getInt64Field("created_at");
+		f.createdById = static_cast<uint32_t>(r.getInt64Field("created_by_id"));
+		f.createdBy = r.getStringField("created_by");
+		f.status = r.getStringField("status");
+		f.channel = r.getStringField("channel");
+		f.characterId = r.getInt64Field("character_id");
+		f.characterName = r.getStringField("character_name");
+		f.accountId = static_cast<uint32_t>(r.getInt64Field("account_id"));
+		f.firstMessageId = static_cast<uint64_t>(r.getInt64Field("first_message_id"));
+		f.lastMessageId = static_cast<uint64_t>(r.getInt64Field("last_message_id"));
+		f.firstTime = r.getInt64Field("first_time");
+		f.lastTime = r.getInt64Field("last_time");
+		f.excerpt = r.getStringField("excerpt");
+		f.note = r.getStringField("note");
+		if (withMessages) f.messages = r.getStringField("messages");
+		f.playerReportId = static_cast<uint64_t>(r.getInt64Field("player_report_id"));
+		f.updatedAt = r.getInt64Field("updated_at");
+		f.updatedBy = r.getStringField("updated_by");
+		return f;
+	}
+}
+
+#define CHAT_FLAG_QUERY_PARAMS(q) (q).includePrivate, (q).includeWhispers, (q).status, (q).status, (q).characterId, (q).characterId, (q).accountId, (q).accountId
+
+uint64_t SQLiteDatabase::InsertChatFlag(const ChatFlag& f, const std::vector<uint64_t>& messageIds) {
+	ExecuteInsert("INSERT INTO chat_flags (created_at, created_by_id, created_by, status, channel, character_id, character_name, account_id, first_message_id, "
+		"last_message_id, first_time, last_time, excerpt, note, messages, player_report_id, updated_at, updated_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
+		f.createdAt, f.createdById, f.createdBy, f.status, f.channel, f.characterId, f.characterName, f.accountId, f.firstMessageId, f.lastMessageId,
+		f.firstTime, f.lastTime, f.excerpt, f.note, f.messages, f.playerReportId, f.updatedAt, f.updatedBy);
+	auto [_, last] = ExecuteSelect("SELECT last_insert_rowid() AS id;");
+	const uint64_t id = last.eof() ? 0 : static_cast<uint64_t>(last.getInt64Field("id"));
+	for (const auto messageId : messageIds) {
+		ExecuteInsert("INSERT OR IGNORE INTO chat_flag_messages (flag_id, message_id) VALUES (?, ?);", id, messageId);
+	}
+	return id;
+}
+
+std::optional<IChatFlags::ChatFlag> SQLiteDatabase::GetChatFlag(const uint64_t id) {
+	auto [_, result] = ExecuteSelect("SELECT * FROM chat_flags WHERE id = ?;", id);
+	if (result.eof()) return std::nullopt;
+	return Row(result, true);
+}
+
+std::vector<IChatFlags::ChatFlag> SQLiteDatabase::GetChatFlags(const ChatFlagQuery& q) {
+	std::vector<ChatFlag> flags;
+	auto [_, result] = ExecuteSelect(std::string("SELECT ") + LIST_COLUMNS + " FROM chat_flags" + Where(q) + " ORDER BY id DESC LIMIT ? OFFSET ?;",
+		CHAT_FLAG_QUERY_PARAMS(q), q.limit, q.offset);
+	for (; !result.eof(); result.nextRow()) flags.push_back(Row(result, false));
+	return flags;
+}
+
+uint64_t SQLiteDatabase::CountChatFlags(const ChatFlagQuery& q) {
+	auto [_, result] = ExecuteSelect("SELECT COUNT(*) AS count FROM chat_flags" + Where(q) + ";", CHAT_FLAG_QUERY_PARAMS(q));
+	return result.eof() ? 0 : static_cast<uint64_t>(result.getInt64Field("count"));
+}
+
+bool SQLiteDatabase::UpdateChatFlag(const uint64_t id, const std::string& status, const std::string& note, const uint64_t playerReportId, const int64_t time, const std::string& updatedBy) {
+	return ExecuteUpdate("UPDATE chat_flags SET status = ?, note = ?, player_report_id = ?, updated_at = ?, updated_by = ? WHERE id = ?;", status, note, playerReportId, time, updatedBy, id) > 0;
+}
+
+void SQLiteDatabase::InsertChatFlagEvent(const ChatFlagEvent& e) {
+	ExecuteInsert("INSERT INTO chat_flag_events (flag_id, time, account_id, actor, action, detail) VALUES (?, ?, ?, ?, ?, ?);",
+		e.flagId, e.time, e.accountId, e.actor, e.action, e.detail);
+}
+
+std::vector<IChatFlags::ChatFlagEvent> SQLiteDatabase::GetChatFlagEvents(const uint64_t flagId) {
+	std::vector<ChatFlagEvent> events;
+	auto [_, result] = ExecuteSelect("SELECT * FROM chat_flag_events WHERE flag_id = ? ORDER BY id ASC;", flagId);
+	for (; !result.eof(); result.nextRow()) {
+		events.push_back({ static_cast<uint64_t>(result.getInt64Field("id")), static_cast<uint64_t>(result.getInt64Field("flag_id")), result.getInt64Field("time"),
+			static_cast<uint32_t>(result.getInt64Field("account_id")), result.getStringField("actor"), result.getStringField("action"), result.getStringField("detail") });
+	}
+	return events;
+}
+
+std::vector<std::pair<uint64_t, uint64_t>> SQLiteDatabase::GetFlaggedMessages(const std::vector<uint64_t>& messageIds) {
+	std::vector<std::pair<uint64_t, uint64_t>> flagged;
+	if (messageIds.empty()) return flagged;
+	// Numbers only, written by us: safe to put in the query
+	std::string list;
+	for (const auto id : messageIds) list += (list.empty() ? "" : ",") + std::to_string(id);
+	auto [_, result] = ExecuteSelect("SELECT message_id, MAX(flag_id) AS flag_id FROM chat_flag_messages WHERE message_id IN (" + list + ") GROUP BY message_id ORDER BY message_id;");
+	for (; !result.eof(); result.nextRow()) flagged.emplace_back(static_cast<uint64_t>(result.getInt64Field("message_id")), static_cast<uint64_t>(result.getInt64Field("flag_id")));
+	return flagged;
+}

dDatabase/GameDatabase/SQLite/Tables/ChatLog.cpp

@@ -1,48 +1,123 @@
 #include "SQLiteDatabase.h"
 #include "GeneralUtils.h"
 
+#include <algorithm>
+
 // The filters are all optional. Each binds the same parameters whether it is on or off, so both queries below take one
 // argument list. A filter that is off only compares its own parameters (worked out once, not per row); one that is on is
-// a plain comparison, so the character and account filters are answered from chat_log's indexes instead of a full scan.
+// a plain comparison, so the character, account, guild and team filters are answered from chat_log's indexes instead of
+// a full scan.
 namespace {
 	std::string Where(const IChatLog::ChatQuery& q) {
 		std::string where = " WHERE id > ?";
+		where += q.beforeId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND id < ?)";
 		where += q.channel.empty() ? " AND (? = '' AND ? = '')" : " AND (? <> '' AND channel = ?)";
-		where += q.includePrivate ? " AND (? = 1)" : " AND (? = 1 OR channel IN ('zone', 'web'))";
+		// Zone and web chat for everyone who may read chat; team and guild chat, and whispers, each behind their own permission
+		where += " AND (channel IN ('zone', 'web') OR (? = 1 AND channel IN ('team', 'guild')) OR (? = 1 AND channel = 'whisper'))";
 		where += q.characterId == 0 ? " AND (? = 0 AND ? = 0 AND ? = 0)" : " AND (? <> 0 AND (sender_id = ? OR recipient_id = ?))";
+		where += q.otherCharacterId == 0 ? " AND (? = 0 AND ? = 0 AND ? = 0)" : " AND (? <> 0 AND (sender_id = ? OR recipient_id = ?))";
 		where += q.accountId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND account_id = ?)";
 		where += q.zoneId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND zone_id = ?)";
 		where += q.instanceId < 0 ? " AND (? < 0 AND ? < 0)" : " AND (? >= 0 AND instance_id = ?)";
 		where += q.since == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND time >= ?)";
+		where += q.until == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND time < ?)";
+		where += q.guildId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND guild_id = ?)";
+		where += q.teamId == 0 ? " AND (? = 0 AND ? = 0)" : " AND (? <> 0 AND team_id = ?)";
 		where += q.search.empty() ? " AND (? = '' AND ? = '' AND ? = '' AND ? = '')"
 			: " AND (? <> '' AND (message LIKE '%' || ? || '%' ESCAPE '!' OR sender_name LIKE '%' || ? || '%' ESCAPE '!' OR recipient_name LIKE '%' || ? || '%' ESCAPE '!'))";
 		where += q.blockedOnly ? " AND (? <> 0 AND blocked = 1)" : " AND (? = 0)";
 		return where;
 	}
+
+	IChatLog::ChatMessage Row(CppSQLite3Query& result) {
+		IChatLog::ChatMessage m{ static_cast<uint64_t>(result.getInt64Field("id")), result.getInt64Field("time"), result.getStringField("channel"),
+			result.getInt64Field("sender_id"), result.getStringField("sender_name"), static_cast<uint32_t>(result.getIntField("account_id")),
+			result.getInt64Field("recipient_id"), result.getStringField("recipient_name"), static_cast<uint32_t>(result.getIntField("zone_id")),
+			static_cast<uint32_t>(result.getIntField("instance_id")), static_cast<uint32_t>(result.getIntField("clone_id")), result.getStringField("message"),
+			result.getIntField("blocked") != 0 };
+		m.guildId = result.getInt64Field("guild_id");
+		m.teamId = result.getInt64Field("team_id");
+		m.filtered = result.getIntField("filtered") != 0;
+		return m;
+	}
+
+	// GROUP_CONCAT lists names in whatever order the database likes; sorted, so both databases agree
+	std::string SortedNames(const std::string& concatenated) {
+		auto names = GeneralUtils::SplitString(concatenated, ',');
+		std::ranges::sort(names);
+		std::string out;
+		for (const auto& name : names) {
+			if (name.empty()) continue;
+			out += (out.empty() ? "" : ",") + name;
+		}
+		return out;
+	}
 }
 
+// Every query ending in Where(q) takes the filter's parameters in this order, then its own
+#define CHAT_QUERY_PARAMS(q, search) static_cast<int64_t>((q).afterId), static_cast<int64_t>((q).beforeId), static_cast<int64_t>((q).beforeId), (q).channel, (q).channel, \
+	(q).includePrivate, (q).includeWhispers, (q).characterId, (q).characterId, (q).characterId, (q).otherCharacterId, (q).otherCharacterId, (q).otherCharacterId, \
+	(q).accountId, (q).accountId, (q).zoneId, (q).zoneId, (q).instanceId, (q).instanceId, (q).since, (q).since, (q).until, (q).until, (q).guildId, (q).guildId, \
+	(q).teamId, (q).teamId, (q).search, search, search, search, (q).blockedOnly
+
 uint64_t SQLiteDatabase::InsertChatMessage(const ChatMessage& m) {
-	ExecuteInsert("INSERT INTO chat_log (time, channel, sender_id, sender_name, account_id, recipient_id, recipient_name, zone_id, instance_id, clone_id, message, blocked) "
+	ExecuteInsert("INSERT INTO chat_log (time, channel, sender_id, sender_name, account_id, recipient_id, recipient_name, zone_id, instance_id, clone_id, message, blocked, guild_id, team_id, filtered) "
-		"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
+		"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
-		m.time, m.channel, m.senderId, m.senderName, m.accountId, m.recipientId, m.recipientName, m.zoneId, m.instanceId, m.cloneId, m.message, m.blocked);
+		m.time, m.channel, m.senderId, m.senderName, m.accountId, m.recipientId, m.recipientName, m.zoneId, m.instanceId, m.cloneId, m.message, m.blocked,
+		m.guildId, m.teamId, m.filtered || m.blocked);
 	auto [_, last] = ExecuteSelect("SELECT last_insert_rowid() AS id;"); // this connection's insert, not another server's
 	return last.eof() ? 0 : static_cast<uint64_t>(last.getInt64Field("id"));
 }
 
 std::vector<IChatLog::ChatMessage> SQLiteDatabase::GetChatMessages(const ChatQuery& q) {
 	std::vector<ChatMessage> messages;
-	auto [_, result] = ExecuteSelect("SELECT * FROM chat_log" + Where(q) + " ORDER BY id " + std::string(q.newestFirst ? "DESC" : "ASC") + " LIMIT ? OFFSET ?;", static_cast<int64_t>(q.afterId), q.channel, q.channel, q.includePrivate, q.characterId, q.characterId, q.characterId, q.accountId, q.accountId, q.zoneId, q.zoneId, q.instanceId, q.instanceId, q.since, q.since, q.search, GeneralUtils::LikeEscape(q.search, '!'), GeneralUtils::LikeEscape(q.search, '!'), GeneralUtils::LikeEscape(q.search, '!'), q.blockedOnly, q.limit, q.offset);
+	const auto search = GeneralUtils::LikeEscape(q.search, '!');
-	for (; !result.eof(); result.nextRow()) {
+	auto [_, result] = ExecuteSelect("SELECT * FROM chat_log" + Where(q) + " ORDER BY id " + std::string(q.newestFirst ? "DESC" : "ASC") + " LIMIT ? OFFSET ?;",
-		messages.push_back({ static_cast<uint64_t>(result.getInt64Field("id")), result.getInt64Field("time"), result.getStringField("channel"),
+		CHAT_QUERY_PARAMS(q, search), q.limit, q.offset);
-			result.getInt64Field("sender_id"), result.getStringField("sender_name"), static_cast<uint32_t>(result.getIntField("account_id")),
+	for (; !result.eof(); result.nextRow()) messages.push_back(Row(result));
-			result.getInt64Field("recipient_id"), result.getStringField("recipient_name"), static_cast<uint32_t>(result.getIntField("zone_id")),
-			static_cast<uint32_t>(result.getIntField("instance_id")), static_cast<uint32_t>(result.getIntField("clone_id")), result.getStringField("message"),
-			result.getIntField("blocked") != 0 });
-	}
 	return messages;
 }
 
 uint64_t SQLiteDatabase::CountChatMessages(const ChatQuery& q) {
-	auto [_, result] = ExecuteSelect("SELECT COUNT(*) AS count FROM chat_log" + Where(q) + ";", static_cast<int64_t>(q.afterId), q.channel, q.channel, q.includePrivate, q.characterId, q.characterId, q.characterId, q.accountId, q.accountId, q.zoneId, q.zoneId, q.instanceId, q.instanceId, q.since, q.since, q.search, GeneralUtils::LikeEscape(q.search, '!'), GeneralUtils::LikeEscape(q.search, '!'), GeneralUtils::LikeEscape(q.search, '!'), q.blockedOnly);
+	const auto search = GeneralUtils::LikeEscape(q.search, '!');
+	auto [_, result] = ExecuteSelect("SELECT COUNT(*) AS count FROM chat_log" + Where(q) + ";", CHAT_QUERY_PARAMS(q, search));
+	return result.eof() ? 0 : static_cast<uint64_t>(result.getInt64Field("count"));
+}
+
+std::vector<IChatLog::WhisperPartner> SQLiteDatabase::GetWhisperPartners(const LWOOBJID characterId, const uint32_t offset, const uint32_t limit) {
+	std::vector<WhisperPartner> partners;
+	auto [_, result] = ExecuteSelect(
+		"SELECT g.partner, g.messages, g.first_time, g.last_time, CASE WHEN l.sender_id = g.partner THEN l.sender_name ELSE l.recipient_name END AS name FROM "
+		"(SELECT CASE WHEN sender_id = ? THEN recipient_id ELSE sender_id END AS partner, COUNT(*) AS messages, MIN(time) AS first_time, MAX(time) AS last_time, MAX(id) AS last_id "
+		"FROM chat_log WHERE channel = 'whisper' AND (sender_id = ? OR recipient_id = ?) GROUP BY partner) g "
+		"JOIN chat_log l ON l.id = g.last_id ORDER BY g.last_id DESC LIMIT ? OFFSET ?;", characterId, characterId, characterId, limit, offset);
+	for (; !result.eof(); result.nextRow()) {
+		partners.push_back({ result.getInt64Field("partner"), result.getStringField("name"), static_cast<uint64_t>(result.getInt64Field("messages")),
+			result.getInt64Field("first_time"), result.getInt64Field("last_time") });
+	}
+	return partners;
+}
+
+uint64_t SQLiteDatabase::CountWhisperPartners(const LWOOBJID characterId) {
+	auto [_, result] = ExecuteSelect("SELECT COUNT(DISTINCT CASE WHEN sender_id = ? THEN recipient_id ELSE sender_id END) AS count FROM chat_log "
+		"WHERE channel = 'whisper' AND (sender_id = ? OR recipient_id = ?);", characterId, characterId, characterId);
+	return result.eof() ? 0 : static_cast<uint64_t>(result.getInt64Field("count"));
+}
+
+std::vector<IChatLog::ChatTeam> SQLiteDatabase::GetChatTeams(const LWOOBJID characterId, const uint32_t offset, const uint32_t limit) {
+	std::vector<ChatTeam> teams;
+	auto [_, result] = ExecuteSelect("SELECT team_id, COUNT(*) AS messages, MIN(time) AS first_time, MAX(time) AS last_time, MAX(id) AS last_id, GROUP_CONCAT(DISTINCT sender_name) AS senders "
+		"FROM chat_log WHERE channel = 'team' AND team_id <> 0 AND (? = 0 OR team_id IN (SELECT team_id FROM chat_log WHERE channel = 'team' AND sender_id = ?)) "
+		"GROUP BY team_id ORDER BY last_id DESC LIMIT ? OFFSET ?;", characterId, characterId, limit, offset);
+	for (; !result.eof(); result.nextRow()) {
+		teams.push_back({ result.getInt64Field("team_id"), static_cast<uint64_t>(result.getInt64Field("messages")), result.getInt64Field("first_time"),
+			result.getInt64Field("last_time"), SortedNames(result.getStringField("senders")) });
+	}
+	return teams;
+}
+
+uint64_t SQLiteDatabase::CountChatTeams(const LWOOBJID characterId) {
+	auto [_, result] = ExecuteSelect("SELECT COUNT(DISTINCT team_id) AS count FROM chat_log WHERE channel = 'team' AND team_id <> 0 "
+		"AND (? = 0 OR team_id IN (SELECT team_id FROM chat_log WHERE channel = 'team' AND sender_id = ?));", characterId, characterId);
 	return result.eof() ? 0 : static_cast<uint64_t>(result.getInt64Field("count"));
 }

dDatabase/GameDatabase/SQLite/Tables/Dashboard.cpp

@@ -26,7 +26,10 @@ namespace {
 		"(SELECT COALESCE(MAX(id), 0) FROM audit_log) AS audit_log_max_id, "
 		"(SELECT COALESCE(MAX(id), 0) FROM mail) AS mail_max_id, "
 		"(SELECT COUNT(*) FROM economy_flags WHERE status = 0) AS open_economy_flags, "
-		"(SELECT COALESCE(MAX(id), 0) FROM economy_flags) AS economy_flags_max_id;";
+		"(SELECT COALESCE(MAX(id), 0) FROM economy_flags) AS economy_flags_max_id, "
+		"(SELECT COUNT(*) FROM chat_flags WHERE status = 'open') AS open_chat_flags, "
+		"(SELECT COALESCE(MAX(id), 0) FROM chat_flags) AS chat_flags_max_id, "
+		"(SELECT COALESCE(MAX(updated_at), 0) FROM chat_flags) AS chat_flags_updated_at;";
 }
 
 namespace {
@@ -269,6 +272,9 @@ IDashboardStats::Snapshot SQLiteDatabase::GetDashboardSnapshot() {
 	snapshot.mailMaxId = result.getInt64Field("mail_max_id");
 	snapshot.openEconomyFlags = result.getInt64Field("open_economy_flags");
 	snapshot.economyFlagsMaxId = result.getInt64Field("economy_flags_max_id");
+	snapshot.openChatFlags = result.getInt64Field("open_chat_flags");
+	snapshot.chatFlagsMaxId = result.getInt64Field("chat_flags_max_id");
+	snapshot.chatFlagsUpdatedAt = result.getInt64Field("chat_flags_updated_at");
 	return snapshot;
 }
 

dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h

@@ -369,6 +369,18 @@ class TestSQLDatabase : public GameDatabase {
 	uint64_t InsertChatMessage(const ChatMessage& message) override { return 0; }
 	std::vector<ChatMessage> GetChatMessages(const ChatQuery& query) override { return {}; }
 	uint64_t CountChatMessages(const ChatQuery& query) override { return 0; }
+	std::vector<WhisperPartner> GetWhisperPartners(LWOOBJID characterId, uint32_t offset, uint32_t limit) override { return {}; }
+	uint64_t CountWhisperPartners(LWOOBJID characterId) override { return 0; }
+	std::vector<ChatTeam> GetChatTeams(LWOOBJID characterId, uint32_t offset, uint32_t limit) override { return {}; }
+	uint64_t CountChatTeams(LWOOBJID characterId) override { return 0; }
+	uint64_t InsertChatFlag(const ChatFlag& flag, const std::vector<uint64_t>& messageIds) override { return 0; }
+	std::optional<ChatFlag> GetChatFlag(uint64_t id) override { return std::nullopt; }
+	std::vector<ChatFlag> GetChatFlags(const ChatFlagQuery& query) override { return {}; }
+	uint64_t CountChatFlags(const ChatFlagQuery& query) override { return 0; }
+	bool UpdateChatFlag(uint64_t id, const std::string& status, const std::string& note, uint64_t playerReportId, int64_t time, const std::string& updatedBy) override { return false; }
+	void InsertChatFlagEvent(const ChatFlagEvent& event) override {}
+	std::vector<ChatFlagEvent> GetChatFlagEvents(uint64_t flagId) override { return {}; }
+	std::vector<std::pair<uint64_t, uint64_t>> GetFlaggedMessages(const std::vector<uint64_t>& messageIds) override { return {}; }
 	void InsertModerationDecision(const std::string& kind, int64_t subjectId, const std::string& subject, bool approved, const std::string& reason, int64_t time) override {}
 	nlohmann::json GetModerationDecisions(const std::string& kind, int64_t subjectId, uint32_t limit) override { return nlohmann::json::array(); }
 	Totp GetTotp(uint32_t accountId) override { return {}; }

migrations/dlu/mysql/100_chat_log_scope.sql

@@ -0,0 +1,10 @@
+/* Where a chat message went, for the guild and team chat histories: guild_id (channel guild) is guilds.id, team_id
+   (channel team) is the chat server's team, unique across chat server restarts. filtered: the chat filter found words it
+   doesn't allow. Zone chat that is filtered is also blocked (nobody saw it); whispers, team and guild chat are delivered
+   either way, so filtered only records what the filter thought of them. */
+ALTER TABLE chat_log ADD COLUMN guild_id BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE chat_log ADD COLUMN team_id BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE chat_log ADD COLUMN filtered TINYINT NOT NULL DEFAULT 0;
+UPDATE chat_log SET filtered = blocked;
+CREATE INDEX chat_log_guild ON chat_log (guild_id);
+CREATE INDEX chat_log_team ON chat_log (team_id);

migrations/dlu/mysql/101_chat_flags.sql

@@ -0,0 +1,46 @@
+/* Chat messages staff flagged for review (the dashboard's Chat Flags page). status: open, actioned or dismissed.
+   character_id/account_id: who the flag is about. messages: JSON copy of the flagged messages and the chat around them,
+   kept after chat_log prunes them. player_report_id: a player report the flag goes with (0: none).
+   chat_flag_messages: which chat_log rows a flag covers. chat_flag_events: who did what with a flag. */
+CREATE TABLE IF NOT EXISTS chat_flags (
+    id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+    created_at BIGINT NOT NULL,
+    created_by_id INT UNSIGNED NOT NULL DEFAULT 0,
+    created_by VARCHAR(64) NOT NULL DEFAULT '',
+    status VARCHAR(16) NOT NULL DEFAULT 'open',
+    channel VARCHAR(16) NOT NULL,
+    character_id BIGINT NOT NULL DEFAULT 0,
+    character_name VARCHAR(64) NOT NULL DEFAULT '',
+    account_id INT UNSIGNED NOT NULL DEFAULT 0,
+    first_message_id BIGINT UNSIGNED NOT NULL DEFAULT 0,
+    last_message_id BIGINT UNSIGNED NOT NULL DEFAULT 0,
+    first_time BIGINT NOT NULL DEFAULT 0,
+    last_time BIGINT NOT NULL DEFAULT 0,
+    excerpt VARCHAR(512) NOT NULL DEFAULT '',
+    note TEXT NOT NULL,
+    messages MEDIUMTEXT NOT NULL,
+    player_report_id BIGINT UNSIGNED NOT NULL DEFAULT 0,
+    updated_at BIGINT NOT NULL DEFAULT 0,
+    updated_by VARCHAR(64) NOT NULL DEFAULT '',
+    INDEX chat_flags_status (status),
+    INDEX chat_flags_character (character_id),
+    INDEX chat_flags_account (account_id)
+);
+
+CREATE TABLE IF NOT EXISTS chat_flag_messages (
+    flag_id BIGINT UNSIGNED NOT NULL,
+    message_id BIGINT UNSIGNED NOT NULL,
+    PRIMARY KEY (flag_id, message_id),
+    INDEX chat_flag_messages_message (message_id)
+);
+
+CREATE TABLE IF NOT EXISTS chat_flag_events (
+    id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+    flag_id BIGINT UNSIGNED NOT NULL,
+    time BIGINT NOT NULL,
+    account_id INT UNSIGNED NOT NULL DEFAULT 0,
+    actor VARCHAR(64) NOT NULL DEFAULT '',
+    action VARCHAR(16) NOT NULL,
+    detail TEXT NOT NULL,
+    INDEX chat_flag_events_flag (flag_id)
+);

migrations/dlu/sqlite/83_chat_log_scope.sql

@@ -0,0 +1,7 @@
+/* chat_log guild_id, team_id and filtered. See the MySQL migration. */
+ALTER TABLE chat_log ADD COLUMN guild_id BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE chat_log ADD COLUMN team_id BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE chat_log ADD COLUMN filtered INTEGER NOT NULL DEFAULT 0;
+UPDATE chat_log SET filtered = blocked;
+CREATE INDEX IF NOT EXISTS chat_log_guild ON chat_log (guild_id);
+CREATE INDEX IF NOT EXISTS chat_log_team ON chat_log (team_id);

migrations/dlu/sqlite/84_chat_flags.sql

@@ -0,0 +1,43 @@
+/* Chat flags, the chat_log rows they cover and their history. See the MySQL migration. */
+CREATE TABLE IF NOT EXISTS chat_flags (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    created_at BIGINT NOT NULL,
+    created_by_id INTEGER NOT NULL DEFAULT 0,
+    created_by TEXT NOT NULL DEFAULT '',
+    status TEXT NOT NULL DEFAULT 'open',
+    channel TEXT NOT NULL,
+    character_id BIGINT NOT NULL DEFAULT 0,
+    character_name TEXT NOT NULL DEFAULT '',
+    account_id INTEGER NOT NULL DEFAULT 0,
+    first_message_id BIGINT NOT NULL DEFAULT 0,
+    last_message_id BIGINT NOT NULL DEFAULT 0,
+    first_time BIGINT NOT NULL DEFAULT 0,
+    last_time BIGINT NOT NULL DEFAULT 0,
+    excerpt TEXT NOT NULL DEFAULT '',
+    note TEXT NOT NULL DEFAULT '',
+    messages TEXT NOT NULL DEFAULT '[]',
+    player_report_id BIGINT NOT NULL DEFAULT 0,
+    updated_at BIGINT NOT NULL DEFAULT 0,
+    updated_by TEXT NOT NULL DEFAULT ''
+);
+CREATE INDEX IF NOT EXISTS chat_flags_status ON chat_flags (status);
+CREATE INDEX IF NOT EXISTS chat_flags_character ON chat_flags (character_id);
+CREATE INDEX IF NOT EXISTS chat_flags_account ON chat_flags (account_id);
+
+CREATE TABLE IF NOT EXISTS chat_flag_messages (
+    flag_id BIGINT NOT NULL,
+    message_id BIGINT NOT NULL,
+    PRIMARY KEY (flag_id, message_id)
+);
+CREATE INDEX IF NOT EXISTS chat_flag_messages_message ON chat_flag_messages (message_id);
+
+CREATE TABLE IF NOT EXISTS chat_flag_events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    flag_id BIGINT NOT NULL,
+    time BIGINT NOT NULL,
+    account_id INTEGER NOT NULL DEFAULT 0,
+    actor TEXT NOT NULL DEFAULT '',
+    action TEXT NOT NULL,
+    detail TEXT NOT NULL DEFAULT ''
+);
+CREATE INDEX IF NOT EXISTS chat_flag_events_flag ON chat_flag_events (flag_id);

tests/dDatabaseTests/CMakeLists.txt

@@ -14,3 +14,19 @@ target_include_directories(dDatabaseTests PRIVATE
 target_link_libraries(dDatabaseTests ${COMMON_LIBRARIES} MariaDB::ConnCpp sqlite3 GTest::gtest_main)
 
 gtest_discover_tests(dDatabaseTests)
+
+# The chat history and chat flag tables on SQLite alone (no MariaDB/MySQL needed), so they always run.
+add_executable(dDatabaseSqliteTests "ChatHistorySqliteTests.cpp")
+
+target_include_directories(dDatabaseSqliteTests PRIVATE
+	"${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase/MySQL"
+	"${PROJECT_SOURCE_DIR}/dDatabase/GameDatabase/SQLite"
+	"${PROJECT_SOURCE_DIR}/dNet"
+	"${PROJECT_SOURCE_DIR}/thirdparty/nlohmann"
+	"${PROJECT_SOURCE_DIR}/thirdparty/SQLite"
+)
+
+# dCommon uses raknet without linking it: link them as a group so the order of the static libraries does not matter
+target_link_libraries(dDatabaseSqliteTests glm::glm "dNet" "magic_enum" "$<LINK_GROUP:RESCAN,dDatabase,dDatabaseGame,dCommon,raknet>" MariaDB::ConnCpp sqlite3 GTest::gtest_main ${CMAKE_DL_LIBS})
+
+gtest_discover_tests(dDatabaseSqliteTests)

tests/dDatabaseTests/ChatHistorySqliteTests.cpp

@@ -0,0 +1,309 @@
+/**
+ * The chat history and chat flag tables on SQLite, with no other database needed: a fresh SQLite file gets every
+ * migration (MigrationRunner::RunMigrations, as the servers run them), then the IChatLog and IChatFlags methods are
+ * checked against what was written. DatabaseParityTests compares the same methods with MySQL when one is set up.
+ */
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <cstdlib>
+#include <filesystem>
+#include <unistd.h>
+
+#include "Database.h"
+#include "dConfig.h"
+#include "Game.h"
+#include "Logger.h"
+#include "MigrationRunner.h"
+#include "SQLiteDatabase.h"
+
+namespace Game {
+	Logger* logger = nullptr;
+	dConfig* config = nullptr;
+}
+
+namespace {
+	constexpr LWOOBJID ALICE = 1152921510000000001;
+	constexpr LWOOBJID BOB = 1152921510000000002;
+	constexpr LWOOBJID CAROL = 1152921510000000003;
+	constexpr LWOOBJID TEAM = 1700000000000001;
+
+	std::filesystem::path g_Path;
+	std::filesystem::path g_Log;
+	std::unique_ptr<SQLiteDatabase> g_Db;
+
+	class SqliteEnvironment : public ::testing::Environment {
+	public:
+		void SetUp() override {
+			const auto base = std::filesystem::temp_directory_path() / ("dDatabaseSqliteTests_" + std::to_string(getpid()));
+			g_Path = base.string() + ".sqlite";
+			g_Log = base.string() + ".log";
+			for (const auto* suffix : { "", "-wal", "-shm" }) std::filesystem::remove(g_Path.string() + suffix);
+			Game::logger = new Logger(g_Log.string(), false, false);
+			setenv("DATABASE_TYPE", "sqlite", 1);
+			setenv("SQLITE_DATABASE_PATH", g_Path.string().c_str(), 1);
+			Game::config = new dConfig("dDatabaseSqliteTests.ini");
+			Database::Connect();
+			MigrationRunner::RunMigrations();
+			Database::Destroy("dDatabaseSqliteTests migrations");
+			g_Db = std::make_unique<SQLiteDatabase>();
+			g_Db->Connect();
+		}
+
+		void TearDown() override {
+			if (g_Db) g_Db->Destroy("dDatabaseSqliteTests");
+			g_Db.reset();
+			for (const auto* suffix : { "", "-wal", "-shm" }) std::filesystem::remove(g_Path.string() + suffix);
+			std::filesystem::remove(g_Log);
+			delete Game::config;
+			Game::config = nullptr;
+			delete Game::logger;
+			Game::logger = nullptr;
+		}
+	};
+
+	const auto* const g_Environment = ::testing::AddGlobalTestEnvironment(new SqliteEnvironment());
+
+	IChatLog::ChatMessage Message(int64_t time, const std::string& channel, LWOOBJID sender, const std::string& senderName, uint32_t account, const std::string& text) {
+		IChatLog::ChatMessage m;
+		m.time = time;
+		m.channel = channel;
+		m.senderId = sender;
+		m.senderName = senderName;
+		m.accountId = account;
+		m.zoneId = 1100;
+		m.instanceId = 2;
+		m.message = text;
+		return m;
+	}
+
+	IChatLog::ChatMessage Whisper(int64_t time, LWOOBJID from, const std::string& fromName, LWOOBJID to, const std::string& toName, const std::string& text) {
+		auto m = Message(time, "whisper", from, fromName, from == ALICE ? 1 : from == BOB ? 2 : 3, text);
+		m.recipientId = to;
+		m.recipientName = toName;
+		return m;
+	}
+
+	std::vector<std::string> Texts(const std::vector<IChatLog::ChatMessage>& messages) {
+		std::vector<std::string> out;
+		for (const auto& m : messages) out.push_back(m.message);
+		return out;
+	}
+
+	// Every test gets the same chat: zone chat, a guild, a team and whispers between three characters
+	class ChatHistorySqlite : public ::testing::Test {
+	protected:
+		static inline std::vector<uint64_t> ids;
+
+		static void SetUpTestSuite() {
+			ASSERT_TRUE(g_Db);
+			auto& db = *g_Db;
+			ids.push_back(db.InsertChatMessage(Message(1000, "zone", ALICE, "Alice", 1, "hello zone")));
+			auto guild = Message(1010, "guild", ALICE, "Alice", 1, "guild hi");
+			guild.guildId = 7;
+			ids.push_back(db.InsertChatMessage(guild));
+			auto team = Message(1020, "team", BOB, "Bob", 2, "team darn");
+			team.teamId = TEAM;
+			team.filtered = true;
+			ids.push_back(db.InsertChatMessage(team));
+			ids.push_back(db.InsertChatMessage(Whisper(1030, ALICE, "Alice", BOB, "Bob", "psst bob")));
+			ids.push_back(db.InsertChatMessage(Whisper(1040, BOB, "Bob", ALICE, "Alice", "yes alice")));
+			ids.push_back(db.InsertChatMessage(Whisper(1050, ALICE, "Alice", CAROL, "Carol", "hi carol")));
+			auto blocked = Message(1060, "zone", BOB, "Bob", 2, "bad words");
+			blocked.blocked = true;
+			ids.push_back(db.InsertChatMessage(blocked));
+			team.senderId = ALICE; team.senderName = "Alice"; team.accountId = 1; team.time = 1070; team.message = "team reply"; team.filtered = false;
+			ids.push_back(db.InsertChatMessage(team));
+		}
+	};
+}
+
+TEST_F(ChatHistorySqlite, StoresWhereEachMessageWent) {
+	IChatLog::ChatQuery all;
+	all.includePrivate = true;
+	all.includeWhispers = true;
+	const auto messages = g_Db->GetChatMessages(all);
+	ASSERT_EQ(messages.size(), 8u);
+	EXPECT_EQ(messages[1].guildId, 7);
+	EXPECT_EQ(messages[2].teamId, TEAM);
+	EXPECT_TRUE(messages[2].filtered);
+	EXPECT_FALSE(messages[2].blocked);
+	EXPECT_EQ(messages[3].recipientId, BOB);
+	// Stopped by the filter is always filtered too
+	EXPECT_TRUE(messages[6].blocked);
+	EXPECT_TRUE(messages[6].filtered);
+}
+
+TEST_F(ChatHistorySqlite, WhispersAndGroupChatNeedTheirOwnPermission) {
+	IChatLog::ChatQuery zoneOnly;
+	EXPECT_EQ(Texts(g_Db->GetChatMessages(zoneOnly)), (std::vector<std::string>{ "hello zone", "bad words" }));
+	IChatLog::ChatQuery group;
+	group.includePrivate = true;
+	EXPECT_EQ(g_Db->CountChatMessages(group), 5u); // no whispers
+	IChatLog::ChatQuery whispers;
+	whispers.includeWhispers = true;
+	EXPECT_EQ(g_Db->CountChatMessages(whispers), 5u); // no team or guild chat
+	// Asking for a channel doesn't get round it
+	whispers.channel = "team";
+	EXPECT_EQ(g_Db->CountChatMessages(whispers), 0u);
+}
+
+TEST_F(ChatHistorySqlite, ConversationBetweenTwoCharacters) {
+	IChatLog::ChatQuery q;
+	q.includeWhispers = true;
+	q.channel = "whisper";
+	q.characterId = ALICE;
+	q.otherCharacterId = BOB;
+	EXPECT_EQ(Texts(g_Db->GetChatMessages(q)), (std::vector<std::string>{ "psst bob", "yes alice" }));
+	q.otherCharacterId = CAROL;
+	EXPECT_EQ(Texts(g_Db->GetChatMessages(q)), (std::vector<std::string>{ "hi carol" }));
+}
+
+TEST_F(ChatHistorySqlite, GuildTeamTimeAndPaging) {
+	IChatLog::ChatQuery guild;
+	guild.includePrivate = true;
+	guild.guildId = 7;
+	EXPECT_EQ(Texts(g_Db->GetChatMessages(guild)), (std::vector<std::string>{ "guild hi" }));
+	IChatLog::ChatQuery team;
+	team.includePrivate = true;
+	team.teamId = TEAM;
+	EXPECT_EQ(Texts(g_Db->GetChatMessages(team)), (std::vector<std::string>{ "team darn", "team reply" }));
+
+	IChatLog::ChatQuery range;
+	range.includePrivate = true;
+	range.includeWhispers = true;
+	range.since = 1020;
+	range.until = 1050;
+	EXPECT_EQ(Texts(g_Db->GetChatMessages(range)), (std::vector<std::string>{ "team darn", "psst bob", "yes alice" }));
+
+	// Paging back through a conversation: the newest page, then the one before it
+	IChatLog::ChatQuery page;
+	page.includePrivate = true;
+	page.includeWhispers = true;
+	page.newestFirst = true;
+	page.limit = 3;
+	const auto newest = g_Db->GetChatMessages(page);
+	ASSERT_EQ(newest.size(), 3u);
+	EXPECT_EQ(newest.front().message, "team reply");
+	page.beforeId = newest.back().id;
+	EXPECT_EQ(Texts(g_Db->GetChatMessages(page)), (std::vector<std::string>{ "yes alice", "psst bob", "team darn" }));
+
+	IChatLog::ChatQuery account;
+	account.includeWhispers = true;
+	account.includePrivate = true;
+	account.accountId = 2;
+	EXPECT_EQ(g_Db->CountChatMessages(account), 3u);
+}
+
+TEST_F(ChatHistorySqlite, WhisperPartnersNewestFirst) {
+	const auto partners = g_Db->GetWhisperPartners(ALICE, 0, 10);
+	ASSERT_EQ(partners.size(), 2u);
+	EXPECT_EQ(partners[0].characterId, CAROL);
+	EXPECT_EQ(partners[0].name, "Carol");
+	EXPECT_EQ(partners[0].messages, 1u);
+	EXPECT_EQ(partners[1].characterId, BOB);
+	EXPECT_EQ(partners[1].name, "Bob");
+	EXPECT_EQ(partners[1].messages, 2u);
+	EXPECT_EQ(partners[1].firstTime, 1030);
+	EXPECT_EQ(partners[1].lastTime, 1040);
+	EXPECT_EQ(g_Db->CountWhisperPartners(ALICE), 2u);
+	EXPECT_EQ(g_Db->CountWhisperPartners(BOB), 1u);
+	EXPECT_EQ(g_Db->GetWhisperPartners(ALICE, 1, 10).size(), 1u);
+}
+
+TEST_F(ChatHistorySqlite, TeamsWithWhoTalked) {
+	const auto teams = g_Db->GetChatTeams(0, 0, 10);
+	ASSERT_EQ(teams.size(), 1u);
+	EXPECT_EQ(teams[0].teamId, TEAM);
+	EXPECT_EQ(teams[0].messages, 2u);
+	EXPECT_EQ(teams[0].senders, "Alice,Bob");
+	EXPECT_EQ(g_Db->CountChatTeams(ALICE), 1u);
+	EXPECT_EQ(g_Db->CountChatTeams(CAROL), 0u);
+	EXPECT_TRUE(g_Db->GetChatTeams(CAROL, 0, 10).empty());
+}
+
+TEST_F(ChatHistorySqlite, FlagsKeepTheirMessagesAndHistory) {
+	IChatFlags::ChatFlag flag;
+	flag.createdAt = 2000;
+	flag.createdById = 3;
+	flag.createdBy = "gm";
+	flag.channel = "whisper";
+	flag.characterId = ALICE;
+	flag.characterName = "Alice";
+	flag.accountId = 1;
+	flag.firstMessageId = ids[3];
+	flag.lastMessageId = ids[4];
+	flag.excerpt = "Alice: psst bob / Bob: yes alice";
+	flag.note = "rude";
+	flag.messages = R"([{"id":4,"message":"psst bob","flagged":true}])";
+	const auto id = g_Db->InsertChatFlag(flag, { ids[3], ids[4] });
+	ASSERT_GT(id, 0u);
+
+	auto zoneFlag = flag;
+	zoneFlag.channel = "zone";
+	zoneFlag.characterId = BOB;
+	zoneFlag.accountId = 2;
+	const auto zoneId = g_Db->InsertChatFlag(zoneFlag, { ids[6] });
+
+	const auto stored = g_Db->GetChatFlag(id);
+	ASSERT_TRUE(stored.has_value());
+	EXPECT_EQ(stored->status, "open");
+	EXPECT_EQ(stored->messages, flag.messages);
+	EXPECT_EQ(stored->characterId, ALICE);
+	EXPECT_FALSE(g_Db->GetChatFlag(id + 100).has_value());
+
+	// Which shown messages are flagged, with the newest flag on each
+	const auto flagged = g_Db->GetFlaggedMessages({ ids[0], ids[3], ids[4], ids[6] });
+	EXPECT_EQ(flagged, (std::vector<std::pair<uint64_t, uint64_t>>{ { ids[3], id }, { ids[4], id }, { ids[6], zoneId } }));
+	EXPECT_TRUE(g_Db->GetFlaggedMessages({}).empty());
+
+	// A viewer without chat_dms doesn't get flags on whispers in the queue
+	IChatFlags::ChatFlagQuery q;
+	EXPECT_EQ(g_Db->CountChatFlags(q), 1u);
+	q.includeWhispers = true;
+	EXPECT_EQ(g_Db->CountChatFlags(q), 2u);
+	const auto list = g_Db->GetChatFlags(q);
+	ASSERT_EQ(list.size(), 2u);
+	EXPECT_EQ(list[0].id, zoneId); // newest first
+	EXPECT_TRUE(list[0].messages.empty()); // the list leaves the copy of the chat out
+	q.characterId = ALICE;
+	EXPECT_EQ(g_Db->CountChatFlags(q), 1u);
+	q.characterId = 0;
+	q.accountId = 2;
+	EXPECT_EQ(g_Db->CountChatFlags(q), 1u);
+
+	EXPECT_TRUE(g_Db->UpdateChatFlag(id, "actioned", "muted 3 days", 42, 2100, "gm"));
+	EXPECT_FALSE(g_Db->UpdateChatFlag(id + 100, "actioned", "", 0, 2100, "gm"));
+	const auto updated = g_Db->GetChatFlag(id);
+	EXPECT_EQ(updated->status, "actioned");
+	EXPECT_EQ(updated->note, "muted 3 days");
+	EXPECT_EQ(updated->playerReportId, 42u);
+	EXPECT_EQ(updated->updatedBy, "gm");
+	IChatFlags::ChatFlagQuery open;
+	open.includeWhispers = true;
+	open.status = "open";
+	EXPECT_EQ(g_Db->CountChatFlags(open), 1u);
+
+	g_Db->InsertChatFlagEvent({ 0, id, 2000, 3, "gm", "created", "2 message(s)" });
+	g_Db->InsertChatFlagEvent({ 0, id, 2100, 3, "gm", "actioned", "muted" });
+	const auto events = g_Db->GetChatFlagEvents(id);
+	ASSERT_EQ(events.size(), 2u);
+	EXPECT_EQ(events[0].action, "created");
+	EXPECT_EQ(events[1].action, "actioned");
+	EXPECT_EQ(events[1].detail, "muted");
+	EXPECT_TRUE(g_Db->GetChatFlagEvents(zoneId).empty());
+
+	// The badge count
+	EXPECT_EQ(g_Db->GetDashboardSnapshot().openChatFlags, 1u);
+}
+
+TEST_F(ChatHistorySqlite, PruningOldChatKeepsFlags) {
+	IChatLog::ChatQuery all;
+	all.includePrivate = true;
+	all.includeWhispers = true;
+	const auto before = g_Db->CountChatMessages(all);
+	const auto flags = g_Db->CountChatFlags({ .includePrivate = true, .includeWhispers = true });
+	EXPECT_EQ(g_Db->PruneLog(IDashboardAdmin::eLog::CHAT, 1035), 4u);
+	EXPECT_EQ(g_Db->CountChatMessages(all), before - 4);
+	EXPECT_EQ(g_Db->CountChatFlags({ .includePrivate = true, .includeWhispers = true }), flags);
+}

tests/dDatabaseTests/DatabaseParityTests.cpp

@@ -73,7 +73,9 @@ NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IGuilds::GuildSummary, guild, memberCount, le
 NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IGuilds::GuildPage, total, filtered, guilds);
 NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IApiKeys::ApiKey, id, accountId, name, note, keyHash, keyPrefix, permissions, readOnly, allowedIps, allowedPaths, rateLimit, dailyQuota, createdAt, createdBy, issuedAt, expiresAt, revokedAt, revokedBy, lastUsedAt, lastIp, requestCount, quotaDay, dayCount);
 NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ICharacterSnapshots::CharacterSnapshot, id, characterId, takenAt, reason, actor, size, hash, compressed);
-NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IChatLog::ChatMessage, id, time, channel, senderId, senderName, accountId, recipientId, recipientName, zoneId, instanceId, cloneId, message, blocked);
+NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IChatLog::ChatMessage, id, time, channel, senderId, senderName, accountId, recipientId, recipientName, zoneId, instanceId, cloneId, message, blocked, guildId, teamId, filtered);
+NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IChatLog::WhisperPartner, characterId, name, messages, firstTime, lastTime);
+NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IChatLog::ChatTeam, teamId, messages, firstTime, lastTime, senders);
 NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IDashboardAdmin::Webhook, id, name, url, format, events, secret, enabled, createdAt, lastSentAt, lastStatus, lastError);
 NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IDashboardAdmin::Totp, encryptedSecret, enabledAt, lastStep);
 NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(IDashboardStats::Snapshot, accounts, accountsMaxId, characters, pendingNames, properties, pendingProperties, playKeys, bugReports, unresolvedBugReports, petNames, pendingPetNames, activityLogMaxId, chatLogMaxId, commandLogMaxId, auditLogMaxId, mailMaxId, openEconomyFlags, economyFlagsMaxId);
@@ -1459,30 +1461,50 @@ TEST_F(ParitySeeded, ChatLog) {
 		ids.push_back(db.InsertChatMessage({ 0, 1700000010, "whisper", CHAR_BOB, "Bob", 2, CHAR_ALICE, "Alice", 1100, 1, 0, "psst 100% secret", false }));
 		ids.push_back(db.InsertChatMessage({ 0, 1700000020, "zone", CHAR_BOB, "Bob", 2, 0, "", 1200, 0, 0, "bad word", true }));
 		ids.push_back(db.InsertChatMessage({ 0, 1700000030, "web", 0, "Dashboard", 0, 0, "", 0, 0, 0, "Server restarting", false }));
+		IChatLog::ChatMessage guild{ 0, 1700000040, "guild", CHAR_ALICE, "Alice", 1, 0, "", 1100, 1, 0, "guild hello", false };
+		guild.guildId = 7;
+		ids.push_back(db.InsertChatMessage(guild));
+		IChatLog::ChatMessage team{ 0, 1700000050, "team", CHAR_BOB, "Bob", 2, 0, "", 1100, 1, 0, "team darn", false };
+		team.teamId = 1700000000000001;
+		team.filtered = true;
+		ids.push_back(db.InsertChatMessage(team));
+		team.senderId = CHAR_ALICE; team.senderName = "Alice"; team.time = 1700000055; team.filtered = false; team.message = "team reply";
+		ids.push_back(db.InsertChatMessage(team));
+		ids.push_back(db.InsertChatMessage({ 0, 1700000060, "whisper", CHAR_ALICE, "Alice", 1, CHAR_BOB, "Bob", 1100, 1, 0, "reply to bob", false }));
+		ids.push_back(db.InsertChatMessage({ 0, 1700000070, "whisper", CHAR_ALICE, "Alice", 1, CHAR_GM, "GameMaster", 1100, 1, 0, "hi gm", false }));
 		return ids;
 	});
 	const auto queries = [] {
 		std::vector<std::pair<std::string, IChatLog::ChatQuery>> list;
 		IChatLog::ChatQuery query;
 		list.emplace_back("default", query);
-		query.includePrivate = true; list.emplace_back("private", query);
+		query.includePrivate = true; query.includeWhispers = true; list.emplace_back("private", query);
 		query.newestFirst = true; list.emplace_back("newest", query);
-		IChatLog::ChatQuery after; after.afterId = 1; after.includePrivate = true; list.emplace_back("after", after);
+		IChatLog::ChatQuery after; after.afterId = 1; after.includePrivate = true; after.includeWhispers = true; list.emplace_back("after", after);
 		IChatLog::ChatQuery channel; channel.channel = "zone"; list.emplace_back("channel", channel);
-		IChatLog::ChatQuery character; character.characterId = CHAR_ALICE; character.includePrivate = true; list.emplace_back("character", character);
+		IChatLog::ChatQuery character; character.characterId = CHAR_ALICE; character.includePrivate = true; character.includeWhispers = true; list.emplace_back("character", character);
 		IChatLog::ChatQuery account; account.accountId = 2; list.emplace_back("account", account);
 		IChatLog::ChatQuery zone; zone.zoneId = 1100; zone.instanceId = 1; list.emplace_back("zone", zone);
 		IChatLog::ChatQuery instance0; instance0.instanceId = 0; list.emplace_back("instance 0", instance0);
-		IChatLog::ChatQuery since; since.since = 1700000015; since.includePrivate = true; list.emplace_back("since", since);
+		IChatLog::ChatQuery since; since.since = 1700000015; since.includePrivate = true; since.includeWhispers = true; list.emplace_back("since", since);
-		IChatLog::ChatQuery search; search.search = "100%"; search.includePrivate = true; list.emplace_back("search percent", search);
+		IChatLog::ChatQuery search; search.search = "100%"; search.includePrivate = true; search.includeWhispers = true; list.emplace_back("search percent", search);
-		IChatLog::ChatQuery searchName; searchName.search = "bob"; searchName.includePrivate = true; list.emplace_back("search name", searchName);
+		IChatLog::ChatQuery searchName; searchName.search = "bob"; searchName.includePrivate = true; searchName.includeWhispers = true; list.emplace_back("search name", searchName);
 		IChatLog::ChatQuery blocked; blocked.blockedOnly = true; list.emplace_back("blocked", blocked);
-		IChatLog::ChatQuery page; page.limit = 1; page.offset = 1; page.includePrivate = true; list.emplace_back("page", page);
+		IChatLog::ChatQuery page; page.limit = 1; page.offset = 1; page.includePrivate = true; page.includeWhispers = true; list.emplace_back("page", page);
+		IChatLog::ChatQuery groupOnly; groupOnly.includePrivate = true; list.emplace_back("team and guild without whispers", groupOnly);
+		IChatLog::ChatQuery whispersOnly; whispersOnly.includeWhispers = true; list.emplace_back("whispers without team and guild", whispersOnly);
+		IChatLog::ChatQuery pair; pair.channel = "whisper"; pair.includeWhispers = true; pair.characterId = CHAR_ALICE; pair.otherCharacterId = CHAR_BOB; list.emplace_back("conversation", pair);
+		IChatLog::ChatQuery guildQ; guildQ.includePrivate = true; guildQ.guildId = 7; list.emplace_back("guild", guildQ);
+		IChatLog::ChatQuery teamQ; teamQ.includePrivate = true; teamQ.teamId = 1700000000000001; list.emplace_back("team", teamQ);
+		IChatLog::ChatQuery until; until.until = 1700000020; until.includePrivate = true; until.includeWhispers = true; list.emplace_back("until", until);
+		IChatLog::ChatQuery before; before.beforeId = 3; before.includePrivate = true; before.includeWhispers = true; before.newestFirst = true; list.emplace_back("before", before);
 		return list;
 	}();
 	for (const auto& [name, query] : queries) {
 		Both("GetChatMessages " + name, [&](GameDatabase& db) { return json{ db.GetChatMessages(query), db.CountChatMessages(query) }; });
 	}
+	Both("GetWhisperPartners", [](GameDatabase& db) { return json{ db.GetWhisperPartners(CHAR_ALICE, 0, 10), db.CountWhisperPartners(CHAR_ALICE), db.GetWhisperPartners(CHAR_ALICE, 1, 1), db.GetWhisperPartners(CHAR_GM, 0, 10) }; });
+	Both("GetChatTeams", [](GameDatabase& db) { return json{ db.GetChatTeams(0, 0, 10), db.CountChatTeams(0), db.GetChatTeams(CHAR_ALICE, 0, 10), db.CountChatTeams(CHAR_GM) }; });
 	Both("PruneLog chat", [](GameDatabase& db) { return db.PruneLog(IDashboardAdmin::eLog::CHAT, 1700000015); });
 	Both("GetDashboardSnapshot", [](GameDatabase& db) { return db.GetDashboardSnapshot(); });
 }
@@ -1632,7 +1654,7 @@ TEST_F(ParitySeeded, SearchEscaping) {
 	Both("GetShowcaseProperties wildcard search", [](GameDatabase& db) { IProperty::ShowcaseQuery query; query.search = "%"; return db.GetShowcaseProperties(query); });
 	Both("GetShowcaseProperties underscore search", [](GameDatabase& db) { IProperty::ShowcaseQuery query; query.search = "_"; return db.GetShowcaseProperties(query); });
 	Both("GetChatMessages wildcard search", [](GameDatabase& db) {
-		IChatLog::ChatQuery query; query.includePrivate = true;
+		IChatLog::ChatQuery query; query.includePrivate = true; query.includeWhispers = true;
 		json out = json::array();
 		for (const auto* text : { "%", "_", "!", "100%", "'" }) {
 			query.search = text;