mirror of
https://github.com/DarkflameUniverse/DarkflameServer.git
synced 2026-10-02 10:53:44 +00:00
fix(chat-filter): portable .dcf hashing, block list phrases
The filter stored and compared words by std::hash<std::string> in size_t, which differs between standard libraries and platforms, so a .dcf made on one system never matched on another and the block list never worked there (issue 215). - ChatFilterCore.h: 64-bit FNV-1a over the entry's bytes, ASCII lower case, fixed-width uint64_t everywhere stored or compared. - .dcf version 3: little-endian magic, version, longest entry in words, uint64 count and sorted uint64 hashes. Version 2 files are refused: the allowed words cache is rebuilt from its .txt, an old blocklist.dcf is logged as unreadable. - The servers build blocklist.dcf from a plain blocklist.txt next to them (one word or phrase per line) when it is newer. - Blocked entries can be phrases: runs of consecutive words up to the longest entry, the whole run marked. Whitelist chat still checks one word at a time, as the client does. - Dashboard: the chat filter API reads blocklist.dcf the same way (status, phrase length), accepts blocked phrases, refuses allowed ones, and explains phrase matches in its message test. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
@@ -39,6 +39,7 @@ set(DCOMMONTEST_SOURCES
|
||||
"FdbReaderTests.cpp"
|
||||
"FdbSnapshotTests.cpp"
|
||||
"WorldFileWatchTests.cpp"
|
||||
"ChatFilterCoreTests.cpp"
|
||||
)
|
||||
|
||||
add_subdirectory(dEnumsTests)
|
||||
@@ -59,6 +60,8 @@ endif()
|
||||
target_link_libraries(dCommonTests ${COMMON_LIBRARIES} MD5 GTest::gtest_main)
|
||||
# SpareBackoff.h (header only)
|
||||
target_include_directories(dCommonTests PRIVATE "${PROJECT_SOURCE_DIR}/dMasterServer")
|
||||
# ChatFilterCore.h (header only)
|
||||
target_include_directories(dCommonTests PRIVATE "${PROJECT_SOURCE_DIR}/dChatFilter")
|
||||
|
||||
# Copy test files to testing directory
|
||||
add_subdirectory(TestBitStreams)
|
||||
|
||||
149
tests/dCommonTests/ChatFilterCoreTests.cpp
Normal file
149
tests/dCommonTests/ChatFilterCoreTests.cpp
Normal file
@@ -0,0 +1,149 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "ChatFilterCore.h"
|
||||
|
||||
using namespace ChatFilterWords;
|
||||
using Spans = std::set<std::pair<uint8_t, uint8_t>>;
|
||||
|
||||
// The hash is a compile-time constant: the same value on every compiler, standard library and platform
|
||||
static_assert(Hash("") == 0xcbf29ce484222325ULL);
|
||||
static_assert(Hash("a") == 0xaf63dc4c8601ec8cULL);
|
||||
static_assert(Hash("foobar") == 0x85944171f73967e8ULL);
|
||||
|
||||
TEST(ChatFilterCoreTest, HashIsFnv1a64) {
|
||||
// The standard FNV-1a 64 test vectors
|
||||
EXPECT_EQ(Hash(""), 0xcbf29ce484222325ULL);
|
||||
EXPECT_EQ(Hash("a"), 0xaf63dc4c8601ec8cULL);
|
||||
EXPECT_EQ(Hash("foobar"), 0x85944171f73967e8ULL);
|
||||
// Words of the client's chatplus_en_us.txt, as the filter hashes them (lower case)
|
||||
EXPECT_EQ(Hash("hello"), 0xa430d84680aabd0bULL);
|
||||
EXPECT_EQ(Hash("brick"), 0xf9236d1e24832c9aULL);
|
||||
EXPECT_EQ(Hash(AsciiLower("Brick")), Hash("brick"));
|
||||
// A phrase is its words joined by one space
|
||||
EXPECT_EQ(Hash("bad phrase"), 0x72e9fce49c1b0875ULL);
|
||||
// Bytes, not chars: a high byte hashes as 0x80-0xFF whether char is signed or not
|
||||
EXPECT_EQ(Hash("\xC3\xA9"), 0x0ac21707b7181e01ULL);
|
||||
}
|
||||
|
||||
TEST(ChatFilterCoreTest, NormalizeEntry) {
|
||||
EXPECT_EQ(NormalizeWord("Hello!?"), "hello");
|
||||
EXPECT_EQ(NormalizeEntry(" Bad \t PHRASE! "), "bad phrase");
|
||||
EXPECT_EQ(NormalizeEntry("word , here"), "word here");
|
||||
EXPECT_EQ(NormalizeEntry("..."), "");
|
||||
EXPECT_EQ(WordCount("bad phrase here"), 3u);
|
||||
EXPECT_EQ(WordCount("word"), 1u);
|
||||
EXPECT_EQ(WordCount(""), 0u);
|
||||
// ASCII only: other bytes are left alone whatever the locale
|
||||
EXPECT_EQ(AsciiLower("\xC3\x89Z"), "\xC3\x89z");
|
||||
}
|
||||
|
||||
TEST(ChatFilterCoreTest, DcfBytesAreFixed) {
|
||||
WordList list;
|
||||
list.AddEntry("a");
|
||||
list.AddEntry("bad phrase");
|
||||
const auto bytes = dChatFilterDCF::Serialize(list);
|
||||
// magic DCFB, version 3, 2 words at most, 2 hashes, then the hashes sorted, all little-endian
|
||||
const std::string expected(
|
||||
"DCFB" "\x03\x00\x00\x00" "\x02\x00\x00\x00" "\x02\x00\x00\x00\x00\x00\x00\x00"
|
||||
"\x75\x08\x1b\x9c\xe4\xfc\xe9\x72" "\x8c\xec\x01\x86\x4c\xdc\x63\xaf", 4 + 4 + 4 + 8 + 16);
|
||||
EXPECT_EQ(bytes, expected);
|
||||
|
||||
const auto parsed = dChatFilterDCF::Parse(bytes);
|
||||
ASSERT_EQ(parsed.status, dChatFilterDCF::eStatus::OK);
|
||||
EXPECT_EQ(parsed.list.maxWords, 2u);
|
||||
EXPECT_EQ(parsed.list.hashes, list.hashes);
|
||||
}
|
||||
|
||||
TEST(ChatFilterCoreTest, DcfRejectsOldAndBadFiles) {
|
||||
// Version 2 (std::hash, platform dependent) is refused, not guessed at
|
||||
const std::string old("DCFB" "\x02\x00\x00\x00" "\x01\x00\x00\x00\x00\x00\x00\x00" "\x11\x22\x33\x44\x55\x66\x77\x88", 24);
|
||||
EXPECT_EQ(dChatFilterDCF::Parse(old).status, dChatFilterDCF::eStatus::OLD_FORMAT);
|
||||
EXPECT_EQ(dChatFilterDCF::Parse("XCFB\x03\x00\x00\x00").status, dChatFilterDCF::eStatus::NOT_DCF);
|
||||
EXPECT_EQ(dChatFilterDCF::Parse(std::string("DCFB\x09\x00\x00\x00", 8)).status, dChatFilterDCF::eStatus::UNKNOWN);
|
||||
WordList list;
|
||||
list.AddEntry("word");
|
||||
auto bytes = dChatFilterDCF::Serialize(list);
|
||||
bytes.pop_back();
|
||||
EXPECT_EQ(dChatFilterDCF::Parse(bytes).status, dChatFilterDCF::eStatus::TRUNCATED);
|
||||
EXPECT_EQ(dChatFilterDCF::ReadFile("no such blocklist.dcf").status, dChatFilterDCF::eStatus::MISSING);
|
||||
}
|
||||
|
||||
TEST(ChatFilterCoreTest, BlockListFromText) {
|
||||
const auto list = dChatFilterDCF::BlockListFromText("Badword\r\n\r\n Very BAD phrase!\nbadword\n...\n");
|
||||
EXPECT_EQ(list.Size(), 2u);
|
||||
EXPECT_TRUE(list.Contains("badword"));
|
||||
EXPECT_TRUE(list.Contains("very bad phrase"));
|
||||
EXPECT_EQ(list.maxWords, 3u);
|
||||
// Written and read back, the same list
|
||||
const auto parsed = dChatFilterDCF::Parse(dChatFilterDCF::Serialize(list));
|
||||
ASSERT_EQ(parsed.status, dChatFilterDCF::eStatus::OK);
|
||||
EXPECT_EQ(parsed.list.hashes, list.hashes);
|
||||
EXPECT_EQ(parsed.list.maxWords, 3u);
|
||||
}
|
||||
|
||||
namespace {
|
||||
Lists FreeChatLists(std::string_view blockText) {
|
||||
Lists lists;
|
||||
// Through the file format, as the servers load it
|
||||
lists.denied = dChatFilterDCF::Parse(dChatFilterDCF::Serialize(dChatFilterDCF::BlockListFromText(blockText))).list;
|
||||
lists.approved = dChatFilterDCF::AllowListFromText("hello\nthere\nfriend\nbad\nphrase\n");
|
||||
return lists;
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ChatFilterCoreTest, BlockedWordFromFileIsStopped) {
|
||||
const auto lists = FreeChatLists("badword\n");
|
||||
EXPECT_EQ(CheckMessage("hello badword there", false, lists), (Spans{ { 6, 7 } }));
|
||||
EXPECT_EQ(CheckMessage("hello BadWord!", false, lists), (Spans{ { 6, 8 } }));
|
||||
EXPECT_TRUE(CheckMessage("hello there", false, lists).empty());
|
||||
// Whitelist chat doesn't use the block list: the word is simply not allowed there
|
||||
EXPECT_EQ(CheckMessage("hello badword", true, lists), (Spans{ { 6, 7 } }));
|
||||
// No block list: free chat stops everything
|
||||
EXPECT_EQ(CheckMessage("hello", false, Lists{}), (Spans{ { 0, 5 } }));
|
||||
}
|
||||
|
||||
TEST(ChatFilterCoreTest, PhrasesAtStartMiddleEnd) {
|
||||
const auto lists = FreeChatLists("bad phrase\n");
|
||||
EXPECT_EQ(CheckMessage("bad phrase hello", false, lists), (Spans{ { 0, 10 } }));
|
||||
EXPECT_EQ(CheckMessage("hello bad phrase there", false, lists), (Spans{ { 6, 10 } }));
|
||||
EXPECT_EQ(CheckMessage("hello bad phrase", false, lists), (Spans{ { 6, 10 } }));
|
||||
// The words alone are fine
|
||||
EXPECT_TRUE(CheckMessage("bad hello phrase", false, lists).empty());
|
||||
EXPECT_TRUE(CheckMessage("phrase bad", false, lists).empty());
|
||||
}
|
||||
|
||||
TEST(ChatFilterCoreTest, PhrasesWithExtraSpacesAndPunctuation) {
|
||||
const auto lists = FreeChatLists("bad phrase\n");
|
||||
// Two spaces: the span covers both words and the gap
|
||||
EXPECT_EQ(CheckMessage("hi Bad Phrase!", false, lists), (Spans{ { 4, 12 } }));
|
||||
// A piece that is only punctuation is skipped
|
||||
EXPECT_EQ(CheckMessage("bad ... phrase", false, lists), (Spans{ { 0, 14 } }));
|
||||
EXPECT_EQ(CheckMessage("bad, phrase.", false, lists), (Spans{ { 0, 12 } }));
|
||||
}
|
||||
|
||||
TEST(ChatFilterCoreTest, PhrasesOverlapWithWords) {
|
||||
const auto lists = FreeChatLists("bad phrase\nphrase here\nbadword\nfriend\n");
|
||||
// Two phrases sharing a word become one span
|
||||
EXPECT_EQ(CheckMessage("a bad phrase here b", false, lists), (Spans{ { 2, 15 } }));
|
||||
// A blocked word inside a blocked phrase: one span for the phrase
|
||||
EXPECT_EQ(CheckMessage("bad phrase", false, FreeChatLists("bad phrase\nphrase\n")), (Spans{ { 0, 10 } }));
|
||||
// A blocked word right after a phrase: its own span
|
||||
EXPECT_EQ(CheckMessage("bad phrase badword", false, lists), (Spans{ { 0, 10 }, { 11, 7 } }));
|
||||
// Longest match wins where it starts
|
||||
EXPECT_EQ(CheckMessage("friend bad phrase", false, FreeChatLists("friend\nfriend bad\nbad phrase\n")), (Spans{ { 0, 17 } }));
|
||||
}
|
||||
|
||||
TEST(ChatFilterCoreTest, DashboardPhrasesInWhitelistChat) {
|
||||
auto lists = FreeChatLists("");
|
||||
lists.customBlocked.AddEntry(NormalizeEntry("Bad Phrase"));
|
||||
// Blocked on the dashboard: stopped in whitelist chat too, even though each word is allowed
|
||||
EXPECT_EQ(CheckMessage("hello bad phrase", true, lists), (Spans{ { 6, 10 } }));
|
||||
EXPECT_TRUE(CheckMessage("hello bad there phrase", true, lists).empty());
|
||||
// Whitelist chat checks one word at a time, as the client does
|
||||
EXPECT_EQ(CheckMessage("hello stranger", true, lists), (Spans{ { 6, 8 } }));
|
||||
lists.customAllowed.AddEntry("stranger");
|
||||
EXPECT_TRUE(CheckMessage("hello stranger", true, lists).empty());
|
||||
}
|
||||
@@ -43,7 +43,10 @@ TEST(ChatFilterWordsTest, FilterWord) {
|
||||
EXPECT_EQ(ModerationTools::FilterWord(" Hello! "), "hello");
|
||||
EXPECT_EQ(ModerationTools::FilterWord("W.o,r;d?"), "word");
|
||||
EXPECT_FALSE(ModerationTools::FilterWord(""));
|
||||
EXPECT_FALSE(ModerationTools::FilterWord("two words"));
|
||||
// Phrases: words normalized and joined by one space
|
||||
EXPECT_EQ(ModerationTools::FilterWord(" Two Words! "), "two words");
|
||||
EXPECT_TRUE(ModerationTools::IsPhrase("two words"));
|
||||
EXPECT_FALSE(ModerationTools::IsPhrase("word"));
|
||||
EXPECT_FALSE(ModerationTools::FilterWord("!!!"));
|
||||
EXPECT_FALSE(ModerationTools::FilterWord(std::string(65, 'a')));
|
||||
}
|
||||
@@ -53,6 +56,11 @@ TEST(ChatFilterWordsTest, HasFilterWord) {
|
||||
EXPECT_TRUE(ModerationTools::HasFilterWord("bad", "bad"));
|
||||
EXPECT_FALSE(ModerationTools::HasFilterWord("badger badminton", "bad"));
|
||||
EXPECT_FALSE(ModerationTools::HasFilterWord("", "bad"));
|
||||
// Phrases: the words in a row, whatever the spaces and punctuation between them
|
||||
EXPECT_TRUE(ModerationTools::HasFilterWord("well, Bad Phrase!", "bad phrase"));
|
||||
EXPECT_TRUE(ModerationTools::HasFilterWord("bad ... phrase", "bad phrase"));
|
||||
EXPECT_FALSE(ModerationTools::HasFilterWord("bad other phrase", "bad phrase"));
|
||||
EXPECT_FALSE(ModerationTools::HasFilterWord("phrase bad", "bad phrase"));
|
||||
}
|
||||
|
||||
TEST(ChatFilterWordsTest, FileWords) {
|
||||
@@ -61,27 +69,6 @@ TEST(ChatFilterWordsTest, FileWords) {
|
||||
ASSERT_TRUE(ModerationTools::FileWords("").empty());
|
||||
}
|
||||
|
||||
TEST(ChatFilterWordsTest, DcfHashes) {
|
||||
const std::vector<size_t> hashes{ ModerationTools::WordHash("badword"), 42 };
|
||||
std::string bytes(sizeof(dChatFilterDCF::fileHeader) + sizeof(size_t) * (hashes.size() + 1), '\0');
|
||||
const dChatFilterDCF::fileHeader header{ dChatFilterDCF::header, dChatFilterDCF::formatVersion };
|
||||
const size_t count = hashes.size();
|
||||
std::memcpy(bytes.data(), &header, sizeof(header));
|
||||
std::memcpy(bytes.data() + sizeof(header), &count, sizeof(count));
|
||||
std::memcpy(bytes.data() + sizeof(header) + sizeof(count), hashes.data(), sizeof(size_t) * count);
|
||||
ASSERT_EQ(ModerationTools::DcfHashes(bytes), hashes);
|
||||
|
||||
// Wrong header, other version, or fewer hashes than it says
|
||||
auto wrong = bytes;
|
||||
wrong[0] = 'X';
|
||||
ASSERT_FALSE(ModerationTools::DcfHashes(wrong).has_value());
|
||||
auto version = bytes;
|
||||
version[sizeof(uint32_t)] = 9;
|
||||
ASSERT_FALSE(ModerationTools::DcfHashes(version).has_value());
|
||||
ASSERT_FALSE(ModerationTools::DcfHashes(bytes.substr(0, bytes.size() - sizeof(size_t) * 2)).has_value());
|
||||
ASSERT_FALSE(ModerationTools::DcfHashes("DCFB").has_value());
|
||||
}
|
||||
|
||||
namespace {
|
||||
ModerationTools::WordSources Sources(bool blockFileLoaded = true) {
|
||||
ModerationTools::WordSources sources;
|
||||
@@ -121,3 +108,22 @@ TEST(ChatFilterWordsTest, ExplainFreeChat) {
|
||||
EXPECT_EQ(Reasons(ModerationTools::ExplainMessage("zzz darn", false, Sources(false))),
|
||||
(std::vector<std::string>{ "x no_block_file", "x no_block_file" }));
|
||||
}
|
||||
|
||||
TEST(ChatFilterWordsTest, ExplainPhrases) {
|
||||
auto sources = Sources();
|
||||
sources.dashboard = [](const std::string& w) -> std::optional<bool> {
|
||||
if (w == "no way") return false;
|
||||
return std::nullopt;
|
||||
};
|
||||
sources.allowFile = [](const std::string& w) { return w == "hello" || w == "no" || w == "way" || w == "rude"; };
|
||||
sources.blockFile = [](const std::string& w) { return w == "very rude"; };
|
||||
sources.maxWords = 2;
|
||||
// A phrase blocked here stops each of its words (and the empty piece between two spaces inside it), in normal chat too
|
||||
auto verdicts = ModerationTools::ExplainMessage("hello No way!", true, sources);
|
||||
EXPECT_EQ(Reasons(verdicts), (std::vector<std::string>{ "ok allow_file", "x blocked_here", "x blocked_here", "x blocked_here" }));
|
||||
EXPECT_EQ(verdicts[1].phrase, "no way");
|
||||
EXPECT_EQ(verdicts[3].text, "way!");
|
||||
// The block file's phrases only in free chat
|
||||
EXPECT_EQ(Reasons(ModerationTools::ExplainMessage("very rude", false, sources)), (std::vector<std::string>{ "x block_file", "x block_file" }));
|
||||
EXPECT_EQ(Reasons(ModerationTools::ExplainMessage("rude very", false, sources)), (std::vector<std::string>{ "ok not_in_block_file", "ok not_in_block_file" }));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user