Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-02 10:53:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(chat-filter): portable .dcf hashing, block list phrases

Browse Source 
The filter stored and compared words by std::hash<std::string> in size_t,
which differs between standard libraries and platforms, so a .dcf made on
one system never matched on another and the block list never worked
there (issue 215).

- ChatFilterCore.h: 64-bit FNV-1a over the entry's bytes, ASCII lower
  case, fixed-width uint64_t everywhere stored or compared.
- .dcf version 3: little-endian magic, version, longest entry in words,
  uint64 count and sorted uint64 hashes. Version 2 files are refused:
  the allowed words cache is rebuilt from its .txt, an old
  blocklist.dcf is logged as unreadable.
- The servers build blocklist.dcf from a plain blocklist.txt next to
  them (one word or phrase per line) when it is newer.
- Blocked entries can be phrases: runs of consecutive words up to the
  longest entry, the whole run marked. Whitelist chat still checks one
  word at a time, as the client does.
- Dashboard: the chat filter API reads blocklist.dcf the same way
  (status, phrase length), accepts blocked phrases, refuses allowed
  ones, and explains phrase matches in its message test.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-30 08:05:00 -05:00
parent 3018312f29
commit c1bcda8dd1
8 changed files with 691 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
tests/dWebTests/ModerationToolsTests.cpp
View File

@@ -43,7 +43,10 @@ TEST(ChatFilterWordsTest, FilterWord) {
EXPECT_EQ(ModerationTools::FilterWord(" Hello! "), "hello");
EXPECT_EQ(ModerationTools::FilterWord("W.o,r;d?"), "word");
EXPECT_FALSE(ModerationTools::FilterWord(""));
EXPECT_FALSE(ModerationTools::FilterWord("two words"));
// Phrases: words normalized and joined by one space
EXPECT_EQ(ModerationTools::FilterWord(" Two Words! "), "two words");
EXPECT_TRUE(ModerationTools::IsPhrase("two words"));
EXPECT_FALSE(ModerationTools::IsPhrase("word"));
EXPECT_FALSE(ModerationTools::FilterWord("!!!"));
EXPECT_FALSE(ModerationTools::FilterWord(std::string(65, 'a')));
}
@@ -53,6 +56,11 @@ TEST(ChatFilterWordsTest, HasFilterWord) {
EXPECT_TRUE(ModerationTools::HasFilterWord("bad", "bad"));
EXPECT_FALSE(ModerationTools::HasFilterWord("badger badminton", "bad"));
EXPECT_FALSE(ModerationTools::HasFilterWord("", "bad"));
// Phrases: the words in a row, whatever the spaces and punctuation between them
EXPECT_TRUE(ModerationTools::HasFilterWord("well, Bad Phrase!", "bad phrase"));
EXPECT_TRUE(ModerationTools::HasFilterWord("bad ... phrase", "bad phrase"));
EXPECT_FALSE(ModerationTools::HasFilterWord("bad other phrase", "bad phrase"));
EXPECT_FALSE(ModerationTools::HasFilterWord("phrase bad", "bad phrase"));
}
TEST(ChatFilterWordsTest, FileWords) {
@@ -61,27 +69,6 @@ TEST(ChatFilterWordsTest, FileWords) {
ASSERT_TRUE(ModerationTools::FileWords("").empty());
}
TEST(ChatFilterWordsTest, DcfHashes) {
const std::vector<size_t> hashes{ ModerationTools::WordHash("badword"), 42 };
std::string bytes(sizeof(dChatFilterDCF::fileHeader) + sizeof(size_t) * (hashes.size() + 1), '\0');
const dChatFilterDCF::fileHeader header{ dChatFilterDCF::header, dChatFilterDCF::formatVersion };
const size_t count = hashes.size();
std::memcpy(bytes.data(), &header, sizeof(header));
std::memcpy(bytes.data() + sizeof(header), &count, sizeof(count));
std::memcpy(bytes.data() + sizeof(header) + sizeof(count), hashes.data(), sizeof(size_t) * count);
ASSERT_EQ(ModerationTools::DcfHashes(bytes), hashes);
// Wrong header, other version, or fewer hashes than it says
auto wrong = bytes;
wrong[0] = 'X';
ASSERT_FALSE(ModerationTools::DcfHashes(wrong).has_value());
auto version = bytes;
version[sizeof(uint32_t)] = 9;
ASSERT_FALSE(ModerationTools::DcfHashes(version).has_value());
ASSERT_FALSE(ModerationTools::DcfHashes(bytes.substr(0, bytes.size() - sizeof(size_t) * 2)).has_value());
ASSERT_FALSE(ModerationTools::DcfHashes("DCFB").has_value());
}
namespace {
ModerationTools::WordSources Sources(bool blockFileLoaded = true) {
ModerationTools::WordSources sources;
@@ -121,3 +108,22 @@ TEST(ChatFilterWordsTest, ExplainFreeChat) {
EXPECT_EQ(Reasons(ModerationTools::ExplainMessage("zzz darn", false, Sources(false))),
(std::vector<std::string>{ "x no_block_file", "x no_block_file" }));
}
TEST(ChatFilterWordsTest, ExplainPhrases) {
auto sources = Sources();
sources.dashboard = [](const std::string& w) -> std::optional<bool> {
if (w == "no way") return false;
return std::nullopt;
};
sources.allowFile = [](const std::string& w) { return w == "hello" || w == "no" || w == "way" || w == "rude"; };
sources.blockFile = [](const std::string& w) { return w == "very rude"; };
sources.maxWords = 2;
// A phrase blocked here stops each of its words (and the empty piece between two spaces inside it), in normal chat too
auto verdicts = ModerationTools::ExplainMessage("hello No way!", true, sources);
EXPECT_EQ(Reasons(verdicts), (std::vector<std::string>{ "ok allow_file", "x blocked_here", "x blocked_here", "x blocked_here" }));
EXPECT_EQ(verdicts[1].phrase, "no way");
EXPECT_EQ(verdicts[3].text, "way!");
// The block file's phrases only in free chat
EXPECT_EQ(Reasons(ModerationTools::ExplainMessage("very rude", false, sources)), (std::vector<std::string>{ "x block_file", "x block_file" }));
EXPECT_EQ(Reasons(ModerationTools::ExplainMessage("rude very", false, sources)), (std::vector<std::string>{ "ok not_in_block_file", "ok not_in_block_file" }));
}