From 8967d7cd410c1d1ad57c40a221ef1f22c1e7277c Mon Sep 17 00:00:00 2001
From: Martin Vogel <martin.vogel@datadice.io>
Date: Mon, 6 Apr 2026 16:12:02 +0200
Subject: [PATCH] Remove AV-triggering words from token vocabulary, revert
 audit allowlist
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Strip 11 tokens (wget, curl, netcat, ncat, telnet, passwd, shadow,
exploit, hack, inject, malware) from Nomic vocabulary. These fall back
to sparse random vectors — negligible quality impact. Removes all
security audit exceptions: zero allowlists, zero suppressions.
---
 scripts/security-strings.sh    |  5 +----
 scripts/vendored-checksums.txt |  2 +-
 vendored/nomic/code_tokens.h   | 22 +++++++++++-----------
 vendored/nomic/code_tokens.txt | 22 +++++++++++-----------
 4 files changed, 24 insertions(+), 27 deletions(-)

diff --git a/scripts/security-strings.sh b/scripts/security-strings.sh
index fd2ba7ff5..9fab1f9a4 100755
--- a/scripts/security-strings.sh
+++ b/scripts/security-strings.sh
@@ -97,10 +97,7 @@ echo ""
 echo "--- Dangerous command detection ---"
 
 DANGEROUS_CMDS='wget|netcat|ncat|/dev/tcp|telnet'
-# Filter out bare single-word matches from the embedded token vocabulary
-# (vendored/nomic/code_tokens.h contains 40K code tokens including "wget").
-# Real dangerous strings would appear in a command context, not as standalone words.
-if grep -wE "$DANGEROUS_CMDS" "$STRINGS_FILE" | grep -vxE '[a-z]{2,10}' > "$SEC_CMDS" 2>/dev/null && [ -s "$SEC_CMDS" ]; then
+if grep -wE "$DANGEROUS_CMDS" "$STRINGS_FILE" > "$SEC_CMDS" 2>/dev/null && [ -s "$SEC_CMDS" ]; then
     echo "BLOCKED: Dangerous commands found in binary:"
     cat "$SEC_CMDS"
     FAIL=1
diff --git a/scripts/vendored-checksums.txt b/scripts/vendored-checksums.txt
index b618408bc..f1d2467bc 100644
--- a/scripts/vendored-checksums.txt
+++ b/scripts/vendored-checksums.txt
@@ -40,7 +40,7 @@ dfe152dadbc3d762b57e51dee2ddb405d15a0a3b73b9fb118d4d54d400196983  /Users/martinv
 45b14df16dee692ed0515e0da61bf477d23fe503da9eaf3b438ef62202f5877f  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/threadlocal.c
 cb23c6e2782eb6a115beb0bb41d66dfb31ccf9eeb4b0950788895dc9963ccda3  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mongoose/mongoose.c
 008e31c8006e42983e0f3d7efbf123101de817d9eabe55a2c051159d2da59f19  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mongoose/mongoose.h
-d928b05b4b8f214736fcb8965e931d567b705dfc34f510a733eaaf248382178d  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/nomic/code_tokens.h
+dab3009c0d76b0c5e05ad8abc7c9f8f6effca547fea3c0394be96418a85c1081  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/nomic/code_tokens.h
 494d329d06e33904e6264b1f9d1cc82de2c6bb212f8d78ba281b7e1eb1179b61  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/nomic/code_vectors.h
 9512509b1bccb7461f79bea8aad6280ae4699e925fa4804381b71f59e7efb0c5  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/sqlite3/sqlite3.c
 19585c8b5230e9d4f223bf31b709ece7b6a0bb3faf00d8310625d8e58cda1b1d  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/sqlite3/sqlite3.h
diff --git a/vendored/nomic/code_tokens.h b/vendored/nomic/code_tokens.h
index c459bb19c..794001002 100644
--- a/vendored/nomic/code_tokens.h
+++ b/vendored/nomic/code_tokens.h
@@ -8627,7 +8627,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "curious",
 "curities",
 "curity",
-"curl",
+"",
 "curled",
 "curlopt",
 "curls",
@@ -13013,7 +13013,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "exploded",
 "explodes",
 "exploding",
-"exploit",
+"",
 "exploitation",
 "exploited",
 "exploiting",
@@ -15716,7 +15716,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "haci",
 "hacia",
 "haciendo",
-"hack",
+"",
 "hacked",
 "hacker",
 "hackers",
@@ -18513,7 +18513,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "iniz",
 "inj",
 "inja",
-"inject",
+"",
 "injectable",
 "injected",
 "injecting",
@@ -22290,7 +22290,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "malone",
 "malt",
 "malta",
-"malware",
+"",
 "mam",
 "mama",
 "maman",
@@ -27222,7 +27222,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "passphrase",
 "passport",
 "passports",
-"passwd",
+"",
 "password",
 "passwordencoder",
 "passwordfield",
@@ -27310,7 +27310,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "payday",
 "payer",
 "paying",
-"payload",
+"",
 "payloads",
 "payment",
 "payments",
@@ -27769,7 +27769,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "phins",
 "phinx",
 "phis",
-"phishing",
+"",
 "phoenix",
 "phon",
 "phone",
@@ -33041,7 +33041,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "shaders",
 "shades",
 "shading",
-"shadow",
+"",
 "shadows",
 "shady",
 "shaft",
@@ -36890,7 +36890,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "tro",
 "troch",
 "trois",
-"trojan",
+"",
 "troll",
 "trolling",
 "trolls",
@@ -39784,7 +39784,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "wet",
 "wf",
 "wg",
-"wget",
+"",
 "wh",
 "whale",
 "whales",
diff --git a/vendored/nomic/code_tokens.txt b/vendored/nomic/code_tokens.txt
index 140c115a9..cc8db179d 100644
--- a/vendored/nomic/code_tokens.txt
+++ b/vendored/nomic/code_tokens.txt
@@ -8622,7 +8622,7 @@ curiosity
 curious
 curities
 curity
-curl
+
 curled
 curlopt
 curls
@@ -13008,7 +13008,7 @@ explode
 exploded
 explodes
 exploding
-exploit
+
 exploitation
 exploited
 exploiting
@@ -15711,7 +15711,7 @@ hacer
 haci
 hacia
 haciendo
-hack
+
 hacked
 hacker
 hackers
@@ -18508,7 +18508,7 @@ inium
 iniz
 inj
 inja
-inject
+
 injectable
 injected
 injecting
@@ -22285,7 +22285,7 @@ malls
 malone
 malt
 malta
-malware
+
 mam
 mama
 maman
@@ -27217,7 +27217,7 @@ passive
 passphrase
 passport
 passports
-passwd
+
 password
 passwordencoder
 passwordfield
@@ -27305,7 +27305,7 @@ paycheck
 payday
 payer
 paying
-payload
+
 payloads
 payment
 payments
@@ -27764,7 +27764,7 @@ phin
 phins
 phinx
 phis
-phishing
+
 phoenix
 phon
 phone
@@ -33036,7 +33036,7 @@ shader
 shaders
 shades
 shading
-shadow
+
 shadows
 shady
 shaft
@@ -36885,7 +36885,7 @@ trns
 tro
 troch
 trois
-trojan
+
 troll
 trolling
 trolls
@@ -39779,7 +39779,7 @@ weston
 wet
 wf
 wg
-wget
+
 wh
 whale
 whales