mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-11-02 13:12:50 +00:00
fix(posting_guard): decode text spelled in invisible characters
clean_text removed zero-width, bidi, soft-hyphen and BOM characters, but Unicode tag characters (U+E0000-E007F, which shadow ASCII one for one) and runs of variation selectors (one byte each) went into the cleaned text untouched. An instruction spelled in them renders as nothing on the page or in the terminal and still reads as text to the model, so the gate passed it as clean. Both are now removed and counted. What a tag run or a selector run spells is decoded into a new invisible_text finding, scanned like hidden content (so instruction_like and hidden_urls see it), and raises the verdict's warning. Emoji subdivision flags and a lone variation selector are counted, not flagged. /apply's list of findings to show gains the new one. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
67d1b84003
commit
07a77acbb0
4 files changed
+123
-15
No files matched your search
+48
-12
@@ -24,6 +24,10 @@ evaluates anything:
|
||||
`tag`, and always flagged, because no legitimate posting contains one -
|
||||
other than the three UK subdivision flag emoji, which are matched in
|
||||
full and kept;
|
||||
* text spelled in invisible characters (a run of tag characters, a run of
|
||||
variation selectors carrying one byte each) - decoded, listed under
|
||||
`invisible_text` and scanned like hidden content, so an instruction
|
||||
spelled that way is quoted instead of merely counted;
|
||||
* instruction-shaped lines aimed at an agent rather than a reader - quoted,
|
||||
never removed (an SSH administrator posting legitimately says "SSH");
|
||||
* URLs that only exist in hidden content - listed;
|
||||
@@ -33,7 +37,7 @@ evaluates anything:
|
||||
Removal is the defense: hidden content never reaches the cleaned text the
|
||||
workflow reads. The ⚠ in the verdict is for the user and fires only on signals
|
||||
that indicate targeting (instruction-shaped lines, tampering characters,
|
||||
mixed-script words); hidden blocks and hidden URLs are reported as context,
|
||||
invisible text, mixed-script words); hidden blocks and hidden URLs are reported as context,
|
||||
because navigation chrome is hidden on nearly every real page. It advises; it
|
||||
never blocks. The archive gets counts only (see /apply Step 6b); excerpts are
|
||||
for the terminal.
|
||||
@@ -72,10 +76,12 @@ FORMAT = "
" # soft hyphen, line/paragraph separators
|
||||
BOM = ""
|
||||
# U+E0000-U+E007F: the Unicode "tags" block, one invisible code point per ASCII
|
||||
# character. A payload written in it survives copy-paste, never renders, and
|
||||
# is decoded by a model as if it were the ASCII it mirrors; a single one
|
||||
# inside "ignore" also splits the word past every instruction pattern.
|
||||
# is decoded by a model as if it were the ASCII it mirrors ("ASCII smuggling");
|
||||
# a single one inside "ignore" also splits the word past every instruction
|
||||
# pattern.
|
||||
TAG_CHARS = "".join(chr(c) for c in range(0xE0000, 0xE0080))
|
||||
INVISIBLE = ZERO_WIDTH + BIDI + FORMAT + BOM + TAG_CHARS
|
||||
TAG_RUN = re.compile("[\U000E0000-\U000E007F]+")
|
||||
# The one legitimate use of tag characters: the England, Scotland and Wales
|
||||
# flags (black flag, tag letters gbeng/gbsct/gbwls, cancel tag). Matched in
|
||||
# full and left intact; a payload inside a well-formed tag sequence, extra
|
||||
@@ -84,6 +90,11 @@ RGI_TAG_FLAG = re.compile(
|
||||
"\U0001F3F4\U000E0067\U000E0062(?:\U000E0065\U000E006E\U000E0067"
|
||||
"|\U000E0073\U000E0063\U000E0074|\U000E0077\U000E006C\U000E0073)\U000E007F"
|
||||
)
|
||||
# Variation selectors (U+FE00-FE0F, U+E0100-E01EF) pick a glyph variant and
|
||||
# render as nothing on their own; a run of them carries one byte each. A lone
|
||||
# one (the VS16 in ❤️) is encoding noise; a run is a payload.
|
||||
VARIATION = re.compile("[︀-️\U000E0100-\U000E01EF]")
|
||||
VARIATION_RUN = re.compile("[︀-️\U000E0100-\U000E01EF]{2,}")
|
||||
|
||||
HIDDEN_STYLE = re.compile(
|
||||
r"display\s*:\s*none"
|
||||
@@ -157,11 +168,14 @@ def _fold(text: str) -> str:
|
||||
def clean_text(text: str):
|
||||
"""Remove invisible characters.
|
||||
|
||||
Returns (text, {"invisible_chars": {category: count}, "invisible_flagged": n})
|
||||
where the flagged count covers a zero-width or BOM character between two
|
||||
letters (splitting a word to dodge a scan) and any bidi override or isolate.
|
||||
A BOM at offset 0, emoji joiners, RTL marks and soft hyphens are counted
|
||||
but not flagged: they are encoding noise, not tampering.
|
||||
Returns (text, {"invisible_chars": {category: count}, "invisible_flagged": n,
|
||||
"invisible_text": [decoded, ...]}) where the flagged count covers a
|
||||
zero-width or BOM character between two letters (splitting a word to dodge
|
||||
a scan) and any bidi override or isolate. A BOM at offset 0, emoji joiners,
|
||||
RTL marks, soft hyphens, emoji flag tags and a lone variation selector are
|
||||
counted but not flagged: they are encoding noise, not tampering. Tag
|
||||
character runs and variation selector runs are decoded into
|
||||
invisible_text instead, since what they spell is the payload.
|
||||
"""
|
||||
counts = {}
|
||||
flagged = 0
|
||||
@@ -170,9 +184,13 @@ def clean_text(text: str):
|
||||
if n:
|
||||
counts[key] = n
|
||||
flags = RGI_TAG_FLAG.findall(text)
|
||||
tags = sum(1 for ch in RGI_TAG_FLAG.sub("", text) if "\U000E0000" <= ch <= "\U000E007F")
|
||||
unflagged = RGI_TAG_FLAG.sub("", text) # the text minus its legitimate flags
|
||||
tags = sum(1 for ch in unflagged if "\U000E0000" <= ch <= "\U000E007F")
|
||||
if tags:
|
||||
counts["tag"] = tags
|
||||
variation = len(VARIATION.findall(text))
|
||||
if variation:
|
||||
counts["variation"] = variation
|
||||
for i, ch in enumerate(text):
|
||||
if ch in BIDI_TAMPER:
|
||||
flagged += 1
|
||||
@@ -181,6 +199,17 @@ def clean_text(text: str):
|
||||
after = text[i + 1] if i + 1 < len(text) else ""
|
||||
if before.isalpha() and after.isalpha():
|
||||
flagged += 1
|
||||
invisible_text = []
|
||||
for run in TAG_RUN.findall(unflagged):
|
||||
decoded = "".join(chr(ord(c) - 0xE0000) for c in run if 0x20 <= ord(c) - 0xE0000 < 0x7F).strip()
|
||||
if decoded:
|
||||
invisible_text.append(decoded[:LINE_MAX])
|
||||
for run in VARIATION_RUN.findall(text):
|
||||
data = bytes(ord(c) - 0xFE00 if ord(c) <= 0xFE0F else ord(c) - 0xE0100 + 16 for c in run)
|
||||
decoded = "".join(c for c in data.decode("utf-8", errors="replace") if c.isprintable()).strip()
|
||||
if decoded:
|
||||
invisible_text.append(decoded[:LINE_MAX])
|
||||
text = VARIATION.sub("", text)
|
||||
if flags:
|
||||
# Strip around the flags so the visible emoji survive intact.
|
||||
strip = {ord(c): None for c in INVISIBLE}
|
||||
@@ -193,7 +222,7 @@ def clean_text(text: str):
|
||||
text = "".join(pieces)
|
||||
else:
|
||||
text = text.translate({ord(c): None for c in INVISIBLE})
|
||||
return _fold(text), {"invisible_chars": counts, "invisible_flagged": flagged}
|
||||
return _fold(text), {"invisible_chars": counts, "invisible_flagged": flagged, "invisible_text": invisible_text}
|
||||
|
||||
|
||||
def _attrs(raw: str) -> dict:
|
||||
@@ -375,6 +404,9 @@ def verdict(findings: dict) -> str:
|
||||
n = int((findings.get("invisible_chars") or {}).get("tag") or 0)
|
||||
if n:
|
||||
alarms.append(_plural(n, "tag character (text hidden from the reader)", "tag characters (text hidden from the reader)"))
|
||||
n = len(findings.get("invisible_text") or [])
|
||||
if n:
|
||||
alarms.append(_plural(n, "string spelled in invisible characters", "strings spelled in invisible characters"))
|
||||
n = len(findings.get("homoglyph_runs") or [])
|
||||
if n:
|
||||
alarms.append(_plural(n, "mixed-script word", "mixed-script words"))
|
||||
@@ -406,7 +438,11 @@ def guard(raw: str, mode: str):
|
||||
else:
|
||||
text, hidden_text, hidden_blocks = raw, "", []
|
||||
text, tf = clean_text(text)
|
||||
hidden_text, _ = clean_text(hidden_text)
|
||||
hidden_text, hidden_tf = clean_text(hidden_text)
|
||||
# What invisible characters spell is hidden content: scanned and reported,
|
||||
# never part of the cleaned text.
|
||||
tf["invisible_text"] += hidden_tf["invisible_text"]
|
||||
hidden_text = "\n".join([hidden_text, *tf["invisible_text"]])
|
||||
findings = {"hidden_blocks": hidden_blocks, **tf, **scan(text, hidden_text)}
|
||||
findings["verdict"] = verdict(findings)
|
||||
return text, findings
|
||||
@@ -414,7 +450,7 @@ def guard(raw: str, mode: str):
|
||||
|
||||
def _plain_findings(findings: dict) -> str:
|
||||
lines = []
|
||||
for key in ("hidden_blocks", "instruction_like", "hidden_urls", "homoglyph_runs"):
|
||||
for key in ("hidden_blocks", "instruction_like", "invisible_text", "hidden_urls", "homoglyph_runs"):
|
||||
items = findings[key]
|
||||
lines.append(f"{key} ({len(items)}):")
|
||||
lines.extend(f' - "{item}"' for item in items)
|
||||
|
||||
Reference in new issue
Block a user