fix(tools): decode saved non-UTF-8 pages by their declared charset (#690)

* fix(tools): decode saved non-UTF-8 pages by their declared charset

fetch_page.py saves a body exactly as the server sent it, but its readers,
posting_guard.py --html and fetch_page.py --text, decoded the file as UTF-8
with replacement. A posting served in EUC-KR, Shift_JIS or windows-1252
reached the integrity gate and the scorer as replacement characters.

Valid UTF-8 still wins, so any page that decoded cleanly before decodes to the
same string. Bytes that are not valid UTF-8 now fall back to the charset the
page declares (UTF-16 BOM, then <meta charset> or http-equiv Content-Type in
the first 1024 bytes), mapped to the codec browsers use for that label. Missing
or unknown labels keep the previous UTF-8-with-replacement result.

* changelog: place the #690 entry under [Unreleased]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Mads Lorentzen <madslorentzen_17@hotmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -242,6 +242,17 @@ per-file diff commands.
   runs a bare `python`, or a spec that runs a tool carries no note, a second
   note, or one after its first command.
 
+- **A saved page that is not UTF-8 is read in the charset it declares** (#690, `tools/page_decode.py`,
+  `tools/posting_guard.py`, `tools/fetch_page.py`, `tests/test_page_decode.py`) - `posting_guard.py --html`
+  and `fetch_page.py --text` decoded a page saved by `fetch_page.py` as UTF-8 with replacement, so a
+  posting served in EUC-KR, Shift_JIS, windows-1251 or another legacy charset reached the gate and the
+  scorer as replacement characters while the gate still reported it clean (a live job.incruit.com
+  posting came out as 1206 U+FFFD and 4 Hangul characters). Bytes that are valid UTF-8 decode exactly
+  as before; otherwise a UTF-16 byte order mark or a `<meta>` charset in the first 1024 bytes picks the
+  codec browsers use for that label, from a fixed list (so a page cannot select UTF-7), and a missing
+  or unknown label keeps the old result. The homoglyph check now sees Cyrillic letters on a
+  windows-1251 page. Text input to the gate (`--text`) is unchanged.
+
 ### Security
 
 - **CI no longer leaves the job token in `.git/config` or expands a PR-controlled name into a

tests/test_fetch_page.py

@@ -30,6 +30,11 @@ PAGE = (
     "<noscript>noscript text</noscript><svg><text>svg text</text></svg>"
     "</body></html>"
 )
+KOREAN_PAGE = (
+    '<html><head><meta http-equiv="Content-Type" content="text/html; charset=euc-kr">'
+    "<title>채용공고</title></head><body><h1>백엔드 개발자 채용</h1></body></html>"
+)
+CONTENT_TYPES = {"/euc-kr.html": "text/html; charset=euc-kr"}
 
 
 class _Handler(http.server.BaseHTTPRequestHandler):
@@ -37,6 +42,9 @@ class _Handler(http.server.BaseHTTPRequestHandler):
         if self.path == "/ok.html":
             body = PAGE.encode("utf-8")
             self.send_response(200)
+        elif self.path == "/euc-kr.html":
+            body = KOREAN_PAGE.encode("cp949")
+            self.send_response(200)
         elif self.path == "/forbidden":
             body = b"<html><body>denied</body></html>"
             self.send_response(403)
@@ -54,7 +62,7 @@ class _Handler(http.server.BaseHTTPRequestHandler):
         else:
             body = b"nope"
             self.send_response(404)
-        self.send_header("Content-Type", "text/html; charset=utf-8")
+        self.send_header("Content-Type", CONTENT_TYPES.get(self.path, "text/html; charset=utf-8"))
         self.send_header("Content-Length", str(len(body)))
         self.end_headers()
         self.wfile.write(body)
@@ -152,6 +160,14 @@ class Fetching(FetchPageFixture):
         for leaked in ("alert(", "color:red", "hidden comment", "noscript text", "svg text", "<p>"):
             self.assertNotIn(leaked, text)
 
+    def test_text_decodes_a_page_by_its_declared_charset(self):
+        code, out, err = self.run_main(f"{self.base}/euc-kr.html", "--out", str(self.out), "--text")
+        self.assertEqual(code, 0, err)
+        self.assertEqual(self.out.read_bytes(), KOREAN_PAGE.encode("cp949"))
+        text = out.split("--- text ---", 1)[1]
+        self.assertIn("백엔드 개발자 채용", text)
+        self.assertNotIn("\ufffd", text)
+
     def test_max_chars_caps_text(self):
         code, out, err = self.run_main(
             f"{self.base}/ok.html", "--out", str(self.out), "--text", "--max-chars", "7"

tests/test_page_decode.py

@@ -0,0 +1,82 @@
+"""tools/page_decode.py decodes a saved page by the charset it declares.
+
+Why this test exists: fetch_page.py saves bodies as the server sent them, and
+posting_guard.py --html and fetch_page.py --text used to decode every file as
+UTF-8 with replacement. A posting served in EUC-KR (job.incruit.com serves its
+postings that way) reached the gate and the scorer as replacement characters.
+These tests pin that declared legacy charsets are honoured, and that anything
+which decoded cleanly before still decodes to exactly the same string.
+"""
+import sys
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "tools"))
+from page_decode import decode_page, declared_codec  # noqa: E402
+
+
+def page(body: str, head: str = "") -> str:
+    return f"<!doctype html><html><head>{head}<title>Posting</title></head><body>{body}</body></html>"
+
+
+class LegacyCharsets(unittest.TestCase):
+    def test_http_equiv_euc_kr_is_read_as_cp949(self):
+        # 똠 is outside strict EUC-KR; browsers read the euc-kr label as CP949.
+        text = page("<h1>백엔드 개발자 채용</h1><p>똠방각하 우대</p>",
+                    '<meta http-equiv="Content-Type" content="text/html; charset=euc-kr">')
+        decoded = decode_page(text.encode("cp949"))
+        self.assertIn("백엔드 개발자 채용", decoded)
+        self.assertIn("똠방각하", decoded)
+        self.assertNotIn("�", decoded)
+
+    def test_meta_charset_shift_jis(self):
+        text = page("<h1>データエンジニア募集</h1>", '<meta charset="Shift_JIS">')
+        self.assertIn("データエンジニア募集", decode_page(text.encode("cp932")))
+
+    def test_iso_8859_1_label_is_read_as_windows_1252(self):
+        text = page("<p>Løn: 50.000 € om måneden</p>", "<meta charset=iso-8859-1>")
+        self.assertIn("Løn: 50.000 € om måneden", decode_page(text.encode("cp1252")))
+
+    def test_windows_1251_cyrillic(self):
+        text = page("<p>Инженер данных</p>", '<meta charset="windows-1251">')
+        self.assertIn("Инженер данных", decode_page(text.encode("cp1251")))
+
+    def test_utf16_byte_order_mark(self):
+        text = page("<p>Ingeniør</p>")
+        self.assertIn("Ingeniør", decode_page(text.encode("utf-16")))
+
+
+class UnchangedBehaviour(unittest.TestCase):
+    def test_valid_utf8_is_unchanged(self):
+        raw = page("<p>Ingeniør søges, 백엔드, データ</p>").encode("utf-8")
+        self.assertEqual(decode_page(raw), raw.decode("utf-8", errors="replace"))
+
+    def test_valid_utf8_wins_over_a_wrong_meta_label(self):
+        # Browsers would follow a utf-8 HTTP header here; the saved file does not
+        # keep headers, so valid UTF-8 has to win over a stale <meta>.
+        raw = page("<p>Café</p>", '<meta charset="iso-8859-1">').encode("utf-8")
+        self.assertIn("Café", decode_page(raw))
+
+    def test_utf8_bom_is_kept_for_the_invisible_character_count(self):
+        raw = b"\xef\xbb\xbf" + page("<p>Plain posting.</p>").encode("utf-8")
+        self.assertTrue(decode_page(raw).startswith(""))
+
+    def test_undeclared_legacy_bytes_fall_back_to_utf8_with_replacement(self):
+        raw = page("<p>Café</p>").encode("latin-1")
+        self.assertEqual(decode_page(raw), raw.decode("utf-8", errors="replace"))
+
+    def test_unknown_and_unsafe_labels_are_ignored(self):
+        for label in ("x-unknown", "utf-7", "utf-16"):
+            raw = page("<p>Café</p>", f'<meta charset="{label}">').encode("latin-1")
+            with self.subTest(label=label):
+                self.assertIsNone(declared_codec(raw))
+                self.assertEqual(decode_page(raw), raw.decode("utf-8", errors="replace"))
+
+    def test_meta_after_the_prescan_window_is_ignored(self):
+        raw = (" " * 1100 + page("<p>Café</p>", '<meta charset="windows-1252">')).encode("cp1252")
+        self.assertIsNone(declared_codec(raw))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_posting_guard.py

@@ -228,6 +228,37 @@ class InputHandling(GuardFixture):
         self.assertEqual(code, 0, err)
         self.assertIn("Caf� manager wanted", self.text_of(out))
 
+    def test_html_mode_reads_a_page_saved_in_its_declared_charset(self):
+        # job.incruit.com serves postings as EUC-KR; fetch_page.py saves the bytes as sent.
+        page = (
+            '<html><head><meta http-equiv="Content-Type" content="text/html; charset=euc-kr">'
+            "</head><body><h1>백엔드 개발자 채용</h1><p>Python, SQL 경력 3년 이상</p></body></html>"
+        )
+        p = self.write("page.html", page.encode("cp949"))
+        code, out, err = self.run_main("--in", str(p), "--html", "--json")
+        self.assertEqual(code, 0, err)
+        self.assertIn("백엔드 개발자 채용", self.text_of(out))
+        self.assertNotIn("\ufffd", self.text_of(out))
+
+    def test_html_mode_scans_a_legacy_cyrillic_page_for_homoglyphs(self):
+        # Cyrillic о and а inside Latin words, saved as windows-1251.
+        page = (
+            '<html><head><meta charset="windows-1251"></head>'
+            "<body><p>Join Nоvo Nаrdisk as an engineer.</p></body></html>"
+        )
+        p = self.write("page.html", page.encode("cp1251"))
+        code, out, err = self.run_main("--in", str(p), "--html", "--json")
+        self.assertEqual(code, 0, err)
+        self.assertEqual(len(self.findings_of(out)["homoglyph_runs"]), 2)
+        self.assertIn("⚠", out.splitlines()[0])
+
+    def test_html_mode_keeps_valid_utf8_when_meta_names_another_charset(self):
+        page = '<html><head><meta charset="iso-8859-1"></head><body><p>Café manager</p></body></html>'
+        p = self.write("page.html", page)
+        code, out, err = self.run_main("--in", str(p), "--html", "--json")
+        self.assertEqual(code, 0, err)
+        self.assertIn("Café manager", self.text_of(out))
+
     def test_missing_in_file_exits_2(self):
         code, out, err = self.run_main("--in", str(self.tmp / "nope.txt"))
         self.assertEqual(code, 2)

tools/fetch_page.py

@@ -31,7 +31,8 @@ Prints `HTTP <code> size=<bytes> saved=<path>` and exits 0 whatever the HTTP
 status - the status is information for the escalation order, not a failure.
 With --text, also prints the tag-stripped page text (script/style/noscript/svg
 and comments removed, entities unescaped, whitespace folded), capped at
---max-chars (default 6000), after a `--- text ---` line.
+--max-chars (default 6000), after a `--- text ---` line. A saved body that is
+not valid UTF-8 is decoded by the charset the page declares (page_decode.py).
 
 Exit codes: 0 fetched; 1 refused by robots policy (go to escalation step 4);
 2 usage or transport error.
@@ -53,6 +54,7 @@ from urllib.parse import urlsplit
 from urllib.request import HTTPRedirectHandler, Request, build_opener
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
+from page_decode import decode_page  # noqa: E402
 from robots_check import BROWSER, gate  # noqa: E402
 
 ROOT = Path(__file__).resolve().parent.parent
@@ -213,7 +215,7 @@ def main(argv=None) -> int:
     print(f"{status} saved={out}")
     if args.text:
         print("--- text ---")
-        print(strip_tags(out.read_text(encoding="utf-8", errors="replace"), args.max_chars))
+        print(strip_tags(decode_page(out.read_bytes()), args.max_chars))
     return 0
 
 

tools/page_decode.py

@@ -0,0 +1,64 @@
+"""Decode a saved web page by the charset it declares, the way a browser does.
+
+fetch_page.py saves the body exactly as the server sent it. Its readers,
+`posting_guard.py --html` and `fetch_page.py --text`, decoded that file as
+UTF-8 with replacement, so a posting served in EUC-KR, Shift_JIS or
+windows-1252 reached the gate and the scorer as replacement characters.
+
+Valid UTF-8 always wins, so a page that decoded cleanly before decodes to the
+same string. Only bytes that are not valid UTF-8 fall back to what the page
+declares: a UTF-16 byte order mark, then `<meta charset>` or the http-equiv
+Content-Type in the first 1024 bytes (the window the HTML prescan reads). A
+label is mapped to the codec browsers use for it ("euc-kr" is read as CP949,
+"iso-8859-1" as windows-1252). A missing or unknown label keeps UTF-8 with
+replacement, and only the labels below are honoured, so a page cannot select
+a codec such as UTF-7.
+"""
+
+import re
+
+PRESCAN_BYTES = 1024
+META_CHARSET = re.compile(rb"<meta\b[^>]*?charset\s*=\s*[\"']?\s*([A-Za-z0-9._:-]+)", re.I)
+
+_CODECS = {
+    "cp949": ("euc-kr", "cp949", "windows-949", "ks_c_5601-1987", "ks_c_5601-1989",
+              "ksc5601", "ksc_5601", "korean", "iso-ir-149", "csksc56011987"),
+    "cp932": ("shift_jis", "shift-jis", "sjis", "x-sjis", "ms_kanji", "ms932",
+              "windows-31j", "csshiftjis"),
+    "euc_jp": ("euc-jp", "x-euc-jp", "cseucpkdfmtjapanese"),
+    "iso2022_jp": ("iso-2022-jp", "csiso2022jp"),
+    "gb18030": ("gb18030", "gbk", "gb2312", "x-gbk", "chinese", "csgb2312", "iso-ir-58"),
+    "big5hkscs": ("big5", "big5-hkscs", "x-x-big5", "cn-big5", "csbig5"),
+    "cp1252": ("windows-1252", "cp1252", "x-cp1252", "iso-8859-1", "iso8859-1",
+               "iso_8859-1", "latin1", "l1", "ascii", "us-ascii", "cp819", "ibm819"),
+    "cp1250": ("windows-1250", "cp1250", "x-cp1250"),
+    "cp1251": ("windows-1251", "cp1251", "x-cp1251"),
+    "cp1253": ("windows-1253", "cp1253"),
+    "cp1254": ("windows-1254", "cp1254", "iso-8859-9", "latin5"),
+    "cp1257": ("windows-1257", "cp1257"),
+    "iso8859_2": ("iso-8859-2", "latin2", "l2"),
+    "iso8859_5": ("iso-8859-5", "cyrillic"),
+    "iso8859_15": ("iso-8859-15", "latin9", "l9"),
+    "koi8_r": ("koi8-r", "koi8", "koi"),
+    "koi8_u": ("koi8-u",),
+}
+LABEL_TO_CODEC = {label: codec for codec, labels in _CODECS.items() for label in labels}
+
+
+def declared_codec(raw: bytes):
+    """The codec a page declares through a UTF-16 BOM or a <meta> tag, or None."""
+    if raw.startswith((b"\xff\xfe", b"\xfe\xff")):
+        return "utf-16"
+    match = META_CHARSET.search(raw[:PRESCAN_BYTES])
+    if match is None:
+        return None
+    return LABEL_TO_CODEC.get(match.group(1).decode("ascii").lower())
+
+
+def decode_page(raw: bytes) -> str:
+    """Text of a saved page: UTF-8 when valid, else its declared charset, else UTF-8 with replacement."""
+    try:
+        return raw.decode("utf-8")
+    except UnicodeDecodeError:
+        pass
+    return raw.decode(declared_codec(raw) or "utf-8", errors="replace")

tools/posting_guard.py

@@ -33,7 +33,10 @@ for the terminal.
 Usage:
   python3 tools/posting_guard.py [--in FILE] [--html|--text] [--out FILE] [--json]
 
-Input is `--in FILE` or stdin, decoded as UTF-8 with replacement. Pass --html
+Input is `--in FILE` or stdin, decoded as UTF-8 with replacement. With --html,
+bytes that are not valid UTF-8 are decoded by the charset the page declares
+(see page_decode.py), so a page saved in EUC-KR, Shift_JIS or windows-1252 is
+read as text instead of replacement characters. Pass --html
 for a saved page and --text for portal CLI output, pasted or archived text;
 without either the mode is guessed from the content, which is a fallback, not
 a documented path. Output: the verdict line, a `--- findings ---` block (JSON
@@ -48,6 +51,9 @@ import re
 import sys
 from pathlib import Path
 
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from page_decode import decode_page  # noqa: E402
+
 EXCERPT = 120
 LINE_MAX = 200
 
@@ -331,13 +337,17 @@ def _force_utf8_output() -> None:
             reconfigure(encoding="utf-8")
 
 
-def _read_input(path):
+def _read_input(path, mode):
     if path is not None:
-        return path.read_bytes().decode("utf-8", errors="replace")
+        data = path.read_bytes()
-    buffer = getattr(sys.stdin, "buffer", None)
+    else:
-    if buffer is not None:
+        buffer = getattr(sys.stdin, "buffer", None)
-        return buffer.read().decode("utf-8", errors="replace")
+        if buffer is None:
-    return sys.stdin.read()
+            return sys.stdin.read()
+        data = buffer.read()
+    if mode == "html":
+        return decode_page(data)
+    return data.decode("utf-8", errors="replace")
 
 
 def main(argv=None) -> int:
@@ -353,7 +363,7 @@ def main(argv=None) -> int:
     args = parser.parse_args(argv)
 
     try:
-        raw = _read_input(args.infile)
+        raw = _read_input(args.infile, args.mode)
     except OSError as exc:
         print(f"posting_guard: cannot read input: {exc}", file=sys.stderr)
         return 2