From f97a7599450933091534114bffb2ceff63f01ee4 Mon Sep 17 00:00:00 2001 From: Dongmin Kim Date: Tue, 20 Oct 2026 15:28:58 +0900 Subject: [PATCH] fix(tools): decode saved non-UTF-8 pages by their declared charset (#690) * fix(tools): decode saved non-UTF-8 pages by their declared charset fetch_page.py saves a body exactly as the server sent it, but its readers, posting_guard.py --html and fetch_page.py --text, decoded the file as UTF-8 with replacement. A posting served in EUC-KR, Shift_JIS or windows-1252 reached the integrity gate and the scorer as replacement characters. Valid UTF-8 still wins, so any page that decoded cleanly before decodes to the same string. Bytes that are not valid UTF-8 now fall back to the charset the page declares (UTF-16 BOM, then or http-equiv Content-Type in the first 1024 bytes), mapped to the codec browsers use for that label. Missing or unknown labels keep the previous UTF-8-with-replacement result. * changelog: place the #690 entry under [Unreleased] Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Mads Lorentzen Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 11 +++++ tests/test_fetch_page.py | 18 +++++++- tests/test_page_decode.py | 82 +++++++++++++++++++++++++++++++++++++ tests/test_posting_guard.py | 31 ++++++++++++++ tools/fetch_page.py | 6 ++- tools/page_decode.py | 64 +++++++++++++++++++++++++++++ tools/posting_guard.py | 26 ++++++++---- 7 files changed, 227 insertions(+), 11 deletions(-) create mode 100644 tests/test_page_decode.py create mode 100644 tools/page_decode.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ce1064f..d5b7427d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,6 +242,17 @@ per-file diff commands. runs a bare `python`, or a spec that runs a tool carries no note, a second note, or one after its first command. +- **A saved page that is not UTF-8 is read in the charset it declares** (#690, `tools/page_decode.py`, + `tools/posting_guard.py`, `tools/fetch_page.py`, `tests/test_page_decode.py`) - `posting_guard.py --html` + and `fetch_page.py --text` decoded a page saved by `fetch_page.py` as UTF-8 with replacement, so a + posting served in EUC-KR, Shift_JIS, windows-1251 or another legacy charset reached the gate and the + scorer as replacement characters while the gate still reported it clean (a live job.incruit.com + posting came out as 1206 U+FFFD and 4 Hangul characters). Bytes that are valid UTF-8 decode exactly + as before; otherwise a UTF-16 byte order mark or a `` charset in the first 1024 bytes picks the + codec browsers use for that label, from a fixed list (so a page cannot select UTF-7), and a missing + or unknown label keeps the old result. The homoglyph check now sees Cyrillic letters on a + windows-1251 page. Text input to the gate (`--text`) is unchanged. + ### Security - **CI no longer leaves the job token in `.git/config` or expands a PR-controlled name into a diff --git a/tests/test_fetch_page.py b/tests/test_fetch_page.py index e58733b8..bd72905a 100644 --- a/tests/test_fetch_page.py +++ b/tests/test_fetch_page.py @@ -30,6 +30,11 @@ PAGE = ( "svg text" "" ) +KOREAN_PAGE = ( + '' + "채용공고

백엔드 개발자 채용

" +) +CONTENT_TYPES = {"/euc-kr.html": "text/html; charset=euc-kr"} class _Handler(http.server.BaseHTTPRequestHandler): @@ -37,6 +42,9 @@ class _Handler(http.server.BaseHTTPRequestHandler): if self.path == "/ok.html": body = PAGE.encode("utf-8") self.send_response(200) + elif self.path == "/euc-kr.html": + body = KOREAN_PAGE.encode("cp949") + self.send_response(200) elif self.path == "/forbidden": body = b"denied" self.send_response(403) @@ -54,7 +62,7 @@ class _Handler(http.server.BaseHTTPRequestHandler): else: body = b"nope" self.send_response(404) - self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Type", CONTENT_TYPES.get(self.path, "text/html; charset=utf-8")) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) @@ -152,6 +160,14 @@ class Fetching(FetchPageFixture): for leaked in ("alert(", "color:red", "hidden comment", "noscript text", "svg text", "

"): self.assertNotIn(leaked, text) + def test_text_decodes_a_page_by_its_declared_charset(self): + code, out, err = self.run_main(f"{self.base}/euc-kr.html", "--out", str(self.out), "--text") + self.assertEqual(code, 0, err) + self.assertEqual(self.out.read_bytes(), KOREAN_PAGE.encode("cp949")) + text = out.split("--- text ---", 1)[1] + self.assertIn("백엔드 개발자 채용", text) + self.assertNotIn("\ufffd", text) + def test_max_chars_caps_text(self): code, out, err = self.run_main( f"{self.base}/ok.html", "--out", str(self.out), "--text", "--max-chars", "7" diff --git a/tests/test_page_decode.py b/tests/test_page_decode.py new file mode 100644 index 00000000..c0bc082d --- /dev/null +++ b/tests/test_page_decode.py @@ -0,0 +1,82 @@ +"""tools/page_decode.py decodes a saved page by the charset it declares. + +Why this test exists: fetch_page.py saves bodies as the server sent them, and +posting_guard.py --html and fetch_page.py --text used to decode every file as +UTF-8 with replacement. A posting served in EUC-KR (job.incruit.com serves its +postings that way) reached the gate and the scorer as replacement characters. +These tests pin that declared legacy charsets are honoured, and that anything +which decoded cleanly before still decodes to exactly the same string. +""" +import sys +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "tools")) +from page_decode import decode_page, declared_codec # noqa: E402 + + +def page(body: str, head: str = "") -> str: + return f"{head}Posting{body}" + + +class LegacyCharsets(unittest.TestCase): + def test_http_equiv_euc_kr_is_read_as_cp949(self): + # 똠 is outside strict EUC-KR; browsers read the euc-kr label as CP949. + text = page("

백엔드 개발자 채용

똠방각하 우대

", + '') + decoded = decode_page(text.encode("cp949")) + self.assertIn("백엔드 개발자 채용", decoded) + self.assertIn("똠방각하", decoded) + self.assertNotIn("�", decoded) + + def test_meta_charset_shift_jis(self): + text = page("

データエンジニア募集

", '') + self.assertIn("データエンジニア募集", decode_page(text.encode("cp932"))) + + def test_iso_8859_1_label_is_read_as_windows_1252(self): + text = page("

Løn: 50.000 € om måneden

", "") + self.assertIn("Løn: 50.000 € om måneden", decode_page(text.encode("cp1252"))) + + def test_windows_1251_cyrillic(self): + text = page("

Инженер данных

", '') + self.assertIn("Инженер данных", decode_page(text.encode("cp1251"))) + + def test_utf16_byte_order_mark(self): + text = page("

Ingeniør

") + self.assertIn("Ingeniør", decode_page(text.encode("utf-16"))) + + +class UnchangedBehaviour(unittest.TestCase): + def test_valid_utf8_is_unchanged(self): + raw = page("

Ingeniør søges, 백엔드, データ

").encode("utf-8") + self.assertEqual(decode_page(raw), raw.decode("utf-8", errors="replace")) + + def test_valid_utf8_wins_over_a_wrong_meta_label(self): + # Browsers would follow a utf-8 HTTP header here; the saved file does not + # keep headers, so valid UTF-8 has to win over a stale . + raw = page("

Café

", '').encode("utf-8") + self.assertIn("Café", decode_page(raw)) + + def test_utf8_bom_is_kept_for_the_invisible_character_count(self): + raw = b"\xef\xbb\xbf" + page("

Plain posting.

").encode("utf-8") + self.assertTrue(decode_page(raw).startswith("")) + + def test_undeclared_legacy_bytes_fall_back_to_utf8_with_replacement(self): + raw = page("

Café

").encode("latin-1") + self.assertEqual(decode_page(raw), raw.decode("utf-8", errors="replace")) + + def test_unknown_and_unsafe_labels_are_ignored(self): + for label in ("x-unknown", "utf-7", "utf-16"): + raw = page("

Café

", f'').encode("latin-1") + with self.subTest(label=label): + self.assertIsNone(declared_codec(raw)) + self.assertEqual(decode_page(raw), raw.decode("utf-8", errors="replace")) + + def test_meta_after_the_prescan_window_is_ignored(self): + raw = (" " * 1100 + page("

Café

", '')).encode("cp1252") + self.assertIsNone(declared_codec(raw)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_posting_guard.py b/tests/test_posting_guard.py index 38dc3642..21ac5b8b 100644 --- a/tests/test_posting_guard.py +++ b/tests/test_posting_guard.py @@ -228,6 +228,37 @@ class InputHandling(GuardFixture): self.assertEqual(code, 0, err) self.assertIn("Caf� manager wanted", self.text_of(out)) + def test_html_mode_reads_a_page_saved_in_its_declared_charset(self): + # job.incruit.com serves postings as EUC-KR; fetch_page.py saves the bytes as sent. + page = ( + '' + "

백엔드 개발자 채용

Python, SQL 경력 3년 이상

" + ) + p = self.write("page.html", page.encode("cp949")) + code, out, err = self.run_main("--in", str(p), "--html", "--json") + self.assertEqual(code, 0, err) + self.assertIn("백엔드 개발자 채용", self.text_of(out)) + self.assertNotIn("\ufffd", self.text_of(out)) + + def test_html_mode_scans_a_legacy_cyrillic_page_for_homoglyphs(self): + # Cyrillic о and а inside Latin words, saved as windows-1251. + page = ( + '' + "

Join Nоvo Nаrdisk as an engineer.

" + ) + p = self.write("page.html", page.encode("cp1251")) + code, out, err = self.run_main("--in", str(p), "--html", "--json") + self.assertEqual(code, 0, err) + self.assertEqual(len(self.findings_of(out)["homoglyph_runs"]), 2) + self.assertIn("⚠", out.splitlines()[0]) + + def test_html_mode_keeps_valid_utf8_when_meta_names_another_charset(self): + page = '

Café manager

' + p = self.write("page.html", page) + code, out, err = self.run_main("--in", str(p), "--html", "--json") + self.assertEqual(code, 0, err) + self.assertIn("Café manager", self.text_of(out)) + def test_missing_in_file_exits_2(self): code, out, err = self.run_main("--in", str(self.tmp / "nope.txt")) self.assertEqual(code, 2) diff --git a/tools/fetch_page.py b/tools/fetch_page.py index 955fe8e2..95db9bb2 100644 --- a/tools/fetch_page.py +++ b/tools/fetch_page.py @@ -31,7 +31,8 @@ Prints `HTTP size= saved=` and exits 0 whatever the HTTP status - the status is information for the escalation order, not a failure. With --text, also prints the tag-stripped page text (script/style/noscript/svg and comments removed, entities unescaped, whitespace folded), capped at ---max-chars (default 6000), after a `--- text ---` line. +--max-chars (default 6000), after a `--- text ---` line. A saved body that is +not valid UTF-8 is decoded by the charset the page declares (page_decode.py). Exit codes: 0 fetched; 1 refused by robots policy (go to escalation step 4); 2 usage or transport error. @@ -53,6 +54,7 @@ from urllib.parse import urlsplit from urllib.request import HTTPRedirectHandler, Request, build_opener sys.path.insert(0, str(Path(__file__).resolve().parent)) +from page_decode import decode_page # noqa: E402 from robots_check import BROWSER, gate # noqa: E402 ROOT = Path(__file__).resolve().parent.parent @@ -213,7 +215,7 @@ def main(argv=None) -> int: print(f"{status} saved={out}") if args.text: print("--- text ---") - print(strip_tags(out.read_text(encoding="utf-8", errors="replace"), args.max_chars)) + print(strip_tags(decode_page(out.read_bytes()), args.max_chars)) return 0 diff --git a/tools/page_decode.py b/tools/page_decode.py new file mode 100644 index 00000000..a7ca0af3 --- /dev/null +++ b/tools/page_decode.py @@ -0,0 +1,64 @@ +"""Decode a saved web page by the charset it declares, the way a browser does. + +fetch_page.py saves the body exactly as the server sent it. Its readers, +`posting_guard.py --html` and `fetch_page.py --text`, decoded that file as +UTF-8 with replacement, so a posting served in EUC-KR, Shift_JIS or +windows-1252 reached the gate and the scorer as replacement characters. + +Valid UTF-8 always wins, so a page that decoded cleanly before decodes to the +same string. Only bytes that are not valid UTF-8 fall back to what the page +declares: a UTF-16 byte order mark, then `` or the http-equiv +Content-Type in the first 1024 bytes (the window the HTML prescan reads). A +label is mapped to the codec browsers use for it ("euc-kr" is read as CP949, +"iso-8859-1" as windows-1252). A missing or unknown label keeps UTF-8 with +replacement, and only the labels below are honoured, so a page cannot select +a codec such as UTF-7. +""" + +import re + +PRESCAN_BYTES = 1024 +META_CHARSET = re.compile(rb"]*?charset\s*=\s*[\"']?\s*([A-Za-z0-9._:-]+)", re.I) + +_CODECS = { + "cp949": ("euc-kr", "cp949", "windows-949", "ks_c_5601-1987", "ks_c_5601-1989", + "ksc5601", "ksc_5601", "korean", "iso-ir-149", "csksc56011987"), + "cp932": ("shift_jis", "shift-jis", "sjis", "x-sjis", "ms_kanji", "ms932", + "windows-31j", "csshiftjis"), + "euc_jp": ("euc-jp", "x-euc-jp", "cseucpkdfmtjapanese"), + "iso2022_jp": ("iso-2022-jp", "csiso2022jp"), + "gb18030": ("gb18030", "gbk", "gb2312", "x-gbk", "chinese", "csgb2312", "iso-ir-58"), + "big5hkscs": ("big5", "big5-hkscs", "x-x-big5", "cn-big5", "csbig5"), + "cp1252": ("windows-1252", "cp1252", "x-cp1252", "iso-8859-1", "iso8859-1", + "iso_8859-1", "latin1", "l1", "ascii", "us-ascii", "cp819", "ibm819"), + "cp1250": ("windows-1250", "cp1250", "x-cp1250"), + "cp1251": ("windows-1251", "cp1251", "x-cp1251"), + "cp1253": ("windows-1253", "cp1253"), + "cp1254": ("windows-1254", "cp1254", "iso-8859-9", "latin5"), + "cp1257": ("windows-1257", "cp1257"), + "iso8859_2": ("iso-8859-2", "latin2", "l2"), + "iso8859_5": ("iso-8859-5", "cyrillic"), + "iso8859_15": ("iso-8859-15", "latin9", "l9"), + "koi8_r": ("koi8-r", "koi8", "koi"), + "koi8_u": ("koi8-u",), +} +LABEL_TO_CODEC = {label: codec for codec, labels in _CODECS.items() for label in labels} + + +def declared_codec(raw: bytes): + """The codec a page declares through a UTF-16 BOM or a tag, or None.""" + if raw.startswith((b"\xff\xfe", b"\xfe\xff")): + return "utf-16" + match = META_CHARSET.search(raw[:PRESCAN_BYTES]) + if match is None: + return None + return LABEL_TO_CODEC.get(match.group(1).decode("ascii").lower()) + + +def decode_page(raw: bytes) -> str: + """Text of a saved page: UTF-8 when valid, else its declared charset, else UTF-8 with replacement.""" + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + pass + return raw.decode(declared_codec(raw) or "utf-8", errors="replace") diff --git a/tools/posting_guard.py b/tools/posting_guard.py index d48ab472..81ac8043 100644 --- a/tools/posting_guard.py +++ b/tools/posting_guard.py @@ -33,7 +33,10 @@ for the terminal. Usage: python3 tools/posting_guard.py [--in FILE] [--html|--text] [--out FILE] [--json] -Input is `--in FILE` or stdin, decoded as UTF-8 with replacement. Pass --html +Input is `--in FILE` or stdin, decoded as UTF-8 with replacement. With --html, +bytes that are not valid UTF-8 are decoded by the charset the page declares +(see page_decode.py), so a page saved in EUC-KR, Shift_JIS or windows-1252 is +read as text instead of replacement characters. Pass --html for a saved page and --text for portal CLI output, pasted or archived text; without either the mode is guessed from the content, which is a fallback, not a documented path. Output: the verdict line, a `--- findings ---` block (JSON @@ -48,6 +51,9 @@ import re import sys from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from page_decode import decode_page # noqa: E402 + EXCERPT = 120 LINE_MAX = 200 @@ -331,13 +337,17 @@ def _force_utf8_output() -> None: reconfigure(encoding="utf-8") -def _read_input(path): +def _read_input(path, mode): if path is not None: - return path.read_bytes().decode("utf-8", errors="replace") - buffer = getattr(sys.stdin, "buffer", None) - if buffer is not None: - return buffer.read().decode("utf-8", errors="replace") - return sys.stdin.read() + data = path.read_bytes() + else: + buffer = getattr(sys.stdin, "buffer", None) + if buffer is None: + return sys.stdin.read() + data = buffer.read() + if mode == "html": + return decode_page(data) + return data.decode("utf-8", errors="replace") def main(argv=None) -> int: @@ -353,7 +363,7 @@ def main(argv=None) -> int: args = parser.parse_args(argv) try: - raw = _read_input(args.infile) + raw = _read_input(args.infile, args.mode) except OSError as exc: print(f"posting_guard: cannot read input: {exc}", file=sys.stderr) return 2