fix(tools): decode saved non-UTF-8 pages by their declared charset (#690)

* fix(tools): decode saved non-UTF-8 pages by their declared charset

fetch_page.py saves a body exactly as the server sent it, but its readers,
posting_guard.py --html and fetch_page.py --text, decoded the file as UTF-8
with replacement. A posting served in EUC-KR, Shift_JIS or windows-1252
reached the integrity gate and the scorer as replacement characters.

Valid UTF-8 still wins, so any page that decoded cleanly before decodes to the
same string. Bytes that are not valid UTF-8 now fall back to the charset the
page declares (UTF-16 BOM, then <meta charset> or http-equiv Content-Type in
the first 1024 bytes), mapped to the codec browsers use for that label. Missing
or unknown labels keep the previous UTF-8-with-replacement result.

* changelog: place the #690 entry under [Unreleased]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Mads Lorentzen <madslorentzen_17@hotmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_fetch_page.py

@@ -30,6 +30,11 @@ PAGE = (
     "<noscript>noscript text</noscript><svg><text>svg text</text></svg>"
     "</body></html>"
 )
+KOREAN_PAGE = (
+    '<html><head><meta http-equiv="Content-Type" content="text/html; charset=euc-kr">'
+    "<title>채용공고</title></head><body><h1>백엔드 개발자 채용</h1></body></html>"
+)
+CONTENT_TYPES = {"/euc-kr.html": "text/html; charset=euc-kr"}
 
 
 class _Handler(http.server.BaseHTTPRequestHandler):
@@ -37,6 +42,9 @@ class _Handler(http.server.BaseHTTPRequestHandler):
         if self.path == "/ok.html":
             body = PAGE.encode("utf-8")
             self.send_response(200)
+        elif self.path == "/euc-kr.html":
+            body = KOREAN_PAGE.encode("cp949")
+            self.send_response(200)
         elif self.path == "/forbidden":
             body = b"<html><body>denied</body></html>"
             self.send_response(403)
@@ -54,7 +62,7 @@ class _Handler(http.server.BaseHTTPRequestHandler):
         else:
             body = b"nope"
             self.send_response(404)
-        self.send_header("Content-Type", "text/html; charset=utf-8")
+        self.send_header("Content-Type", CONTENT_TYPES.get(self.path, "text/html; charset=utf-8"))
         self.send_header("Content-Length", str(len(body)))
         self.end_headers()
         self.wfile.write(body)
@@ -152,6 +160,14 @@ class Fetching(FetchPageFixture):
         for leaked in ("alert(", "color:red", "hidden comment", "noscript text", "svg text", "<p>"):
             self.assertNotIn(leaked, text)
 
+    def test_text_decodes_a_page_by_its_declared_charset(self):
+        code, out, err = self.run_main(f"{self.base}/euc-kr.html", "--out", str(self.out), "--text")
+        self.assertEqual(code, 0, err)
+        self.assertEqual(self.out.read_bytes(), KOREAN_PAGE.encode("cp949"))
+        text = out.split("--- text ---", 1)[1]
+        self.assertIn("백엔드 개발자 채용", text)
+        self.assertNotIn("\ufffd", text)
+
     def test_max_chars_caps_text(self):
         code, out, err = self.run_main(
             f"{self.base}/ok.html", "--out", str(self.out), "--text", "--max-chars", "7"