fix(robots-check): decode curl output as UTF-8 so non-ASCII responses stop reading as UNCONFIRMED on Windows (#506)

_fetch() ran curl with text=True and no encoding, so Python decoded the
response with the Windows ANSI code page. On a byte that code page leaves
undefined (0x81 in cp1252 and cp1254, 0x9e in cp1254) subprocess's pipe
reader thread died, stdout came back None, rpartition() raised
AttributeError, and the gate printed "UNCONFIRMED (AttributeError)". That
fails closed, but it blocked the browser-header retry 09-web-research.md
permits on hosts such as tr.indeed.com (a UTF-8 CJK Disallow rule) and
kap.org.tr (a UTF-8 Turkish 404 page).

Pin encoding='utf-8', errors='replace' on the call (RFC 9309: robots.txt
is UTF-8), as verify_pdf.py and verify_layout.py already do. Decision
rules and the fail-closed handling of real read failures are unchanged.

The regression tests swap curl for a child process that prints the real
bytes, so the actual subprocess decode path runs with _fetch's own
kwargs, and pin cp1254 when no encoding is passed so a UTF-8 host cannot
pass the bug by luck. Both fail on master.

Co-authored-by: Can Mugan <217335573+canmugan@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -105,6 +105,23 @@ per-file diff commands.
 
 ### Fixed
 
+- **`robots_check` decodes curl output as UTF-8, so a non-ASCII response no longer reads as
+  UNCONFIRMED on Windows** (`tools/robots_check.py`, `tests/test_robots_check.py`) -
+  `_fetch()` ran curl with `text=True` and no encoding, so Python decoded the response with
+  the ANSI code page. On a byte that code page leaves undefined (0x81 in cp1252 and cp1254,
+  0x9e in cp1254) subprocess's pipe reader thread died, `stdout` came back `None`, and the
+  gate printed `UNCONFIRMED (AttributeError)` - failing closed, but blocking the
+  browser-header retry `09-web-research.md` permits. Reproduced through the CLI on real
+  hosts: tr.indeed.com's robots.txt carries `Disallow: /職涯貼士/` (職 is `e8 81 b7`), and
+  kap.org.tr answers `/robots.txt` with a UTF-8 HTML 404 whose "A.Ş." carries `c5 9e`. The
+  call now pins `encoding='utf-8', errors='replace'` (RFC 9309: robots.txt is UTF-8), as
+  `verify_pdf.py` and `verify_layout.py` already do. That also closes a silent fail-open: a
+  UTF-8 rule whose bytes the code page does define (`Disallow: /şirket/` under cp1254)
+  decoded as `/ÅŸirket/`, never matched, and read as allowed. Decision rules are unchanged.
+  Two new tests swap curl for a child process that prints those bytes and pin cp1254 when no
+  encoding is passed, so the failure does not depend on the host's locale; both fail without
+  the fix.
+
 - **The compile-warning check covers `tests/*.py` too** (`tests/test_verify_pdf.py`,
   `tests/test_verify_layout.py`) - `FindNonAsciiDateRangesTests`' docstring quotes the regex
   `\s*` unescaped, so Python 3.12+ prints a `SyntaxWarning` for the invalid escape `\s` when

tests/test_robots_check.py

@@ -259,6 +259,60 @@ class TestArgumentHardening(unittest.TestCase):
         self.assertEqual(seen[0], "https://x.example/robots.txt")
 
 
+class TestNonAsciiCurlOutput(unittest.TestCase):
+    """curl's output is decoded as UTF-8, never the locale's code page.
+
+    With text=True and no encoding, Windows decoded with the locale's code page.
+    Under cp1254 (Turkish) the bytes 0x81 and 0x9e are undefined, so the pipe
+    reader thread died, stdout came back None, and the gate printed
+    "UNCONFIRMED (AttributeError)" for policies it never read - blocking a
+    retry those policies allow (reproduced 2026-09-29).
+    """
+
+    def _gate_with_curl_output(self, url, body, status):
+        """gate(url) with curl swapped for a process that prints `body` and the
+        status the way curl's -w does, decoded however _fetch asks."""
+        import robots_check
+
+        # A fixture cp1254 can decode would pass without the fix.
+        self.assertRaises(UnicodeDecodeError, body.decode, "cp1254")
+        raw = body + b"\n" + str(status).encode()
+        real_run = subprocess.run
+
+        def fake_curl(argv, **kwargs):
+            # Unpinned text mode takes the host locale's code page. Pin the
+            # reporter's, so a UTF-8 host (CI) cannot pass the bug by luck.
+            if kwargs.get("text") and not kwargs.get("encoding"):
+                kwargs["encoding"] = "cp1254"
+            write = "import sys; sys.stdout.buffer.write(bytes.fromhex(sys.argv[1]))"
+            return real_run([sys.executable, "-c", write, raw.hex()], **kwargs)
+
+        robots_check.subprocess.run = fake_curl
+        try:
+            return robots_check.gate(url)
+        finally:
+            robots_check.subprocess.run = real_run
+
+    def test_rule_holding_a_cp1254_undefined_byte_is_read_and_obeyed(self):
+        """The rule is verbatim from tr.indeed.com/robots.txt: 職 is e8 81 b7.
+        DISALLOWED on its path proves it was decoded, not merely survived."""
+        body = "User-agent: *\nDisallow: /職涯貼士/\n".encode("utf-8")
+        rc, msg = self._gate_with_curl_output("https://tr.indeed.example/cmp/x/reviews", body, 200)
+        self.assertEqual(rc, 0, msg)
+        self.assertIn("robots.txt permits this path", msg)
+        rc, msg = self._gate_with_curl_output("https://tr.indeed.example/職涯貼士/x", body, 200)
+        self.assertEqual(rc, 1, msg)
+        self.assertIn("DISALLOWED", msg)
+
+    def test_404_page_holding_a_cp1254_undefined_byte_still_means_no_policy(self):
+        """kap.org.tr answers /robots.txt with a UTF-8 HTML 404 that names
+        "Merkezi Kayıt Kuruluşu A.Ş." - Ş is c5 9e."""
+        body = '<html lang="tr"><body>Merkezi Kayıt Kuruluşu A.Ş.</body></html>'.encode("utf-8")
+        rc, msg = self._gate_with_curl_output("https://kap.example/tr/sirket-bilgileri", body, 404)
+        self.assertEqual(rc, 0, msg)
+        self.assertIn("no robots.txt published", msg)
+
+
 
 if __name__ == "__main__":
     unittest.main()

tools/robots_check.py

@@ -37,7 +37,10 @@ def _fetch(url, ua):
     r = subprocess.run(
         ['curl', '-sS', '-L', '--max-redirs', '5', '--max-time', '12', '-A', ua,
          '-H', 'Accept: text/plain,*/*', '-w', '\n%{http_code}', '--', url],
-        capture_output=True, text=True, timeout=20)
+        # robots.txt is UTF-8 (RFC 9309). Unpinned, Turkish Windows decoded it as
+        # cp1254, where bytes like 0x81 are undefined: the pipe reader thread died,
+        # stdout came back None, and tr.indeed.com read as UNCONFIRMED.
+        capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=20)
     if r.returncode != 0:
         raise RuntimeError('curl exit %d' % r.returncode)
     body, _, code = r.stdout.rpartition('\n')