diff --git a/CHANGELOG.md b/CHANGELOG.md index edee454b..cbcd6970 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,23 @@ per-file diff commands. ### Fixed +- **`robots_check` decodes curl output as UTF-8, so a non-ASCII response no longer reads as + UNCONFIRMED on Windows** (`tools/robots_check.py`, `tests/test_robots_check.py`) - + `_fetch()` ran curl with `text=True` and no encoding, so Python decoded the response with + the ANSI code page. On a byte that code page leaves undefined (0x81 in cp1252 and cp1254, + 0x9e in cp1254) subprocess's pipe reader thread died, `stdout` came back `None`, and the + gate printed `UNCONFIRMED (AttributeError)` - failing closed, but blocking the + browser-header retry `09-web-research.md` permits. Reproduced through the CLI on real + hosts: tr.indeed.com's robots.txt carries `Disallow: /職涯貼士/` (職 is `e8 81 b7`), and + kap.org.tr answers `/robots.txt` with a UTF-8 HTML 404 whose "A.Ş." carries `c5 9e`. The + call now pins `encoding='utf-8', errors='replace'` (RFC 9309: robots.txt is UTF-8), as + `verify_pdf.py` and `verify_layout.py` already do. That also closes a silent fail-open: a + UTF-8 rule whose bytes the code page does define (`Disallow: /şirket/` under cp1254) + decoded as `/ÅŸirket/`, never matched, and read as allowed. Decision rules are unchanged. + Two new tests swap curl for a child process that prints those bytes and pin cp1254 when no + encoding is passed, so the failure does not depend on the host's locale; both fail without + the fix. + - **The compile-warning check covers `tests/*.py` too** (`tests/test_verify_pdf.py`, `tests/test_verify_layout.py`) - `FindNonAsciiDateRangesTests`' docstring quotes the regex `\s*` unescaped, so Python 3.12+ prints a `SyntaxWarning` for the invalid escape `\s` when diff --git a/tests/test_robots_check.py b/tests/test_robots_check.py index beb9286d..66787481 100644 --- a/tests/test_robots_check.py +++ b/tests/test_robots_check.py @@ -259,6 +259,60 @@ class TestArgumentHardening(unittest.TestCase): self.assertEqual(seen[0], "https://x.example/robots.txt") +class TestNonAsciiCurlOutput(unittest.TestCase): + """curl's output is decoded as UTF-8, never the locale's code page. + + With text=True and no encoding, Windows decoded with the locale's code page. + Under cp1254 (Turkish) the bytes 0x81 and 0x9e are undefined, so the pipe + reader thread died, stdout came back None, and the gate printed + "UNCONFIRMED (AttributeError)" for policies it never read - blocking a + retry those policies allow (reproduced 2026-09-29). + """ + + def _gate_with_curl_output(self, url, body, status): + """gate(url) with curl swapped for a process that prints `body` and the + status the way curl's -w does, decoded however _fetch asks.""" + import robots_check + + # A fixture cp1254 can decode would pass without the fix. + self.assertRaises(UnicodeDecodeError, body.decode, "cp1254") + raw = body + b"\n" + str(status).encode() + real_run = subprocess.run + + def fake_curl(argv, **kwargs): + # Unpinned text mode takes the host locale's code page. Pin the + # reporter's, so a UTF-8 host (CI) cannot pass the bug by luck. + if kwargs.get("text") and not kwargs.get("encoding"): + kwargs["encoding"] = "cp1254" + write = "import sys; sys.stdout.buffer.write(bytes.fromhex(sys.argv[1]))" + return real_run([sys.executable, "-c", write, raw.hex()], **kwargs) + + robots_check.subprocess.run = fake_curl + try: + return robots_check.gate(url) + finally: + robots_check.subprocess.run = real_run + + def test_rule_holding_a_cp1254_undefined_byte_is_read_and_obeyed(self): + """The rule is verbatim from tr.indeed.com/robots.txt: 職 is e8 81 b7. + DISALLOWED on its path proves it was decoded, not merely survived.""" + body = "User-agent: *\nDisallow: /職涯貼士/\n".encode("utf-8") + rc, msg = self._gate_with_curl_output("https://tr.indeed.example/cmp/x/reviews", body, 200) + self.assertEqual(rc, 0, msg) + self.assertIn("robots.txt permits this path", msg) + rc, msg = self._gate_with_curl_output("https://tr.indeed.example/職涯貼士/x", body, 200) + self.assertEqual(rc, 1, msg) + self.assertIn("DISALLOWED", msg) + + def test_404_page_holding_a_cp1254_undefined_byte_still_means_no_policy(self): + """kap.org.tr answers /robots.txt with a UTF-8 HTML 404 that names + "Merkezi Kayıt Kuruluşu A.Ş." - Ş is c5 9e.""" + body = 'Merkezi Kayıt Kuruluşu A.Ş.'.encode("utf-8") + rc, msg = self._gate_with_curl_output("https://kap.example/tr/sirket-bilgileri", body, 404) + self.assertEqual(rc, 0, msg) + self.assertIn("no robots.txt published", msg) + + if __name__ == "__main__": unittest.main() diff --git a/tools/robots_check.py b/tools/robots_check.py index b59c9723..c17cab65 100644 --- a/tools/robots_check.py +++ b/tools/robots_check.py @@ -37,7 +37,10 @@ def _fetch(url, ua): r = subprocess.run( ['curl', '-sS', '-L', '--max-redirs', '5', '--max-time', '12', '-A', ua, '-H', 'Accept: text/plain,*/*', '-w', '\n%{http_code}', '--', url], - capture_output=True, text=True, timeout=20) + # robots.txt is UTF-8 (RFC 9309). Unpinned, Turkish Windows decoded it as + # cp1254, where bytes like 0x81 are undefined: the pipe reader thread died, + # stdout came back None, and tr.indeed.com read as UNCONFIRMED. + capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=20) if r.returncode != 0: raise RuntimeError('curl exit %d' % r.returncode) body, _, code = r.stdout.rpartition('\n')