diff --git a/CHANGELOG.md b/CHANGELOG.md index c705ac02..1f708ebf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -213,6 +213,13 @@ per-file diff commands. `[WARNING]` that the result used cached data, and never with `[OK]`. A successful fetch and `--no-fetch` are unchanged. +- **`fetch_page.py` and `robots_check.py` force UTF-8 stdout to avoid crash on non-Latin web text on Windows** + (#686, fixes #685, `tools/fetch_page.py`, `tools/robots_check.py`, `tests/test_tools_utf8_output.py`) - like the other + Python tools, both CLI utilities now call `_force_utf8_output()` on startup. Under Windows default ANSI + code pages (such as `cp1252`), running `fetch_page.py --text` on postings containing characters outside + that code page (e.g. Polish, Cyrillic, or CJK) raised `UnicodeEncodeError` when writing stdout. + Similarly, `robots_check.py` now guarantees UTF-8 standard streams across platforms. + ### Security - **CI no longer leaves the job token in `.git/config` or expands a PR-controlled name into a diff --git a/tests/test_tools_utf8_output.py b/tests/test_tools_utf8_output.py index ecf99cb3..d0689eac 100644 --- a/tests/test_tools_utf8_output.py +++ b/tests/test_tools_utf8_output.py @@ -171,6 +171,65 @@ class ToolsWriteUtf8(unittest.TestCase): proc = run_legacy_stdout([repo / "tools" / "upstream_triage.py"], cwd=repo) self.assert_clean_utf8(proc, 0, CYRILLIC) + def test_fetch_page_prints_non_latin_text(self): + import http.server + import threading + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + if self.path == "/robots.txt": + self.send_response(404) + self.end_headers() + return + body = f"

{POLISH}

{CYRILLIC}

".encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + port = server.server_address[1] + out = self.tmp / "saved.html" + proc = run_legacy_stdout([ + TOOLS / "fetch_page.py", f"http://127.0.0.1:{port}/posting", "--text", "--out", out, + ]) + self.assert_clean_utf8(proc, 0, POLISH, CYRILLIC) + finally: + server.shutdown() + server.server_close() + + def test_robots_check_runs_cleanly(self): + import http.server + import threading + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + self.send_response(404) + self.end_headers() + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + port = server.server_address[1] + proc = run_legacy_stdout([ + TOOLS / "robots_check.py", f"http://127.0.0.1:{port}/jobs/{POLISH}", + ]) + self.assert_clean_utf8(proc, 0, "ALLOWED") + finally: + server.shutdown() + server.server_close() + if __name__ == "__main__": unittest.main() diff --git a/tools/fetch_page.py b/tools/fetch_page.py index c9e62007..955fe8e2 100644 --- a/tools/fetch_page.py +++ b/tools/fetch_page.py @@ -153,7 +153,21 @@ def _inside_repo(path: Path) -> bool: return True +def _force_utf8_output() -> None: + """Write UTF-8 whatever the host's default encoding is. + + A piped stdout on Windows defaults to the ANSI code page (cp1252 on most + Western installs), so printing a page with characters outside it raised + UnicodeEncodeError before the workflow saw any output. + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) # absent on a StringIO under test + if reconfigure: + reconfigure(encoding="utf-8") + + def main(argv=None) -> int: + _force_utf8_output() parser = argparse.ArgumentParser( prog="fetch_page.py", description="GET a page with browser headers (robots.txt enforced); see module docstring.", diff --git a/tools/robots_check.py b/tools/robots_check.py index 2c507728..cf3daf27 100644 --- a/tools/robots_check.py +++ b/tools/robots_check.py @@ -173,7 +173,21 @@ def gate(url): return 1, f'DISALLOWED for {a} - do not retry, go to step 3' return 0, 'ALLOWED - robots.txt permits this path' +def _force_utf8_output() -> None: + """Write UTF-8 whatever the host's default encoding is. + + A piped stdout on Windows defaults to the ANSI code page (cp1252 on most + Western installs), so printing messages or paths outside it raised + UnicodeEncodeError before the workflow saw any output. + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) # absent on a StringIO under test + if reconfigure: + reconfigure(encoding="utf-8") + + if __name__ == '__main__': + _force_utf8_output() if len(sys.argv) != 2: print('usage: python3 tools/robots_check.py ', file=sys.stderr) sys.exit(2)