fix(tools): force UTF-8 output on fetch_page & robots_check (closes #685) (#686)

* fix(tools): force UTF-8 output on fetch_page & robots_check (closes #685)

On Windows, a piped stdout (used by CLI runners like Claude Code)
defaults to the host ANSI code page (cp1252 on Western installs).
When fetch_page.py --text outputs a webpage containing non-Latin
or accented characters outside cp1252 (e.g. Polish, Danish, Cyrillic,
or CJK), printing the tag-stripped text failed with UnicodeEncodeError.
Similarly, robots_check.py lacked explicit UTF-8 stream configuration.

Address this by forcing UTF-8 encoding on standard output and error:
- tools/fetch_page.py: call _force_utf8_output() on startup
- tools/robots_check.py: add _force_utf8_output() and call it in CLI
- tests/test_tools_utf8_output.py: add regression tests for both tools
- CHANGELOG.md: document fix under Unreleased

All 770 unit tests pass, along with lint_skills and security_guards.

* changelog: place the #686 entry under [Unreleased]

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Mads Lorentzen <madslorentzen_17@hotmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -213,6 +213,13 @@ per-file diff commands.
   `[WARNING]` that the result used cached data, and never with `[OK]`. A successful fetch and
   `--no-fetch` are unchanged.
 
+- **`fetch_page.py` and `robots_check.py` force UTF-8 stdout to avoid crash on non-Latin web text on Windows**
+  (#686, fixes #685, `tools/fetch_page.py`, `tools/robots_check.py`, `tests/test_tools_utf8_output.py`) - like the other
+  Python tools, both CLI utilities now call `_force_utf8_output()` on startup. Under Windows default ANSI
+  code pages (such as `cp1252`), running `fetch_page.py --text` on postings containing characters outside
+  that code page (e.g. Polish, Cyrillic, or CJK) raised `UnicodeEncodeError` when writing stdout.
+  Similarly, `robots_check.py` now guarantees UTF-8 standard streams across platforms.
+
 ### Security
 
 - **CI no longer leaves the job token in `.git/config` or expands a PR-controlled name into a

tests/test_tools_utf8_output.py

@@ -171,6 +171,65 @@ class ToolsWriteUtf8(unittest.TestCase):
         proc = run_legacy_stdout([repo / "tools" / "upstream_triage.py"], cwd=repo)
         self.assert_clean_utf8(proc, 0, CYRILLIC)
 
+    def test_fetch_page_prints_non_latin_text(self):
+        import http.server
+        import threading
+
+        class Handler(http.server.BaseHTTPRequestHandler):
+            def do_GET(self):  # noqa: N802
+                if self.path == "/robots.txt":
+                    self.send_response(404)
+                    self.end_headers()
+                    return
+                body = f"<html><body><h1>{POLISH}</h1><p>{CYRILLIC}</p></body></html>".encode("utf-8")
+                self.send_response(200)
+                self.send_header("Content-Type", "text/html; charset=utf-8")
+                self.send_header("Content-Length", str(len(body)))
+                self.end_headers()
+                self.wfile.write(body)
+
+            def log_message(self, *args):
+                pass
+
+        server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+        thread = threading.Thread(target=server.serve_forever, daemon=True)
+        thread.start()
+        try:
+            port = server.server_address[1]
+            out = self.tmp / "saved.html"
+            proc = run_legacy_stdout([
+                TOOLS / "fetch_page.py", f"http://127.0.0.1:{port}/posting", "--text", "--out", out,
+            ])
+            self.assert_clean_utf8(proc, 0, POLISH, CYRILLIC)
+        finally:
+            server.shutdown()
+            server.server_close()
+
+    def test_robots_check_runs_cleanly(self):
+        import http.server
+        import threading
+
+        class Handler(http.server.BaseHTTPRequestHandler):
+            def do_GET(self):  # noqa: N802
+                self.send_response(404)
+                self.end_headers()
+
+            def log_message(self, *args):
+                pass
+
+        server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+        thread = threading.Thread(target=server.serve_forever, daemon=True)
+        thread.start()
+        try:
+            port = server.server_address[1]
+            proc = run_legacy_stdout([
+                TOOLS / "robots_check.py", f"http://127.0.0.1:{port}/jobs/{POLISH}",
+            ])
+            self.assert_clean_utf8(proc, 0, "ALLOWED")
+        finally:
+            server.shutdown()
+            server.server_close()
+
 
 if __name__ == "__main__":
     unittest.main()

tools/fetch_page.py

@@ -153,7 +153,21 @@ def _inside_repo(path: Path) -> bool:
     return True
 
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing a page with characters outside it raised
+    UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 def main(argv=None) -> int:
+    _force_utf8_output()
     parser = argparse.ArgumentParser(
         prog="fetch_page.py",
         description="GET a page with browser headers (robots.txt enforced); see module docstring.",

tools/robots_check.py

@@ -173,7 +173,21 @@ def gate(url):
             return 1, f'DISALLOWED for {a} - do not retry, go to step 3'
     return 0, 'ALLOWED - robots.txt permits this path'
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing messages or paths outside it raised
+    UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 if __name__ == '__main__':
+    _force_utf8_output()
     if len(sys.argv) != 2:
         print('usage: python3 tools/robots_check.py <url>', file=sys.stderr)
         sys.exit(2)