fix(tools): write UTF-8 on stdout so non-Latin postings stop crashing on Windows (#490)

A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
Western installs). rank_state.py printed titles and companies with
ensure_ascii=False, so one Cyrillic, CJK, Devanagari, Polish or Turkish
posting ended /rank with UnicodeEncodeError. job_key --audit, salary_lookup,
convert_salary_excel and verify_layout failed the same way.

Each tool now reconfigures stdout and stderr to UTF-8 at entry. The new
tests run every tool in a child process with a cp1252 stdout forced through
PYTHONIOENCODING, so Linux CI reproduces the Windows failure.


Claude-Session: https://claude.ai/code/session_01WPXraLZ4i7UW4xGn9tVcox

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

CHANGELOG.md

@@ -84,6 +84,22 @@ per-file diff commands.
 
 ### Fixed
 
+- **The Python tools no longer crash on Windows when a posting, company, CV line or file
+  name falls outside the ANSI code page** (`tools/rank_state.py`, `tools/job_key.py`,
+  `tools/verify_pdf.py`, `tools/verify_layout.py`, `tools/convert_salary_excel.py`,
+  `salary_lookup.py`, `tests/test_tools_utf8_output.py`) - a piped stdout on Windows
+  defaults to the ANSI code page (cp1252 on most Western installs), and that is how Claude
+  Code runs every tool. `/rank`'s candidate listing printed titles and companies with
+  `ensure_ascii=False`, so a single Cyrillic, CJK, Devanagari, Polish or Turkish posting
+  ended the run with `UnicodeEncodeError` before any output reached the workflow; the key
+  audit, the salary lookup, the salary converter and the layout report failed the same way.
+  Each tool now switches stdout and stderr to UTF-8 at entry, which also stops Danish and
+  other Western accents from arriving as cp1252 bytes. The regression tests run every
+  tool in a child process with a cp1252 stdout forced through `PYTHONIOENCODING`, so the
+  Linux CI job reproduces the Windows failure; all seven fail without the fix. The
+  subprocess helpers in `tests/test_rank_state.py` and `tests/test_job_key.py` now decode
+  child output as UTF-8 to match.
+
 - **`/rank` rejects invalid score dimensions before updating an entry**
   (`tools/rank_state.py`, `tests/test_rank_state.py`) - enforce the rubric's
   inclusive 0-100 range and reject booleans, NaN, and infinities. Invalid results

salary_lookup.py

@@ -385,7 +385,21 @@ def print_validation_report(errors, warnings):
     return 0
 
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing a company, title or file name outside it
+    raised UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 def main():
+    _force_utf8_output()
     parser = argparse.ArgumentParser(description="Salary Benchmark Lookup")
     parser.add_argument("company", nargs="?", help="Company name to search for")
     parser.add_argument("--city", help="Filter by city name")

tests/test_job_key.py

@@ -76,7 +76,7 @@ class CompanyFallbackCLI(unittest.TestCase):
         proc = subprocess.run(
             [sys.executable, str(TOOL), "--company", company,
              "--title", "Software Engineer", "--url", url],
-            capture_output=True, text=True, check=True,
+            capture_output=True, text=True, encoding="utf-8", check=True,
         )
         return proc.stdout.strip()
 
@@ -126,7 +126,7 @@ class AuditCLI(unittest.TestCase):
             json.dump({"seen": seen}, fh)
             path = fh.name
         proc = subprocess.run(
-            [sys.executable, str(TOOL), "--audit", path], capture_output=True, text=True
+            [sys.executable, str(TOOL), "--audit", path], capture_output=True, text=True, encoding="utf-8"
         )
         return json.loads(proc.stdout), proc.returncode
 

tests/test_rank_state.py

@@ -52,6 +52,7 @@ class RankStateCase(unittest.TestCase):
             [sys.executable, str(TOOL), *args, "--state", str(self.state), "--today", TODAY],
             capture_output=True,
             text=True,
+            encoding="utf-8",
         )
         self.assertEqual(proc.returncode, expect, proc.stderr)
         return json.loads(proc.stdout)
@@ -148,7 +149,7 @@ class Candidates(RankStateCase):
         proc = subprocess.run(
             [sys.executable, str(TOOL), "candidates", "--state", str(self.tmp / "nope.json"),
              "--tracker", str(self.tmp / "n.csv")],
-            capture_output=True, text=True,
+            capture_output=True, text=True, encoding="utf-8",
         )
         self.assertNotEqual(proc.returncode, 0)
         self.assertIn("not found", proc.stderr + proc.stdout)

tests/test_tools_utf8_output.py

@@ -0,0 +1,154 @@
+"""The Python tools must write UTF-8 whatever the host's default encoding is.
+
+On Windows a piped stdout (which is how Claude Code runs every tool) defaults
+to the ANSI code page, cp1252 on most Western installs. A tool that prints a
+posting title, company, CV line or file name outside that code page then dies
+with UnicodeEncodeError before the workflow sees any output: `/rank` could not
+list a single Cyrillic, CJK, Devanagari, Polish or Turkish posting.
+
+Each case runs the real CLI in a child process with a legacy stdout forced via
+PYTHONIOENCODING, so the Linux CI job reproduces what Windows users hit. The
+child must exit cleanly and its bytes must decode as UTF-8.
+"""
+import json
+import os
+import shutil
+import subprocess
+import sys
+import unittest
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+try:
+    import openpyxl
+except ImportError:
+    openpyxl = None
+
+REPO = Path(__file__).resolve().parent.parent
+TOOLS = REPO / "tools"
+
+# One name per script family that cp1252 cannot encode.
+CYRILLIC = "Яндекс"
+CJK = "腾讯"
+POLISH = "Żabka Łódź"
+
+
+def run_legacy_stdout(argv, cwd=None):
+    """Run a tool the way a Western-locale Windows host pipes it."""
+    env = dict(os.environ, PYTHONIOENCODING="cp1252", PYTHONUTF8="0")
+    return subprocess.run([sys.executable, *map(str, argv)], cwd=cwd, env=env, capture_output=True)
+
+
+class ToolsWriteUtf8(unittest.TestCase):
+    def setUp(self):
+        directory = TemporaryDirectory()
+        self.addCleanup(directory.cleanup)
+        self.tmp = Path(directory.name)
+
+    def assert_clean_utf8(self, proc, expect_code, *needles, stream="stdout"):
+        err = proc.stderr.decode("utf-8", "replace")
+        self.assertNotIn("UnicodeEncodeError", err)
+        self.assertEqual(proc.returncode, expect_code, err)
+        text = getattr(proc, stream).decode("utf-8")
+        for needle in needles:
+            self.assertIn(needle, text)
+        return text
+
+    def write_state(self, seen):
+        state = self.tmp / "seen_jobs.json"
+        state.write_text(json.dumps({"version": 1, "seen": seen}, ensure_ascii=False), encoding="utf-8")
+        return state
+
+    @staticmethod
+    def entry(company, title):
+        return {
+            "title": title, "company": company, "url": "https://example.com/jobs/123456",
+            "portal": "example", "status": "new", "first_seen": "2026-09-20",
+        }
+
+    def test_rank_candidates_lists_non_latin_postings(self):
+        state = self.write_state({"a": self.entry(CYRILLIC, "Инженер"), "b": self.entry(CJK, "工程师")})
+        proc = run_legacy_stdout([TOOLS / "rank_state.py", "candidates", "--state", state, "--today", "2026-09-21"])
+        out = json.loads(self.assert_clean_utf8(proc, 0, CYRILLIC, CJK))
+        self.assertEqual(out["eligible"], 2)
+
+    def test_rank_apply_prints_the_ranking_for_non_latin_postings(self):
+        state = self.write_state({"a": self.entry(CJK, "工程师")})
+        results = self.tmp / "results.json"
+        results.write_text(json.dumps([{
+            "key": "a", "status": "scored",
+            "scores": {"technical": 80, "experience": 80, "behavioral": 80, "career": 80},
+        }]), encoding="utf-8")
+        proc = run_legacy_stdout([
+            TOOLS / "rank_state.py", "apply", "--results", results, "--state", state, "--today", "2026-09-21",
+        ])
+        out = json.loads(self.assert_clean_utf8(proc, 0, CJK))
+        self.assertEqual([row["key"] for row in out["ranked"]], ["a"])
+
+    def test_job_key_audit_reports_non_latin_entries(self):
+        state = self.write_state({f"{CYRILLIC}_Инженер": self.entry(CYRILLIC, "Инженер")})
+        proc = run_legacy_stdout([TOOLS / "job_key.py", "--audit", state])
+        self.assertNotIn(b"UnicodeEncodeError", proc.stderr)
+        self.assertIn(CYRILLIC, proc.stdout.decode("utf-8"))
+
+    def test_salary_lookup_prints_a_company_outside_cp1252(self):
+        shutil.copy(REPO / "salary_lookup.py", self.tmp / "salary_lookup.py")
+        (self.tmp / "salary_data.json").write_text(json.dumps({
+            "metadata": {"source": "fixture", "index_baseline": 100, "index_label": "Index",
+                         "baseline_description": "Index 100 = baseline"},
+            "companies": [{"company": POLISH, "city": "Łódź",
+                           "categories": {"all_employees": {"count": 50, "index": 104.0}}}],
+        }, ensure_ascii=False), encoding="utf-8")
+        proc = run_legacy_stdout([self.tmp / "salary_lookup.py", "Żabka"], cwd=self.tmp)
+        self.assert_clean_utf8(proc, 0, POLISH)
+
+    def test_verify_pdf_names_a_non_latin_file_in_its_error(self):
+        missing = self.tmp / f"main_{CJK}_engineer.pdf"
+        proc = run_legacy_stdout([TOOLS / "verify_pdf.py", missing])
+        self.assert_clean_utf8(proc, 1, CJK, stream="stderr")
+
+    def test_verify_layout_reports_a_non_latin_cv_line(self):
+        # Poppler is not installed in the Python CI job, so feed main() synthetic
+        # page geometry (as test_verify_layout.py does) but still in a child
+        # process, because the encoding under test belongs to the process.
+        driver = self.tmp / "drive_verify_layout.py"
+        driver.write_text(
+            "import sys\n"
+            "from unittest.mock import patch\n"
+            f"sys.path.insert(0, {str(REPO)!r})\n"
+            "from tools import verify_layout as v\n"
+            "A4 = 842.0\n"
+            "def line(text, top, left=70.0, height=10.0):\n"
+            "    return v.Line(top=top, bottom=top + height, left=left, height=height, text=text)\n"
+            "pages = [\n"
+            f"    v.Page(height=A4, lines=[line('Profil', 100.0), line({POLISH!r}, 700.0, left=60.0)]),\n"
+            "    v.Page(height=A4, lines=[line('kontynuacja punktu', 80.0, left=90.0), line('dalej', 700.0, left=90.0)]),\n"
+            "]\n"
+            "with patch.object(v, 'parse_pdf', return_value=pages):\n"
+            "    sys.argv = ['verify_layout.py', __file__]\n"
+            "    sys.exit(v.main())\n",
+            encoding="utf-8",
+        )
+        proc = run_legacy_stdout([driver])
+        self.assertNotIn(b"UnicodeEncodeError", proc.stderr)
+        self.assertIn(POLISH, proc.stdout.decode("utf-8"))
+
+    @unittest.skipUnless(openpyxl, "requires optional openpyxl (installed in CI)")
+    def test_salary_converter_names_a_non_latin_worksheet(self):
+        workbook = self.tmp / "salary.xlsx"
+        book = openpyxl.Workbook()
+        try:
+            book.active.title = CYRILLIC
+            book.active.append(["Company", "City", "Count", "Index"])
+            book.active.append([CYRILLIC, "Москва", 40, 101.5])
+            book.save(workbook)
+        finally:
+            book.close()
+        proc = run_legacy_stdout([
+            TOOLS / "convert_salary_excel.py", workbook, "--output", self.tmp / "out.json",
+        ])
+        self.assert_clean_utf8(proc, 0, CYRILLIC)
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/convert_salary_excel.py

@@ -327,7 +327,21 @@ def parse_sheet(ws, sheet_label=None):
     return companies
 
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing a company, title or file name outside it
+    raised UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 def main():
+    _force_utf8_output()
     parser = argparse.ArgumentParser(
         description="Convert salary Excel data to JSON"
     )

tools/job_key.py

@@ -164,7 +164,21 @@ def audit(path: Path) -> int:
     return 1 if (malformed or duplicates) else 0
 
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing a company, title or file name outside it
+    raised UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 def main() -> int:
+    _force_utf8_output()
     ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
     ap.add_argument("--company")
     ap.add_argument("--title")

tools/rank_state.py

@@ -349,7 +349,21 @@ def cmd_apply(args) -> int:
     return 1 if errors else 0
 
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing a company, title or file name outside it
+    raised UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 def main() -> int:
+    _force_utf8_output()
     common = argparse.ArgumentParser(add_help=False)
     common.add_argument("--state", type=Path, default=STATE)
     common.add_argument("--today", type=date.fromisoformat, default=date.today())

tools/verify_layout.py

@@ -304,7 +304,21 @@ def report(path: Path, pages: list[Page]) -> list[str]:
     return problems
 
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing a company, title or file name outside it
+    raised UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 def main() -> int:
+    _force_utf8_output()
     ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     ap.add_argument("pdf", nargs="?", type=Path)
     args = ap.parse_args()

tools/verify_pdf.py

@@ -191,7 +191,21 @@ def build_parser():
     return parser
 
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing a company, title or file name outside it
+    raised UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 def main(argv=None):
+    _force_utf8_output()
     args = build_parser().parse_args(argv)
     try:
         extractor, text, pages = verify_pdf(