fix(tools): write UTF-8 on stdout so non-Latin postings stop crashing on Windows (#490)

A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
Western installs). rank_state.py printed titles and companies with
ensure_ascii=False, so one Cyrillic, CJK, Devanagari, Polish or Turkish
posting ended /rank with UnicodeEncodeError. job_key --audit, salary_lookup,
convert_salary_excel and verify_layout failed the same way.

Each tool now reconfigures stdout and stderr to UTF-8 at entry. The new
tests run every tool in a child process with a cp1252 stdout forced through
PYTHONIOENCODING, so Linux CI reproduces the Windows failure.


Claude-Session: https://claude.ai/code/session_01WPXraLZ4i7UW4xGn9tVcox

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

tools/convert_salary_excel.py

@@ -327,7 +327,21 @@ def parse_sheet(ws, sheet_label=None):
     return companies
 
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing a company, title or file name outside it
+    raised UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 def main():
+    _force_utf8_output()
     parser = argparse.ArgumentParser(
         description="Convert salary Excel data to JSON"
     )

tools/job_key.py

@@ -164,7 +164,21 @@ def audit(path: Path) -> int:
     return 1 if (malformed or duplicates) else 0
 
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing a company, title or file name outside it
+    raised UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 def main() -> int:
+    _force_utf8_output()
     ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
     ap.add_argument("--company")
     ap.add_argument("--title")

tools/rank_state.py

@@ -349,7 +349,21 @@ def cmd_apply(args) -> int:
     return 1 if errors else 0
 
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing a company, title or file name outside it
+    raised UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 def main() -> int:
+    _force_utf8_output()
     common = argparse.ArgumentParser(add_help=False)
     common.add_argument("--state", type=Path, default=STATE)
     common.add_argument("--today", type=date.fromisoformat, default=date.today())

tools/verify_layout.py

@@ -304,7 +304,21 @@ def report(path: Path, pages: list[Page]) -> list[str]:
     return problems
 
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing a company, title or file name outside it
+    raised UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 def main() -> int:
+    _force_utf8_output()
     ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     ap.add_argument("pdf", nargs="?", type=Path)
     args = ap.parse_args()

tools/verify_pdf.py

@@ -191,7 +191,21 @@ def build_parser():
     return parser
 
 
+def _force_utf8_output() -> None:
+    """Write UTF-8 whatever the host's default encoding is.
+
+    A piped stdout on Windows defaults to the ANSI code page (cp1252 on most
+    Western installs), so printing a company, title or file name outside it
+    raised UnicodeEncodeError before the workflow saw any output.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)  # absent on a StringIO under test
+        if reconfigure:
+            reconfigure(encoding="utf-8")
+
+
 def main(argv=None):
+    _force_utf8_output()
     args = build_parser().parse_args(argv)
     try:
         extractor, text, pages = verify_pdf(