diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ecee0e3..7ec21657 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,22 @@ per-file diff commands. ### Fixed +- **The Python tools no longer crash on Windows when a posting, company, CV line or file + name falls outside the ANSI code page** (`tools/rank_state.py`, `tools/job_key.py`, + `tools/verify_pdf.py`, `tools/verify_layout.py`, `tools/convert_salary_excel.py`, + `salary_lookup.py`, `tests/test_tools_utf8_output.py`) - a piped stdout on Windows + defaults to the ANSI code page (cp1252 on most Western installs), and that is how Claude + Code runs every tool. `/rank`'s candidate listing printed titles and companies with + `ensure_ascii=False`, so a single Cyrillic, CJK, Devanagari, Polish or Turkish posting + ended the run with `UnicodeEncodeError` before any output reached the workflow; the key + audit, the salary lookup, the salary converter and the layout report failed the same way. + Each tool now switches stdout and stderr to UTF-8 at entry, which also stops Danish and + other Western accents from arriving as cp1252 bytes. The regression tests run every + tool in a child process with a cp1252 stdout forced through `PYTHONIOENCODING`, so the + Linux CI job reproduces the Windows failure; all seven fail without the fix. The + subprocess helpers in `tests/test_rank_state.py` and `tests/test_job_key.py` now decode + child output as UTF-8 to match. + - **`/rank` rejects invalid score dimensions before updating an entry** (`tools/rank_state.py`, `tests/test_rank_state.py`) - enforce the rubric's inclusive 0-100 range and reject booleans, NaN, and infinities. Invalid results diff --git a/salary_lookup.py b/salary_lookup.py index 3a9b7fa3..6c921627 100644 --- a/salary_lookup.py +++ b/salary_lookup.py @@ -385,7 +385,21 @@ def print_validation_report(errors, warnings): return 0 +def _force_utf8_output() -> None: + """Write UTF-8 whatever the host's default encoding is. + + A piped stdout on Windows defaults to the ANSI code page (cp1252 on most + Western installs), so printing a company, title or file name outside it + raised UnicodeEncodeError before the workflow saw any output. + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) # absent on a StringIO under test + if reconfigure: + reconfigure(encoding="utf-8") + + def main(): + _force_utf8_output() parser = argparse.ArgumentParser(description="Salary Benchmark Lookup") parser.add_argument("company", nargs="?", help="Company name to search for") parser.add_argument("--city", help="Filter by city name") diff --git a/tests/test_job_key.py b/tests/test_job_key.py index c0c4a3f1..b56779b2 100644 --- a/tests/test_job_key.py +++ b/tests/test_job_key.py @@ -76,7 +76,7 @@ class CompanyFallbackCLI(unittest.TestCase): proc = subprocess.run( [sys.executable, str(TOOL), "--company", company, "--title", "Software Engineer", "--url", url], - capture_output=True, text=True, check=True, + capture_output=True, text=True, encoding="utf-8", check=True, ) return proc.stdout.strip() @@ -126,7 +126,7 @@ class AuditCLI(unittest.TestCase): json.dump({"seen": seen}, fh) path = fh.name proc = subprocess.run( - [sys.executable, str(TOOL), "--audit", path], capture_output=True, text=True + [sys.executable, str(TOOL), "--audit", path], capture_output=True, text=True, encoding="utf-8" ) return json.loads(proc.stdout), proc.returncode diff --git a/tests/test_rank_state.py b/tests/test_rank_state.py index 9ccc8606..097689c7 100644 --- a/tests/test_rank_state.py +++ b/tests/test_rank_state.py @@ -52,6 +52,7 @@ class RankStateCase(unittest.TestCase): [sys.executable, str(TOOL), *args, "--state", str(self.state), "--today", TODAY], capture_output=True, text=True, + encoding="utf-8", ) self.assertEqual(proc.returncode, expect, proc.stderr) return json.loads(proc.stdout) @@ -148,7 +149,7 @@ class Candidates(RankStateCase): proc = subprocess.run( [sys.executable, str(TOOL), "candidates", "--state", str(self.tmp / "nope.json"), "--tracker", str(self.tmp / "n.csv")], - capture_output=True, text=True, + capture_output=True, text=True, encoding="utf-8", ) self.assertNotEqual(proc.returncode, 0) self.assertIn("not found", proc.stderr + proc.stdout) diff --git a/tests/test_tools_utf8_output.py b/tests/test_tools_utf8_output.py new file mode 100644 index 00000000..ea8de69b --- /dev/null +++ b/tests/test_tools_utf8_output.py @@ -0,0 +1,154 @@ +"""The Python tools must write UTF-8 whatever the host's default encoding is. + +On Windows a piped stdout (which is how Claude Code runs every tool) defaults +to the ANSI code page, cp1252 on most Western installs. A tool that prints a +posting title, company, CV line or file name outside that code page then dies +with UnicodeEncodeError before the workflow sees any output: `/rank` could not +list a single Cyrillic, CJK, Devanagari, Polish or Turkish posting. + +Each case runs the real CLI in a child process with a legacy stdout forced via +PYTHONIOENCODING, so the Linux CI job reproduces what Windows users hit. The +child must exit cleanly and its bytes must decode as UTF-8. +""" +import json +import os +import shutil +import subprocess +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +try: + import openpyxl +except ImportError: + openpyxl = None + +REPO = Path(__file__).resolve().parent.parent +TOOLS = REPO / "tools" + +# One name per script family that cp1252 cannot encode. +CYRILLIC = "Яндекс" +CJK = "腾讯" +POLISH = "Żabka Łódź" + + +def run_legacy_stdout(argv, cwd=None): + """Run a tool the way a Western-locale Windows host pipes it.""" + env = dict(os.environ, PYTHONIOENCODING="cp1252", PYTHONUTF8="0") + return subprocess.run([sys.executable, *map(str, argv)], cwd=cwd, env=env, capture_output=True) + + +class ToolsWriteUtf8(unittest.TestCase): + def setUp(self): + directory = TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.tmp = Path(directory.name) + + def assert_clean_utf8(self, proc, expect_code, *needles, stream="stdout"): + err = proc.stderr.decode("utf-8", "replace") + self.assertNotIn("UnicodeEncodeError", err) + self.assertEqual(proc.returncode, expect_code, err) + text = getattr(proc, stream).decode("utf-8") + for needle in needles: + self.assertIn(needle, text) + return text + + def write_state(self, seen): + state = self.tmp / "seen_jobs.json" + state.write_text(json.dumps({"version": 1, "seen": seen}, ensure_ascii=False), encoding="utf-8") + return state + + @staticmethod + def entry(company, title): + return { + "title": title, "company": company, "url": "https://example.com/jobs/123456", + "portal": "example", "status": "new", "first_seen": "2026-09-20", + } + + def test_rank_candidates_lists_non_latin_postings(self): + state = self.write_state({"a": self.entry(CYRILLIC, "Инженер"), "b": self.entry(CJK, "工程师")}) + proc = run_legacy_stdout([TOOLS / "rank_state.py", "candidates", "--state", state, "--today", "2026-09-21"]) + out = json.loads(self.assert_clean_utf8(proc, 0, CYRILLIC, CJK)) + self.assertEqual(out["eligible"], 2) + + def test_rank_apply_prints_the_ranking_for_non_latin_postings(self): + state = self.write_state({"a": self.entry(CJK, "工程师")}) + results = self.tmp / "results.json" + results.write_text(json.dumps([{ + "key": "a", "status": "scored", + "scores": {"technical": 80, "experience": 80, "behavioral": 80, "career": 80}, + }]), encoding="utf-8") + proc = run_legacy_stdout([ + TOOLS / "rank_state.py", "apply", "--results", results, "--state", state, "--today", "2026-09-21", + ]) + out = json.loads(self.assert_clean_utf8(proc, 0, CJK)) + self.assertEqual([row["key"] for row in out["ranked"]], ["a"]) + + def test_job_key_audit_reports_non_latin_entries(self): + state = self.write_state({f"{CYRILLIC}_Инженер": self.entry(CYRILLIC, "Инженер")}) + proc = run_legacy_stdout([TOOLS / "job_key.py", "--audit", state]) + self.assertNotIn(b"UnicodeEncodeError", proc.stderr) + self.assertIn(CYRILLIC, proc.stdout.decode("utf-8")) + + def test_salary_lookup_prints_a_company_outside_cp1252(self): + shutil.copy(REPO / "salary_lookup.py", self.tmp / "salary_lookup.py") + (self.tmp / "salary_data.json").write_text(json.dumps({ + "metadata": {"source": "fixture", "index_baseline": 100, "index_label": "Index", + "baseline_description": "Index 100 = baseline"}, + "companies": [{"company": POLISH, "city": "Łódź", + "categories": {"all_employees": {"count": 50, "index": 104.0}}}], + }, ensure_ascii=False), encoding="utf-8") + proc = run_legacy_stdout([self.tmp / "salary_lookup.py", "Żabka"], cwd=self.tmp) + self.assert_clean_utf8(proc, 0, POLISH) + + def test_verify_pdf_names_a_non_latin_file_in_its_error(self): + missing = self.tmp / f"main_{CJK}_engineer.pdf" + proc = run_legacy_stdout([TOOLS / "verify_pdf.py", missing]) + self.assert_clean_utf8(proc, 1, CJK, stream="stderr") + + def test_verify_layout_reports_a_non_latin_cv_line(self): + # Poppler is not installed in the Python CI job, so feed main() synthetic + # page geometry (as test_verify_layout.py does) but still in a child + # process, because the encoding under test belongs to the process. + driver = self.tmp / "drive_verify_layout.py" + driver.write_text( + "import sys\n" + "from unittest.mock import patch\n" + f"sys.path.insert(0, {str(REPO)!r})\n" + "from tools import verify_layout as v\n" + "A4 = 842.0\n" + "def line(text, top, left=70.0, height=10.0):\n" + " return v.Line(top=top, bottom=top + height, left=left, height=height, text=text)\n" + "pages = [\n" + f" v.Page(height=A4, lines=[line('Profil', 100.0), line({POLISH!r}, 700.0, left=60.0)]),\n" + " v.Page(height=A4, lines=[line('kontynuacja punktu', 80.0, left=90.0), line('dalej', 700.0, left=90.0)]),\n" + "]\n" + "with patch.object(v, 'parse_pdf', return_value=pages):\n" + " sys.argv = ['verify_layout.py', __file__]\n" + " sys.exit(v.main())\n", + encoding="utf-8", + ) + proc = run_legacy_stdout([driver]) + self.assertNotIn(b"UnicodeEncodeError", proc.stderr) + self.assertIn(POLISH, proc.stdout.decode("utf-8")) + + @unittest.skipUnless(openpyxl, "requires optional openpyxl (installed in CI)") + def test_salary_converter_names_a_non_latin_worksheet(self): + workbook = self.tmp / "salary.xlsx" + book = openpyxl.Workbook() + try: + book.active.title = CYRILLIC + book.active.append(["Company", "City", "Count", "Index"]) + book.active.append([CYRILLIC, "Москва", 40, 101.5]) + book.save(workbook) + finally: + book.close() + proc = run_legacy_stdout([ + TOOLS / "convert_salary_excel.py", workbook, "--output", self.tmp / "out.json", + ]) + self.assert_clean_utf8(proc, 0, CYRILLIC) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/convert_salary_excel.py b/tools/convert_salary_excel.py index 24c7186c..4cf58b30 100644 --- a/tools/convert_salary_excel.py +++ b/tools/convert_salary_excel.py @@ -327,7 +327,21 @@ def parse_sheet(ws, sheet_label=None): return companies +def _force_utf8_output() -> None: + """Write UTF-8 whatever the host's default encoding is. + + A piped stdout on Windows defaults to the ANSI code page (cp1252 on most + Western installs), so printing a company, title or file name outside it + raised UnicodeEncodeError before the workflow saw any output. + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) # absent on a StringIO under test + if reconfigure: + reconfigure(encoding="utf-8") + + def main(): + _force_utf8_output() parser = argparse.ArgumentParser( description="Convert salary Excel data to JSON" ) diff --git a/tools/job_key.py b/tools/job_key.py index fd7894b4..7effd2f3 100644 --- a/tools/job_key.py +++ b/tools/job_key.py @@ -164,7 +164,21 @@ def audit(path: Path) -> int: return 1 if (malformed or duplicates) else 0 +def _force_utf8_output() -> None: + """Write UTF-8 whatever the host's default encoding is. + + A piped stdout on Windows defaults to the ANSI code page (cp1252 on most + Western installs), so printing a company, title or file name outside it + raised UnicodeEncodeError before the workflow saw any output. + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) # absent on a StringIO under test + if reconfigure: + reconfigure(encoding="utf-8") + + def main() -> int: + _force_utf8_output() ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--company") ap.add_argument("--title") diff --git a/tools/rank_state.py b/tools/rank_state.py index a10f9c07..454907cc 100644 --- a/tools/rank_state.py +++ b/tools/rank_state.py @@ -349,7 +349,21 @@ def cmd_apply(args) -> int: return 1 if errors else 0 +def _force_utf8_output() -> None: + """Write UTF-8 whatever the host's default encoding is. + + A piped stdout on Windows defaults to the ANSI code page (cp1252 on most + Western installs), so printing a company, title or file name outside it + raised UnicodeEncodeError before the workflow saw any output. + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) # absent on a StringIO under test + if reconfigure: + reconfigure(encoding="utf-8") + + def main() -> int: + _force_utf8_output() common = argparse.ArgumentParser(add_help=False) common.add_argument("--state", type=Path, default=STATE) common.add_argument("--today", type=date.fromisoformat, default=date.today()) diff --git a/tools/verify_layout.py b/tools/verify_layout.py index 0a330aa1..b8eb34aa 100644 --- a/tools/verify_layout.py +++ b/tools/verify_layout.py @@ -304,7 +304,21 @@ def report(path: Path, pages: list[Page]) -> list[str]: return problems +def _force_utf8_output() -> None: + """Write UTF-8 whatever the host's default encoding is. + + A piped stdout on Windows defaults to the ANSI code page (cp1252 on most + Western installs), so printing a company, title or file name outside it + raised UnicodeEncodeError before the workflow saw any output. + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) # absent on a StringIO under test + if reconfigure: + reconfigure(encoding="utf-8") + + def main() -> int: + _force_utf8_output() ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("pdf", nargs="?", type=Path) args = ap.parse_args() diff --git a/tools/verify_pdf.py b/tools/verify_pdf.py index 25fa2b33..d72239d5 100644 --- a/tools/verify_pdf.py +++ b/tools/verify_pdf.py @@ -191,7 +191,21 @@ def build_parser(): return parser +def _force_utf8_output() -> None: + """Write UTF-8 whatever the host's default encoding is. + + A piped stdout on Windows defaults to the ANSI code page (cp1252 on most + Western installs), so printing a company, title or file name outside it + raised UnicodeEncodeError before the workflow saw any output. + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) # absent on a StringIO under test + if reconfigure: + reconfigure(encoding="utf-8") + + def main(argv=None): + _force_utf8_output() args = build_parser().parse_args(argv) try: extractor, text, pages = verify_pdf(