diff --git a/tests/test_tools_utf8_output.py b/tests/test_tools_utf8_output.py index 32ca3029..7515b130 100644 --- a/tests/test_tools_utf8_output.py +++ b/tests/test_tools_utf8_output.py @@ -85,6 +85,20 @@ class ToolsWriteUtf8(unittest.TestCase): out = json.loads(self.assert_clean_utf8(proc, 0, CJK)) self.assertEqual([row["key"] for row in out["ranked"]], ["a"]) + def test_scrape_candidates_lists_non_latin_hits(self): + # Copy the tool so its repo-relative history and tracker paths point at the temp dir. + tools_dir = self.tmp / "tools" + tools_dir.mkdir() + for name in ("scrape_state.py", "rank_state.py", "job_key.py"): + shutil.copy(TOOLS / name, tools_dir / name) + results = self.tmp / "hits.json" + results.write_text(json.dumps([ + {"title": "Инженер", "company": CYRILLIC, "url": "https://example.com/jobs/1"}, + ], ensure_ascii=False), encoding="utf-8") + proc = run_legacy_stdout([tools_dir / "scrape_state.py", "candidates", "--results", results]) + out = json.loads(self.assert_clean_utf8(proc, 0, CYRILLIC)) + self.assertEqual(len(out["selected"]), 1) + def test_job_key_audit_reports_non_latin_entries(self): state = self.write_state({f"{CYRILLIC}_Инженер": self.entry(CYRILLIC, "Инженер")}) proc = run_legacy_stdout([TOOLS / "job_key.py", "--audit", state]) diff --git a/tools/scrape_state.py b/tools/scrape_state.py index 814703ea..a6727443 100644 --- a/tools/scrape_state.py +++ b/tools/scrape_state.py @@ -13,7 +13,7 @@ from pathlib import Path from urllib.parse import urlsplit from job_key import make_key -from rank_state import STATE, TRACKER, load_state, norm, save_state, tracker_pairs +from rank_state import STATE, TRACKER, _force_utf8_output, load_state, norm, save_state, tracker_pairs def read_jobs(path: Path) -> list: @@ -127,6 +127,7 @@ def run(args) -> int: def main() -> int: + _force_utf8_output() parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("command", choices=("candidates", "store", "history")) parser.add_argument("--results", type=Path)