fix(scrape): force UTF-8 output in scrape_state.py (#744)

`tools/scrape_state.py` was the only workflow tool whose `main()` did not force UTF-8 output, so on Windows hosts whose piped stdout defaults to cp1252 the `/scrape` candidates step exited 1 with `'charmap' codec can't encode characters` as soon as one hit had a non-Latin company or title. `main()` now calls `_force_utf8_output()` from `rank_state` first, like the other tools. One real-CLI test on a Cyrillic hit fails on master and passes here.

Reproduced on a Windows host before merging. Rebased onto master by the maintainer to drop the CHANGELOG hunk (the file is frozen at 1.7.4). #814 by @ahmed-husssain reproduced the same failure independently and was closed in favour of this PR.

Fixes #743

tests/test_tools_utf8_output.py

@@ -85,6 +85,20 @@ class ToolsWriteUtf8(unittest.TestCase):
         out = json.loads(self.assert_clean_utf8(proc, 0, CJK))
         self.assertEqual([row["key"] for row in out["ranked"]], ["a"])
 
+    def test_scrape_candidates_lists_non_latin_hits(self):
+        # Copy the tool so its repo-relative history and tracker paths point at the temp dir.
+        tools_dir = self.tmp / "tools"
+        tools_dir.mkdir()
+        for name in ("scrape_state.py", "rank_state.py", "job_key.py"):
+            shutil.copy(TOOLS / name, tools_dir / name)
+        results = self.tmp / "hits.json"
+        results.write_text(json.dumps([
+            {"title": "Инженер", "company": CYRILLIC, "url": "https://example.com/jobs/1"},
+        ], ensure_ascii=False), encoding="utf-8")
+        proc = run_legacy_stdout([tools_dir / "scrape_state.py", "candidates", "--results", results])
+        out = json.loads(self.assert_clean_utf8(proc, 0, CYRILLIC))
+        self.assertEqual(len(out["selected"]), 1)
+
     def test_job_key_audit_reports_non_latin_entries(self):
         state = self.write_state({f"{CYRILLIC}_Инженер": self.entry(CYRILLIC, "Инженер")})
         proc = run_legacy_stdout([TOOLS / "job_key.py", "--audit", state])

tools/scrape_state.py

@@ -13,7 +13,7 @@ from pathlib import Path
 from urllib.parse import urlsplit
 
 from job_key import make_key
-from rank_state import STATE, TRACKER, load_state, norm, save_state, tracker_pairs
+from rank_state import STATE, TRACKER, _force_utf8_output, load_state, norm, save_state, tracker_pairs
 
 
 def read_jobs(path: Path) -> list:
@@ -127,6 +127,7 @@ def run(args) -> int:
 
 
 def main() -> int:
+    _force_utf8_output()
     parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
     parser.add_argument("command", choices=("candidates", "store", "history"))
     parser.add_argument("--results", type=Path)