fix(scrape): distinguish non-Latin companies in fallback job keys (#487)

CHANGELOG.md

@@ -84,6 +84,14 @@ per-file diff commands.
 
 ### Fixed
 
+- **Distinct non-Latin company names no longer share the unknown-company job key**
+  (`tools/job_key.py`, `tests/test_job_key.py`) - when a non-empty company name
+  has no ASCII slug, derive its fallback from a hash of the normalized name.
+  Missing names retain `unknown-company`, and existing ASCII keys are unchanged.
+  CLI tests cover distinct companies, case and canonical Unicode equivalence,
+  and stable keys across posting URLs. Existing state is not rewritten; `/scrape`
+  already recognizes stored postings by URL regardless of their previous key.
+
 - **`/rank` tracker exclusion handles UTF-8 BOMs on reordered CSV headers**
   (`tools/rank_state.py`, `tests/test_rank_state.py`) - when `company` or `role`
   is the first column, a leading BOM becomes part of the header name and an

tests/test_job_key.py

@@ -71,6 +71,34 @@ class MakeKey(unittest.TestCase):
         self.assertFalse(key.startswith("_"))
 
 
+class CompanyFallbackCLI(unittest.TestCase):
+    def key_for(self, company, url="https://example.com/jobs/123456"):
+        proc = subprocess.run(
+            [sys.executable, str(TOOL), "--company", company,
+             "--title", "Software Engineer", "--url", url],
+            capture_output=True, text=True, check=True,
+        )
+        return proc.stdout.strip()
+
+    def test_distinct_non_latin_companies_have_distinct_keys(self):
+        keys = [self.key_for(company) for company in ("腾讯", "阿里巴巴", "")]
+        self.assertEqual(len(set(keys)), 3)
+        self.assertTrue(all(is_canonical(key) for key in keys))
+
+    def test_company_fallback_is_stable_across_case_normalization_and_urls(self):
+        for first, second in (("КОМПАНИЯ", "компания"), ("ガンホー", "カ\u3099ンホー")):
+            with self.subTest(first=first, second=second):
+                self.assertEqual(
+                    self.key_for(first),
+                    self.key_for(second, "https://example.com/jobs/654321"),
+                )
+
+    def test_missing_company_and_existing_ascii_keys_are_unchanged(self):
+        self.assertEqual(self.key_for(""), "unknown-company_software-engineer")
+        self.assertEqual(self.key_for("  "), "unknown-company_software-engineer")
+        self.assertEqual(self.key_for("Acme Corp"), "acme-corp_software-engineer")
+
+
 class CanonicalAndLegacyShape(unittest.TestCase):
     def test_canonical_accepts_company_underscore_title(self):
         self.assertTrue(is_canonical("acme-corp_soc-analyst"))

tools/job_key.py

@@ -84,7 +84,12 @@ def _cap(slug: str, limit: int) -> str:
 
 def make_key(company: str, title: str, url: str = "") -> str:
     """The canonical seen_jobs.json key for one posting."""
-    company_slug = _cap(slugify(company), COMPANY_MAX) or "unknown-company"
+    company_slug = _cap(slugify(company), COMPANY_MAX)
+    if not company_slug:
+        name = unicodedata.normalize("NFC", str(company or "").strip().casefold())
+        # An absent name stays unknown; a non-Latin name still has an identity.
+        digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:HASH_LEN]
+        company_slug = f"company-{digest}" if name else "unknown-company"
     title_slug = _cap(slugify(title), TITLE_MAX)
     if not title_slug:
         # No Latin characters in the title. The portal's own numeric id is the