fix(job_key): hash mixed-script names instead of keying on their Latin fragment (#514)

The hash fallbacks from #487 and #502 fired only when the slug was
completely empty. A mixed-script name kept its Latin or digit fragment
and skipped them: "Программист 1С" and "Аналитик 1С" at one company both
keyed as company-6d769a_1, "Python-разработчик" and "Python-аналитик" as
..._python, "Сбер AI" and "Яндекс AI" as ai_... /scrape Step 4 then
dropped the second posting as already seen, and because the dict key is
overwritten, --audit could not report the collision as a duplicate URL.

Treat the fold as lossy whenever it drops a letter outside the Latin
script, and let the existing fallbacks take over: the title half uses
the portal's numeric id or the URL hash with the fragment kept as a
readable prefix, the company half uses the NFC-casefold name hash with
the fragment as prefix. Latin letters that also lack a decomposition
(ø, æ, ß, ł) are deliberately not counted, so "Ørsted" still keys as
"rsted" and a live Danish seen_jobs.json does not re-key.

Seven new cases; the four collision cases fail on master.

CHANGELOG.md

@@ -15,6 +15,25 @@ per-file diff commands.
 
 ### Fixed
 
+- **`job_key.py` no longer gives two postings one key when a title or company is partly
+  non-Latin** (`tools/job_key.py`, `tests/test_job_key.py`) - the hash fallbacks from #487
+  and #502 fired only when the slug was completely empty, so a mixed-script name kept its
+  Latin or digit fragment and skipped them: `Программист 1С` and `Аналитик 1С` at one company
+  both keyed as `company-6d769a_1`, `Python-разработчик` and `Python-аналитик` as `..._python`,
+  `Сбер AI` and `Яндекс AI` as `ai_...`. `/scrape` Step 4 then dropped the second posting as
+  already seen, and because the dict key is overwritten, `--audit` could not report the
+  collision as a duplicate URL either. "1С" titles and "N категории" grade suffixes are
+  everyday Russian listings on freehire, the shipped multi-market portal. The fold is now
+  treated as lossy whenever it drops a letter outside the Latin script, and the existing
+  fallbacks take over: the title half uses the portal's numeric id or the URL hash with the
+  fragment kept as a readable prefix (`1-4461771225`, `python-bb31e9`), the company half uses
+  the NFC-casefold name hash with the fragment as prefix (`ai-d35210`). Latin letters that
+  also lack a decomposition (`ø`, `æ`, `ß`, `ł`) are deliberately not counted, so `Ørsted`
+  still keys as `rsted` and a live Danish `seen_jobs.json` does not re-key; an existing
+  mixed-script entry re-keys once on the next scrape and `--audit` lists it under
+  `keys_not_matching_current_rule`, the same one-time drift #502 accepted. Seven new cases;
+  the four collision cases fail on master.
+
 - **`verify_pdf.py --ascii-dates` no longer reads a year-like run inside a longer number as a
   date** - the year pattern had no digit boundaries, so `2000` inside `120000` or `12000` made
   `Grew budget DKK 120000–200000` and `12000–15000 events/s` fail `/apply` Step 5d as

tests/test_job_key.py

@@ -100,6 +100,60 @@ class MakeKey(unittest.TestCase):
         self.assertNotEqual(a, b)
 
 
+class MixedScriptNamesKeepTheirIdentity(unittest.TestCase):
+    """A name that folds to a fragment is as lossy as one that folds to nothing.
+
+    #487 and #502 added the hash fallbacks for titles and companies that
+    slugify to '', but a mixed-script name keeps its Latin or digit fragment and
+    skipped them: "Программист 1С" and "Аналитик 1С" both became "1", "Сбер AI"
+    and "Яндекс AI" both became "ai", so two postings shared one key and
+    `/scrape` Step 4 dropped the second as already seen. "1С" titles and
+    "N категории" grade suffixes are everyday Russian listings on freehire.
+    """
+
+    def test_titles_sharing_a_digit_fragment_get_distinct_keys(self):
+        company = "Яндекс"
+        a = make_key(company, "Программист 1С", url="https://freehire.me/jobs/programmist-1s-aaa")
+        b = make_key(company, "Аналитик 1С", url="https://freehire.me/jobs/analitik-1s-bbb")
+        self.assertNotEqual(a, b)
+        self.assertTrue(is_canonical(a) and is_canonical(b))
+
+    def test_titles_sharing_a_latin_word_get_distinct_keys(self):
+        company = "ООО Чен"
+        a = make_key(company, "Python-разработчик", url="https://freehire.me/jobs/python-razrabotchik-x1")
+        b = make_key(company, "Python-аналитик", url="https://freehire.me/jobs/python-analitik-x2")
+        self.assertNotEqual(a, b)
+        # The fragment survives as a readable prefix; the URL carries the identity.
+        self.assertTrue(a.split("_", 1)[1].startswith("python-"))
+
+    def test_lossy_title_prefers_the_portal_numeric_id(self):
+        key = make_key("Acme", "Инженер 1 категории", url="https://kr.linkedin.com/jobs/view/x-4461771225")
+        self.assertEqual(key, "acme_1-4461771225")
+
+    def test_companies_sharing_a_latin_word_get_distinct_keys(self):
+        a = make_key("Сбер AI", "ML Engineer", url="https://example.com/1")
+        b = make_key("Яндекс AI", "ML Engineer", url="https://example.com/2")
+        self.assertNotEqual(a, b)
+        self.assertTrue(a.startswith("ai-") and b.startswith("ai-"))
+        self.assertTrue(is_canonical(a) and is_canonical(b))
+
+    def test_lossy_company_is_stable_across_case_and_urls(self):
+        a = make_key("Сбер AI", "ML Engineer", url="https://example.com/1")
+        b = make_key("сбер ai", "ML Engineer", url="https://example.com/2")
+        self.assertEqual(a.split("_", 1)[0], b.split("_", 1)[0])
+
+    def test_latin_names_with_accents_and_ligatures_are_not_lossy(self):
+        # NFKD folds these without dropping a letter, so existing keys stay put.
+        self.assertEqual(make_key("Zürich Versicherung", "Ingénieur finance"), "zurich-versicherung_ingenieur-finance")
+
+    def test_latin_letters_without_a_decomposition_do_not_re_key(self):
+        # "ø" and "æ" have no NFKD decomposition and are dropped by the fold,
+        # but they are Latin letters: "Ørsted" has keyed as "rsted" since the
+        # rule existed, and a live Danish seen_jobs.json must not re-key.
+        self.assertEqual(make_key("Ørsted A/S", "Senior Engineer"), "rsted-a-s_senior-engineer")
+        self.assertEqual(make_key("Mærsk", "Søfarende"), "mrsk_sfarende")
+
+
 class CompanyFallbackCLI(unittest.TestCase):
     def key_for(self, company, url="https://example.com/jobs/123456"):
         proc = subprocess.run(

tools/job_key.py

@@ -27,7 +27,11 @@ collide.
 A title that slugifies to nothing (a posting written in a non-Latin script) has
 no usable key half at all - "securion_" was a real entry, and it would have
 collided with every future non-Latin posting from that company. Those fall back
-to the portal's numeric id from the URL.
+to the portal's numeric id from the URL. A title or company that slugifies to
+*less* than it says is the same problem one step on: "Программист 1С" and
+"Аналитик 1С" both fold to "1", "Сбер AI" and "Яндекс AI" both fold to "ai",
+so whenever the ASCII fold drops letters the surviving fragment is only a
+readable prefix and the identity comes from the same fallback.
 
 Usage:
   python3 tools/job_key.py --company "Acme Corp" --title "SOC Analyst (L2)"
@@ -68,6 +72,29 @@ def slugify(text: str) -> str:
     return _NON_SLUG.sub("-", ascii_only.lower()).strip("-")
 
 
+def _lost_letters(text: str) -> bool:
+    """True when the ASCII fold dropped non-Latin letters, so the slug under-identifies.
+
+    NFKD turns "ü" into "u" plus a combining mark and "fi" into "fi", so most
+    Latin text keeps every letter. A Cyrillic, Greek, CJK or Arabic letter has
+    no ASCII decomposition and vanishes; a mixed-script name then keeps only
+    its Latin or digit fragment, and two different names can share it.
+
+    Latin letters that also lack a decomposition ("ø", "æ", "ß", "ł") are
+    deliberately NOT counted: "Ørsted" has keyed as "rsted" since the rule
+    existed, and treating it as lossy would re-key every Danish company in a
+    live seen_jobs.json for a collision that does not happen in practice.
+    """
+    if not text:
+        return False
+    for ch in unicodedata.normalize("NFKD", str(text)):
+        if ord(ch) < 128 or not ch.isalpha():
+            continue
+        if not unicodedata.name(ch, "LATIN").startswith("LATIN"):
+            return True
+    return False
+
+
 def _cap(slug: str, limit: int) -> str:
     """Cap length without making truncation lossy across runs.
 
@@ -85,18 +112,26 @@ def _cap(slug: str, limit: int) -> str:
 def make_key(company: str, title: str, url: str = "") -> str:
     """The canonical seen_jobs.json key for one posting."""
     company_slug = _cap(slugify(company), COMPANY_MAX)
-    if not company_slug:
+    if not company_slug or _lost_letters(company):
         name = unicodedata.normalize("NFC", str(company or "").strip().casefold())
         # An absent name stays unknown; a non-Latin name still has an identity.
+        # A mixed-script name keeps its Latin fragment as a readable prefix, but
+        # the identity is the hash: "Сбер AI" and "Яндекс AI" both fold to "ai".
         digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:HASH_LEN]
-        company_slug = f"company-{digest}" if name else "unknown-company"
+        if not name:
+            company_slug = "unknown-company"
+        else:
+            company_slug = f"{company_slug}-{digest}" if company_slug else f"company-{digest}"
     title_slug = _cap(slugify(title), TITLE_MAX)
-    if not title_slug:
+    if not title_slug or _lost_letters(title):
-        # No Latin characters in the title. The portal's own numeric id is the
+        # No Latin characters in the title, or not enough of them to identify
-        # only stable handle left; never emit a bare "company_" prefix.
+        # it: "Программист 1С" and "Аналитик 1С" both fold to "1". The portal's
+        # own numeric id is the stable handle; a surviving fragment stays as a
+        # readable prefix only. Never emit a bare "company_" prefix.
+        fragment = title_slug
         match = _JOB_ID.search(url or "")
         if match:
-            title_slug = match.group(1)
+            title_slug = f"{fragment}-{match.group(1)}" if fragment else match.group(1)
         else:
             basis = slugify(unicodedata.normalize("NFKD", str(title or url or "")))
             # Hash the URL alone when there is one. The URL is the posting's
@@ -111,7 +146,7 @@ def make_key(company: str, title: str, url: str = "") -> str:
             # is left to key on.
             digest_basis = str(url) if url else str(title)
             digest = hashlib.sha1(digest_basis.encode("utf-8")).hexdigest()[:HASH_LEN]
-            title_slug = basis or f"untitled-{digest}"
+            title_slug = f"{fragment or basis or 'untitled'}-{digest}"
     return f"{company_slug}_{title_slug}"