From 0c9bb3cb073af1af47c2afe9bad9386f209256e3 Mon Sep 17 00:00:00 2001 From: Ayobami Adegoke Date: Mon, 5 Oct 2026 18:20:49 +0100 Subject: [PATCH] fix(job_key): hash mixed-script names instead of keying on their Latin fragment (#514) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hash fallbacks from #487 and #502 fired only when the slug was completely empty. A mixed-script name kept its Latin or digit fragment and skipped them: "Программист 1С" and "Аналитик 1С" at one company both keyed as company-6d769a_1, "Python-разработчик" and "Python-аналитик" as ..._python, "Сбер AI" and "Яндекс AI" as ai_... /scrape Step 4 then dropped the second posting as already seen, and because the dict key is overwritten, --audit could not report the collision as a duplicate URL. Treat the fold as lossy whenever it drops a letter outside the Latin script, and let the existing fallbacks take over: the title half uses the portal's numeric id or the URL hash with the fragment kept as a readable prefix, the company half uses the NFC-casefold name hash with the fragment as prefix. Latin letters that also lack a decomposition (ø, æ, ß, ł) are deliberately not counted, so "Ørsted" still keys as "rsted" and a live Danish seen_jobs.json does not re-key. Seven new cases; the four collision cases fail on master. --- CHANGELOG.md | 19 +++++++++++++++ tests/test_job_key.py | 54 +++++++++++++++++++++++++++++++++++++++++++ tools/job_key.py | 51 +++++++++++++++++++++++++++++++++------- 3 files changed, 116 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10f14cbe..c47515f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,25 @@ per-file diff commands. ### Fixed +- **`job_key.py` no longer gives two postings one key when a title or company is partly + non-Latin** (`tools/job_key.py`, `tests/test_job_key.py`) - the hash fallbacks from #487 + and #502 fired only when the slug was completely empty, so a mixed-script name kept its + Latin or digit fragment and skipped them: `Программист 1С` and `Аналитик 1С` at one company + both keyed as `company-6d769a_1`, `Python-разработчик` and `Python-аналитик` as `..._python`, + `Сбер AI` and `Яндекс AI` as `ai_...`. `/scrape` Step 4 then dropped the second posting as + already seen, and because the dict key is overwritten, `--audit` could not report the + collision as a duplicate URL either. "1С" titles and "N категории" grade suffixes are + everyday Russian listings on freehire, the shipped multi-market portal. The fold is now + treated as lossy whenever it drops a letter outside the Latin script, and the existing + fallbacks take over: the title half uses the portal's numeric id or the URL hash with the + fragment kept as a readable prefix (`1-4461771225`, `python-bb31e9`), the company half uses + the NFC-casefold name hash with the fragment as prefix (`ai-d35210`). Latin letters that + also lack a decomposition (`ø`, `æ`, `ß`, `ł`) are deliberately not counted, so `Ørsted` + still keys as `rsted` and a live Danish `seen_jobs.json` does not re-key; an existing + mixed-script entry re-keys once on the next scrape and `--audit` lists it under + `keys_not_matching_current_rule`, the same one-time drift #502 accepted. Seven new cases; + the four collision cases fail on master. + - **`verify_pdf.py --ascii-dates` no longer reads a year-like run inside a longer number as a date** - the year pattern had no digit boundaries, so `2000` inside `120000` or `12000` made `Grew budget DKK 120000–200000` and `12000–15000 events/s` fail `/apply` Step 5d as diff --git a/tests/test_job_key.py b/tests/test_job_key.py index b37d8545..8d8fbeae 100644 --- a/tests/test_job_key.py +++ b/tests/test_job_key.py @@ -100,6 +100,60 @@ class MakeKey(unittest.TestCase): self.assertNotEqual(a, b) +class MixedScriptNamesKeepTheirIdentity(unittest.TestCase): + """A name that folds to a fragment is as lossy as one that folds to nothing. + + #487 and #502 added the hash fallbacks for titles and companies that + slugify to '', but a mixed-script name keeps its Latin or digit fragment and + skipped them: "Программист 1С" and "Аналитик 1С" both became "1", "Сбер AI" + and "Яндекс AI" both became "ai", so two postings shared one key and + `/scrape` Step 4 dropped the second as already seen. "1С" titles and + "N категории" grade suffixes are everyday Russian listings on freehire. + """ + + def test_titles_sharing_a_digit_fragment_get_distinct_keys(self): + company = "Яндекс" + a = make_key(company, "Программист 1С", url="https://freehire.me/jobs/programmist-1s-aaa") + b = make_key(company, "Аналитик 1С", url="https://freehire.me/jobs/analitik-1s-bbb") + self.assertNotEqual(a, b) + self.assertTrue(is_canonical(a) and is_canonical(b)) + + def test_titles_sharing_a_latin_word_get_distinct_keys(self): + company = "ООО Чен" + a = make_key(company, "Python-разработчик", url="https://freehire.me/jobs/python-razrabotchik-x1") + b = make_key(company, "Python-аналитик", url="https://freehire.me/jobs/python-analitik-x2") + self.assertNotEqual(a, b) + # The fragment survives as a readable prefix; the URL carries the identity. + self.assertTrue(a.split("_", 1)[1].startswith("python-")) + + def test_lossy_title_prefers_the_portal_numeric_id(self): + key = make_key("Acme", "Инженер 1 категории", url="https://kr.linkedin.com/jobs/view/x-4461771225") + self.assertEqual(key, "acme_1-4461771225") + + def test_companies_sharing_a_latin_word_get_distinct_keys(self): + a = make_key("Сбер AI", "ML Engineer", url="https://example.com/1") + b = make_key("Яндекс AI", "ML Engineer", url="https://example.com/2") + self.assertNotEqual(a, b) + self.assertTrue(a.startswith("ai-") and b.startswith("ai-")) + self.assertTrue(is_canonical(a) and is_canonical(b)) + + def test_lossy_company_is_stable_across_case_and_urls(self): + a = make_key("Сбер AI", "ML Engineer", url="https://example.com/1") + b = make_key("сбер ai", "ML Engineer", url="https://example.com/2") + self.assertEqual(a.split("_", 1)[0], b.split("_", 1)[0]) + + def test_latin_names_with_accents_and_ligatures_are_not_lossy(self): + # NFKD folds these without dropping a letter, so existing keys stay put. + self.assertEqual(make_key("Zürich Versicherung", "Ingénieur finance"), "zurich-versicherung_ingenieur-finance") + + def test_latin_letters_without_a_decomposition_do_not_re_key(self): + # "ø" and "æ" have no NFKD decomposition and are dropped by the fold, + # but they are Latin letters: "Ørsted" has keyed as "rsted" since the + # rule existed, and a live Danish seen_jobs.json must not re-key. + self.assertEqual(make_key("Ørsted A/S", "Senior Engineer"), "rsted-a-s_senior-engineer") + self.assertEqual(make_key("Mærsk", "Søfarende"), "mrsk_sfarende") + + class CompanyFallbackCLI(unittest.TestCase): def key_for(self, company, url="https://example.com/jobs/123456"): proc = subprocess.run( diff --git a/tools/job_key.py b/tools/job_key.py index 42b8a279..99445816 100644 --- a/tools/job_key.py +++ b/tools/job_key.py @@ -27,7 +27,11 @@ collide. A title that slugifies to nothing (a posting written in a non-Latin script) has no usable key half at all - "securion_" was a real entry, and it would have collided with every future non-Latin posting from that company. Those fall back -to the portal's numeric id from the URL. +to the portal's numeric id from the URL. A title or company that slugifies to +*less* than it says is the same problem one step on: "Программист 1С" and +"Аналитик 1С" both fold to "1", "Сбер AI" and "Яндекс AI" both fold to "ai", +so whenever the ASCII fold drops letters the surviving fragment is only a +readable prefix and the identity comes from the same fallback. Usage: python3 tools/job_key.py --company "Acme Corp" --title "SOC Analyst (L2)" @@ -68,6 +72,29 @@ def slugify(text: str) -> str: return _NON_SLUG.sub("-", ascii_only.lower()).strip("-") +def _lost_letters(text: str) -> bool: + """True when the ASCII fold dropped non-Latin letters, so the slug under-identifies. + + NFKD turns "ü" into "u" plus a combining mark and "fi" into "fi", so most + Latin text keeps every letter. A Cyrillic, Greek, CJK or Arabic letter has + no ASCII decomposition and vanishes; a mixed-script name then keeps only + its Latin or digit fragment, and two different names can share it. + + Latin letters that also lack a decomposition ("ø", "æ", "ß", "ł") are + deliberately NOT counted: "Ørsted" has keyed as "rsted" since the rule + existed, and treating it as lossy would re-key every Danish company in a + live seen_jobs.json for a collision that does not happen in practice. + """ + if not text: + return False + for ch in unicodedata.normalize("NFKD", str(text)): + if ord(ch) < 128 or not ch.isalpha(): + continue + if not unicodedata.name(ch, "LATIN").startswith("LATIN"): + return True + return False + + def _cap(slug: str, limit: int) -> str: """Cap length without making truncation lossy across runs. @@ -85,18 +112,26 @@ def _cap(slug: str, limit: int) -> str: def make_key(company: str, title: str, url: str = "") -> str: """The canonical seen_jobs.json key for one posting.""" company_slug = _cap(slugify(company), COMPANY_MAX) - if not company_slug: + if not company_slug or _lost_letters(company): name = unicodedata.normalize("NFC", str(company or "").strip().casefold()) # An absent name stays unknown; a non-Latin name still has an identity. + # A mixed-script name keeps its Latin fragment as a readable prefix, but + # the identity is the hash: "Сбер AI" and "Яндекс AI" both fold to "ai". digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:HASH_LEN] - company_slug = f"company-{digest}" if name else "unknown-company" + if not name: + company_slug = "unknown-company" + else: + company_slug = f"{company_slug}-{digest}" if company_slug else f"company-{digest}" title_slug = _cap(slugify(title), TITLE_MAX) - if not title_slug: - # No Latin characters in the title. The portal's own numeric id is the - # only stable handle left; never emit a bare "company_" prefix. + if not title_slug or _lost_letters(title): + # No Latin characters in the title, or not enough of them to identify + # it: "Программист 1С" and "Аналитик 1С" both fold to "1". The portal's + # own numeric id is the stable handle; a surviving fragment stays as a + # readable prefix only. Never emit a bare "company_" prefix. + fragment = title_slug match = _JOB_ID.search(url or "") if match: - title_slug = match.group(1) + title_slug = f"{fragment}-{match.group(1)}" if fragment else match.group(1) else: basis = slugify(unicodedata.normalize("NFKD", str(title or url or ""))) # Hash the URL alone when there is one. The URL is the posting's @@ -111,7 +146,7 @@ def make_key(company: str, title: str, url: str = "") -> str: # is left to key on. digest_basis = str(url) if url else str(title) digest = hashlib.sha1(digest_basis.encode("utf-8")).hexdigest()[:HASH_LEN] - title_slug = basis or f"untitled-{digest}" + title_slug = f"{fragment or basis or 'untitled'}-{digest}" return f"{company_slug}_{title_slug}"