From da2aee7bf1285bff35b010fd4acb8f40d397ae00 Mon Sep 17 00:00:00 2001 From: Ayobami Adegoke Date: Thu, 8 Oct 2026 20:18:32 +0100 Subject: [PATCH] fix(job_key): keep the bare basis for an empty title whose URL has no six-digit run (#523) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mixed-script rule from #514 appended the digest to `basis` as well, so an empty title with a URL that carries no six-digit run moved from "acme_https-example-com-jobs-abc-xyz" to "..._5ca8c6". Unreachable from /scrape, which always passes a title, but a key is a pure function of the posting and should not move for a case the rule was never about. A lossy fragment still takes the digest; an empty title keeps the bare basis or "untitled-" exactly as before #514. One new case pins the pre-#514 output and fails on current master. The CHANGELOG example for the Python-разработчик key is corrected to its real digest (python-fb29d7). Follow-up suggested in the #514 merge note. --- CHANGELOG.md | 8 ++++++-- tests/test_job_key.py | 13 +++++++++++++ tools/job_key.py | 5 ++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 957569b5..2523daa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,13 +63,17 @@ per-file diff commands. everyday Russian listings on freehire, the shipped multi-market portal. The fold is now treated as lossy whenever it drops a letter outside the Latin script, and the existing fallbacks take over: the title half uses the portal's numeric id or the URL hash with the - fragment kept as a readable prefix (`1-4461771225`, `python-bb31e9`), the company half uses + fragment kept as a readable prefix (`1-4461771225`, `python-fb29d7`), the company half uses the NFC-casefold name hash with the fragment as prefix (`ai-d35210`). Latin letters that also lack a decomposition (`ø`, `æ`, `ß`, `ł`) are deliberately not counted, so `Ørsted` still keys as `rsted` and a live Danish `seen_jobs.json` does not re-key; an existing mixed-script entry re-keys once on the next scrape and `--audit` lists it under `keys_not_matching_current_rule`, the same one-time drift #502 accepted. Seven new cases; - the four collision cases fail on master. + the four collision cases fail on master. A follow-up keeps the rule inside its own case: + an empty title whose URL carries no six-digit run keys on the URL-derived basis alone + (`acme_https-example-com-jobs-abc-xyz`) exactly as before, where the first cut had started + appending the digest - unreachable from `/scrape`, which always passes a title, but a key + is a pure function of the posting and should not move for a case the rule was never about. - **`verify_pdf.py --ascii-dates` no longer reads a year-like run inside a longer number as a date** - the year pattern had no digit boundaries, so `2000` inside `120000` or `12000` made diff --git a/tests/test_job_key.py b/tests/test_job_key.py index 8d8fbeae..16f5a862 100644 --- a/tests/test_job_key.py +++ b/tests/test_job_key.py @@ -142,6 +142,19 @@ class MixedScriptNamesKeepTheirIdentity(unittest.TestCase): b = make_key("сбер ai", "ML Engineer", url="https://example.com/2") self.assertEqual(a.split("_", 1)[0], b.split("_", 1)[0]) + def test_empty_title_with_a_non_numeric_url_keeps_its_pre_fragment_key(self): + # The fragment rule must not reach past its own case: an empty title + # with a URL that carries no six-digit run keyed on the URL-derived + # basis alone before #514, and the first cut of the rule had started + # appending the digest there too. Unreachable from /scrape, which always + # passes a title, but the key is a pure function of the posting and + # should not move for a case the rule was never about. + self.assertEqual( + make_key("Acme", "", url="https://example.com/jobs/abc-xyz"), + "acme_https-example-com-jobs-abc-xyz", + ) + self.assertTrue(make_key("Acme", "", url="").startswith("acme_untitled-")) + def test_latin_names_with_accents_and_ligatures_are_not_lossy(self): # NFKD folds these without dropping a letter, so existing keys stay put. self.assertEqual(make_key("Zürich Versicherung", "Ingénieur finance"), "zurich-versicherung_ingenieur-finance") diff --git a/tools/job_key.py b/tools/job_key.py index 99445816..64964113 100644 --- a/tools/job_key.py +++ b/tools/job_key.py @@ -146,7 +146,10 @@ def make_key(company: str, title: str, url: str = "") -> str: # is left to key on. digest_basis = str(url) if url else str(title) digest = hashlib.sha1(digest_basis.encode("utf-8")).hexdigest()[:HASH_LEN] - title_slug = f"{fragment or basis or 'untitled'}-{digest}" + # A lossy fragment needs the digest to tell two postings apart; an + # empty title keeps the bare basis (a URL-derived slug, or + # "untitled-") exactly as before the fragment rule existed. + title_slug = f"{fragment}-{digest}" if fragment else (basis or f"untitled-{digest}") return f"{company_slug}_{title_slug}"