From a56b8d0414cc3232390a8ee8ca9fa26972ba7d65 Mon Sep 17 00:00:00 2001 From: Ahmed Ashraf <51435340+ahmedashraffcih@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:45:19 +0400 Subject: [PATCH] fix(job_key): hash the URL alone so a re-listed non-Latin title keeps one key (#502) * fix(job_key): hash the URL alone so a re-listed non-Latin title keeps one key * docs(changelog): record the job_key URL-identity fix Adds the [Unreleased] / Fixed entry for #501, naming tools/job_key.py and tests/test_job_key_url_identity.py. The entry states the migration consequence in bold: an entry written before this fix, for a non-Latin title on a portal without a numeric id, re-keys once on the next scrape. It also names where those entries surface, keys_not_matching_current_rule, and that --audit never rewrites them. Verified that claim rather than asserting it. A seen_jobs.json holding one entry under its pre-fix key for a Cyrillic-titled freehire URL reports keys_not_matching_current_rule: 1 under the fixed audit. The same audit over a real 884-entry seen_jobs.json reports 0, because those keys are the company_title form that is_canonical() excludes from the drift check. --- CHANGELOG.md | 15 +++++++++++++++ tests/test_job_key.py | 29 +++++++++++++++++++++++++++++ tools/job_key.py | 13 ++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7487b7ef..5c755084 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,21 @@ per-file diff commands. ## [Unreleased] +### Fixed + +- **One posting on a portal without a numeric id no longer gets a second key when its + title is re-listed with different casing** (#501, `tools/job_key.py`, + `tests/test_job_key.py`) - the key hashed a slugified title alongside the URL whenever + `normalizeId` found no six-digit run in the path, which is every freehire posting. The + live API returns both `Инженер` and `инженер` across rows, and the slugifier maps those + to different slugs, so one URL could be stored under several keys. `/rank` builds its + exclusion set from `seen_jobs.json`, so a posting already seen could be presented again. + The key is now a hash of the URL alone: the URL is the identity `/scrape` stores, and + the title was never a stable half of it. **Entries written before this fix for a + non-Latin title on a portal without a numeric id will re-key once on the next scrape.** + `python3 tools/job_key.py --audit` reports those entries under + `keys_not_matching_current_rule`; it never rewrites them. + ## [1.7.2] - 2026-09-29 ### Added diff --git a/tests/test_job_key.py b/tests/test_job_key.py index b56779b2..b37d8545 100644 --- a/tests/test_job_key.py +++ b/tests/test_job_key.py @@ -70,6 +70,35 @@ class MakeKey(unittest.TestCase): self.assertTrue(is_canonical(key)) self.assertFalse(key.startswith("_")) + def test_one_url_keeps_one_key_when_a_non_latin_title_is_re_listed(self): + # freehire is the shipped multi-market portal, and its public API + # returns Cyrillic and Greek titles. Its real slugs carry no run of six + # digits, so the numeric-id branch above never fires for them and the + # hash is the only key half left. Hashing the title alongside the URL + # made that hash move whenever a portal re-listed the same posting with + # the title altered - including a mere case change, which this portal + # really does emit ("Инженер" and "инженер" both appear). + # + # URL and slug are real values from freehire's public API, not + # constructed: https://freehire.me/api/v1/agent/jobs/search?q=инженер + url = "https://freehire.me/jobs/inzhener-ooo-chen-hlk3qjfg" + company = "ООО Чен" + first = make_key(company, "Инженер", url=url) + recased = make_key(company, "инженер", url=url) + retitled = make_key(company, "Инженер-механик", url=url) + self.assertTrue(is_canonical(first)) + self.assertEqual(first, recased) + self.assertEqual(first, retitled) + + def test_distinct_urls_still_get_distinct_keys(self): + # The counterpart to the above: collapsing title variants must not + # collapse two genuinely different postings from one company. Both + # slugs are real freehire values. + company = "ООО Чен" + a = make_key(company, "Инженер", url="https://freehire.me/jobs/inzhener-ooo-chen-hlk3qjfg") + b = make_key(company, "Инженер", url="https://freehire.me/jobs/inzhener-mup-g-khabarovska-tep") + self.assertNotEqual(a, b) + class CompanyFallbackCLI(unittest.TestCase): def key_for(self, company, url="https://example.com/jobs/123456"): diff --git a/tools/job_key.py b/tools/job_key.py index 7effd2f3..42b8a279 100644 --- a/tools/job_key.py +++ b/tools/job_key.py @@ -99,7 +99,18 @@ def make_key(company: str, title: str, url: str = "") -> str: title_slug = match.group(1) else: basis = slugify(unicodedata.normalize("NFKD", str(title or url or ""))) - digest = hashlib.sha1((str(title) + str(url)).encode("utf-8")).hexdigest()[:HASH_LEN] + # Hash the URL alone when there is one. The URL is the posting's + # identity; the title is not. Including the title made the key + # change whenever a portal re-listed the same posting with the + # title altered, which stores one job twice - the failure this + # whole helper exists to prevent. Portals whose ids carry no run + # of six digits never reach the branch above, so for them this + # hash is the only key half there is: freehire's real slugs look + # like "inzhener-ooo-chen-hlk3qjfg", and its Cyrillic and Greek + # titles slugify to nothing. With no URL, the title is all that + # is left to key on. + digest_basis = str(url) if url else str(title) + digest = hashlib.sha1(digest_basis.encode("utf-8")).hexdigest()[:HASH_LEN] title_slug = basis or f"untitled-{digest}" return f"{company_slug}_{title_slug}"