mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-11-02 13:12:50 +00:00
fix(job_key): keep the bare basis for an empty title whose URL has no six-digit run (#523)
The mixed-script rule from #514 appended the digest to `basis` as well, so an empty title with a URL that carries no six-digit run moved from "acme_https-example-com-jobs-abc-xyz" to "..._5ca8c6". Unreachable from /scrape, which always passes a title, but a key is a pure function of the posting and should not move for a case the rule was never about. A lossy fragment still takes the digest; an empty title keeps the bare basis or "untitled-<hash>" exactly as before #514. One new case pins the pre-#514 output and fails on current master. The CHANGELOG example for the Python-разработчик key is corrected to its real digest (python-fb29d7). Follow-up suggested in the #514 merge note.
This commit is contained in:
1 parent
760bbc34cf
commit
da2aee7bf1
3 files changed
+23
-3
No files matched your search
+6
-2
@@ -63,13 +63,17 @@ per-file diff commands.
|
|||||||
everyday Russian listings on freehire, the shipped multi-market portal. The fold is now
|
everyday Russian listings on freehire, the shipped multi-market portal. The fold is now
|
||||||
treated as lossy whenever it drops a letter outside the Latin script, and the existing
|
treated as lossy whenever it drops a letter outside the Latin script, and the existing
|
||||||
fallbacks take over: the title half uses the portal's numeric id or the URL hash with the
|
fallbacks take over: the title half uses the portal's numeric id or the URL hash with the
|
||||||
fragment kept as a readable prefix (`1-4461771225`, `python-bb31e9`), the company half uses
|
fragment kept as a readable prefix (`1-4461771225`, `python-fb29d7`), the company half uses
|
||||||
the NFC-casefold name hash with the fragment as prefix (`ai-d35210`). Latin letters that
|
the NFC-casefold name hash with the fragment as prefix (`ai-d35210`). Latin letters that
|
||||||
also lack a decomposition (`ø`, `æ`, `ß`, `ł`) are deliberately not counted, so `Ørsted`
|
also lack a decomposition (`ø`, `æ`, `ß`, `ł`) are deliberately not counted, so `Ørsted`
|
||||||
still keys as `rsted` and a live Danish `seen_jobs.json` does not re-key; an existing
|
still keys as `rsted` and a live Danish `seen_jobs.json` does not re-key; an existing
|
||||||
mixed-script entry re-keys once on the next scrape and `--audit` lists it under
|
mixed-script entry re-keys once on the next scrape and `--audit` lists it under
|
||||||
`keys_not_matching_current_rule`, the same one-time drift #502 accepted. Seven new cases;
|
`keys_not_matching_current_rule`, the same one-time drift #502 accepted. Seven new cases;
|
||||||
the four collision cases fail on master.
|
the four collision cases fail on master. A follow-up keeps the rule inside its own case:
|
||||||
|
an empty title whose URL carries no six-digit run keys on the URL-derived basis alone
|
||||||
|
(`acme_https-example-com-jobs-abc-xyz`) exactly as before, where the first cut had started
|
||||||
|
appending the digest - unreachable from `/scrape`, which always passes a title, but a key
|
||||||
|
is a pure function of the posting and should not move for a case the rule was never about.
|
||||||
|
|
||||||
- **`verify_pdf.py --ascii-dates` no longer reads a year-like run inside a longer number as a
|
- **`verify_pdf.py --ascii-dates` no longer reads a year-like run inside a longer number as a
|
||||||
date** - the year pattern had no digit boundaries, so `2000` inside `120000` or `12000` made
|
date** - the year pattern had no digit boundaries, so `2000` inside `120000` or `12000` made
|
||||||
|
|||||||
@@ -142,6 +142,19 @@ class MixedScriptNamesKeepTheirIdentity(unittest.TestCase):
|
|||||||
b = make_key("сбер ai", "ML Engineer", url="https://example.com/2")
|
b = make_key("сбер ai", "ML Engineer", url="https://example.com/2")
|
||||||
self.assertEqual(a.split("_", 1)[0], b.split("_", 1)[0])
|
self.assertEqual(a.split("_", 1)[0], b.split("_", 1)[0])
|
||||||
|
|
||||||
|
def test_empty_title_with_a_non_numeric_url_keeps_its_pre_fragment_key(self):
|
||||||
|
# The fragment rule must not reach past its own case: an empty title
|
||||||
|
# with a URL that carries no six-digit run keyed on the URL-derived
|
||||||
|
# basis alone before #514, and the first cut of the rule had started
|
||||||
|
# appending the digest there too. Unreachable from /scrape, which always
|
||||||
|
# passes a title, but the key is a pure function of the posting and
|
||||||
|
# should not move for a case the rule was never about.
|
||||||
|
self.assertEqual(
|
||||||
|
make_key("Acme", "", url="https://example.com/jobs/abc-xyz"),
|
||||||
|
"acme_https-example-com-jobs-abc-xyz",
|
||||||
|
)
|
||||||
|
self.assertTrue(make_key("Acme", "", url="").startswith("acme_untitled-"))
|
||||||
|
|
||||||
def test_latin_names_with_accents_and_ligatures_are_not_lossy(self):
|
def test_latin_names_with_accents_and_ligatures_are_not_lossy(self):
|
||||||
# NFKD folds these without dropping a letter, so existing keys stay put.
|
# NFKD folds these without dropping a letter, so existing keys stay put.
|
||||||
self.assertEqual(make_key("Zürich Versicherung", "Ingénieur finance"), "zurich-versicherung_ingenieur-finance")
|
self.assertEqual(make_key("Zürich Versicherung", "Ingénieur finance"), "zurich-versicherung_ingenieur-finance")
|
||||||
|
|||||||
+4
-1
@@ -146,7 +146,10 @@ def make_key(company: str, title: str, url: str = "") -> str:
|
|||||||
# is left to key on.
|
# is left to key on.
|
||||||
digest_basis = str(url) if url else str(title)
|
digest_basis = str(url) if url else str(title)
|
||||||
digest = hashlib.sha1(digest_basis.encode("utf-8")).hexdigest()[:HASH_LEN]
|
digest = hashlib.sha1(digest_basis.encode("utf-8")).hexdigest()[:HASH_LEN]
|
||||||
title_slug = f"{fragment or basis or 'untitled'}-{digest}"
|
# A lossy fragment needs the digest to tell two postings apart; an
|
||||||
|
# empty title keeps the bare basis (a URL-derived slug, or
|
||||||
|
# "untitled-<hash>") exactly as before the fragment rule existed.
|
||||||
|
title_slug = f"{fragment}-{digest}" if fragment else (basis or f"untitled-{digest}")
|
||||||
return f"{company_slug}_{title_slug}"
|
return f"{company_slug}_{title_slug}"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user