mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-11-02 21:22:51 +00:00
fix(job_key): hash the URL alone so a re-listed non-Latin title keeps one key (#502)
* fix(job_key): hash the URL alone so a re-listed non-Latin title keeps one key * docs(changelog): record the job_key URL-identity fix Adds the [Unreleased] / Fixed entry for #501, naming tools/job_key.py and tests/test_job_key_url_identity.py. The entry states the migration consequence in bold: an entry written before this fix, for a non-Latin title on a portal without a numeric id, re-keys once on the next scrape. It also names where those entries surface, keys_not_matching_current_rule, and that --audit never rewrites them. Verified that claim rather than asserting it. A seen_jobs.json holding one entry under its pre-fix key for a Cyrillic-titled freehire URL reports keys_not_matching_current_rule: 1 under the fixed audit. The same audit over a real 884-entry seen_jobs.json reports 0, because those keys are the company_title form that is_canonical() excludes from the drift check.
This commit is contained in:
1 parent
d7287ec834
commit
a56b8d0414
3 files changed
+56
-1
No files matched your search
@@ -70,6 +70,35 @@ class MakeKey(unittest.TestCase):
|
||||
self.assertTrue(is_canonical(key))
|
||||
self.assertFalse(key.startswith("_"))
|
||||
|
||||
def test_one_url_keeps_one_key_when_a_non_latin_title_is_re_listed(self):
|
||||
# freehire is the shipped multi-market portal, and its public API
|
||||
# returns Cyrillic and Greek titles. Its real slugs carry no run of six
|
||||
# digits, so the numeric-id branch above never fires for them and the
|
||||
# hash is the only key half left. Hashing the title alongside the URL
|
||||
# made that hash move whenever a portal re-listed the same posting with
|
||||
# the title altered - including a mere case change, which this portal
|
||||
# really does emit ("Инженер" and "инженер" both appear).
|
||||
#
|
||||
# URL and slug are real values from freehire's public API, not
|
||||
# constructed: https://freehire.me/api/v1/agent/jobs/search?q=инженер
|
||||
url = "https://freehire.me/jobs/inzhener-ooo-chen-hlk3qjfg"
|
||||
company = "ООО Чен"
|
||||
first = make_key(company, "Инженер", url=url)
|
||||
recased = make_key(company, "инженер", url=url)
|
||||
retitled = make_key(company, "Инженер-механик", url=url)
|
||||
self.assertTrue(is_canonical(first))
|
||||
self.assertEqual(first, recased)
|
||||
self.assertEqual(first, retitled)
|
||||
|
||||
def test_distinct_urls_still_get_distinct_keys(self):
|
||||
# The counterpart to the above: collapsing title variants must not
|
||||
# collapse two genuinely different postings from one company. Both
|
||||
# slugs are real freehire values.
|
||||
company = "ООО Чен"
|
||||
a = make_key(company, "Инженер", url="https://freehire.me/jobs/inzhener-ooo-chen-hlk3qjfg")
|
||||
b = make_key(company, "Инженер", url="https://freehire.me/jobs/inzhener-mup-g-khabarovska-tep")
|
||||
self.assertNotEqual(a, b)
|
||||
|
||||
|
||||
class CompanyFallbackCLI(unittest.TestCase):
|
||||
def key_for(self, company, url="https://example.com/jobs/123456"):
|
||||
|
||||
Reference in new issue
Block a user