diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c2d9492..72ce6e8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,14 @@ per-file diff commands. ### Fixed +- **Distinct non-Latin company names no longer share the unknown-company job key** + (`tools/job_key.py`, `tests/test_job_key.py`) - when a non-empty company name + has no ASCII slug, derive its fallback from a hash of the normalized name. + Missing names retain `unknown-company`, and existing ASCII keys are unchanged. + CLI tests cover distinct companies, case and canonical Unicode equivalence, + and stable keys across posting URLs. Existing state is not rewritten; `/scrape` + already recognizes stored postings by URL regardless of their previous key. + - **`/rank` tracker exclusion handles UTF-8 BOMs on reordered CSV headers** (`tools/rank_state.py`, `tests/test_rank_state.py`) - when `company` or `role` is the first column, a leading BOM becomes part of the header name and an diff --git a/tests/test_job_key.py b/tests/test_job_key.py index 61e9551a..c0c4a3f1 100644 --- a/tests/test_job_key.py +++ b/tests/test_job_key.py @@ -71,6 +71,34 @@ class MakeKey(unittest.TestCase): self.assertFalse(key.startswith("_")) +class CompanyFallbackCLI(unittest.TestCase): + def key_for(self, company, url="https://example.com/jobs/123456"): + proc = subprocess.run( + [sys.executable, str(TOOL), "--company", company, + "--title", "Software Engineer", "--url", url], + capture_output=True, text=True, check=True, + ) + return proc.stdout.strip() + + def test_distinct_non_latin_companies_have_distinct_keys(self): + keys = [self.key_for(company) for company in ("腾讯", "阿里巴巴", "")] + self.assertEqual(len(set(keys)), 3) + self.assertTrue(all(is_canonical(key) for key in keys)) + + def test_company_fallback_is_stable_across_case_normalization_and_urls(self): + for first, second in (("КОМПАНИЯ", "компания"), ("ガンホー", "カ\u3099ンホー")): + with self.subTest(first=first, second=second): + self.assertEqual( + self.key_for(first), + self.key_for(second, "https://example.com/jobs/654321"), + ) + + def test_missing_company_and_existing_ascii_keys_are_unchanged(self): + self.assertEqual(self.key_for(""), "unknown-company_software-engineer") + self.assertEqual(self.key_for(" "), "unknown-company_software-engineer") + self.assertEqual(self.key_for("Acme Corp"), "acme-corp_software-engineer") + + class CanonicalAndLegacyShape(unittest.TestCase): def test_canonical_accepts_company_underscore_title(self): self.assertTrue(is_canonical("acme-corp_soc-analyst")) diff --git a/tools/job_key.py b/tools/job_key.py index d84af9f4..fd7894b4 100644 --- a/tools/job_key.py +++ b/tools/job_key.py @@ -84,7 +84,12 @@ def _cap(slug: str, limit: int) -> str: def make_key(company: str, title: str, url: str = "") -> str: """The canonical seen_jobs.json key for one posting.""" - company_slug = _cap(slugify(company), COMPANY_MAX) or "unknown-company" + company_slug = _cap(slugify(company), COMPANY_MAX) + if not company_slug: + name = unicodedata.normalize("NFC", str(company or "").strip().casefold()) + # An absent name stays unknown; a non-Latin name still has an identity. + digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:HASH_LEN] + company_slug = f"company-{digest}" if name else "unknown-company" title_slug = _cap(slugify(title), TITLE_MAX) if not title_slug: # No Latin characters in the title. The portal's own numeric id is the