From 44f95de6e9e4e8f8d2f65cbd36dcab88a3a70d91 Mon Sep 17 00:00:00 2001 From: Jallepalli Harsha Vardhan <87539405+harshajv7981@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:30:25 +0530 Subject: [PATCH] fix(scrape): keep C++, C# and C job titles distinct in dedup and keys (#830) norm() dropped "+" and "#", so "C++ Developer", "C# Developer" and "C Developer" at one company normalized alike: /scrape counted a new posting as already seen and /rank excluded it once the user had applied to the other. slugify() did the same for canonical keys, so keep both characters (spelled out as plus/sharp in keys). Keys stored under the old rule are re-keyed by job_key.py --audit --migrate; the store collision error now says so. --- tests/test_scrape_state.py | 12 ++++++++++++ tools/job_key.py | 2 ++ tools/rank_state.py | 3 ++- tools/scrape_state.py | 5 ++++- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/test_scrape_state.py b/tests/test_scrape_state.py index d2dbe2c0..89bfd057 100644 --- a/tests/test_scrape_state.py +++ b/tests/test_scrape_state.py @@ -70,6 +70,18 @@ class ScrapeStateTests(unittest.TestCase): self.assertEqual(stored["legacy_key"], old["legacy_key"]) self.assertEqual(stored["fresh_builder"], fresh) + def test_c_sharp_and_c_plus_plus_titles_are_different_postings(self): + base = { + "company": "Acme", "first_seen": "2026-10-21", "posted_date": None, "deadline": None, + "fit": "high", "status": "new", "portal": "example-search", "source": "cli", + } + cpp = {**base, "title": "C++ Developer", "url": "https://jobs.test/1111111"} + self.assertEqual(len(self.run_tool("store", [cpp])["added"]), 1) + csharp = {**base, "title": "C# Developer", "url": "https://jobs.test/2222222"} + out = self.run_tool("candidates", [csharp]) + self.assertEqual(out["selected"], [csharp]) + self.assertEqual(len(self.run_tool("store", [csharp])["added"]), 1) + def test_store_creates_missing_history_and_rejects_partial_entries_without_writing(self): good = { "title": "Builder", "company": "Fresh", "url": "https://jobs.test/fresh", diff --git a/tools/job_key.py b/tools/job_key.py index 8e17ad1f..6738338d 100644 --- a/tools/job_key.py +++ b/tools/job_key.py @@ -105,6 +105,8 @@ def slugify(text: str) -> str: decomposed = unicodedata.normalize("NFKD", str(text)) folded = decomposed.translate(_LATIN_FOLD_TABLE) ascii_only = folded.encode("ascii", "ignore").decode("ascii") + # Spell out "+" and "#" so "C++", "C#" and "C" titles keep distinct keys. + ascii_only = ascii_only.replace("+", "plus").replace("#", "sharp") return _NON_SLUG.sub("-", ascii_only.lower()).strip("-") diff --git a/tools/rank_state.py b/tools/rank_state.py index 8deecc9a..ed92a54b 100644 --- a/tools/rank_state.py +++ b/tools/rank_state.py @@ -146,7 +146,8 @@ def norm(text) -> str: # Combining marks can distinguish names even after NFC (e.g. Indic vowels). return "".join( char for char in text - if char.isalnum() or unicodedata.category(char).startswith("M") + # "+" and "#" carry identity: "C++", "C#" and "C" are different jobs. + if char.isalnum() or char in "+#" or unicodedata.category(char).startswith("M") ) diff --git a/tools/scrape_state.py b/tools/scrape_state.py index a6727443..dd8f8bb4 100644 --- a/tools/scrape_state.py +++ b/tools/scrape_state.py @@ -117,7 +117,10 @@ def run(args) -> int: raise ValueError("job has an invalid portal or source") key = make_key(job["company"], job["title"], job["url"]) if key in seen: - raise ValueError(f"canonical key already exists for another posting: {key}") + raise ValueError( + f"canonical key already exists for another posting: {key} " + "(keys stored before a key-rule change: run python3 tools/job_key.py --audit --migrate)" + ) seen[key] = job if selected: args.state.parent.mkdir(parents=True, exist_ok=True)