fix(scrape): make seen_jobs.json keys a pure function of the posting (#441)

* fix(scraper): make seen_jobs.json keys a pure function of the posting, and clean up the drift

The dedup key was prose only ("<url_or_company_title_key>"), so different
/scrape runs slugified company+title differently and the state file
accumulated two failures: keys carrying "&", "/", "," and ":" that break
the archive-folder path /apply and /outcome derive from company+role
(documents/README.md's subfolder rule exists because of exactly this),
and the same posting stored twice under two different truncations of a
long title (two Deloitte entries, one job, one URL).

tools/job_key.py makes the key a pure, deterministic function of
company+title+url: a strict allowlist slug, length-capped with a hash of
the full slug so truncation never collides across runs, and a fallback
to the portal's numeric job id when a non-Latin title slugifies to
nothing (a real prior entry, "securion_", would have collided with
every future non-Latin posting from that company).

--audit finds both failure classes in an existing seen_jobs.json without
guessing at a fix: malformed keys (real damage), a legacy three-part
company_title_location shape (harmless but not what the current rule
produces, so it silently re-duplicates on the next scrape), and
duplicate URLs. Ran it against this workspace's file and re-keyed the 15
entries it found - 7 malformed, 8 legacy-shape - verified byte-for-byte
against the pre-cleanup copy that no entry's data changed, only its key.

tests/test_job_key.py (16 tests) covers the slugify rules, the
truncation-hash behavior, both non-Latin fallback paths, and the audit
CLI's exit codes.

* fix(scrape): call the key helper from Step 4 instead of slugifying ad hoc

The helper added in the previous commit is only load-bearing if the spec
calls it. Step 4 described the key as prose ("<url_or_company_title_key>"),
which is what let each run slugify its own way. Step 4 now names the
command, and the schema shows the key's provenance.

Renumbers the trailing list item; no other behaviour in the step changes.

* docs(changelog): record the job-key rule under Unreleased

* fix(scrape): preserve dedup continuity across key rule

* changelog: note that existing seen_jobs.json entries need no migration (#441)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013fqqLgQSnwgWkv98twQhHi

---------

Co-authored-by: nox <nox@Mac.home>
Co-authored-by: Mads Lorentzen <madslorentzen_17@hotmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

tests/test_job_key.py

@@ -0,0 +1,137 @@
+"""Tests for tools/job_key.py - the canonical seen_jobs.json key function.
+
+/scrape's key rule was prose only, so runs slugified inconsistently and the
+state file accumulated two failures: keys carrying "/", "," and "&" that break
+the archive-folder path `/apply`/`/outcome` derive from company+role, and the
+same job stored twice under two different truncations of a long title. These
+pin the fix - a pure, deterministic function of company+title(+url) - and the
+audit that finds both failure classes in an existing file.
+"""
+import json
+import subprocess
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))
+from job_key import is_canonical, is_legacy_shape, make_key, slugify  # noqa: E402
+
+REPO = Path(__file__).resolve().parent.parent
+TOOL = REPO / "tools" / "job_key.py"
+
+
+class Slugify(unittest.TestCase):
+    def test_basic(self):
+        self.assertEqual(slugify("Acme Corp"), "acme-corp")
+
+    def test_strips_punctuation_that_breaks_paths(self):
+        self.assertEqual(slugify("Ops Consulting, LLC"), "ops-consulting-llc")
+        self.assertEqual(slugify("Penetration Tester / Red Teamer"), "penetration-tester-red-teamer")
+        self.assertEqual(slugify("Junior Cybersecurity Analyst (OT/IoT)"), "junior-cybersecurity-analyst-ot-iot")
+
+    def test_non_latin_script_reduces_to_empty(self):
+        self.assertEqual(slugify("시큐리온"), "")
+        self.assertEqual(slugify("Код Безопасности"), "")
+
+
+class MakeKey(unittest.TestCase):
+    def test_shape(self):
+        key = make_key("Acme Corp", "SOC Analyst (L2)")
+        self.assertEqual(key, "acme-corp_soc-analyst-l2")
+        self.assertTrue(is_canonical(key))
+
+    def test_deterministic_across_calls(self):
+        title = "Cyber Intelligence Center Security Analyst with an unusually long title"
+        self.assertEqual(make_key("Deloitte", title), make_key("Deloitte", title))
+
+    def test_long_titles_never_collide_after_truncation(self):
+        """The bug that produced two Deloitte entries for one posting: two
+        runs truncated the same long title at different points. A hash of the
+        full slug makes truncation deterministic instead of lossy."""
+        a = make_key("Deloitte", "Cyber Intelligence Center Security Analyst with trailing text A")
+        b = make_key("Deloitte", "Cyber Intelligence Center Security Analyst with trailing text B")
+        self.assertNotEqual(a, b)
+
+    def test_non_latin_title_falls_back_to_the_portal_job_id(self):
+        key = make_key(
+            "SecuriON",
+            "안드로이드 앱(악성코드) 분석가 채용",
+            url="https://kr.linkedin.com/jobs/view/x-4461771225",
+        )
+        self.assertEqual(key, "securion_4461771225")
+
+    def test_non_latin_title_with_no_url_id_still_produces_a_canonical_key(self):
+        key = make_key("SecuriON", "안드로이드 앱 분석가", url="")
+        self.assertTrue(is_canonical(key))
+        self.assertNotEqual(key, "securion_")
+
+    def test_non_latin_company_falls_back_without_producing_a_bare_prefix(self):
+        key = make_key("Код Безопасности", "Malware Analytic", url="")
+        self.assertTrue(is_canonical(key))
+        self.assertFalse(key.startswith("_"))
+
+
+class CanonicalAndLegacyShape(unittest.TestCase):
+    def test_canonical_accepts_company_underscore_title(self):
+        self.assertTrue(is_canonical("acme-corp_soc-analyst"))
+
+    def test_canonical_rejects_path_breaking_characters(self):
+        for bad in ("deloitte_junior-cybersecurity-analyst-(ot/iot)",
+                    "neverhack-estonia_penetration-tester-/-red-teamer",
+                    "ops-consulting,-llc_malware-analyst",
+                    "",
+                    "securion_"):
+            self.assertFalse(is_canonical(bad), f"{bad!r} should not be canonical")
+
+    def test_legacy_three_part_shape_is_flagged_separately_from_malformed(self):
+        self.assertTrue(is_legacy_shape("nviso-security_soc-analyst_athens"))
+        self.assertFalse(is_canonical("nviso-security_soc-analyst_athens"))
+        # A malformed key (bad characters) is never also reported as legacy shape.
+        self.assertFalse(is_legacy_shape("deloitte_junior-cybersecurity-analyst-(ot/iot)"))
+
+
+class AuditCLI(unittest.TestCase):
+    def run_audit(self, seen: dict) -> tuple[dict, int]:
+        import tempfile
+
+        with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fh:
+            json.dump({"seen": seen}, fh)
+            path = fh.name
+        proc = subprocess.run(
+            [sys.executable, str(TOOL), "--audit", path], capture_output=True, text=True
+        )
+        return json.loads(proc.stdout), proc.returncode
+
+    def test_clean_state_exits_zero(self):
+        report, code = self.run_audit({"acme_soc-analyst": {"company": "Acme", "title": "SOC Analyst"}})
+        self.assertEqual(code, 0)
+        self.assertEqual(report["malformed_keys"], [])
+        self.assertEqual(report["duplicate_urls"], {})
+
+    def test_malformed_key_exits_nonzero(self):
+        report, code = self.run_audit(
+            {"deloitte_junior-cybersecurity-analyst-(ot/iot)": {"company": "Deloitte", "title": "x"}}
+        )
+        self.assertEqual(code, 1)
+        self.assertIn("deloitte_junior-cybersecurity-analyst-(ot/iot)", report["malformed_keys"])
+
+    def test_duplicate_url_exits_nonzero(self):
+        report, code = self.run_audit(
+            {
+                "a": {"company": "Acme", "title": "x", "url": "https://x/1"},
+                "b": {"company": "Acme", "title": "y", "url": "https://x/1"},
+            }
+        )
+        self.assertEqual(code, 1)
+        self.assertIn("https://x/1", report["duplicate_urls"])
+
+    def test_legacy_shape_alone_does_not_fail_the_audit(self):
+        """Harmless drift, not damage - the sweep-worthy rewrite is a decision
+        the maintainer makes, not something the audit enforces."""
+        report, code = self.run_audit({"acme_soc-analyst_athens": {"company": "Acme", "title": "x"}})
+        self.assertEqual(code, 0)
+        self.assertIn("acme_soc-analyst_athens", report["legacy_three_part_keys"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_scrape_contract.py

@@ -136,5 +136,21 @@ class SeenJobsPostingDateTests(unittest.TestCase):
         )
 
 
+class SeenJobsDedupContinuityTests(unittest.TestCase):
+    """The new key rule must not replay jobs stored under legacy keys."""
+
+    def test_existing_urls_are_seen_regardless_of_key(self):
+        text = SCRAPER_SKILL.read_text(encoding="utf-8")
+        self.assertRegex(
+            text,
+            r"URL matches any existing `seen_jobs\.json` entry, regardless of\s+that entry's key",
+            "legacy seen_jobs entries must be matched by URL during the key-rule transition",
+        )
+
+    def test_step4_presentation_mentions_url_deduplication(self):
+        text = SCRAPER_SKILL.read_text(encoding="utf-8")
+        self.assertRegex(text, r"matched by URL or\s+company\+title")
+
+
 if __name__ == "__main__":
     unittest.main()