fix(job-key): fold Latin letters without an NFKD decomposition and add --audit --migrate (#637)

* fix(job-key): fold Latin letters without an NFKD decomposition and add --audit --migrate

Closes #537

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

* refactor(job-key): drop the redundant NFC before NFKD

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

* fix(job-key): extend the fold to ı ħ ĸ ŋ ŧ, ignore migrate backups, correct NFC wording

Fold the remaining Latin Extended-A letters without an NFKD decomposition
(ı→i, ħ/Ħ→h, ĸ→k, ŋ/Ŋ→ng, ŧ/Ŧ→t, per Unidecode 1.4.0), so "Kılıç" keys
as "kilic" instead of "klc". Gitignore the seen_jobs.json.bak-* backups
written by --audit --migrate and require the rule in security_guards.
The CHANGELOG and a test comment still described an NFC pass; the fold
runs after NFKD and there is no NFC.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* chore: drop the CHANGELOG hunk, frozen by #800 (release notes come from the PR title)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-authored-by: Mads Lorentzen <madslorentzen_17@hotmail.com>

tools/job_key.py

@@ -36,6 +36,11 @@ readable prefix and the identity comes from the same fallback.
 Usage:
   python3 tools/job_key.py --company "Acme Corp" --title "SOC Analyst (L2)"
   python3 tools/job_key.py --audit job_scraper/seen_jobs.json
+  python3 tools/job_key.py --audit --migrate
+
+`--audit --migrate` rewrites keys that no longer match the current rule, in
+place: backup first, old->new reported, a new key that already exists is
+reported and left alone, and a second run changes nothing.
 
 Exit 0 when a key is produced, or when an audit finds nothing. Exit 1 when an
 audit finds violations.
@@ -44,9 +49,13 @@ audit finds violations.
 import argparse
 import hashlib
 import json
+import os
 import re
+import shutil
 import sys
+import tempfile
 import unicodedata
+from datetime import datetime
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parent.parent
@@ -63,12 +72,39 @@ _NON_SLUG = re.compile(r"[^a-z0-9]+")
 _JOB_ID = re.compile(r"(\d{6,})")
 
 
+# Latin letters with no NFKD decomposition, which the ASCII fold would otherwise
+# drop ("Ørsted" keyed as "rsted", "Mærsk" as "mrsk", "Kılıç" as "klc"). Values
+# follow Unidecode 1.4.0; ICU's Latin-ASCII differs only on "ĸ" (q) and "ŋ" (n).
+# "ĸ" and "ı" have no table capital: "ĸ" has none and "ı"'s is plain "I".
+# "å" is deliberately absent: it decomposes to "a" plus a ring without losing a
+# letter, and "å"->"aa" would move keys that work today for no gain (#537).
+_LATIN_FOLD = {
+    "ø": "o", "Ø": "O",
+    "æ": "ae", "Æ": "AE",
+    "œ": "oe", "Œ": "OE",
+    "ß": "ss", "ẞ": "SS",
+    "þ": "th", "Þ": "TH",
+    "ð": "d", "Ð": "D",
+    "đ": "d", "Đ": "D",
+    "ł": "l", "Ł": "L",
+    "ı": "i",
+    "ħ": "h", "Ħ": "H",
+    "ĸ": "k",
+    "ŋ": "ng", "Ŋ": "NG",
+    "ŧ": "t", "Ŧ": "T",
+}
+_LATIN_FOLD_TABLE = str.maketrans(_LATIN_FOLD)
+
+
 def slugify(text: str) -> str:
     """Lowercase ASCII slug. Non-Latin scripts legitimately reduce to ''."""
     if not text:
         return ""
+    # Fold after NFKD so a letter that decomposes into a fold-table letter plus
+    # a mark ("ǿ" -> "ø" + acute) is folded too.
     decomposed = unicodedata.normalize("NFKD", str(text))
-    ascii_only = decomposed.encode("ascii", "ignore").decode("ascii")
+    folded = decomposed.translate(_LATIN_FOLD_TABLE)
+    ascii_only = folded.encode("ascii", "ignore").decode("ascii")
     return _NON_SLUG.sub("-", ascii_only.lower()).strip("-")
 
 
@@ -81,9 +117,8 @@ def _lost_letters(text: str) -> bool:
     its Latin or digit fragment, and two different names can share it.
 
     Latin letters that also lack a decomposition ("ø", "æ", "ß", "ł") are
-    deliberately NOT counted: "Ørsted" has keyed as "rsted" since the rule
-    existed, and treating it as lossy would re-key every Danish company in a
-    live seen_jobs.json for a collision that does not happen in practice.
+    not counted: slugify folds them through _LATIN_FOLD ("ø" -> "o"), so they
+    keep their letter and are not lossy.
     """
     if not text:
         return False
@@ -177,24 +212,125 @@ def is_legacy_shape(key: str) -> bool:
     )
 
 
-def audit(path: Path) -> int:
+def _load_seen(path: Path):
+    """Return (document, seen) or print the reason and return None."""
     try:
         doc = json.loads(path.read_text(encoding="utf-8"))
     except (OSError, json.JSONDecodeError) as exc:
         print(f"cannot read {path}: {exc}", file=sys.stderr)
-        return 1
+        return None
     seen = doc.get("seen", doc) if isinstance(doc, dict) else doc
     if not isinstance(seen, dict):
         print(f"{path}: expected an object of job entries", file=sys.stderr)
-        return 1
+        return None
     not_objects = [k for k, v in seen.items() if not isinstance(v, dict)]
     if not_objects:
         print(f"{path}: entry is not an object: {not_objects[0]}", file=sys.stderr)
-        return 1
+        return None
     bad_url = [k for k, v in seen.items() if v.get("url") is not None and not isinstance(v["url"], str)]
     if bad_url:
         print(f"{path}: entry url is not a string: {bad_url[0]}", file=sys.stderr)
+        return None
+    return doc, seen
+
+
+def _drifted(seen: dict) -> list[str]:
+    """Canonical keys that differ from what make_key would produce today."""
+    return [
+        k for k, v in seen.items()
+        if is_canonical(k) and k != make_key(v.get("company", ""), v.get("title", ""), v.get("url", ""))
+    ]
+
+
+def _rename_pass(seen: dict) -> tuple[dict[str, str], list[dict]]:
+    """One pass: which drifted keys can move, and which are blocked.
+
+    A key is blocked when its new key is held by an entry that does not itself
+    move in this pass, or when an earlier entry (file order) claims the same
+    new key. A blocked entry stays put, which can in turn block another, so
+    the movers are narrowed until the set is stable. An entry whose new key is
+    held by a mover is fine: the chain A -> x, B(on x) -> y moves together.
+    """
+    target = {
+        k: make_key(seen[k].get("company", ""), seen[k].get("title", ""), seen[k].get("url", ""))
+        for k in _drifted(seen)
+    }
+    movers = dict(target)
+    blocked: dict[str, str] = {}
+    changed = True
+    while changed:
+        changed = False
+        claimed: set[str] = set()
+        for old, new in list(movers.items()):
+            if (new in seen and new not in movers) or new in claimed:
+                blocked[old] = new
+                del movers[old]
+                changed = True
+            else:
+                claimed.add(new)
+    return movers, [{"old": o, "new": n} for o, n in blocked.items()]
+
+
+def migrate(path: Path, doc, seen: dict) -> dict:
+    """Rewrite drifted keys in place; return the report.
+
+    Runs passes until nothing renames. Deterministic (file order decides which
+    of two entries claims a new key), never overwrites an existing key, and
+    idempotent: once every rewritable key is current, a second run finds
+    nothing to change and writes nothing. Collisions in the report are the
+    final ones only.
+    """
+    current = dict(seen)
+    migrated: list[dict] = []
+    collisions: list[dict] = []
+    for _ in range(len(seen) + 1):
+        renames, collisions = _rename_pass(current)
+        if not renames:
+            break
+        migrated.extend({"old": o, "new": n} for o, n in renames.items())
+        current = {renames.get(k, k): v for k, v in current.items()}
+    report = {
+        "migrated": migrated,
+        "migrate_collisions": collisions,
+        "backup": None,
+    }
+    if not migrated:
+        return report
+
+    backup = path.with_name(f"{path.name}.bak-{datetime.now():%Y%m%d-%H%M%S}")
+    shutil.copy2(path, backup)
+    report["backup"] = str(backup)
+
+    out = dict(doc, seen=current) if isinstance(doc, dict) and "seen" in doc else current
+    # Same dump settings as tools/rank_state.save_state, so a migration changes
+    # only the lines that carry a renamed key. Atomic replace: a half-written
+    # seen_jobs.json loses the scrape history.
+    fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            json.dump(out, fh, indent=2, ensure_ascii=False)
+            fh.write("\n")
+        os.replace(tmp, path)
+    except BaseException:
+        Path(tmp).unlink(missing_ok=True)
+        raise
+    return report
+
+
+def audit(path: Path, do_migrate: bool = False) -> int:
+    loaded = _load_seen(path)
+    if loaded is None:
         return 1
+    doc, seen = loaded
+
+    migration = {}
+    if do_migrate:
+        migration = migrate(path, doc, seen)
+        if migration["migrated"]:
+            loaded = _load_seen(path)
+            if loaded is None:
+                return 1
+            doc, seen = loaded
 
     malformed = [k for k in seen if not is_canonical(k) and not is_legacy_shape(k)]
     legacy = [k for k in seen if is_legacy_shape(k)]
@@ -205,11 +341,8 @@ def audit(path: Path) -> int:
             by_url.setdefault(url, []).append(key)
     duplicates = {u: ks for u, ks in by_url.items() if len(ks) > 1}
     # A key that does not match what make_key would produce today is drift, not
-    # damage: reported separately so a rename is a choice, never automatic.
-    drift = [
-        k for k, v in seen.items()
-        if is_canonical(k) and k != make_key(v.get("company", ""), v.get("title", ""), v.get("url", ""))
-    ]
+    # damage: reported separately so a rename is a choice (--migrate), never automatic.
+    drift = _drifted(seen)
 
     print(json.dumps({
         "entries": len(seen),
@@ -217,6 +350,7 @@ def audit(path: Path) -> int:
         "legacy_three_part_keys": legacy,
         "duplicate_urls": duplicates,
         "keys_not_matching_current_rule": len(drift),
+        **migration,
     }, indent=2, ensure_ascii=False))
     return 1 if (malformed or duplicates) else 0
 
@@ -241,10 +375,13 @@ def main() -> int:
     ap.add_argument("--title")
     ap.add_argument("--url", default="")
     ap.add_argument("--audit", nargs="?", const=str(STATE), metavar="STATE_JSON")
+    ap.add_argument("--migrate", action="store_true", help="with --audit: rewrite drifted keys in place (backup first)")
     args = ap.parse_args()
 
+    if args.migrate and not args.audit:
+        ap.error("--migrate needs --audit")
     if args.audit:
-        return audit(Path(args.audit))
+        return audit(Path(args.audit), args.migrate)
     if args.company is None or args.title is None:
         ap.error("give --company and --title, or --audit")
     print(make_key(args.company, args.title, args.url))

tools/security_guards.py

@@ -168,6 +168,7 @@ REQUIRED_IGNORE_RULES = [
     # to its own directory, so the state file lands under .claude/skills/... and
     # a repo-rooted rule silently fails to match it.
     "**/job_scraper/seen_jobs.json",
+    "**/job_scraper/seen_jobs.json.bak-*",
     "**/job_scraper/notion_sync.json",
     "**/job_scraper/*.md",
     "*_BehavioralReport.pdf",