From 120f476a089358363ceaf2528f52edf2854994bd Mon Sep 17 00:00:00 2001
From: Ayobami Adegoke <adegokeayobamiseun@gmail.com>
Date: Mon, 21 Sep 2026 07:41:22 +0100
Subject: [PATCH] fix(rank): preserve Unicode identity in tracker matching
 (#486)

---
 CHANGELOG.md             |  9 +++++++++
 tests/test_rank_state.py | 37 +++++++++++++++++++++++++++++++++++++
 tools/rank_state.py      |  9 ++++++++-
 3 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7ec21657..ccef9f2a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -84,6 +84,15 @@ per-file diff commands.
 
 ### Fixed
 
+- **`/rank` tracker matching preserves Unicode company and role names**
+  (`tools/rank_state.py`, `tests/test_rank_state.py`) - ASCII-only normalization
+  collapsed distinct non-Latin roles to the same empty value and dropped
+  entirely non-Latin companies from tracker exclusions. Match using Unicode
+  case folding and NFC normalization, retaining letters, numbers, and combining
+  marks while continuing to ignore punctuation and spacing. CLI regressions
+  use the standard tracker header and cover distinct names, tracked matches,
+  equivalent accent encodings, and the existing ASCII matching behavior.
+
 - **The Python tools no longer crash on Windows when a posting, company, CV line or file
   name falls outside the ANSI code page** (`tools/rank_state.py`, `tools/job_key.py`,
   `tools/verify_pdf.py`, `tools/verify_layout.py`, `tools/convert_salary_excel.py`,
diff --git a/tests/test_rank_state.py b/tests/test_rank_state.py
index 097689c7..01b09829 100644
--- a/tests/test_rank_state.py
+++ b/tests/test_rank_state.py
@@ -10,6 +10,7 @@ matches Step 4's existing rules exactly - the location_verdict legacy
 migration, the deadline null-is-not-a-correction rule, and verbatim
 strengths/gaps persistence.
 """
+import csv
 import json
 import subprocess
 import sys
@@ -122,6 +123,42 @@ class Candidates(RankStateCase):
                     self.assertEqual([row["key"] for row in out["selected"]], ["b"])
                     self.assertEqual(out["excluded_by_tracker"], 1)
 
+    def test_tracker_exclusion_preserves_unicode_identity(self):
+        # Use the standard /outcome header and the real candidates CLI.
+        header = (
+            "date,company,sector,role,role_type,channel,status,contact_person,"
+            "fit_rating,notes,cv_file,cover_letter_file,source,deadline"
+        ).split(",")
+        cases = (
+            # Tracked company/role, candidate company/title, expected exclusion.
+            ("Acme", "设计师", "Acme", "工程师", False),
+            ("Acme", "工程师", "Acme", "工程师", True),
+            ("腾讯", "Engineer", "腾讯", "Engineer", True),
+            ("腾讯", "Engineer", "阿里巴巴", "Engineer", False),
+            ("Компания", "Инженер", "КОМПАНИЯ", "ИНЖЕНЕР", True),
+            ("Café", "Engineer", "Cafe\u0301", "Engineer", True),
+            ("Straße", "Engineer", "STRASSE", "Engineer", True),
+            ("कला Labs", "Engineer", "कल Labs", "Engineer", False),
+            ("Acme, Inc.", "SOC Analyst", "ACME_INC", "soc-analyst", True),
+        )
+        tracker = self.tmp / "tracker.csv"
+        for company, role, candidate_company, title, excluded in cases:
+            with self.subTest(tracked=(company, role), candidate=(candidate_company, title)):
+                self.write_state({
+                    "candidate": entry(company=candidate_company, title=title),
+                    "other": entry(company="Other", title="Untracked role"),
+                })
+                with tracker.open("w", encoding="utf-8", newline="") as fh:
+                    writer = csv.DictWriter(fh, fieldnames=header)
+                    writer.writeheader()
+                    writer.writerow({"date": TODAY, "company": company, "role": role, "status": "applied"})
+                out = self.run_tool("candidates", "--tracker", str(tracker))
+                self.assertEqual(
+                    [row["key"] for row in out["selected"]],
+                    ["other"] if excluded else ["candidate", "other"],
+                )
+                self.assertEqual(out["excluded_by_tracker"], int(excluded))
+
     def test_focus_filters_on_title_company_and_stored_fit_notes(self):
         self.write_state(
             {
diff --git a/tools/rank_state.py b/tools/rank_state.py
index 454907cc..ea00cb2a 100644
--- a/tools/rank_state.py
+++ b/tools/rank_state.py
@@ -42,6 +42,7 @@ import os
 import re
 import sys
 import tempfile
+import unicodedata
 from datetime import date, timedelta
 from pathlib import Path
 
@@ -97,7 +98,13 @@ def parse_iso(value) -> date | None:
 
 
 def norm(text) -> str:
-    return re.sub(r"[^a-z0-9]", "", str(text or "").lower())
+    """Ignore case and separators without discarding non-Latin identity."""
+    text = unicodedata.normalize("NFC", str(text or "").casefold())
+    # Combining marks can distinguish names even after NFC (e.g. Indic vowels).
+    return "".join(
+        char for char in text
+        if char.isalnum() or unicodedata.category(char).startswith("M")
+    )
 
 
 def tracker_pairs(path: Path) -> set[tuple[str, str]]: