From 894e3917ba8ea2c753acf61a4c0376fec47cbb54 Mon Sep 17 00:00:00 2001
From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com>
Date: Sun, 18 Oct 2026 01:28:42 -0400
Subject: [PATCH] fix(salary): preserve Unicode company identity in lookups
 (#570)

* fix(salary): preserve Unicode company identity in lookups

* fix(salary): restore accent folding after Nordic spellings

Address MadsLorentzen review feedback on #570: keep Unicode identity normalization and restore ASCII-typed accented-name controls on the documented CLI path.
---
 CHANGELOG.md                     |  2 ++
 salary_lookup.py                 | 25 +++++++++----
 tests/test_salary_unicode_cli.py | 62 ++++++++++++++++++++++++++++++++
 3 files changed, 82 insertions(+), 7 deletions(-)
 create mode 100644 tests/test_salary_unicode_cli.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b5a0fde4..7174314f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -463,6 +463,8 @@ per-file diff commands.
   `python3 tools/job_key.py --audit` reports those entries under
   `keys_not_matching_current_rule`; it never rewrites them.
 
+- **Salary lookups preserve Unicode company identities and match ASCII-typed accented names such as `Nestle` to `Nestlé`, while retaining Nordic spelling variants** (#570).
+
 ## [1.7.2] - 2026-09-29
 
 ### Added
diff --git a/salary_lookup.py b/salary_lookup.py
index 33b9eede..ad8779a6 100644
--- a/salary_lookup.py
+++ b/salary_lookup.py
@@ -174,27 +174,38 @@ def load_data():
 
 def normalize(s):
     """Normalize string for robust fuzzy matching."""
-    s = s.lower().strip()
+    s = unicodedata.normalize("NFC", s.casefold()).strip()
     for pat in STRIP_PATTERNS:
         s = re.sub(pat, "", s)
-    s = re.sub(r"[^a-zæøåöäü0-9]", "", s)
-    return s.strip()
+    return "".join(
+        char for char in s
+        if char.isalnum() or unicodedata.category(char).startswith("M")
+    )
 
 
 def anglicize(s):
-    """Convert Danish/Nordic characters to anglicized equivalents."""
+    """Fold Nordic spellings first, then accents for ASCII-typed queries."""
     s = s.lower()
     for danish, english in SPELLING_VARIANTS.items():
         s = s.replace(danish, english)
-    return s
+    return "".join(
+        char for char in unicodedata.normalize("NFKD", s)
+        if not unicodedata.combining(char)
+    )
 
 
 def extract_core_words(s):
     """Extract meaningful words from a company name, ignoring noise."""
-    s = s.lower()
+    s = unicodedata.normalize("NFC", s.casefold())
     for pat in STRIP_PATTERNS:
         s = re.sub(pat, "", s)
-    words = re.findall(r"[a-zæøåöäü0-9]+", s)
+    # Keep the same identity as normalize(), including combining marks in
+    # scripts where NFC cannot compose them into a single letter.
+    s = "".join(
+        char if char.isalnum() or unicodedata.category(char).startswith("M") else " "
+        for char in s
+    )
+    words = s.split()
     return [w for w in words if len(w) > 1]
 
 
diff --git a/tests/test_salary_unicode_cli.py b/tests/test_salary_unicode_cli.py
new file mode 100644
index 00000000..e78cb2b6
--- /dev/null
+++ b/tests/test_salary_unicode_cli.py
@@ -0,0 +1,62 @@
+"""Company identity survives the documented salary-data-file -> CLI path."""
+import json
+import shutil
+import subprocess
+import sys
+import unittest
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+
+TOOL = Path(__file__).resolve().parent.parent / "salary_lookup.py"
+
+
+class UnicodeSalaryCLI(unittest.TestCase):
+    def setUp(self):
+        directory = TemporaryDirectory()
+        self.addCleanup(directory.cleanup)
+        self.root = Path(directory.name)
+        shutil.copyfile(TOOL, self.root / TOOL.name)
+        # Same layout as README_SALARY_TOOL.md's manual-data setup. Keep the
+        # fixture next to an isolated copy of the CLI, never a user's dataset.
+        names = [
+            "腾讯", "東京 AI", "北京 AI", "Acme Corp", "Café Étoile",
+            "Nestlé Danmark A/S", "Ørsted A/S", "Mærsk A/S", "Åland A/S",
+        ]
+        data = {"companies": [
+            {"company": name, "categories": {"all_employees": {"index": 105}}}
+            for name in names
+        ]}
+        (self.root / "salary_data.json").write_text(
+            json.dumps(data, ensure_ascii=False), encoding="utf-8"
+        )
+
+    def lookup(self, query):
+        result = subprocess.run(
+            [sys.executable, str(self.root / TOOL.name), query, "--json"],
+            capture_output=True, text=True, encoding="utf-8",
+        )
+        self.assertEqual(result.returncode, 0, result.stderr + result.stdout)
+        return [row["company"] for row in json.loads(result.stdout)]
+
+    def test_exact_non_latin_company_can_be_found(self):
+        self.assertEqual(self.lookup("腾讯"), ["腾讯"])
+
+    def test_mixed_script_companies_do_not_collapse_to_a_latin_fragment(self):
+        self.assertEqual(self.lookup("東京 AI")[0], "東京 AI")
+
+    def test_canonically_equivalent_spelling_finds_the_same_company(self):
+        self.assertEqual(self.lookup("Cafe\u0301 E\u0301toile"), ["Café Étoile"])
+
+    def test_ascii_control_still_matches(self):
+        self.assertEqual(self.lookup("acme"), ["Acme Corp"])
+
+    def test_ascii_query_finds_accented_company(self):
+        self.assertEqual(self.lookup("Nestle"), ["Nestlé Danmark A/S"])
+
+    def test_nordic_spelling_variants_survive_accent_folding(self):
+        for query, expected in [("Orsted", "Ørsted A/S"),
+                                ("Maersk", "Mærsk A/S"),
+                                ("Aaland", "Åland A/S")]:
+            with self.subTest(query=query):
+                self.assertEqual(self.lookup(query), [expected])