refactor(salary): optimize search match scoring and normalize Excel category keys (#101)

This commit improves the performance and consistency of the salary tools:

- Redundant query normalization and word extraction are eliminated in salary_lookup.py by pre-calculating representations once before the search loop.
- A match_score_optimized helper is introduced to perform the comparison using the pre-calculated query data, preserving full backward compatibility for match_score.
- Normalization in tools/convert_salary_excel.py is unified: paired column headers now consistently substitute spaces and dashes with underscores (e.g. 'software_engineering') to match the single-column formatting.
- Unit test coverage is significantly expanded in tests/test_salary_lookup.py and tests/test_convert_salary_excel.py to cover normalization, anglicization, search filtering, and matching behaviors.

salary_lookup.py

@@ -83,9 +83,8 @@ def extract_core_words(s):
     return [w for w in words if len(w) > 1]
 
 
-def match_score(query, entry_name):
+def match_score_optimized(q_norm, q_ang, q_words_set, q_words_ang_set, query, entry_name):
-    """Compute a match score between 0 and 100 for ranking results."""
+    """Compute a match score between 0 and 100 using precalculated query values."""
-    q_norm = normalize(query)
     n_norm = normalize(entry_name)
 
     if not q_norm or not n_norm:
@@ -97,9 +96,8 @@ def match_score(query, entry_name):
     if q_norm in n_norm:
         ratio = len(q_norm) / len(n_norm)
         if len(q_norm) <= 4 and ratio < 0.5:
-            q_words = set(extract_core_words(query))
             n_words = set(extract_core_words(entry_name))
-            if not q_words & n_words:
+            if not q_words_set & n_words:
                 pass
             else:
                 return 80 + int(ratio * 10)
@@ -112,7 +110,6 @@ def match_score(query, entry_name):
         else:
             return 80 + int(ratio * 10)
 
-    q_ang = anglicize(q_norm)
     n_ang = anglicize(n_norm)
     if q_ang == n_ang:
         return 85
@@ -120,43 +117,57 @@ def match_score(query, entry_name):
         shorter = min(len(q_ang), len(n_ang))
         longer = max(len(q_ang), len(n_ang))
         if shorter <= 4 and shorter / longer < 0.5:
-            q_words_ang = {anglicize(w) for w in extract_core_words(query)}
             n_words_ang = {anglicize(w) for w in extract_core_words(entry_name)}
-            if q_words_ang & n_words_ang:
+            if q_words_ang_set & n_words_ang:
                 return 75
         else:
             return 75
 
-    q_words = set(extract_core_words(query))
     n_words = set(extract_core_words(entry_name))
-    if not q_words or not n_words:
+    if not q_words_set or not n_words:
         return 0
 
-    overlap = q_words & n_words
+    overlap = q_words_set & n_words
     if not overlap:
-        q_words_ang = {anglicize(w) for w in q_words}
         n_words_ang = {anglicize(w) for w in n_words}
-        overlap = q_words_ang & n_words_ang
+        overlap = q_words_ang_set & n_words_ang
 
     if overlap:
-        if len(q_words) == 1:
+        if len(q_words_set) == 1:
-            q_word = list(q_words)[0]
+            q_word = list(q_words_set)[0]
             if q_word in n_words or anglicize(q_word) in {anglicize(w) for w in n_words}:
                 return 70
             else:
                 return 0
 
-        coverage = len(overlap) / len(q_words)
+        coverage = len(overlap) / len(q_words_set)
         return int(30 + coverage * 40)
 
     return 0
 
 
+def match_score(query, entry_name):
+    """Compute a match score between 0 and 100 for ranking results."""
+    q_norm = normalize(query)
+    q_ang = anglicize(q_norm)
+    q_words = extract_core_words(query)
+    q_words_set = set(q_words)
+    q_words_ang_set = {anglicize(w) for w in q_words}
+    return match_score_optimized(q_norm, q_ang, q_words_set, q_words_ang_set, query, entry_name)
+
+
 def search_company(data, query, city=None):
     """Search for a company by name. Returns matching entries sorted by relevance."""
     companies = data.get("companies", [])
     scored = []
 
+    # Pre-calculate query representations once to avoid redundant computations inside the loop
+    q_norm = normalize(query)
+    q_ang = anglicize(q_norm)
+    q_words = extract_core_words(query)
+    q_words_set = set(q_words)
+    q_words_ang_set = {anglicize(w) for w in q_words}
+
     for entry in companies:
         if city:
             city_lower = city.lower()
@@ -164,7 +175,7 @@ def search_company(data, query, city=None):
             if city_lower not in entry_city and anglicize(city_lower) not in anglicize(entry_city):
                 continue
 
-        score = match_score(query, entry["company"])
+        score = match_score_optimized(q_norm, q_ang, q_words_set, q_words_ang_set, query, entry["company"])
         if score > 0:
             scored.append((score, entry))
 

tests/test_convert_salary_excel.py

@@ -77,6 +77,16 @@ class DetectColumnTypeTests(unittest.TestCase):
 
         self.assertEqual(companies[0]["categories"]["accounting"], {"count": 12, "index": 105.5})
 
+    def test_parse_sheet_normalizes_paired_category_name_with_underscores(self):
+        ws = FakeWorksheet([
+            ("Company", "Software Engineering Count", "Software Engineering Index"),
+            ("Example Corp", 8, 110.0),
+        ])
+
+        companies = parse_sheet(ws)
+
+        self.assertEqual(companies[0]["categories"]["software_engineering"], {"count": 8, "index": 110.0})
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_salary_lookup.py

@@ -2,7 +2,14 @@
 
 import unittest
 
-from salary_lookup import format_entry, match_score, search_company
+from salary_lookup import (
+    format_entry,
+    normalize,
+    anglicize,
+    extract_core_words,
+    match_score,
+    search_company,
+)
 
 
 # ---------------------------------------------------------------------------
@@ -158,6 +165,68 @@ class SearchCompanyTests(unittest.TestCase):
         self.assertEqual(results, [])
 
 
+class UtilityTests(unittest.TestCase):
+    def test_normalize_strips_suffix_and_noise(self):
+        self.assertEqual(normalize("Novo Nordisk A/S"), "novonordisk")
+        self.assertEqual(normalize("Ørsted (VG) Holding"), "ørsted")
+        self.assertEqual(normalize("Chr. Hansen, Denmark Division"), "chrhansen")
+        self.assertEqual(normalize("Simple Corp ApS"), "simplecorp")
+
+    def test_anglicize_replaces_danish_chars(self):
+        self.assertEqual(anglicize("ørsted"), "orsted")
+        self.assertEqual(anglicize("mærsk"), "maersk")
+        self.assertEqual(anglicize("ålborg"), "aalborg")
+
+    def test_extract_core_words(self):
+        self.assertEqual(extract_core_words("Novo Nordisk A/S"), ["novo", "nordisk"])
+        self.assertEqual(extract_core_words("A/S"), [])
+        self.assertEqual(extract_core_words("Test Company (Sub-entity)"), ["test", "company"])
+
+
+class MatchScoreTests(unittest.TestCase):
+    def test_exact_match_score(self):
+        self.assertEqual(match_score("Novo Nordisk", "Novo Nordisk"), 100)
+        self.assertEqual(match_score("novo nordisk", "Novo Nordisk A/S"), 100)
+
+    def test_partial_match_score(self):
+        self.assertGreater(match_score("Novo", "Novo Nordisk A/S"), 80)
+        self.assertEqual(match_score("Novo Nordisk", "Novo"), 75)
+
+    def test_anglicized_match_score(self):
+        self.assertEqual(match_score("Orsted", "Ørsted A/S"), 85)
+
+    def test_overlap_match_score(self):
+        # Overlap of multiple words
+        self.assertGreater(match_score("Novo Tech", "Novo Nordisk Tech A/S"), 30)
+
+    def test_no_match_score(self):
+        self.assertEqual(match_score("Google", "Microsoft"), 0)
+
+
+class SearchCompanyRefactoredTests(unittest.TestCase):
+    def setUp(self):
+        self.data = {
+            "companies": [
+                {"company": "Novo Nordisk A/S", "city": "Bagsværd"},
+                {"company": "Ørsted", "city": "Fredericia"},
+                {"company": "Vestas Wind Systems", "city": "Aarhus"},
+            ]
+        }
+
+    def test_search_by_name(self):
+        results = search_company(self.data, "Novo")
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]["company"], "Novo Nordisk A/S")
+
+    def test_search_with_city_filter(self):
+        results = search_company(self.data, "Ørsted", city="Fredericia")
+        self.assertEqual(len(results), 1)
+
+        # Mismatching city
+        results_wrong_city = search_company(self.data, "Ørsted", city="Bagsværd")
+        self.assertEqual(len(results_wrong_city), 0)
+
+
 class TestSearchCompanyBasicMatch(unittest.TestCase):
     def test_exact_name_returns_match(self):
         data = _make_data(_entry("Novo Nordisk", "Bagsværd"))

tools/convert_salary_excel.py

@@ -145,6 +145,8 @@ def parse_sheet(ws, sheet_label=None):
                 cat_name = strip_type_patterns(col_header, COUNT_PATTERNS)
                 if not cat_name:
                     cat_name = f"category_{len(categories)+1}"
+                else:
+                    cat_name = cat_name.replace(" ", "_").replace("-", "_")
                 categories.append({
                     "name": cat_name,
                     "count_col": col_idx,
@@ -156,6 +158,8 @@ def parse_sheet(ws, sheet_label=None):
                 cat_name = strip_type_patterns(col_header, INDEX_PATTERNS)
                 if not cat_name:
                     cat_name = f"category_{len(categories)+1}"
+                else:
+                    cat_name = cat_name.replace(" ", "_").replace("-", "_")
                 categories.append({
                     "name": cat_name,
                     "index_col": col_idx,