py/ai-job-search
1
0
Fork 0
mirror of https://github.com/MadsLorentzen/ai-job-search.git synced 2026-09-17 00:26:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(salary): optimize search match scoring and normalize Excel category keys (#101)

Browse Source 
This commit improves the performance and consistency of the salary tools:

- Redundant query normalization and word extraction are eliminated in salary_lookup.py by pre-calculating representations once before the search loop.
- A match_score_optimized helper is introduced to perform the comparison using the pre-calculated query data, preserving full backward compatibility for match_score.
- Normalization in tools/convert_salary_excel.py is unified: paired column headers now consistently substitute spaces and dashes with underscores (e.g. 'software_engineering') to match the single-column formatting.
- Unit test coverage is significantly expanded in tests/test_salary_lookup.py and tests/test_convert_salary_excel.py to cover normalization, anglicization, search filtering, and matching behaviors.
This commit is contained in:
♦ jabarii♦
2026-07-10 15:24:20 +02:00
committed by GitHub
parent 7e8df35819
commit c134eef553
4 changed files with 112 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
salary_lookup.py
+28 -17
View File
@@ -83,9 +83,8 @@ def extract_core_words(s):
return [w for w in words if len(w) > 1]
def match_score(query, entry_name):
"""Compute a match score between 0 and 100 for ranking results."""
q_norm = normalize(query)
def match_score_optimized(q_norm, q_ang, q_words_set, q_words_ang_set, query, entry_name):
"""Compute a match score between 0 and 100 using precalculated query values."""
n_norm = normalize(entry_name)
if not q_norm or not n_norm:
@@ -97,9 +96,8 @@ def match_score(query, entry_name):
if q_norm in n_norm:
ratio = len(q_norm) / len(n_norm)
if len(q_norm) <= 4 and ratio < 0.5:
q_words = set(extract_core_words(query))
n_words = set(extract_core_words(entry_name))
if not q_words & n_words:
if not q_words_set & n_words:
pass
else:
return 80 + int(ratio * 10)
@@ -112,7 +110,6 @@ def match_score(query, entry_name):
else:
return 80 + int(ratio * 10)
q_ang = anglicize(q_norm)
n_ang = anglicize(n_norm)
if q_ang == n_ang:
return 85
@@ -120,43 +117,57 @@ def match_score(query, entry_name):
shorter = min(len(q_ang), len(n_ang))
longer = max(len(q_ang), len(n_ang))
if shorter <= 4 and shorter / longer < 0.5:
q_words_ang = {anglicize(w) for w in extract_core_words(query)}
n_words_ang = {anglicize(w) for w in extract_core_words(entry_name)}
if q_words_ang & n_words_ang:
if q_words_ang_set & n_words_ang:
return 75
else:
return 75
q_words = set(extract_core_words(query))
n_words = set(extract_core_words(entry_name))
if not q_words or not n_words:
if not q_words_set or not n_words:
return 0
overlap = q_words & n_words
overlap = q_words_set & n_words
if not overlap:
q_words_ang = {anglicize(w) for w in q_words}
n_words_ang = {anglicize(w) for w in n_words}
overlap = q_words_ang & n_words_ang
overlap = q_words_ang_set & n_words_ang
if overlap:
if len(q_words) == 1:
q_word = list(q_words)[0]
if len(q_words_set) == 1:
q_word = list(q_words_set)[0]
if q_word in n_words or anglicize(q_word) in {anglicize(w) for w in n_words}:
return 70
else:
return 0
coverage = len(overlap) / len(q_words)
coverage = len(overlap) / len(q_words_set)
return int(30 + coverage * 40)
return 0
def match_score(query, entry_name):
"""Compute a match score between 0 and 100 for ranking results."""
q_norm = normalize(query)
q_ang = anglicize(q_norm)
q_words = extract_core_words(query)
q_words_set = set(q_words)
q_words_ang_set = {anglicize(w) for w in q_words}
return match_score_optimized(q_norm, q_ang, q_words_set, q_words_ang_set, query, entry_name)
def search_company(data, query, city=None):
"""Search for a company by name. Returns matching entries sorted by relevance."""
companies = data.get("companies", [])
scored = []
# Pre-calculate query representations once to avoid redundant computations inside the loop
q_norm = normalize(query)
q_ang = anglicize(q_norm)
q_words = extract_core_words(query)
q_words_set = set(q_words)
q_words_ang_set = {anglicize(w) for w in q_words}
for entry in companies:
if city:
city_lower = city.lower()
@@ -164,7 +175,7 @@ def search_company(data, query, city=None):
if city_lower not in entry_city and anglicize(city_lower) not in anglicize(entry_city):
continue
score = match_score(query, entry["company"])
score = match_score_optimized(q_norm, q_ang, q_words_set, q_words_ang_set, query, entry["company"])
if score > 0:
scored.append((score, entry))