fix: Fix salary Excel column detection for index headers (#64)

改动点:
修复 [tools/convert_salary_excel.py (line 47)](/Users/alwin/ai-job-search/tools/convert_salary_excel.py:47) 里列类型识别的问题:之前 n 被当作任意子串匹配,导致 Index / Engineering Index 这类列会被误判成 count。
同步修复类别名生成,避免 Engineering Count 里的 n 被删坏。
把 openpyxl 缺失报错延迟到实际运行转换命令时,这样纯函数可以被单元测试导入。
新增 [tests/test_convert_salary_excel.py (line 25)](/Users/alwin/ai-job-search/tests/test_convert_salary_excel.py:25),覆盖 index/count 识别和 worksheet 解析。

Co-authored-by: Alwin.Zhang <alwin.zhang420@gmail.com>
This commit is contained in:
Alwin4Zhang
2026-07-08 17:11:03 +02:00
committed by GitHub
co-authored by Alwin.Zhang
parent 4807cc846b
commit 3c7a1cfdf5
3 changed files with 90 additions and 15 deletions
+1
View File
@@ -0,0 +1 @@
+52
View File
@@ -0,0 +1,52 @@
import unittest
from types import SimpleNamespace
from tools.convert_salary_excel import detect_column_type, parse_sheet
class FakeWorksheet:
title = "Sheet1"
def __init__(self, rows):
self.rows = rows
def iter_rows(self, min_row=1, max_row=None, values_only=False):
rows = self.rows[min_row - 1:max_row]
for row in rows:
if values_only:
yield row
else:
yield [SimpleNamespace(value=value) for value in row]
def __getitem__(self, row_number):
return [SimpleNamespace(value=value) for value in self.rows[row_number - 1]]
class DetectColumnTypeTests(unittest.TestCase):
def test_index_headers_are_not_misclassified_as_count(self):
for header in ("Index", "Salary Index", "Engineering Index", "Median salary"):
with self.subTest(header=header):
self.assertEqual(detect_column_type(header), "index")
def test_single_letter_n_only_matches_as_a_token(self):
self.assertEqual(detect_column_type("Employee n"), "count")
self.assertEqual(detect_column_type("Engineering"), None)
def test_count_headers_still_match_common_labels(self):
for header in ("Count", "Engineering Count", "Antal medarbejdere"):
with self.subTest(header=header):
self.assertEqual(detect_column_type(header), "count")
def test_parse_sheet_preserves_category_name_with_letter_n(self):
ws = FakeWorksheet([
("Company", "Engineering Count", "Engineering Index"),
("Example Corp", 12, 105.5),
])
companies = parse_sheet(ws)
self.assertEqual(companies[0]["categories"]["engineering"], {"count": 12, "index": 105.5})
if __name__ == "__main__":
unittest.main()
+35 -13
View File
@@ -28,13 +28,13 @@ with paired count/index columns per category, it groups them automatically.
import json
import sys
import argparse
import re
from pathlib import Path
try:
import openpyxl
except ImportError:
print("Error: openpyxl is required. Install it with: pip install openpyxl", file=sys.stderr)
sys.exit(1)
openpyxl = None
# Column name patterns for auto-detection
@@ -44,14 +44,36 @@ COUNT_PATTERNS = {"antal", "count", "number", "n", "employees", "medarbejdere"}
INDEX_PATTERNS = {"indeks", "index", "idx", "salary", "løn", "median", "average", "gennemsnit"}
def header_matches(header, patterns):
"""Return True when a header contains a meaningful pattern match."""
h = header.lower().strip()
tokens = set(re.findall(r"[a-zæøåöäü0-9]+", h))
for p in patterns:
if len(p) == 1:
if p in tokens:
return True
elif p in h:
return True
return False
def strip_type_patterns(header, patterns):
"""Remove count/index words from a header to derive a category name."""
name = header.lower()
for p in patterns:
if len(p) == 1:
name = re.sub(rf"\b{re.escape(p)}\b", "", name)
else:
name = name.replace(p, "")
return name.strip(" _-")
def detect_column_type(header):
"""Detect whether a column header refers to count or index data."""
h = header.lower().strip()
for p in COUNT_PATTERNS:
if p in h:
if header_matches(header, COUNT_PATTERNS):
return "count"
for p in INDEX_PATTERNS:
if p in h:
if header_matches(header, INDEX_PATTERNS):
return "index"
return None
@@ -113,9 +135,7 @@ def parse_sheet(ws, sheet_label=None):
# If we have a count/index pair, group them
if col_type == "count" and next_col_type == "index":
# Use the header minus the count/index suffix as category name
cat_name = col_header
for p in COUNT_PATTERNS:
cat_name = cat_name.lower().replace(p, "").strip(" _-")
cat_name = strip_type_patterns(col_header, COUNT_PATTERNS)
if not cat_name:
cat_name = f"category_{len(categories)+1}"
categories.append({
@@ -126,9 +146,7 @@ def parse_sheet(ws, sheet_label=None):
i += 2
continue
elif col_type == "index" and next_col_type == "count":
cat_name = col_header
for p in INDEX_PATTERNS:
cat_name = cat_name.lower().replace(p, "").strip(" _-")
cat_name = strip_type_patterns(col_header, INDEX_PATTERNS)
if not cat_name:
cat_name = f"category_{len(categories)+1}"
categories.append({
@@ -219,6 +237,10 @@ def main():
print(f"Error: File not found: {excel_path}", file=sys.stderr)
sys.exit(1)
if openpyxl is None:
print("Error: openpyxl is required. Install it with: pip install openpyxl", file=sys.stderr)
sys.exit(1)
output_path = Path(args.output) if args.output else Path(__file__).parent.parent / "salary_data.json"
print(f"Reading: {excel_path}")