fix: Fix salary Excel column detection for index headers (#64)

改动点:
修复 [tools/convert_salary_excel.py (line 47)](/Users/alwin/ai-job-search/tools/convert_salary_excel.py:47) 里列类型识别的问题:之前 n 被当作任意子串匹配,导致 Index / Engineering Index 这类列会被误判成 count。
同步修复类别名生成,避免 Engineering Count 里的 n 被删坏。
把 openpyxl 缺失报错延迟到实际运行转换命令时,这样纯函数可以被单元测试导入。
新增 [tests/test_convert_salary_excel.py (line 25)](/Users/alwin/ai-job-search/tests/test_convert_salary_excel.py:25),覆盖 index/count 识别和 worksheet 解析。

Co-authored-by: Alwin.Zhang <alwin.zhang420@gmail.com>
This commit is contained in:
Alwin4Zhang
2026-07-08 17:11:03 +02:00
committed by GitHub
co-authored by Alwin.Zhang
parent 4807cc846b
commit 3c7a1cfdf5
3 changed files with 90 additions and 15 deletions
+1
View File
@@ -0,0 +1 @@
+52
View File
@@ -0,0 +1,52 @@
import unittest
from types import SimpleNamespace
from tools.convert_salary_excel import detect_column_type, parse_sheet
class FakeWorksheet:
title = "Sheet1"
def __init__(self, rows):
self.rows = rows
def iter_rows(self, min_row=1, max_row=None, values_only=False):
rows = self.rows[min_row - 1:max_row]
for row in rows:
if values_only:
yield row
else:
yield [SimpleNamespace(value=value) for value in row]
def __getitem__(self, row_number):
return [SimpleNamespace(value=value) for value in self.rows[row_number - 1]]
class DetectColumnTypeTests(unittest.TestCase):
def test_index_headers_are_not_misclassified_as_count(self):
for header in ("Index", "Salary Index", "Engineering Index", "Median salary"):
with self.subTest(header=header):
self.assertEqual(detect_column_type(header), "index")
def test_single_letter_n_only_matches_as_a_token(self):
self.assertEqual(detect_column_type("Employee n"), "count")
self.assertEqual(detect_column_type("Engineering"), None)
def test_count_headers_still_match_common_labels(self):
for header in ("Count", "Engineering Count", "Antal medarbejdere"):
with self.subTest(header=header):
self.assertEqual(detect_column_type(header), "count")
def test_parse_sheet_preserves_category_name_with_letter_n(self):
ws = FakeWorksheet([
("Company", "Engineering Count", "Engineering Index"),
("Example Corp", 12, 105.5),
])
companies = parse_sheet(ws)
self.assertEqual(companies[0]["categories"]["engineering"], {"count": 12, "index": 105.5})
if __name__ == "__main__":
unittest.main()
+37 -15
View File
@@ -28,13 +28,13 @@ with paired count/index columns per category, it groups them automatically.
import json import json
import sys import sys
import argparse import argparse
import re
from pathlib import Path from pathlib import Path
try: try:
import openpyxl import openpyxl
except ImportError: except ImportError:
print("Error: openpyxl is required. Install it with: pip install openpyxl", file=sys.stderr) openpyxl = None
sys.exit(1)
# Column name patterns for auto-detection # Column name patterns for auto-detection
@@ -44,15 +44,37 @@ COUNT_PATTERNS = {"antal", "count", "number", "n", "employees", "medarbejdere"}
INDEX_PATTERNS = {"indeks", "index", "idx", "salary", "løn", "median", "average", "gennemsnit"} INDEX_PATTERNS = {"indeks", "index", "idx", "salary", "løn", "median", "average", "gennemsnit"}
def header_matches(header, patterns):
"""Return True when a header contains a meaningful pattern match."""
h = header.lower().strip()
tokens = set(re.findall(r"[a-zæøåöäü0-9]+", h))
for p in patterns:
if len(p) == 1:
if p in tokens:
return True
elif p in h:
return True
return False
def strip_type_patterns(header, patterns):
"""Remove count/index words from a header to derive a category name."""
name = header.lower()
for p in patterns:
if len(p) == 1:
name = re.sub(rf"\b{re.escape(p)}\b", "", name)
else:
name = name.replace(p, "")
return name.strip(" _-")
def detect_column_type(header): def detect_column_type(header):
"""Detect whether a column header refers to count or index data.""" """Detect whether a column header refers to count or index data."""
h = header.lower().strip() if header_matches(header, COUNT_PATTERNS):
for p in COUNT_PATTERNS: return "count"
if p in h: if header_matches(header, INDEX_PATTERNS):
return "count" return "index"
for p in INDEX_PATTERNS:
if p in h:
return "index"
return None return None
@@ -113,9 +135,7 @@ def parse_sheet(ws, sheet_label=None):
# If we have a count/index pair, group them # If we have a count/index pair, group them
if col_type == "count" and next_col_type == "index": if col_type == "count" and next_col_type == "index":
# Use the header minus the count/index suffix as category name # Use the header minus the count/index suffix as category name
cat_name = col_header cat_name = strip_type_patterns(col_header, COUNT_PATTERNS)
for p in COUNT_PATTERNS:
cat_name = cat_name.lower().replace(p, "").strip(" _-")
if not cat_name: if not cat_name:
cat_name = f"category_{len(categories)+1}" cat_name = f"category_{len(categories)+1}"
categories.append({ categories.append({
@@ -126,9 +146,7 @@ def parse_sheet(ws, sheet_label=None):
i += 2 i += 2
continue continue
elif col_type == "index" and next_col_type == "count": elif col_type == "index" and next_col_type == "count":
cat_name = col_header cat_name = strip_type_patterns(col_header, INDEX_PATTERNS)
for p in INDEX_PATTERNS:
cat_name = cat_name.lower().replace(p, "").strip(" _-")
if not cat_name: if not cat_name:
cat_name = f"category_{len(categories)+1}" cat_name = f"category_{len(categories)+1}"
categories.append({ categories.append({
@@ -219,6 +237,10 @@ def main():
print(f"Error: File not found: {excel_path}", file=sys.stderr) print(f"Error: File not found: {excel_path}", file=sys.stderr)
sys.exit(1) sys.exit(1)
if openpyxl is None:
print("Error: openpyxl is required. Install it with: pip install openpyxl", file=sys.stderr)
sys.exit(1)
output_path = Path(args.output) if args.output else Path(__file__).parent.parent / "salary_data.json" output_path = Path(args.output) if args.output else Path(__file__).parent.parent / "salary_data.json"
print(f"Reading: {excel_path}") print(f"Reading: {excel_path}")