mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-09-17 00:26:26 +00:00
fix: Fix salary Excel column detection for index headers (#64)
改动点: 修复 [tools/convert_salary_excel.py (line 47)](/Users/alwin/ai-job-search/tools/convert_salary_excel.py:47) 里列类型识别的问题:之前 n 被当作任意子串匹配,导致 Index / Engineering Index 这类列会被误判成 count。 同步修复类别名生成,避免 Engineering Count 里的 n 被删坏。 把 openpyxl 缺失报错延迟到实际运行转换命令时,这样纯函数可以被单元测试导入。 新增 [tests/test_convert_salary_excel.py (line 25)](/Users/alwin/ai-job-search/tests/test_convert_salary_excel.py:25),覆盖 index/count 识别和 worksheet 解析。 Co-authored-by: Alwin.Zhang <alwin.zhang420@gmail.com>
This commit is contained in:
co-authored by
Alwin.Zhang
parent
4807cc846b
commit
3c7a1cfdf5
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from tools.convert_salary_excel import detect_column_type, parse_sheet
|
||||
|
||||
|
||||
class FakeWorksheet:
|
||||
title = "Sheet1"
|
||||
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
|
||||
def iter_rows(self, min_row=1, max_row=None, values_only=False):
|
||||
rows = self.rows[min_row - 1:max_row]
|
||||
for row in rows:
|
||||
if values_only:
|
||||
yield row
|
||||
else:
|
||||
yield [SimpleNamespace(value=value) for value in row]
|
||||
|
||||
def __getitem__(self, row_number):
|
||||
return [SimpleNamespace(value=value) for value in self.rows[row_number - 1]]
|
||||
|
||||
|
||||
class DetectColumnTypeTests(unittest.TestCase):
|
||||
def test_index_headers_are_not_misclassified_as_count(self):
|
||||
for header in ("Index", "Salary Index", "Engineering Index", "Median salary"):
|
||||
with self.subTest(header=header):
|
||||
self.assertEqual(detect_column_type(header), "index")
|
||||
|
||||
def test_single_letter_n_only_matches_as_a_token(self):
|
||||
self.assertEqual(detect_column_type("Employee n"), "count")
|
||||
self.assertEqual(detect_column_type("Engineering"), None)
|
||||
|
||||
def test_count_headers_still_match_common_labels(self):
|
||||
for header in ("Count", "Engineering Count", "Antal medarbejdere"):
|
||||
with self.subTest(header=header):
|
||||
self.assertEqual(detect_column_type(header), "count")
|
||||
|
||||
def test_parse_sheet_preserves_category_name_with_letter_n(self):
|
||||
ws = FakeWorksheet([
|
||||
("Company", "Engineering Count", "Engineering Index"),
|
||||
("Example Corp", 12, 105.5),
|
||||
])
|
||||
|
||||
companies = parse_sheet(ws)
|
||||
|
||||
self.assertEqual(companies[0]["categories"]["engineering"], {"count": 12, "index": 105.5})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -28,13 +28,13 @@ with paired count/index columns per category, it groups them automatically.
|
||||
import json
|
||||
import sys
|
||||
import argparse
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import openpyxl
|
||||
except ImportError:
|
||||
print("Error: openpyxl is required. Install it with: pip install openpyxl", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
openpyxl = None
|
||||
|
||||
|
||||
# Column name patterns for auto-detection
|
||||
@@ -44,14 +44,36 @@ COUNT_PATTERNS = {"antal", "count", "number", "n", "employees", "medarbejdere"}
|
||||
INDEX_PATTERNS = {"indeks", "index", "idx", "salary", "løn", "median", "average", "gennemsnit"}
|
||||
|
||||
|
||||
def header_matches(header, patterns):
|
||||
"""Return True when a header contains a meaningful pattern match."""
|
||||
h = header.lower().strip()
|
||||
tokens = set(re.findall(r"[a-zæøåöäü0-9]+", h))
|
||||
|
||||
for p in patterns:
|
||||
if len(p) == 1:
|
||||
if p in tokens:
|
||||
return True
|
||||
elif p in h:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def strip_type_patterns(header, patterns):
|
||||
"""Remove count/index words from a header to derive a category name."""
|
||||
name = header.lower()
|
||||
for p in patterns:
|
||||
if len(p) == 1:
|
||||
name = re.sub(rf"\b{re.escape(p)}\b", "", name)
|
||||
else:
|
||||
name = name.replace(p, "")
|
||||
return name.strip(" _-")
|
||||
|
||||
|
||||
def detect_column_type(header):
|
||||
"""Detect whether a column header refers to count or index data."""
|
||||
h = header.lower().strip()
|
||||
for p in COUNT_PATTERNS:
|
||||
if p in h:
|
||||
if header_matches(header, COUNT_PATTERNS):
|
||||
return "count"
|
||||
for p in INDEX_PATTERNS:
|
||||
if p in h:
|
||||
if header_matches(header, INDEX_PATTERNS):
|
||||
return "index"
|
||||
return None
|
||||
|
||||
@@ -113,9 +135,7 @@ def parse_sheet(ws, sheet_label=None):
|
||||
# If we have a count/index pair, group them
|
||||
if col_type == "count" and next_col_type == "index":
|
||||
# Use the header minus the count/index suffix as category name
|
||||
cat_name = col_header
|
||||
for p in COUNT_PATTERNS:
|
||||
cat_name = cat_name.lower().replace(p, "").strip(" _-")
|
||||
cat_name = strip_type_patterns(col_header, COUNT_PATTERNS)
|
||||
if not cat_name:
|
||||
cat_name = f"category_{len(categories)+1}"
|
||||
categories.append({
|
||||
@@ -126,9 +146,7 @@ def parse_sheet(ws, sheet_label=None):
|
||||
i += 2
|
||||
continue
|
||||
elif col_type == "index" and next_col_type == "count":
|
||||
cat_name = col_header
|
||||
for p in INDEX_PATTERNS:
|
||||
cat_name = cat_name.lower().replace(p, "").strip(" _-")
|
||||
cat_name = strip_type_patterns(col_header, INDEX_PATTERNS)
|
||||
if not cat_name:
|
||||
cat_name = f"category_{len(categories)+1}"
|
||||
categories.append({
|
||||
@@ -219,6 +237,10 @@ def main():
|
||||
print(f"Error: File not found: {excel_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if openpyxl is None:
|
||||
print("Error: openpyxl is required. Install it with: pip install openpyxl", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
output_path = Path(args.output) if args.output else Path(__file__).parent.parent / "salary_data.json"
|
||||
|
||||
print(f"Reading: {excel_path}")
|
||||
|
||||
Reference in New Issue
Block a user