mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-09-17 00:26:26 +00:00
fix: Fix salary Excel column detection for index headers (#64)
改动点: 修复 [tools/convert_salary_excel.py (line 47)](/Users/alwin/ai-job-search/tools/convert_salary_excel.py:47) 里列类型识别的问题:之前 n 被当作任意子串匹配,导致 Index / Engineering Index 这类列会被误判成 count。 同步修复类别名生成,避免 Engineering Count 里的 n 被删坏。 把 openpyxl 缺失报错延迟到实际运行转换命令时,这样纯函数可以被单元测试导入。 新增 [tests/test_convert_salary_excel.py (line 25)](/Users/alwin/ai-job-search/tests/test_convert_salary_excel.py:25),覆盖 index/count 识别和 worksheet 解析。 Co-authored-by: Alwin.Zhang <alwin.zhang420@gmail.com>
This commit is contained in:
co-authored by
Alwin.Zhang
parent
4807cc846b
commit
3c7a1cfdf5
@@ -0,0 +1 @@
|
|||||||
|
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
import unittest
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
from tools.convert_salary_excel import detect_column_type, parse_sheet
|
||||||
|
|
||||||
|
|
||||||
|
class FakeWorksheet:
|
||||||
|
title = "Sheet1"
|
||||||
|
|
||||||
|
def __init__(self, rows):
|
||||||
|
self.rows = rows
|
||||||
|
|
||||||
|
def iter_rows(self, min_row=1, max_row=None, values_only=False):
|
||||||
|
rows = self.rows[min_row - 1:max_row]
|
||||||
|
for row in rows:
|
||||||
|
if values_only:
|
||||||
|
yield row
|
||||||
|
else:
|
||||||
|
yield [SimpleNamespace(value=value) for value in row]
|
||||||
|
|
||||||
|
def __getitem__(self, row_number):
|
||||||
|
return [SimpleNamespace(value=value) for value in self.rows[row_number - 1]]
|
||||||
|
|
||||||
|
|
||||||
|
class DetectColumnTypeTests(unittest.TestCase):
|
||||||
|
def test_index_headers_are_not_misclassified_as_count(self):
|
||||||
|
for header in ("Index", "Salary Index", "Engineering Index", "Median salary"):
|
||||||
|
with self.subTest(header=header):
|
||||||
|
self.assertEqual(detect_column_type(header), "index")
|
||||||
|
|
||||||
|
def test_single_letter_n_only_matches_as_a_token(self):
|
||||||
|
self.assertEqual(detect_column_type("Employee n"), "count")
|
||||||
|
self.assertEqual(detect_column_type("Engineering"), None)
|
||||||
|
|
||||||
|
def test_count_headers_still_match_common_labels(self):
|
||||||
|
for header in ("Count", "Engineering Count", "Antal medarbejdere"):
|
||||||
|
with self.subTest(header=header):
|
||||||
|
self.assertEqual(detect_column_type(header), "count")
|
||||||
|
|
||||||
|
def test_parse_sheet_preserves_category_name_with_letter_n(self):
|
||||||
|
ws = FakeWorksheet([
|
||||||
|
("Company", "Engineering Count", "Engineering Index"),
|
||||||
|
("Example Corp", 12, 105.5),
|
||||||
|
])
|
||||||
|
|
||||||
|
companies = parse_sheet(ws)
|
||||||
|
|
||||||
|
self.assertEqual(companies[0]["categories"]["engineering"], {"count": 12, "index": 105.5})
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -28,13 +28,13 @@ with paired count/index columns per category, it groups them automatically.
|
|||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import argparse
|
import argparse
|
||||||
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import openpyxl
|
import openpyxl
|
||||||
except ImportError:
|
except ImportError:
|
||||||
print("Error: openpyxl is required. Install it with: pip install openpyxl", file=sys.stderr)
|
openpyxl = None
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
# Column name patterns for auto-detection
|
# Column name patterns for auto-detection
|
||||||
@@ -44,14 +44,36 @@ COUNT_PATTERNS = {"antal", "count", "number", "n", "employees", "medarbejdere"}
|
|||||||
INDEX_PATTERNS = {"indeks", "index", "idx", "salary", "løn", "median", "average", "gennemsnit"}
|
INDEX_PATTERNS = {"indeks", "index", "idx", "salary", "løn", "median", "average", "gennemsnit"}
|
||||||
|
|
||||||
|
|
||||||
|
def header_matches(header, patterns):
|
||||||
|
"""Return True when a header contains a meaningful pattern match."""
|
||||||
|
h = header.lower().strip()
|
||||||
|
tokens = set(re.findall(r"[a-zæøåöäü0-9]+", h))
|
||||||
|
|
||||||
|
for p in patterns:
|
||||||
|
if len(p) == 1:
|
||||||
|
if p in tokens:
|
||||||
|
return True
|
||||||
|
elif p in h:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def strip_type_patterns(header, patterns):
|
||||||
|
"""Remove count/index words from a header to derive a category name."""
|
||||||
|
name = header.lower()
|
||||||
|
for p in patterns:
|
||||||
|
if len(p) == 1:
|
||||||
|
name = re.sub(rf"\b{re.escape(p)}\b", "", name)
|
||||||
|
else:
|
||||||
|
name = name.replace(p, "")
|
||||||
|
return name.strip(" _-")
|
||||||
|
|
||||||
|
|
||||||
def detect_column_type(header):
|
def detect_column_type(header):
|
||||||
"""Detect whether a column header refers to count or index data."""
|
"""Detect whether a column header refers to count or index data."""
|
||||||
h = header.lower().strip()
|
if header_matches(header, COUNT_PATTERNS):
|
||||||
for p in COUNT_PATTERNS:
|
|
||||||
if p in h:
|
|
||||||
return "count"
|
return "count"
|
||||||
for p in INDEX_PATTERNS:
|
if header_matches(header, INDEX_PATTERNS):
|
||||||
if p in h:
|
|
||||||
return "index"
|
return "index"
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -113,9 +135,7 @@ def parse_sheet(ws, sheet_label=None):
|
|||||||
# If we have a count/index pair, group them
|
# If we have a count/index pair, group them
|
||||||
if col_type == "count" and next_col_type == "index":
|
if col_type == "count" and next_col_type == "index":
|
||||||
# Use the header minus the count/index suffix as category name
|
# Use the header minus the count/index suffix as category name
|
||||||
cat_name = col_header
|
cat_name = strip_type_patterns(col_header, COUNT_PATTERNS)
|
||||||
for p in COUNT_PATTERNS:
|
|
||||||
cat_name = cat_name.lower().replace(p, "").strip(" _-")
|
|
||||||
if not cat_name:
|
if not cat_name:
|
||||||
cat_name = f"category_{len(categories)+1}"
|
cat_name = f"category_{len(categories)+1}"
|
||||||
categories.append({
|
categories.append({
|
||||||
@@ -126,9 +146,7 @@ def parse_sheet(ws, sheet_label=None):
|
|||||||
i += 2
|
i += 2
|
||||||
continue
|
continue
|
||||||
elif col_type == "index" and next_col_type == "count":
|
elif col_type == "index" and next_col_type == "count":
|
||||||
cat_name = col_header
|
cat_name = strip_type_patterns(col_header, INDEX_PATTERNS)
|
||||||
for p in INDEX_PATTERNS:
|
|
||||||
cat_name = cat_name.lower().replace(p, "").strip(" _-")
|
|
||||||
if not cat_name:
|
if not cat_name:
|
||||||
cat_name = f"category_{len(categories)+1}"
|
cat_name = f"category_{len(categories)+1}"
|
||||||
categories.append({
|
categories.append({
|
||||||
@@ -219,6 +237,10 @@ def main():
|
|||||||
print(f"Error: File not found: {excel_path}", file=sys.stderr)
|
print(f"Error: File not found: {excel_path}", file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
if openpyxl is None:
|
||||||
|
print("Error: openpyxl is required. Install it with: pip install openpyxl", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
output_path = Path(args.output) if args.output else Path(__file__).parent.parent / "salary_data.json"
|
output_path = Path(args.output) if args.output else Path(__file__).parent.parent / "salary_data.json"
|
||||||
|
|
||||||
print(f"Reading: {excel_path}")
|
print(f"Reading: {excel_path}")
|
||||||
|
|||||||
Reference in New Issue
Block a user