mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-11-09 08:27:39 +00:00
fix(salary): convert the IDA workbook again ("Firmanavn og by" header) and list unreachable in the scrape schema (#615)
"firma" joins COMPOUND_PATTERNS so the glued Danish header "Firmanavn og by" matches again; whole-token matching since #75 skipped both sheets of the repository's own demonstration workbook with "Could not find header row". Every other workbook converts byte-identically. Also folds the `unreachable` status from #564 into the seen_jobs.json schema enum in the scrape spec. Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
8123d88f30
commit
5d9e0977f0
4 files changed
+32
-3
No files matched your search
@@ -161,7 +161,7 @@ It prints one line: the canonical key for that posting. The key must be a pure f
|
|||||||
"posted_date": "YYYY-MM-DD" | null,
|
"posted_date": "YYYY-MM-DD" | null,
|
||||||
"deadline": "YYYY-MM-DD" | null,
|
"deadline": "YYYY-MM-DD" | null,
|
||||||
"fit": "high/medium/low",
|
"fit": "high/medium/low",
|
||||||
"status": "new/skipped/ranked/expired",
|
"status": "new/skipped/ranked/expired/unreachable",
|
||||||
"portal": "<source portal skill, e.g. jobindex-search>",
|
"portal": "<source portal skill, e.g. jobindex-search>",
|
||||||
"source": "cli/websearch"
|
"source": "cli/websearch"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -68,6 +68,17 @@ per-file diff commands.
|
|||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
|
- **`convert_salary_excel.py` converts the IDA workbook again** (#615, `tools/convert_salary_excel.py`,
|
||||||
|
`tests/test_convert_salary_excel.py`) - the workbook heads its company column "Firmanavn og by",
|
||||||
|
and since #75 made header matching whole-token only, "firma" no longer matched inside "Firmanavn",
|
||||||
|
so both sheets were skipped with "Could not find header row" and the converter exited 1 on the
|
||||||
|
repository's own demonstration data. "firma" is now a compound token, like the other glued Danish
|
||||||
|
headers; every other workbook converts byte-identically.
|
||||||
|
|
||||||
|
- **`/scrape`'s entry schema lists the `unreachable` status** (#615, `.claude/skills/job-scraper/SKILL.md`) -
|
||||||
|
follow-up to #564: the `status` enum in the `seen_jobs.json` schema block still read
|
||||||
|
`new/skipped/ranked/expired` while the dedup rule two sections down already named `unreachable`.
|
||||||
|
|
||||||
- **`/scrape` names `job_scraper/seen_jobs.json` by its full path in every step** (#608,
|
- **`/scrape` names `job_scraper/seen_jobs.json` by its full path in every step** (#608,
|
||||||
`.claude/skills/job-scraper/SKILL.md`, `tests/test_scrape_seen_jobs_path.py`) - seven references
|
`.claude/skills/job-scraper/SKILL.md`, `tests/test_scrape_seen_jobs_path.py`) - seven references
|
||||||
after Step 0, including the Step 4 write instruction, used the bare filename, and an assistant
|
after Step 0, including the Step 4 write instruction, used the bare filename, and an assistant
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from salary_lookup import format_entry
|
|||||||
from tools.convert_salary_excel import (
|
from tools.convert_salary_excel import (
|
||||||
INDEX_PATTERNS,
|
INDEX_PATTERNS,
|
||||||
detect_column_type,
|
detect_column_type,
|
||||||
|
COMPANY_PATTERNS,
|
||||||
header_matches,
|
header_matches,
|
||||||
parse_numeric_cell,
|
parse_numeric_cell,
|
||||||
parse_sheet,
|
parse_sheet,
|
||||||
@@ -108,6 +109,22 @@ class DetectColumnTypeTests(unittest.TestCase):
|
|||||||
companies[0]["categories"]["salary"], {"index": 105.5}
|
companies[0]["categories"]["salary"], {"index": 105.5}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_parse_sheet_detects_danish_compound_company_header(self):
|
||||||
|
# The IDA workbook heads its company column "Firmanavn og by": "firma"
|
||||||
|
# glued to "navn". Whole-token matching alone skipped the entire sheet
|
||||||
|
# ("Could not find header row") from #75 until "firma" joined
|
||||||
|
# COMPOUND_PATTERNS, so the shipped demonstration data no longer
|
||||||
|
# converted at all.
|
||||||
|
self.assertTrue(header_matches("Firmanavn og by", COMPANY_PATTERNS))
|
||||||
|
ws = FakeWorksheet([
|
||||||
|
("Firmanavn og by", "Antal", "Indeks"),
|
||||||
|
("Example A/S, Aarhus", 12, 105.0),
|
||||||
|
])
|
||||||
|
companies = parse_sheet(ws)
|
||||||
|
self.assertEqual(len(companies), 1)
|
||||||
|
self.assertEqual(companies[0]["company"], "Example A/S, Aarhus")
|
||||||
|
self.assertEqual(companies[0]["categories"]["all_employees"], {"count": 12, "index": 105.0})
|
||||||
|
|
||||||
def test_parse_sheet_detects_city_column_with_token_header(self):
|
def test_parse_sheet_detects_city_column_with_token_header(self):
|
||||||
# City headers are matched with the same token-based header_matches()
|
# City headers are matched with the same token-based header_matches()
|
||||||
# used for the company column, not exact string equality. Real-world
|
# used for the company column, not exact string equality. Real-world
|
||||||
|
|||||||
@@ -46,8 +46,9 @@ INDEX_PATTERNS = {"indeks", "index", "idx", "salary", "løn", "median", "average
|
|||||||
# header token, for languages that glue words together (e.g. Danish "lønindeks"
|
# header token, for languages that glue words together (e.g. Danish "lønindeks"
|
||||||
# -> løn + indeks). Languages that write headers as separate words need none.
|
# -> løn + indeks). Languages that write headers as separate words need none.
|
||||||
# Ships populated for this repo's Danish demonstration data; a fork targeting
|
# Ships populated for this repo's Danish demonstration data; a fork targeting
|
||||||
# another locale edits this constant.
|
# another locale edits this constant. "firma" is here because the IDA workbook
|
||||||
COMPOUND_PATTERNS = {"antal", "indeks", "løn", "gennemsnit", "medarbejdere"}
|
# heads its company column "Firmanavn og by" (firma + navn glued together).
|
||||||
|
COMPOUND_PATTERNS = {"firma", "antal", "indeks", "løn", "gennemsnit", "medarbejdere"}
|
||||||
# Identifier columns (employee id, Danish "personnummer", etc.) are never salary
|
# Identifier columns (employee id, Danish "personnummer", etc.) are never salary
|
||||||
# data. They are dropped at classification so they are not mistaken for a salary
|
# data. They are dropped at classification so they are not mistaken for a salary
|
||||||
# category. Matched as whole tokens only, like other pattern sets.
|
# category. Matched as whole tokens only, like other pattern sets.
|
||||||
|
|||||||
Reference in new issue
Block a user