mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-09-17 00:26:26 +00:00
fix(jobdanmark-search): normalize the detail command's HTML fallback branch
The JSON-LD and rendered-HTML branches returned structurally different records: the fallback emitted raw DD-MM-YYYY page text as datePosted, free text (including the literal "Loebende") as validThrough, and a hardcoded null addressLocality. Overview dates now convert to ISO, Loebende maps to null, and the locality derives from the workplace address via the same exported extractCity search uses. Review finding F25 (2026-08-19). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
bcba687fbf
commit
dab215073e
@@ -2,6 +2,7 @@ import { defineCommand, option } from "@bunli/core"
|
|||||||
import { z } from "zod"
|
import { z } from "zod"
|
||||||
import { parse } from "node-html-parser"
|
import { parse } from "node-html-parser"
|
||||||
import { BASE_URL, writeError } from "../helpers.js"
|
import { BASE_URL, writeError } from "../helpers.js"
|
||||||
|
import { extractCity, toContractDate } from "./search.js"
|
||||||
|
|
||||||
interface JsonLdJobPosting {
|
interface JsonLdJobPosting {
|
||||||
"@context"?: string
|
"@context"?: string
|
||||||
@@ -119,6 +120,21 @@ function fromJsonLd(jobPosting: JsonLdJobPosting, slug: string, url: string): De
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalize a date read from the rendered page's overview list. The page
|
||||||
|
* writes DD-MM-YYYY (sometimes with a trailing time, "02-08-2026 23.59"),
|
||||||
|
* and the deadline can be the free-text "Løbende" (rolling) - jobbank maps
|
||||||
|
* its equivalent to null, and every consumer does date arithmetic on the
|
||||||
|
* value. The JSON-LD branch gets schema.org ISO dates and needs none of this.
|
||||||
|
*/
|
||||||
|
function normalizeOverviewDate(value: string | null): string | null {
|
||||||
|
if (!value) return null
|
||||||
|
const trimmed = value.trim()
|
||||||
|
if (/^løbende$/iu.test(trimmed)) return null
|
||||||
|
const match = trimmed.match(/^(\d{2})-(\d{2})-(\d{4})/)
|
||||||
|
return match ? `${match[3]}-${match[2]}-${match[1]}` : toContractDate(trimmed)
|
||||||
|
}
|
||||||
|
|
||||||
function overviewValue(root: ReturnType<typeof parse>, label: string): string | null {
|
function overviewValue(root: ReturnType<typeof parse>, label: string): string | null {
|
||||||
const normalizedLabel = label.toLowerCase()
|
const normalizedLabel = label.toLowerCase()
|
||||||
for (const item of root.querySelectorAll(".job-overview li")) {
|
for (const item of root.querySelectorAll(".job-overview li")) {
|
||||||
@@ -174,8 +190,8 @@ function fromRenderedHtml(root: ReturnType<typeof parse>, slug: string, url: str
|
|||||||
slug,
|
slug,
|
||||||
url,
|
url,
|
||||||
title,
|
title,
|
||||||
datePosted: overviewValue(root, "Udgivet") ?? "",
|
datePosted: normalizeOverviewDate(overviewValue(root, "Udgivet")) ?? "",
|
||||||
validThrough: overviewValue(root, "Ansøgningsfrist"),
|
validThrough: normalizeOverviewDate(overviewValue(root, "Ansøgningsfrist")),
|
||||||
employmentType: employmentType ? [employmentType] : [],
|
employmentType: employmentType ? [employmentType] : [],
|
||||||
hiringOrganization: {
|
hiringOrganization: {
|
||||||
name: companyName,
|
name: companyName,
|
||||||
@@ -183,7 +199,7 @@ function fromRenderedHtml(root: ReturnType<typeof parse>, slug: string, url: str
|
|||||||
},
|
},
|
||||||
jobLocation: {
|
jobLocation: {
|
||||||
streetAddress: workplace,
|
streetAddress: workplace,
|
||||||
addressLocality: null,
|
addressLocality: extractCity(workplace),
|
||||||
addressRegion: null,
|
addressRegion: null,
|
||||||
postalCode: null,
|
postalCode: null,
|
||||||
addressCountry: "DK",
|
addressCountry: "DK",
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ interface ApiSearchResponse {
|
|||||||
totalPages: number
|
totalPages: number
|
||||||
}
|
}
|
||||||
|
|
||||||
function toContractDate(value: string | null): string | null {
|
export function toContractDate(value: string | null): string | null {
|
||||||
const match = value?.match(/^(\d{2})-(\d{2})-(\d{4})$/)
|
const match = value?.match(/^(\d{2})-(\d{2})-(\d{4})$/)
|
||||||
return match ? `${match[3]}-${match[2]}-${match[1]}` : (value ?? null)
|
return match ? `${match[3]}-${match[2]}-${match[1]}` : (value ?? null)
|
||||||
}
|
}
|
||||||
@@ -43,7 +43,7 @@ function toContractDate(value: string | null): string | null {
|
|||||||
// "Lautruphoej 2, 2750 Ballerup" or "2670, Greve". The comma fallback
|
// "Lautruphoej 2, 2750 Ballerup" or "2670, Greve". The comma fallback
|
||||||
// requires a non-digit after the comma so a 4-digit street number
|
// requires a non-digit after the comma so a 4-digit street number
|
||||||
// ("Vejlevej 1234, 7100 Vejle") never wins over the real postcode.
|
// ("Vejlevej 1234, 7100 Vejle") never wins over the real postcode.
|
||||||
function extractCity(address: string | null): string | null {
|
export function extractCity(address: string | null): string | null {
|
||||||
if (!address) return null
|
if (!address) return null
|
||||||
const city =
|
const city =
|
||||||
address.match(/\d{4}\s+(.+)$/)?.[1] ?? address.match(/\d{4}\s*,\s*([^\d,].*)$/)?.[1]
|
address.match(/\d{4}\s+(.+)$/)?.[1] ?? address.match(/\d{4}\s*,\s*([^\d,].*)$/)?.[1]
|
||||||
|
|||||||
@@ -40,16 +40,31 @@ describe("parseJobPostingFromHtml", () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
expect(parsed.title).toBe("Journalistisk udvikler søges");
|
expect(parsed.title).toBe("Journalistisk udvikler søges");
|
||||||
expect(parsed.datePosted).toBe("03-07-2026");
|
// The fallback must emit the same shapes as the JSON-LD branch: contract
|
||||||
expect(parsed.validThrough).toBe("02-08-2026 23.59");
|
// dates, not the page's raw DD-MM-YYYY text (review finding F25, 2026-08-19).
|
||||||
|
expect(parsed.datePosted).toBe("2026-07-03");
|
||||||
|
expect(parsed.validThrough).toBe("2026-08-02");
|
||||||
expect(parsed.employmentType).toEqual(["Fuldtid"]);
|
expect(parsed.employmentType).toEqual(["Fuldtid"]);
|
||||||
expect(parsed.hiringOrganization.name).toBe("JFM");
|
expect(parsed.hiringOrganization.name).toBe("JFM");
|
||||||
expect(parsed.hiringOrganization.logo).toBe("https://jobdanmark.dk/media/jfm-logo.png?width=100");
|
expect(parsed.hiringOrganization.logo).toBe("https://jobdanmark.dk/media/jfm-logo.png?width=100");
|
||||||
expect(parsed.jobLocation.streetAddress).toBe("Banegårdspladsen 1, 5000 Odense C");
|
expect(parsed.jobLocation.streetAddress).toBe("Banegårdspladsen 1, 5000 Odense C");
|
||||||
|
expect(parsed.jobLocation.addressLocality).toBe("Odense C");
|
||||||
expect(parsed.description).toContain("identificere relevante datasæt");
|
expect(parsed.description).toContain("identificere relevante datasæt");
|
||||||
expect(parsed.applyUrl).toBe("https://jfm.career.emply.com/da/apply/example");
|
expect(parsed.applyUrl).toBe("https://jfm.career.emply.com/da/apply/example");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("maps a rolling deadline (Løbende) to null in the HTML fallback", () => {
|
||||||
|
// "Løbende" is free text meaning rolling/ongoing - jobbank's parser maps
|
||||||
|
// its equivalent to null, and a stored "Løbende" deadline would hit every
|
||||||
|
// date-arithmetic consumer (review finding F25, 2026-08-19).
|
||||||
|
const html = HTML_WITHOUT_JSON_LD.replace(
|
||||||
|
"<li><strong>Ansøgningsfrist:</strong> 02-08-2026 23.59</li>",
|
||||||
|
"<li><strong>Ansøgningsfrist:</strong> Løbende</li>",
|
||||||
|
);
|
||||||
|
const parsed = parseJobPostingFromHtml(html, "s", "https://jobdanmark.dk/job/s");
|
||||||
|
expect(parsed.validThrough).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
test("does not reject titles containing '404' mid-phrase", () => {
|
test("does not reject titles containing '404' mid-phrase", () => {
|
||||||
const htmlWith404InTitle = HTML_WITHOUT_JSON_LD.replace(
|
const htmlWith404InTitle = HTML_WITHOUT_JSON_LD.replace(
|
||||||
"<title>Journalistisk udvikler søges | jobdanmark</title>",
|
"<title>Journalistisk udvikler søges | jobdanmark</title>",
|
||||||
|
|||||||
@@ -120,6 +120,14 @@ per-file diff commands.
|
|||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
|
- **`jobdanmark-search detail`'s HTML fallback emits the same shapes as its JSON-LD
|
||||||
|
branch** - a posting without JSON-LD returned `datePosted` as the page's raw
|
||||||
|
`DD-MM-YYYY` text, `validThrough` as free text (including the literal `"Løbende"`,
|
||||||
|
which would flow into stored data as a deadline), and a hardcoded `null`
|
||||||
|
`addressLocality`. The fallback now converts overview dates to `YYYY-MM-DD`, maps
|
||||||
|
`Løbende` to `null` (jobbank's precedent for the equivalent), and derives the locality
|
||||||
|
from the workplace address with the same postcode extraction search uses. Pinned in
|
||||||
|
`tests/detail-parsing.test.ts`.
|
||||||
- **`jobnet-search detail` no longer leaks the `1900-01-01` undisclosed-deadline
|
- **`jobnet-search detail` no longer leaks the `1900-01-01` undisclosed-deadline
|
||||||
sentinel** - `search` maps the API's sentinel to `null` (with a test pinning it), but
|
sentinel** - `search` maps the API's sentinel to `null` (with a test pinning it), but
|
||||||
`detail` dumped the raw response, so a posting whose deadline is simply not disclosed
|
`detail` dumped the raw response, so a posting whose deadline is simply not disclosed
|
||||||
|
|||||||
Reference in New Issue
Block a user