mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-09-17 00:26:26 +00:00
fix(jobdanmark): narrow soft-404 detection to avoid rejecting real postings (#206)
The jobdanmark detail parser flagged a soft-404 by testing whether the page title contained the substring '404' anywhere, so a legitimate posting titled e.g. 'Room 404 Cleaner' was wrongly rejected as NOT_FOUND. Narrows title matching to startsWith('404') plus specific error phrases ('page not found', Danish 'siden blev ikke fundet'), keeping the existing body-text backstop. Verified: strictly reduces false-positives, the real 404-page title ('404 | Jobdanmark') still detected, tests pass network-free.
By @oscarbol09.
This commit is contained in:
@@ -134,7 +134,14 @@ function overviewValue(root: ReturnType<typeof parse>, label: string): string |
|
|||||||
|
|
||||||
function fromRenderedHtml(root: ReturnType<typeof parse>, slug: string, url: string): DetailResult {
|
function fromRenderedHtml(root: ReturnType<typeof parse>, slug: string, url: string): DetailResult {
|
||||||
const pageTitle = cleanText(root.querySelector("title")?.text ?? "")
|
const pageTitle = cleanText(root.querySelector("title")?.text ?? "")
|
||||||
if (pageTitle.toLowerCase().includes("404") || root.text.toLowerCase().includes("siden blev ikke fundet")) {
|
const titleLower = pageTitle.toLowerCase()
|
||||||
|
const bodyText = root.text.toLowerCase()
|
||||||
|
if (
|
||||||
|
bodyText.includes("siden blev ikke fundet") ||
|
||||||
|
titleLower.startsWith("404") ||
|
||||||
|
titleLower.includes("page not found") ||
|
||||||
|
titleLower.includes("siden blev ikke fundet")
|
||||||
|
) {
|
||||||
throw new Error("NOT_FOUND")
|
throw new Error("NOT_FOUND")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -49,4 +49,23 @@ describe("parseJobPostingFromHtml", () => {
|
|||||||
expect(parsed.description).toContain("identificere relevante datasæt");
|
expect(parsed.description).toContain("identificere relevante datasæt");
|
||||||
expect(parsed.applyUrl).toBe("https://jfm.career.emply.com/da/apply/example");
|
expect(parsed.applyUrl).toBe("https://jfm.career.emply.com/da/apply/example");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("does not reject titles containing '404' mid-phrase", () => {
|
||||||
|
const htmlWith404InTitle = HTML_WITHOUT_JSON_LD.replace(
|
||||||
|
"<title>Journalistisk udvikler søges | jobdanmark</title>",
|
||||||
|
"<title>HTTP 404 Page Designer | jobdanmark</title>",
|
||||||
|
).replace(
|
||||||
|
'<h3 class="title">Journalistisk udvikler søges</h3>',
|
||||||
|
'<h3 class="title">HTTP 404 Page Designer</h3>',
|
||||||
|
);
|
||||||
|
|
||||||
|
const parsed = parseJobPostingFromHtml(
|
||||||
|
htmlWith404InTitle,
|
||||||
|
"http-404-designer",
|
||||||
|
"https://jobdanmark.dk/job/http-404-designer",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(parsed.title).toBe("HTTP 404 Page Designer");
|
||||||
|
expect(parsed.hiringOrganization.name).toBe("JFM");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user