fix(jobindex-search): rewrite detail parser against current page shapes

Every selector the old parser used (job-text, jix-info,
jix_robotjob--area, jix-toolbar-top__company) is gone from live pages:
detail returned CSS-comment text as the deadline, the external ATS URL
as its own id/url, null company/location/date, and a teaser description
- exit 0, on 4/5 live postings. The rewrite recognises both current
shapes (jobindex-native jd-* layout; external-ATS passthrough), always
keeps the jobindex id and jobannonce URL, anchors the deadline to a real
date in visible markup only (the label also lives in a CSS comment,
which is what the old regex captured), converts Danish long dates to
ISO, decodes Danish named entities, and returns company: null on
passthrough pages instead of the ATS brand. Live: 5/5 postings now
yield full descriptions and correct fields. Review finding F14
(2026-08-19).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Mads Lorentzen
2026-08-19 21:02:29 +02:00
co-authored by Claude Opus 5
parent 9a69309749
commit c85640e30a
4 changed files with 336 additions and 136 deletions
+23 -6
View File
@@ -169,12 +169,29 @@ bun run src/cli.ts detail h1647303 --format plain
```
**Field notes:**
- `deadline` — application deadline date string (`YYYY-MM-DD`); `null` if not listed. Postings flagged "ASAP" by the portal carry no fixed deadline and also map to `null`.
- `employmentType` — e.g. `"Fastansættelse"`, `"Midlertidig ansættelse"`; `null` if not listed.
- `hours` — e.g. `"Fuldtid"`, `"Deltid"`; `null` if not listed.
- `applyUrl` — the external application URL (resolved from the Jobindex redirect link `/c?t=...`); `null` if not available.
- `description` — full plain-text job description (HTML stripped).
- All fields may be `null` if not present in the HTML.
Jobindex serves detail pages in two shapes, and field availability differs:
a **jobindex-native** page (recognisable by its `jd-*` facts blocks) carries
company, location, an ISO deadline, employment type and hours; an **external
ATS passthrough** (the employer's hosted ad, e.g. hr-manager/Talentech, served
through jobindex) has no reliable company anchor, so `company` is `null` there
rather than the ATS brand, and location/deadline come from the ad's own
widgets when present.
- `id` / `url` — always the jobindex id and its `jobannonce` URL, never the
page's `og:url`/canonical (on passthrough pages those point at the external
ATS, not the posting).
- `deadline``YYYY-MM-DD` or `null`; Danish long dates ("13. september
2026") and `DD-MM-YYYY` widget dates are converted.
- `employmentType` / `hours` — from the native facts blocks; `null` on
passthrough pages.
- `companyUrl` — currently always `null`; no page shape carries a usable
company link.
- `applyUrl` — the Jobindex redirect link (`/c?t=...`) when present; `null`
otherwise.
- `description` — plain text of the ad body (HTML stripped), falling back to
the page's meta description when the body is empty.
- All fields except `id`, `title`, and `url` may be `null`.
---
@@ -1,6 +1,6 @@
import { defineCommand, option } from "@bunli/core"
import { z } from "zod"
import { htmlFetch, writeError, extractDivContent } from "../helpers.js"
import { htmlFetch, writeError } from "../helpers.js"
const BASE_URL = "https://www.jobindex.dk"
@@ -39,6 +39,13 @@ function decodeHtmlEntities(text: string): string {
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&apos;/g, "'")
// Danish letters appear as named entities in employer-hosted ad markup.
.replace(/&oslash;/g, "ø")
.replace(/&Oslash;/g, "Ø")
.replace(/&aelig;/g, "æ")
.replace(/&AElig;/g, "Æ")
.replace(/&aring;/g, "å")
.replace(/&Aring;/g, "Å")
// Numeric character references: decimal (&#233;) and hexadecimal (&#xE9;).
.replace(/&#(\d+);/g, (_, dec) => numericEntity(parseInt(dec, 10)))
.replace(/&#[xX]([0-9a-fA-F]+);/g, (_, hex) => numericEntity(parseInt(hex, 16)))
@@ -72,150 +79,169 @@ function buildUrl(idOrUrl: string): { url: string; id: string } {
return { url, id: idOrUrl }
}
/**
* Parse the detail HTML page using regex to avoid node-html-parser nesting bugs.
*/
function parseDetailPage(html: string, url: string, id: string): DetailResult {
// Title: extract from <h1> tag
const h1Match = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)
const title = h1Match ? decodeHtmlEntities(stripTags(h1Match[1])) : ""
const DANISH_MONTHS: Record<string, string> = {
januar: "01", februar: "02", marts: "03", april: "04", maj: "05", juni: "06",
juli: "07", august: "08", september: "09", oktober: "10", november: "11", december: "12",
}
/**
* Normalize a date found on a detail page to YYYY-MM-DD, or null.
* Live pages carry three shapes: ISO, DD-MM-YYYY (the hr-manager widget),
* and Danish long form ("13. september 2026", the jobindex-native facts box).
*/
export function toIsoDate(value: string | null | undefined): string | null {
if (!value) return null
const text = value.trim()
let m = text.match(/^(\d{4})-(\d{2})-(\d{2})/)
if (m) return `${m[1]}-${m[2]}-${m[3]}`
m = text.match(/^(\d{2})-(\d{2})-(\d{4})/)
if (m) return `${m[3]}-${m[2]}-${m[1]}`
m = text.match(/^(\d{1,2})\.?\s+([a-zæøå]+)\s+(\d{4})/i)
if (m) {
const month = DANISH_MONTHS[m[2].toLowerCase()]
if (month) return `${m[3]}-${month}-${m[1].padStart(2, "0")}`
}
return null
}
function metaContent(html: string, matcher: string): string | null {
const re = new RegExp(
`<meta[^>]+(?:property|name|itemprop)="${matcher}"[^>]+content="([^"]*)"|<meta[^>]+content="([^"]*)"[^>]+(?:property|name|itemprop)="${matcher}"`,
"i",
)
const m = html.match(re)
const value = m ? (m[1] ?? m[2]) : null
return value ? decodeHtmlEntities(value).trim() || null : null
}
/** The text of the <p> inside a jobindex-native jd-* facts block. */
function jdBlockValue(html: string, cls: string): string | null {
const m = html.match(new RegExp(`class="${cls}"[^>]*>[\\s\\S]*?<p[^>]*>([\\s\\S]*?)</p>`, "i"))
return m ? decodeHtmlEntities(stripTags(m[1])).replace(/\s+/g, " ").trim() || null : null
}
/** Drop head/script/style content so text scans never read CSS or JS. */
function visibleHtml(html: string): string {
return html
.replace(/<head[\s\S]*?<\/head>/gi, "")
.replace(/<script[\s\S]*?<\/script>/gi, "")
.replace(/<style[\s\S]*?<\/style>/gi, "")
.replace(/<!--[\s\S]*?-->/g, "")
}
function bodyText(html: string): string | null {
const text = decodeHtmlEntities(stripTags(visibleHtml(html).replace(/<(br|\/p|\/div|\/li|\/h[1-6])[^>]*>/gi, "\n")))
.split("\n")
.map((line) => line.replace(/\s+/g, " ").trim())
.filter(Boolean)
.join("\n")
return text || null
}
/**
* Parse a live detail page. Jobindex serves two shapes (verified live
* 2026-08-19; the selectors the previous parser used exist in neither):
*
* - the jobindex-native shape, recognisable by its `jd-*` facts blocks
* (jd-deadline, jd-location, ...), with the company as the `<title>`
* prefix ("COMPANY - Job title");
* - an external ATS passthrough (hr-manager/Talentech and similar), where
* the page IS the employer's hosted ad: `og:url` points at the ATS,
* `og:site_name` is the ATS brand, and there is no reliable company
* anchor at all - so `company` is honestly null there, never the ATS.
*
* `id` and `url` are always the caller's jobindex id and its jobannonce
* URL: the canonical/og:url on these pages is the external ATS, and
* storing that broke /scrape's "store a URL that resolves to the posting".
*/
export function parseDetailPage(html: string, url: string, id: string): DetailResult {
const isNative = html.includes('class="jd-')
const ogTitle = metaContent(html, "og:title")
const itempropName = metaContent(html, "name")
const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)
const h1Title = h1 ? decodeHtmlEntities(stripTags(h1[1])).replace(/\s+/g, " ").trim() : null
const title = ogTitle ?? itempropName ?? h1Title ?? ""
if (!title) {
throw new Error("Failed to parse job listing HTML")
}
// Company and companyUrl from jix-toolbar-top__company section
// Company: only the native shape carries one - as the <title> prefix,
// "VELLIV - Udvikler til Camunda/AWS". Require the suffix to be the job
// title so an unrelated <title> never becomes a company name.
let company: string | null = null
let companyUrl: string | null = null
const companySection = html.match(/class="jix-toolbar-top__company"[^>]*>([\s\S]*?)<\/div>/i)
if (companySection) {
const linkMatch = companySection[1].match(/<[Aa][^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/[Aa]>/i)
if (linkMatch) {
company = decodeHtmlEntities(stripTags(linkMatch[2])) || null
companyUrl = linkMatch[1] || null
if (isNative) {
const titleTag = html.match(/<title>([\s\S]*?)<\/title>/i)
const pageTitle = titleTag ? decodeHtmlEntities(titleTag[1]).replace(/\s+/g, " ").trim() : ""
if (pageTitle.endsWith(` - ${title}`)) {
company = pageTitle.slice(0, -(title.length + 3)).trim() || null
}
}
// Location from jix_robotjob--area span
let location: string | null = null
const locMatch = html.match(/<span[^>]+class="jix_robotjob--area"[^>]*>([\s\S]*?)<\/span>/i)
if (locMatch) {
location = decodeHtmlEntities(stripTags(locMatch[1])) || null
}
// Date from <time datetime="..."> element
let date: string | null = null
const timeMatch = html.match(/<time[^>]+datetime="([^"]+)"/)
if (timeMatch) {
date = timeMatch[1] || null
}
// Employment type and hours from jix-info section
let deadline: string | null = null
let employmentType: string | null = null
let hours: string | null = null
let deadline: string | null = null
const jixInfoMatch = html.match(/class="jix-info"[^>]*>([\s\S]*?)<\/div>/i)
if (jixInfoMatch) {
const jixInfoHtml = jixInfoMatch[1]
// Parse p elements with bold labels
const pMatches = [...jixInfoHtml.matchAll(/<p[^>]*><b>([^<]+)<\/b>\s*([\s\S]*?)<\/p>/gi)]
for (const pm of pMatches) {
const label = pm[1].toLowerCase().trim()
const value = stripTags(pm[2]).trim()
if (label.includes("ansættelsestype") || label.includes("employment type")) {
employmentType = decodeHtmlEntities(value) || null
} else if (label.includes("ugentlig arbejdstid") || label.includes("weekly working time") || label.includes("arbejdstid")) {
hours = decodeHtmlEntities(value) || null
} else if (label.includes("ansøgningsfrist") || label.includes("deadline") || label.includes("application deadline")) {
deadline = decodeHtmlEntities(value) || null
}
}
}
// If not found in jix-info, try broader text patterns
if (!employmentType) {
const emtMatch = html.match(/<b>(?:Ansættelsestype|Employment\s*type):<\/b>\s*([^<\n]+)/i)
if (emtMatch) {
employmentType = decodeHtmlEntities(emtMatch[1].trim()) || null
}
}
if (!hours) {
const hoursMatch = html.match(/<b>(?:Ugentlig\s*arbejdstid|Weekly\s*working\s*time):<\/b>\s*([^<\n]+)/i)
if (hoursMatch) {
hours = decodeHtmlEntities(hoursMatch[1].trim()) || null
}
}
// Deadline from application section
if (!deadline) {
// Look for "senest den" or "Ansøgningsfrist" patterns in text
const deadlineMatch = html.match(/Ansøgningsfrist[^:]*:\s*([^<\n,]+)/i)
if (deadlineMatch) {
deadline = decodeHtmlEntities(deadlineMatch[1].trim()) || null
}
}
// Apply URL: look for /c?t= redirect links in jix_onlineapplication_button
let applyUrl: string | null = null
const applySection = html.match(/class="jix_onlineapplication_button"[^>]*>[\s\S]*?href="([^"]+)"/i)
if (applySection) {
const href = decodeHtmlEntities(applySection[1])
applyUrl = href.startsWith("http") ? href : `${BASE_URL}${href}`
}
// If not found, look for any /c?t= link
if (!applyUrl) {
const ctMatch = html.match(/href="(\/c\?t=[^"]+)"/)
if (ctMatch) {
applyUrl = `${BASE_URL}${decodeHtmlEntities(ctMatch[1])}`
}
}
// Description: job text section
let description: string | null = null
// Try job-text class first
const jobTextHtml = extractDivContent(html, "job-text")
if (jobTextHtml) {
description = decodeHtmlEntities(stripTags(jobTextHtml)).replace(/\s+/g, " ").trim() || null
}
if (isNative) {
location = jdBlockValue(html, "jd-location")
deadline = toIsoDate(jdBlockValue(html, "jd-deadline"))
employmentType = jdBlockValue(html, "jd-type")
hours = jdBlockValue(html, "jd-workhours")
const desc = html.match(/class="jd-description"[^>]*>([\s\S]*?)<\/div>/i)
description = desc
? decodeHtmlEntities(stripTags(desc[1])).replace(/\s+/g, " ").trim() || null
: null
} else {
// hr-manager-style widget: a rowheader label followed by the value span.
const workplace = visibleHtml(html).match(
/class="workplace[^"]*"[\s\S]*?<span class="empty">([\s\S]*?)<\/span>/i,
)
location = workplace
? decodeHtmlEntities(stripTags(workplace[1])).replace(/\s+/g, " ").trim() || null
: null
// Fallback: try og:description meta tag for a brief description
if (!description) {
const ogDescMatch = html.match(/property="og:description"[^>]+content="([^"]+)"/i) ||
html.match(/content="([^"]+)"[^>]+property="og:description"/i)
if (ogDescMatch) {
description = decodeHtmlEntities(ogDescMatch[1]) || null
// Deadline: label + a real date within range, scanned only over visible
// markup - the label also appears inside a CSS comment on these pages,
// which the previous parser captured verbatim as the deadline.
const due = visibleHtml(html).match(
/(?:Ansøgningsfrist|Application\s*due|Frist)[\s\S]{0,300}?(\d{2}-\d{2}-\d{4}|\d{4}-\d{2}-\d{2}|\d{1,2}\.?\s+[a-zæøå]+\s+\d{4})/i,
)
deadline = due ? toIsoDate(due[1]) : null
description = bodyText(html)
if (!description || description.length < 100) {
description = metaContent(html, "og:description") ?? metaContent(html, "description") ?? description
}
}
// Get canonical URL or use the fetched URL
const canonicalMatch = html.match(/<link[^>]+rel="canonical"[^>]+href="([^"]+)"/i) ||
html.match(/property="og:url"[^>]+content="([^"]+)"/i) ||
html.match(/content="([^"]+)"[^>]+property="og:url"/i)
const canonicalUrl = canonicalMatch ? canonicalMatch[1] : url
if (!description) {
description = metaContent(html, "og:description")
}
// Extract ID from canonical URL, fall back to the provided ID
const canonicalId = extractIdFromUrl(canonicalUrl) || id
// Apply URL: jobindex's own /c?t= redirect when present.
let applyUrl: string | null = null
const ctMatch = html.match(/href="(\/c\?t=[^"]+)"/)
if (ctMatch) {
applyUrl = `${BASE_URL}${decodeHtmlEntities(ctMatch[1])}`
}
const timeMatch = html.match(/<time[^>]+datetime="([^"]+)"/)
return {
id: canonicalId,
id,
title,
company: company || null,
companyUrl: companyUrl || null,
location: location || null,
date: date || null,
deadline: deadline || null,
employmentType: employmentType || null,
hours: hours || null,
applyUrl: applyUrl || null,
url: canonicalUrl,
description: description || null,
company,
companyUrl: null,
location,
date: timeMatch ? toIsoDate(timeMatch[1]) : null,
deadline,
employmentType,
hours,
applyUrl,
url,
description,
}
}
@@ -243,13 +269,6 @@ export const detail = defineCommand({
if (signal.aborted) return
// Check if page is not a valid job listing
// A valid job listing has an <h1> tag
if (!html.includes("<h1>") && !html.includes("<h1 ")) {
writeError("Job not found", "NOT_FOUND")
process.exit(1)
}
let data: DetailResult
try {
data = parseDetailPage(html, url, id)
@@ -0,0 +1,152 @@
import { describe, expect, test } from "bun:test";
import { parseDetailPage } from "../src/commands/detail";
// Jobindex redesigned its detail pages; every selector the old parser used
// (job-text, jix-info, jix_robotjob--area, jix-toolbar-top__company) is gone
// from live pages, so detail returned null company/location/date, CSS-comment
// text as the deadline, and an external ATS URL as its own id/url - exit 0,
// nothing signalling breakage (review finding F14, 2026-08-19, measured on
// 5/5 live postings). These fixtures are trimmed from live pages captured
// 2026-08-19: the jobindex-native "jd-*" shape and the external-ATS
// (hr-manager/Talentech) passthrough shape.
const NATIVE_PAGE = `<!DOCTYPE html>
<html lang="da">
<head>
<title>VELLIV - Udvikler til Camunda/AWS</title>
<meta property="og:title" content="Udvikler til Camunda/AWS" />
<meta property="og:url" content="https://www.jobindex.dk/jobannonce/h1690934" />
<meta property="og:description" content="VELLIV" />
</head>
<body>
<div class="jd-appetizer">
<h1>Udvikler til Camunda/AWS</h1>
</div>
<div id="container" class="container">
<div class="row">
<div class="twelve columns jd-details">
<div class="jd-description">
<p class="appetizer">En virksomhed med mere end 100 &aring;rs historie, der samtidig er cloud-only, er ikke hverdagskost.</p>
<p>Hos Velliv f&aring;r du mulighed for at arbejde med Camunda, AWS og automatisering af processer.</p>
</div>
</div>
<div class="four columns jd-facts">
<div class="jd-type">
<h3>Jobtype:</h3>
<p>Fast</p>
</div>
<div class="jd-workhours">
<h3>Arbejdstid:</h3>
<p>Fuldtid</p>
</div>
<div class="jd-worktime">
<h3>Arbejdsdage:</h3>
<p>Dag</p>
</div>
<div class="jd-deadline">
<h3>Ans&oslash;gningsfrist:</h3>
<p>13. september 2026</p>
</div>
<div class="jd-location">
<h3>Arbejdssted:</h3>
<p>Ballerup</p>
</div>
</div>
</div>
</div>
</body>
</html>`;
// The external shape: an employer's ATS-hosted ad served through jobindex.
// og:url points at the ATS (NOT the posting), og:site_name is the ATS brand,
// and the only occurrence of the deadline label outside the widget is inside
// a CSS comment - the exact text the old regex captured as the deadline.
const EXTERNAL_PAGE = `<!DOCTYPE html>
<html>
<head>
<title>
Talentech - C#-udvikler til kritiske analysel&oslash;sninger i elnettet
</title>
<meta name="description" content="Vi s&oslash;ger en udvikler til elnettet" />
<meta itemprop="name" content="C#-udvikler til kritiske analysel&oslash;sninger i elnettet" />
<meta property="og:title" content="C#-udvikler til kritiske analysel&oslash;sninger i elnettet" />
<meta property="og:site_name" content="Talentech" />
<meta property="og:url" content="https://candidate.hr-manager.net/ApplicationInit.aspx?cid=316&amp;ProjectId=188792" />
<style>
/* Defines the style of the Application Due text */
/* DK: Ansøgningsfrist */
/* BOKSTAV: K */
.frist { padding-bottom: 15px; }
</style>
</head>
<body>
<h1>C#-udvikler til kritiske analysel&oslash;sninger i elnettet</h1>
<p>Vil du v&aelig;re med til at udvikle og drifte de systemer, der underst&oslash;tter udbygningen af Danmarks kommende elnet? Vi arbejder i krydsfeltet mellem IT og energi.</p>
<div class="workplacelist emptyparent"><div id="workplacelist_lang" class="rowheader">Workplace</div><div class="widget-line"></div><span class="empty">Fredericia</span><br></div>
<div class="frist emptyparent"><div id="frist_lang" class="rowheader">Application due</div><div class="widget-line"></div><span class="empty">21-09-2026</span><br></div>
</body>
</html>`;
const JOBANNONCE_URL = "https://www.jobindex.dk/jobannonce/";
describe("parseDetailPage - jobindex-native (jd-*) shape", () => {
const job = parseDetailPage(NATIVE_PAGE, `${JOBANNONCE_URL}h1690934`, "h1690934");
test("extracts the contract fields", () => {
expect(job).toMatchObject({
id: "h1690934",
title: "Udvikler til Camunda/AWS",
company: "VELLIV",
location: "Ballerup",
deadline: "2026-09-13",
url: `${JOBANNONCE_URL}h1690934`,
});
});
test("converts the Danish long date to ISO", () => {
expect(job.deadline).toBe("2026-09-13");
});
test("extracts employment metadata and the description body", () => {
expect(job.employmentType).toBe("Fast");
expect(job.hours).toBe("Fuldtid");
expect(job.description).toContain("Camunda, AWS og automatisering");
});
});
describe("parseDetailPage - external ATS passthrough shape", () => {
const job = parseDetailPage(EXTERNAL_PAGE, `${JOBANNONCE_URL}h1690445`, "h1690445");
test("keeps the jobindex id and jobannonce URL, never the ATS og:url", () => {
expect(job.id).toBe("h1690445");
expect(job.url).toBe(`${JOBANNONCE_URL}h1690445`);
expect(job.url).not.toContain("hr-manager.net");
});
test("extracts the title and never reports the ATS brand as the company", () => {
expect(job.title).toBe("C#-udvikler til kritiske analyseløsninger i elnettet");
expect(job.company).toBeNull();
});
test("extracts the workplace widget location", () => {
expect(job.location).toBe("Fredericia");
});
test("finds the real deadline, not the CSS comment", () => {
expect(job.deadline).toBe("2026-09-21");
});
test("description is readable body text, not a stylesheet", () => {
expect(job.description).toContain("udbygningen af Danmarks kommende elnet");
expect(job.description).not.toContain("padding-bottom");
});
test("deadline is null when only the CSS comment mentions the label", () => {
const noDueWidget = EXTERNAL_PAGE.replace(
/<div class="frist emptyparent">[\s\S]*?<br><\/div>/,
"",
);
const parsed = parseDetailPage(noDueWidget, `${JOBANNONCE_URL}h9`, "h9");
expect(parsed.deadline).toBeNull();
});
});