diff --git a/.agents/skills/jobindex-search/cli/README.md b/.agents/skills/jobindex-search/cli/README.md index a13b69a..386ed78 100644 --- a/.agents/skills/jobindex-search/cli/README.md +++ b/.agents/skills/jobindex-search/cli/README.md @@ -169,12 +169,29 @@ bun run src/cli.ts detail h1647303 --format plain ``` **Field notes:** -- `deadline` — application deadline date string (`YYYY-MM-DD`); `null` if not listed. Postings flagged "ASAP" by the portal carry no fixed deadline and also map to `null`. -- `employmentType` — e.g. `"Fastansættelse"`, `"Midlertidig ansættelse"`; `null` if not listed. -- `hours` — e.g. `"Fuldtid"`, `"Deltid"`; `null` if not listed. -- `applyUrl` — the external application URL (resolved from the Jobindex redirect link `/c?t=...`); `null` if not available. -- `description` — full plain-text job description (HTML stripped). -- All fields may be `null` if not present in the HTML. + +Jobindex serves detail pages in two shapes, and field availability differs: +a **jobindex-native** page (recognisable by its `jd-*` facts blocks) carries +company, location, an ISO deadline, employment type and hours; an **external +ATS passthrough** (the employer's hosted ad, e.g. hr-manager/Talentech, served +through jobindex) has no reliable company anchor, so `company` is `null` there +rather than the ATS brand, and location/deadline come from the ad's own +widgets when present. + +- `id` / `url` — always the jobindex id and its `jobannonce` URL, never the + page's `og:url`/canonical (on passthrough pages those point at the external + ATS, not the posting). +- `deadline` — `YYYY-MM-DD` or `null`; Danish long dates ("13. september + 2026") and `DD-MM-YYYY` widget dates are converted. +- `employmentType` / `hours` — from the native facts blocks; `null` on + passthrough pages. +- `companyUrl` — currently always `null`; no page shape carries a usable + company link. +- `applyUrl` — the Jobindex redirect link (`/c?t=...`) when present; `null` + otherwise. +- `description` — plain text of the ad body (HTML stripped), falling back to + the page's meta description when the body is empty. +- All fields except `id`, `title`, and `url` may be `null`. --- diff --git a/.agents/skills/jobindex-search/cli/src/commands/detail.ts b/.agents/skills/jobindex-search/cli/src/commands/detail.ts index 0e82094..f2dab2d 100644 --- a/.agents/skills/jobindex-search/cli/src/commands/detail.ts +++ b/.agents/skills/jobindex-search/cli/src/commands/detail.ts @@ -1,6 +1,6 @@ import { defineCommand, option } from "@bunli/core" import { z } from "zod" -import { htmlFetch, writeError, extractDivContent } from "../helpers.js" +import { htmlFetch, writeError } from "../helpers.js" const BASE_URL = "https://www.jobindex.dk" @@ -39,6 +39,13 @@ function decodeHtmlEntities(text: string): string { .replace(/"/g, '"') .replace(/'/g, "'") .replace(/'/g, "'") + // Danish letters appear as named entities in employer-hosted ad markup. + .replace(/ø/g, "ø") + .replace(/Ø/g, "Ø") + .replace(/æ/g, "æ") + .replace(/Æ/g, "Æ") + .replace(/å/g, "å") + .replace(/Å/g, "Å") // Numeric character references: decimal (é) and hexadecimal (é). .replace(/&#(\d+);/g, (_, dec) => numericEntity(parseInt(dec, 10))) .replace(/&#[xX]([0-9a-fA-F]+);/g, (_, hex) => numericEntity(parseInt(hex, 16))) @@ -72,150 +79,169 @@ function buildUrl(idOrUrl: string): { url: string; id: string } { return { url, id: idOrUrl } } -/** - * Parse the detail HTML page using regex to avoid node-html-parser nesting bugs. - */ -function parseDetailPage(html: string, url: string, id: string): DetailResult { - // Title: extract from

tag - const h1Match = html.match(/]*>([\s\S]*?)<\/h1>/i) - const title = h1Match ? decodeHtmlEntities(stripTags(h1Match[1])) : "" +const DANISH_MONTHS: Record = { + januar: "01", februar: "02", marts: "03", april: "04", maj: "05", juni: "06", + juli: "07", august: "08", september: "09", oktober: "10", november: "11", december: "12", +} +/** + * Normalize a date found on a detail page to YYYY-MM-DD, or null. + * Live pages carry three shapes: ISO, DD-MM-YYYY (the hr-manager widget), + * and Danish long form ("13. september 2026", the jobindex-native facts box). + */ +export function toIsoDate(value: string | null | undefined): string | null { + if (!value) return null + const text = value.trim() + let m = text.match(/^(\d{4})-(\d{2})-(\d{2})/) + if (m) return `${m[1]}-${m[2]}-${m[3]}` + m = text.match(/^(\d{2})-(\d{2})-(\d{4})/) + if (m) return `${m[3]}-${m[2]}-${m[1]}` + m = text.match(/^(\d{1,2})\.?\s+([a-zæøå]+)\s+(\d{4})/i) + if (m) { + const month = DANISH_MONTHS[m[2].toLowerCase()] + if (month) return `${m[3]}-${month}-${m[1].padStart(2, "0")}` + } + return null +} + +function metaContent(html: string, matcher: string): string | null { + const re = new RegExp( + `]+(?:property|name|itemprop)="${matcher}"[^>]+content="([^"]*)"|]+content="([^"]*)"[^>]+(?:property|name|itemprop)="${matcher}"`, + "i", + ) + const m = html.match(re) + const value = m ? (m[1] ?? m[2]) : null + return value ? decodeHtmlEntities(value).trim() || null : null +} + +/** The text of the

inside a jobindex-native jd-* facts block. */ +function jdBlockValue(html: string, cls: string): string | null { + const m = html.match(new RegExp(`class="${cls}"[^>]*>[\\s\\S]*?]*>([\\s\\S]*?)

`, "i")) + return m ? decodeHtmlEntities(stripTags(m[1])).replace(/\s+/g, " ").trim() || null : null +} + +/** Drop head/script/style content so text scans never read CSS or JS. */ +function visibleHtml(html: string): string { + return html + .replace(//gi, "") + .replace(//gi, "") + .replace(//gi, "") + .replace(//g, "") +} + +function bodyText(html: string): string | null { + const text = decodeHtmlEntities(stripTags(visibleHtml(html).replace(/<(br|\/p|\/div|\/li|\/h[1-6])[^>]*>/gi, "\n"))) + .split("\n") + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter(Boolean) + .join("\n") + return text || null +} + +/** + * Parse a live detail page. Jobindex serves two shapes (verified live + * 2026-08-19; the selectors the previous parser used exist in neither): + * + * - the jobindex-native shape, recognisable by its `jd-*` facts blocks + * (jd-deadline, jd-location, ...), with the company as the `` + * prefix ("COMPANY - Job title"); + * - an external ATS passthrough (hr-manager/Talentech and similar), where + * the page IS the employer's hosted ad: `og:url` points at the ATS, + * `og:site_name` is the ATS brand, and there is no reliable company + * anchor at all - so `company` is honestly null there, never the ATS. + * + * `id` and `url` are always the caller's jobindex id and its jobannonce + * URL: the canonical/og:url on these pages is the external ATS, and + * storing that broke /scrape's "store a URL that resolves to the posting". + */ +export function parseDetailPage(html: string, url: string, id: string): DetailResult { + const isNative = html.includes('class="jd-') + + const ogTitle = metaContent(html, "og:title") + const itempropName = metaContent(html, "name") + const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i) + const h1Title = h1 ? decodeHtmlEntities(stripTags(h1[1])).replace(/\s+/g, " ").trim() : null + const title = ogTitle ?? itempropName ?? h1Title ?? "" if (!title) { throw new Error("Failed to parse job listing HTML") } - // Company and companyUrl from jix-toolbar-top__company section + // Company: only the native shape carries one - as the <title> prefix, + // "VELLIV - Udvikler til Camunda/AWS". Require the suffix to be the job + // title so an unrelated <title> never becomes a company name. let company: string | null = null - let companyUrl: string | null = null - - const companySection = html.match(/class="jix-toolbar-top__company"[^>]*>([\s\S]*?)<\/div>/i) - if (companySection) { - const linkMatch = companySection[1].match(/<[Aa][^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/[Aa]>/i) - if (linkMatch) { - company = decodeHtmlEntities(stripTags(linkMatch[2])) || null - companyUrl = linkMatch[1] || null + if (isNative) { + const titleTag = html.match(/<title>([\s\S]*?)<\/title>/i) + const pageTitle = titleTag ? decodeHtmlEntities(titleTag[1]).replace(/\s+/g, " ").trim() : "" + if (pageTitle.endsWith(` - ${title}`)) { + company = pageTitle.slice(0, -(title.length + 3)).trim() || null } } - // Location from jix_robotjob--area span let location: string | null = null - const locMatch = html.match(/<span[^>]+class="jix_robotjob--area"[^>]*>([\s\S]*?)<\/span>/i) - if (locMatch) { - location = decodeHtmlEntities(stripTags(locMatch[1])) || null - } - - // Date from <time datetime="..."> element - let date: string | null = null - const timeMatch = html.match(/<time[^>]+datetime="([^"]+)"/) - if (timeMatch) { - date = timeMatch[1] || null - } - - // Employment type and hours from jix-info section + let deadline: string | null = null let employmentType: string | null = null let hours: string | null = null - let deadline: string | null = null - - const jixInfoMatch = html.match(/class="jix-info"[^>]*>([\s\S]*?)<\/div>/i) - if (jixInfoMatch) { - const jixInfoHtml = jixInfoMatch[1] - - // Parse p elements with bold labels - const pMatches = [...jixInfoHtml.matchAll(/<p[^>]*><b>([^<]+)<\/b>\s*([\s\S]*?)<\/p>/gi)] - for (const pm of pMatches) { - const label = pm[1].toLowerCase().trim() - const value = stripTags(pm[2]).trim() - - if (label.includes("ansættelsestype") || label.includes("employment type")) { - employmentType = decodeHtmlEntities(value) || null - } else if (label.includes("ugentlig arbejdstid") || label.includes("weekly working time") || label.includes("arbejdstid")) { - hours = decodeHtmlEntities(value) || null - } else if (label.includes("ansøgningsfrist") || label.includes("deadline") || label.includes("application deadline")) { - deadline = decodeHtmlEntities(value) || null - } - } - } - - // If not found in jix-info, try broader text patterns - if (!employmentType) { - const emtMatch = html.match(/<b>(?:Ansættelsestype|Employment\s*type):<\/b>\s*([^<\n]+)/i) - if (emtMatch) { - employmentType = decodeHtmlEntities(emtMatch[1].trim()) || null - } - } - - if (!hours) { - const hoursMatch = html.match(/<b>(?:Ugentlig\s*arbejdstid|Weekly\s*working\s*time):<\/b>\s*([^<\n]+)/i) - if (hoursMatch) { - hours = decodeHtmlEntities(hoursMatch[1].trim()) || null - } - } - - // Deadline from application section - if (!deadline) { - // Look for "senest den" or "Ansøgningsfrist" patterns in text - const deadlineMatch = html.match(/Ansøgningsfrist[^:]*:\s*([^<\n,]+)/i) - if (deadlineMatch) { - deadline = decodeHtmlEntities(deadlineMatch[1].trim()) || null - } - } - - // Apply URL: look for /c?t= redirect links in jix_onlineapplication_button - let applyUrl: string | null = null - const applySection = html.match(/class="jix_onlineapplication_button"[^>]*>[\s\S]*?href="([^"]+)"/i) - if (applySection) { - const href = decodeHtmlEntities(applySection[1]) - applyUrl = href.startsWith("http") ? href : `${BASE_URL}${href}` - } - - // If not found, look for any /c?t= link - if (!applyUrl) { - const ctMatch = html.match(/href="(\/c\?t=[^"]+)"/) - if (ctMatch) { - applyUrl = `${BASE_URL}${decodeHtmlEntities(ctMatch[1])}` - } - } - - // Description: job text section let description: string | null = null - // Try job-text class first - const jobTextHtml = extractDivContent(html, "job-text") - if (jobTextHtml) { - description = decodeHtmlEntities(stripTags(jobTextHtml)).replace(/\s+/g, " ").trim() || null - } + if (isNative) { + location = jdBlockValue(html, "jd-location") + deadline = toIsoDate(jdBlockValue(html, "jd-deadline")) + employmentType = jdBlockValue(html, "jd-type") + hours = jdBlockValue(html, "jd-workhours") + const desc = html.match(/class="jd-description"[^>]*>([\s\S]*?)<\/div>/i) + description = desc + ? decodeHtmlEntities(stripTags(desc[1])).replace(/\s+/g, " ").trim() || null + : null + } else { + // hr-manager-style widget: a rowheader label followed by the value span. + const workplace = visibleHtml(html).match( + /class="workplace[^"]*"[\s\S]*?<span class="empty">([\s\S]*?)<\/span>/i, + ) + location = workplace + ? decodeHtmlEntities(stripTags(workplace[1])).replace(/\s+/g, " ").trim() || null + : null - // Fallback: try og:description meta tag for a brief description - if (!description) { - const ogDescMatch = html.match(/property="og:description"[^>]+content="([^"]+)"/i) || - html.match(/content="([^"]+)"[^>]+property="og:description"/i) - if (ogDescMatch) { - description = decodeHtmlEntities(ogDescMatch[1]) || null + // Deadline: label + a real date within range, scanned only over visible + // markup - the label also appears inside a CSS comment on these pages, + // which the previous parser captured verbatim as the deadline. + const due = visibleHtml(html).match( + /(?:Ansøgningsfrist|Application\s*due|Frist)[\s\S]{0,300}?(\d{2}-\d{2}-\d{4}|\d{4}-\d{2}-\d{2}|\d{1,2}\.?\s+[a-zæøå]+\s+\d{4})/i, + ) + deadline = due ? toIsoDate(due[1]) : null + + description = bodyText(html) + if (!description || description.length < 100) { + description = metaContent(html, "og:description") ?? metaContent(html, "description") ?? description } } - // Get canonical URL or use the fetched URL - const canonicalMatch = html.match(/<link[^>]+rel="canonical"[^>]+href="([^"]+)"/i) || - html.match(/property="og:url"[^>]+content="([^"]+)"/i) || - html.match(/content="([^"]+)"[^>]+property="og:url"/i) - const canonicalUrl = canonicalMatch ? canonicalMatch[1] : url + if (!description) { + description = metaContent(html, "og:description") + } - // Extract ID from canonical URL, fall back to the provided ID - const canonicalId = extractIdFromUrl(canonicalUrl) || id + // Apply URL: jobindex's own /c?t= redirect when present. + let applyUrl: string | null = null + const ctMatch = html.match(/href="(\/c\?t=[^"]+)"/) + if (ctMatch) { + applyUrl = `${BASE_URL}${decodeHtmlEntities(ctMatch[1])}` + } + + const timeMatch = html.match(/<time[^>]+datetime="([^"]+)"/) return { - id: canonicalId, + id, title, - company: company || null, - companyUrl: companyUrl || null, - location: location || null, - date: date || null, - deadline: deadline || null, - employmentType: employmentType || null, - hours: hours || null, - applyUrl: applyUrl || null, - url: canonicalUrl, - description: description || null, + company, + companyUrl: null, + location, + date: timeMatch ? toIsoDate(timeMatch[1]) : null, + deadline, + employmentType, + hours, + applyUrl, + url, + description, } } @@ -243,13 +269,6 @@ export const detail = defineCommand({ if (signal.aborted) return - // Check if page is not a valid job listing - // A valid job listing has an <h1> tag - if (!html.includes("<h1>") && !html.includes("<h1 ")) { - writeError("Job not found", "NOT_FOUND") - process.exit(1) - } - let data: DetailResult try { data = parseDetailPage(html, url, id) diff --git a/.agents/skills/jobindex-search/cli/tests/detail-parsing.test.ts b/.agents/skills/jobindex-search/cli/tests/detail-parsing.test.ts new file mode 100644 index 0000000..280c6a6 --- /dev/null +++ b/.agents/skills/jobindex-search/cli/tests/detail-parsing.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, test } from "bun:test"; +import { parseDetailPage } from "../src/commands/detail"; + +// Jobindex redesigned its detail pages; every selector the old parser used +// (job-text, jix-info, jix_robotjob--area, jix-toolbar-top__company) is gone +// from live pages, so detail returned null company/location/date, CSS-comment +// text as the deadline, and an external ATS URL as its own id/url - exit 0, +// nothing signalling breakage (review finding F14, 2026-08-19, measured on +// 5/5 live postings). These fixtures are trimmed from live pages captured +// 2026-08-19: the jobindex-native "jd-*" shape and the external-ATS +// (hr-manager/Talentech) passthrough shape. + +const NATIVE_PAGE = `<!DOCTYPE html> +<html lang="da"> +<head> + <title>VELLIV - Udvikler til Camunda/AWS + + + + + +
+

Udvikler til Camunda/AWS

+
+
+
+
+
+

En virksomhed med mere end 100 års historie, der samtidig er cloud-only, er ikke hverdagskost.

+

Hos Velliv får du mulighed for at arbejde med Camunda, AWS og automatisering af processer.

+
+
+
+
+

Jobtype:

+

Fast

+
+
+

Arbejdstid:

+

Fuldtid

+
+
+

Arbejdsdage:

+

Dag

+
+
+

Ansøgningsfrist:

+

13. september 2026

+
+
+

Arbejdssted:

+

Ballerup

+
+
+
+
+ +`; + +// The external shape: an employer's ATS-hosted ad served through jobindex. +// og:url points at the ATS (NOT the posting), og:site_name is the ATS brand, +// and the only occurrence of the deadline label outside the widget is inside +// a CSS comment - the exact text the old regex captured as the deadline. +const EXTERNAL_PAGE = ` + + + + Talentech - C#-udvikler til kritiske analyseløsninger i elnettet + + + + + + + + + +

C#-udvikler til kritiske analyseløsninger i elnettet

+

Vil du være med til at udvikle og drifte de systemer, der understøtter udbygningen af Danmarks kommende elnet? Vi arbejder i krydsfeltet mellem IT og energi.

+
Workplace
Fredericia
+
Application due
21-09-2026
+ +`; + +const JOBANNONCE_URL = "https://www.jobindex.dk/jobannonce/"; + +describe("parseDetailPage - jobindex-native (jd-*) shape", () => { + const job = parseDetailPage(NATIVE_PAGE, `${JOBANNONCE_URL}h1690934`, "h1690934"); + + test("extracts the contract fields", () => { + expect(job).toMatchObject({ + id: "h1690934", + title: "Udvikler til Camunda/AWS", + company: "VELLIV", + location: "Ballerup", + deadline: "2026-09-13", + url: `${JOBANNONCE_URL}h1690934`, + }); + }); + + test("converts the Danish long date to ISO", () => { + expect(job.deadline).toBe("2026-09-13"); + }); + + test("extracts employment metadata and the description body", () => { + expect(job.employmentType).toBe("Fast"); + expect(job.hours).toBe("Fuldtid"); + expect(job.description).toContain("Camunda, AWS og automatisering"); + }); +}); + +describe("parseDetailPage - external ATS passthrough shape", () => { + const job = parseDetailPage(EXTERNAL_PAGE, `${JOBANNONCE_URL}h1690445`, "h1690445"); + + test("keeps the jobindex id and jobannonce URL, never the ATS og:url", () => { + expect(job.id).toBe("h1690445"); + expect(job.url).toBe(`${JOBANNONCE_URL}h1690445`); + expect(job.url).not.toContain("hr-manager.net"); + }); + + test("extracts the title and never reports the ATS brand as the company", () => { + expect(job.title).toBe("C#-udvikler til kritiske analyseløsninger i elnettet"); + expect(job.company).toBeNull(); + }); + + test("extracts the workplace widget location", () => { + expect(job.location).toBe("Fredericia"); + }); + + test("finds the real deadline, not the CSS comment", () => { + expect(job.deadline).toBe("2026-09-21"); + }); + + test("description is readable body text, not a stylesheet", () => { + expect(job.description).toContain("udbygningen af Danmarks kommende elnet"); + expect(job.description).not.toContain("padding-bottom"); + }); + + test("deadline is null when only the CSS comment mentions the label", () => { + const noDueWidget = EXTERNAL_PAGE.replace( + /
[\s\S]*?
<\/div>/, + "", + ); + const parsed = parseDetailPage(noDueWidget, `${JOBANNONCE_URL}h9`, "h9"); + expect(parsed.deadline).toBeNull(); + }); +}); diff --git a/CHANGELOG.md b/CHANGELOG.md index 20755ec..367d604 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -120,6 +120,18 @@ per-file diff commands. ### Fixed +- **`jobindex-search detail` rewritten against jobindex's current markup** - every + selector the old parser used is gone from live pages, so on 4 of 5 live postings it + returned CSS-comment text as the deadline (`"K \t\t... */"`), an external ATS URL as + its own `id` and `url`, null company/location/date, and a 160-char teaser as the + description - exit 0 every time. The new parser handles both live shapes (the + jobindex-native `jd-*` layout and the external-ATS passthrough), always keeps the + jobindex id and `jobannonce` URL, requires a real date next to the deadline label and + scans only visible markup (killing the CSS-comment capture), converts Danish long + dates to ISO, and reports `company: null` honestly on passthrough pages instead of + the ATS brand. Verified live on 5/5 postings (full descriptions of 5.5-9k chars, 4/5 + ISO deadlines and locations). Fixture tests for both shapes, including the + CSS-comment trap, in the new `tests/detail-parsing.test.ts`. - **`/scrape` gains a recency fallback for portals with no recency flag** - Step 1b.3 told every portal to scope to 14 days "using the portal's supported recency flag", but jobdanmark has none, leaving the instruction unsatisfiable there: the agent either