Add Jobdanmark detail HTML fallback (#115)

2026-09-17 00:26:26 +00:00 · 2026-07-10 08:05:40 +02:00
parent 1bc119dffd
commit d0846aad1d
4 changed files with 229 additions and 71 deletions
@@ -66,7 +66,7 @@ bun run .agents/skills/jobdanmark-search/cli/src/cli.ts detail <slug> [--format
 ```

 `slug` is the URL path segment returned as `slug` in `search` results (e.g. `it-chef-soeges-til-rah`).
-Returns full structured job data from the job page's JSON-LD, including title, organization, location, employment type, deadline, and full HTML description.
+Returns full structured job data from the job page. The CLI prefers Schema.org JSON-LD when present and falls back to parsing the rendered HTML when Jobdanmark omits JSON-LD.

 ### List categories with live counts

@@ -222,7 +222,7 @@ All errors are written to **stderr** as `{ "error": "...", "code": "..." }` and

 - All data is from the public Jobdanmark.dk API — no credentials required.
 - Pagination is 1-indexed (`--page 1` is the first page). 30 items per page, server-enforced.
- The `detail` command fetches the HTML job page and parses the embedded JSON-LD (schema.org/JobPosting). It does not use a separate JSON API.
+- The `detail` command fetches the HTML job page and parses embedded JSON-LD when available, with a rendered-HTML fallback for pages that omit structured data. It does not use a separate JSON API.
 - `slug` in search results is extracted from the API's relative `url` field (the path after `/job/`).
 - `applicationDeadline` in search results can be `null` (no deadline set).
 - Job type values for filters: `fuldtid`, `deltid`, `fleksjob`, `elev`, `studiejob`, `praktik`.
@@ -166,7 +166,7 @@ bun run src/cli.ts search --text "sygeplejerske" --zip 8000 --limit 10

 ## `detail` — Full job posting detail

-**Method**: Fetch HTML from `https://jobdanmark.dk/job/{slug}`, extract `<script type="application/ld+json">` block.
+**Method**: Fetch HTML from `https://jobdanmark.dk/job/{slug}`. The CLI extracts a `<script type="application/ld+json">` JobPosting block when present, and falls back to parsing the rendered job page HTML when Jobdanmark omits JSON-LD.

 ```bash
 bun run src/cli.ts detail <slug> [--format json|plain]
@@ -202,11 +202,12 @@ bun run src/cli.ts detail it-chef-soeges-til-rah --format plain
    "postalCode": "6950",
    "addressCountry": "DK"
  },
-  "description": "<p>Full HTML description...</p>"
+  "description": "<p>Full HTML description...</p>",
+  "applyUrl": "https://example.com/apply"
 }
 ```

-> **Note**: The `hiringOrganization.logo` and `validThrough` fields may be `null` if not present in the JSON-LD. `jobLocation` fields may be `null` if the location data is absent.
+> **Note**: The `hiringOrganization.logo`, `validThrough`, and `applyUrl` fields may be `null` if not present in the structured data or rendered page. `jobLocation` fields may be `null` if the location data is absent.

 ---

@@ -440,7 +441,7 @@ All errors are written to **stderr** in JSON format and exit with code `1`:
 { "error": "Job not found", "code": "NOT_FOUND" }
 { "error": "API request failed: 400 Bad Request", "code": "API_ERROR" }
 { "error": "--query is required", "code": "MISSING_REQUIRED" }
-{ "error": "Failed to parse JSON-LD from job page", "code": "PARSE_ERROR" }
+{ "error": "Failed to parse job page HTML", "code": "PARSE_ERROR" }
 ```

 ---
@@ -29,6 +29,169 @@ interface JsonLdJobPosting {
  description?: string
 }

+interface DetailResult {
+  slug: string
+  url: string
+  title: string
+  datePosted: string
+  validThrough: string | null
+  employmentType: string[]
+  hiringOrganization: {
+    name: string
+    logo: string | null
+  }
+  jobLocation: {
+    streetAddress: string | null
+    addressLocality: string | null
+    addressRegion: string | null
+    postalCode: string | null
+    addressCountry: string | null
+  }
+  description: string
+  applyUrl: string | null
+}
+
+function cleanText(text: string): string {
+  return text.replace(/\s+/g, " ").trim()
+}
+
+function normalizeUrl(value: string | null | undefined): string | null {
+  if (!value) return null
+  const decoded = value.replace(/&amp;/g, "&")
+  if (decoded.startsWith("http")) return decoded
+  if (decoded.startsWith("/")) return `${BASE_URL}${decoded}`
+  return decoded
+}
+
+function findJobPostingJsonLd(root: ReturnType<typeof parse>): JsonLdJobPosting | null {
+  const ldJsonScripts = root.querySelectorAll('script[type="application/ld+json"]')
+
+  for (const script of ldJsonScripts) {
+    try {
+      const parsed = JSON.parse(script.text) as unknown
+      if (isJobPosting(parsed)) return parsed
+      if (Array.isArray(parsed)) {
+        const found = parsed.find(isJobPosting)
+        if (found) return found
+      }
+    } catch {
+      // Continue to next script.
+    }
+  }
+
+  return null
+}
+
+function isJobPosting(value: unknown): value is JsonLdJobPosting {
+  return Boolean(value && typeof value === "object" && (value as JsonLdJobPosting)["@type"] === "JobPosting")
+}
+
+function fromJsonLd(jobPosting: JsonLdJobPosting, slug: string, url: string): DetailResult {
+  const hiringOrg = jobPosting.hiringOrganization
+  const address = jobPosting.jobLocation?.address
+
+  const employmentType = Array.isArray(jobPosting.employmentType)
+    ? jobPosting.employmentType
+    : jobPosting.employmentType
+      ? [jobPosting.employmentType]
+      : []
+
+  return {
+    slug,
+    url,
+    title: jobPosting.title ?? "",
+    datePosted: jobPosting.datePosted ?? "",
+    validThrough: jobPosting.validThrough ?? null,
+    employmentType,
+    hiringOrganization: {
+      name: hiringOrg?.name ?? "",
+      logo: hiringOrg?.logo ?? null,
+    },
+    jobLocation: {
+      streetAddress: address?.streetAddress ?? null,
+      addressLocality: address?.addressLocality ?? null,
+      addressRegion: address?.addressRegion ?? null,
+      postalCode: address?.postalCode ?? null,
+      addressCountry: address?.addressCountry ?? null,
+    },
+    description: jobPosting.description ?? "",
+    applyUrl: null,
+  }
+}
+
+function overviewValue(root: ReturnType<typeof parse>, label: string): string | null {
+  const normalizedLabel = label.toLowerCase()
+  for (const item of root.querySelectorAll(".job-overview li")) {
+    const strong = item.querySelector("strong")
+    const itemLabel = cleanText(strong?.text ?? "").replace(/:$/, "").toLowerCase()
+    if (itemLabel !== normalizedLabel) continue
+
+    const value = cleanText(item.text.replace(strong?.text ?? "", ""))
+    return value.replace(/^:\s*/, "") || null
+  }
+  return null
+}
+
+function fromRenderedHtml(root: ReturnType<typeof parse>, slug: string, url: string): DetailResult {
+  const pageTitle = cleanText(root.querySelector("title")?.text ?? "")
+  if (pageTitle.toLowerCase().includes("404") || root.text.toLowerCase().includes("siden blev ikke fundet")) {
+    throw new Error("NOT_FOUND")
+  }
+
+  const title =
+    cleanText(root.querySelector(".job-list-details .title")?.text ?? "") ||
+    cleanText(root.querySelector("h1")?.text ?? "") ||
+    cleanText(root.querySelector("h3")?.text ?? "").replace(/\s+\|\s+jobdanmark$/, "")
+
+  if (!title) {
+    throw new Error("Failed to parse job page HTML")
+  }
+
+  const companyLink = root.querySelector('.job-details-head a[href^="/virksomheder/"]')
+  const companyName = cleanText(companyLink?.text ?? "")
+  const logoSrc =
+    root.querySelector(".company-logo img")?.getAttribute("src") ??
+    root.querySelector(".company-logo source")?.getAttribute("srcset")?.split(/\s+/)[0]
+  const workplace = overviewValue(root, "Arbejdssted")
+
+  const description = root
+    .querySelectorAll(".job-list-details p, .job-list-details li")
+    .map((node) => cleanText(node.text))
+    .filter(Boolean)
+    .join("\n")
+
+  const employmentType = overviewValue(root, "Jobtype")
+  const applyUrl = normalizeUrl(root.querySelector("a.action.primary")?.getAttribute("href"))
+
+  return {
+    slug,
+    url,
+    title,
+    datePosted: overviewValue(root, "Udgivet") ?? "",
+    validThrough: overviewValue(root, "Ansøgningsfrist"),
+    employmentType: employmentType ? [employmentType] : [],
+    hiringOrganization: {
+      name: companyName,
+      logo: normalizeUrl(logoSrc),
+    },
+    jobLocation: {
+      streetAddress: workplace,
+      addressLocality: null,
+      addressRegion: null,
+      postalCode: null,
+      addressCountry: "DK",
+    },
+    description,
+    applyUrl,
+  }
+}
+
+export function parseJobPostingFromHtml(html: string, slug: string, url: string): DetailResult {
+  const root = parse(html)
+  const jobPosting = findJobPostingJsonLd(root)
+  return jobPosting ? fromJsonLd(jobPosting, slug, url) : fromRenderedHtml(root, slug, url)
+}
+
 export const detail = defineCommand({
  name: "detail",
  description: "Full detail for a single job posting (by slug)",
@@ -67,66 +230,10 @@ export const detail = defineCommand({
      }

      const html = await response.text()
-      const root = parse(html)
-
-      // Find JSON-LD script tag
-      const ldJsonScripts = root.querySelectorAll('script[type="application/ld+json"]')
-      let jobPosting: JsonLdJobPosting | null = null
-
-      for (const script of ldJsonScripts) {
-        try {
-          const parsed = JSON.parse(script.text)
-          if (parsed["@type"] === "JobPosting") {
-            jobPosting = parsed
-            break
-          }
-        } catch {
-          // continue to next script
-        }
-      }
-
-      if (!jobPosting) {
-        // Check for 404 by looking at page content
-        const pageTitle = root.querySelector("title")?.text ?? ""
-        if (pageTitle.toLowerCase().includes("404") || html.toLowerCase().includes("siden blev ikke fundet")) {
-          writeError("Job not found", "NOT_FOUND")
-        } else {
-          writeError("Failed to parse JSON-LD from job page", "PARSE_ERROR")
-        }
-        process.exit(1)
-      }

      if (signal.aborted) return

-      const hiringOrg = jobPosting.hiringOrganization
-      const address = jobPosting.jobLocation?.address
-
-      const employmentType = Array.isArray(jobPosting.employmentType)
-        ? jobPosting.employmentType
-        : jobPosting.employmentType
-          ? [jobPosting.employmentType]
-          : []
-
-      const output = {
-        slug,
-        url,
-        title: jobPosting.title ?? "",
-        datePosted: jobPosting.datePosted ?? "",
-        validThrough: jobPosting.validThrough ?? null,
-        employmentType,
-        hiringOrganization: {
-          name: hiringOrg?.name ?? "",
-          logo: hiringOrg?.logo ?? null,
-        },
-        jobLocation: {
-          streetAddress: address?.streetAddress ?? null,
-          addressLocality: address?.addressLocality ?? null,
-          addressRegion: address?.addressRegion ?? null,
-          postalCode: address?.postalCode ?? null,
-          addressCountry: address?.addressCountry ?? null,
-        },
-        description: jobPosting.description ?? "",
-      }
+      const output = parseJobPostingFromHtml(html, slug, url)

      if (flags.format === "json") {
        console.log(JSON.stringify(output, null, 2))
@@ -136,6 +243,8 @@ export const detail = defineCommand({
    } catch (err) {
      if (err instanceof Error && err.message.includes("NOT_FOUND")) {
        writeError("Job not found", "NOT_FOUND")
+      } else if (err instanceof Error && err.message.includes("Failed to parse")) {
+        writeError(err.message, "PARSE_ERROR")
      } else {
        writeError(err instanceof Error ? err.message : String(err), "API_ERROR")
      }
@@ -144,7 +253,7 @@ export const detail = defineCommand({
  },
 })

-function outputPlain(data: Record<string, unknown>): void {
+function outputPlain(data: DetailResult): void {
  console.log(`slug: ${data.slug}`)
  console.log(`url: ${data.url}`)
  console.log(`title: ${data.title}`)
@@ -152,14 +261,10 @@ function outputPlain(data: Record<string, unknown>): void {
  console.log(`validThrough: ${data.validThrough ?? "N/A"}`)
  const empType = Array.isArray(data.employmentType) ? data.employmentType.join(", ") : "-"
  console.log(`employmentType: ${empType}`)
-  const org = data.hiringOrganization as { name: string; logo: string | null }
+  const org = data.hiringOrganization
  console.log(`company: ${org.name}`)
-  const loc = data.jobLocation as {
-    streetAddress: string | null
-    addressLocality: string | null
-    postalCode: string | null
-    addressCountry: string | null
-  }
+  const loc = data.jobLocation
  console.log(`location: ${[loc.streetAddress, loc.addressLocality, loc.postalCode, loc.addressCountry].filter(Boolean).join(", ")}`)
+  if (data.applyUrl) console.log(`applyUrl: ${data.applyUrl}`)
  console.log(`description: ${data.description}`)
 }
@@ -0,0 +1,52 @@
+import { describe, expect, test } from "bun:test";
+import { parseJobPostingFromHtml } from "../src/commands/detail";
+
+const HTML_WITHOUT_JSON_LD = `<!doctype html>
+<html lang="da">
+  <head>
+    <title>Journalistisk udvikler s&#xF8;ges | jobdanmark</title>
+  </head>
+  <body>
+    <div class="job-list-details">
+      <div class="job-details-head row mx-0">
+        <div class="company-logo col-auto">
+          <img src="/media/jfm-logo.png?width=100" alt="JFM">
+        </div>
+        <h3 class="title">Journalistisk udvikler s&#xF8;ges</h3>
+        <a href="/virksomheder/jfm">JFM</a>
+        <span>Baneg&#xE5;rdspladsen 1, 5000 Odense C</span>
+      </div>
+      <p>Hvad nu hvis du med f&#xE5; klik kunne unders&#xF8;ge dit lokalomr&#xE5;de?</p>
+      <ul>
+        <li>identificere relevante datas&#xE6;t og muligheder</li>
+      </ul>
+    </div>
+    <a href="https://jfm.career.emply.com/da/apply/example" class="action primary count-click">Ans&#xF8;g nu</a>
+    <ul class="job-overview list-unstyled">
+      <li><strong>Udgivet:</strong> 03-07-2026</li>
+      <li><strong>Jobtype:</strong> Fuldtid</li>
+      <li><strong>Arbejdssted:</strong> Baneg&#xE5;rdspladsen 1, 5000 Odense C</li>
+      <li><strong>Ansøgningsfrist:</strong> 02-08-2026 23.59</li>
+    </ul>
+  </body>
+</html>`;
+
+describe("parseJobPostingFromHtml", () => {
+  test("falls back to rendered Jobdanmark HTML when JSON-LD is absent", () => {
+    const parsed = parseJobPostingFromHtml(
+      HTML_WITHOUT_JSON_LD,
+      "journalistisk-udvikler",
+      "https://jobdanmark.dk/job/journalistisk-udvikler",
+    );
+
+    expect(parsed.title).toBe("Journalistisk udvikler søges");
+    expect(parsed.datePosted).toBe("03-07-2026");
+    expect(parsed.validThrough).toBe("02-08-2026 23.59");
+    expect(parsed.employmentType).toEqual(["Fuldtid"]);
+    expect(parsed.hiringOrganization.name).toBe("JFM");
+    expect(parsed.hiringOrganization.logo).toBe("https://jobdanmark.dk/media/jfm-logo.png?width=100");
+    expect(parsed.jobLocation.streetAddress).toBe("Banegårdspladsen 1, 5000 Odense C");
+    expect(parsed.description).toContain("identificere relevante datasæt");
+    expect(parsed.applyUrl).toBe("https://jfm.career.emply.com/da/apply/example");
+  });
+});