fix(portals): depth-track div extraction so nested job descriptions aren't truncated (#204)

The jobindex and linkedin detail parsers matched description containers with a non-greedy regex that stops at the first inner </div>, so any posting whose description contains nested divs was silently truncated (jobindex dropped later sections; linkedin dropped everything after the first block). Replaces the regex with a depth-tracked extractDivContent scanner that walks div open/close markers to the matching close. Verified: truncation bug reproduced against real markup fixtures, depth arithmetic correct (no off-by-one/infinite-loop), 28 tests pass network-free, no regression on non-nested divs. Malformed-HTML over-grabs rather than truncates - the safer failure, cleaned by downstream stripTags/decode.

By @oscarbol09.
This commit is contained in:
Oscar Madera
2026-07-21 08:11:17 +02:00
committed by GitHub
parent 808be3daad
commit d3eea27b90
5 changed files with 179 additions and 11 deletions
@@ -1,6 +1,6 @@
import { defineCommand, option } from "@bunli/core"
import { z } from "zod"
import { htmlFetch, writeError } from "../helpers.js"
import { htmlFetch, writeError, extractDivContent } from "../helpers.js"
const BASE_URL = "https://www.jobindex.dk"
@@ -180,9 +180,9 @@ function parseDetailPage(html: string, url: string, id: string): DetailResult {
let description: string | null = null
// Try job-text class first
const jobTextMatch = html.match(/class="job-text"[^>]*>([\s\S]*?)<\/div>\s*(?:<div|<\/div>)/i)
if (jobTextMatch) {
description = decodeHtmlEntities(stripTags(jobTextMatch[1])).replace(/\s+/g, " ").trim() || null
const jobTextHtml = extractDivContent(html, "job-text")
if (jobTextHtml) {
description = decodeHtmlEntities(stripTags(jobTextHtml)).replace(/\s+/g, " ").trim() || null
}
// Fallback: try og:description meta tag for a brief description