mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-09-17 08:36:25 +00:00
fix(portals): depth-track div extraction so nested job descriptions aren't truncated (#204)
The jobindex and linkedin detail parsers matched description containers with a non-greedy regex that stops at the first inner </div>, so any posting whose description contains nested divs was silently truncated (jobindex dropped later sections; linkedin dropped everything after the first block). Replaces the regex with a depth-tracked extractDivContent scanner that walks div open/close markers to the matching close. Verified: truncation bug reproduced against real markup fixtures, depth arithmetic correct (no off-by-one/infinite-loop), 28 tests pass network-free, no regression on non-nested divs. Malformed-HTML over-grabs rather than truncates - the safer failure, cleaned by downstream stripTags/decode. By @oscarbol09.
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
import { defineCommand, option } from "@bunli/core"
|
||||
import { z } from "zod"
|
||||
import { htmlFetch, writeError } from "../helpers.js"
|
||||
import { htmlFetch, writeError, extractDivContent } from "../helpers.js"
|
||||
|
||||
const BASE_URL = "https://www.jobindex.dk"
|
||||
|
||||
@@ -180,9 +180,9 @@ function parseDetailPage(html: string, url: string, id: string): DetailResult {
|
||||
let description: string | null = null
|
||||
|
||||
// Try job-text class first
|
||||
const jobTextMatch = html.match(/class="job-text"[^>]*>([\s\S]*?)<\/div>\s*(?:<div|<\/div>)/i)
|
||||
if (jobTextMatch) {
|
||||
description = decodeHtmlEntities(stripTags(jobTextMatch[1])).replace(/\s+/g, " ").trim() || null
|
||||
const jobTextHtml = extractDivContent(html, "job-text")
|
||||
if (jobTextHtml) {
|
||||
description = decodeHtmlEntities(stripTags(jobTextHtml)).replace(/\s+/g, " ").trim() || null
|
||||
}
|
||||
|
||||
// Fallback: try og:description meta tag for a brief description
|
||||
|
||||
@@ -307,6 +307,33 @@ export function parseJobCards(html: string): JobCard[] {
|
||||
return results
|
||||
}
|
||||
|
||||
export function extractDivContent(html: string, className: string): string | null {
|
||||
const escaped = className.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
const openRe = new RegExp(`<div[^>]*class="[^"]*${escaped}[^"]*"[^>]*>`, 'i')
|
||||
const open = openRe.exec(html)
|
||||
if (!open) return null
|
||||
|
||||
let i = open.index + open[0].length
|
||||
let depth = 1
|
||||
|
||||
while (depth > 0 && i < html.length) {
|
||||
const nextOpen = html.indexOf('<div', i)
|
||||
const nextClose = html.indexOf('</div>', i)
|
||||
|
||||
if (nextClose === -1) return null
|
||||
|
||||
if (nextOpen !== -1 && nextOpen < nextClose) {
|
||||
depth++
|
||||
i = nextOpen + 4
|
||||
} else {
|
||||
depth--
|
||||
i = nextClose + 6
|
||||
}
|
||||
}
|
||||
|
||||
return html.slice(open.index + open[0].length, i - 6)
|
||||
}
|
||||
|
||||
export function parseHitCount(html: string): number {
|
||||
const match = html.match(/af <strong>([\d.]+)<\/strong>/)
|
||||
if (!match) return 0
|
||||
|
||||
Reference in New Issue
Block a user