From d3eea27b90aeaaeeb2cbf9d702bca3ff4d9f9f87 Mon Sep 17 00:00:00 2001 From: Oscar Madera <80536682+oscarbol09@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:11:17 -0500 Subject: [PATCH] fix(portals): depth-track div extraction so nested job descriptions aren't truncated (#204) The jobindex and linkedin detail parsers matched description containers with a non-greedy regex that stops at the first inner , so any posting whose description contains nested divs was silently truncated (jobindex dropped later sections; linkedin dropped everything after the first block). Replaces the regex with a depth-tracked extractDivContent scanner that walks div open/close markers to the matching close. Verified: truncation bug reproduced against real markup fixtures, depth arithmetic correct (no off-by-one/infinite-loop), 28 tests pass network-free, no regression on non-nested divs. Malformed-HTML over-grabs rather than truncates - the safer failure, cleaned by downstream stripTags/decode. By @oscarbol09. --- .../cli/src/commands/detail.ts | 8 +-- .../skills/jobindex-search/cli/src/helpers.ts | 27 +++++++++ .../jobindex-search/cli/tests/parsing.test.ts | 54 ++++++++++++++++- .../skills/linkedin-search/cli/src/helpers.ts | 41 +++++++++++-- .../linkedin-search/cli/tests/parsing.test.ts | 60 ++++++++++++++++++- 5 files changed, 179 insertions(+), 11 deletions(-) diff --git a/.agents/skills/jobindex-search/cli/src/commands/detail.ts b/.agents/skills/jobindex-search/cli/src/commands/detail.ts index afb3cbd..0e82094 100644 --- a/.agents/skills/jobindex-search/cli/src/commands/detail.ts +++ b/.agents/skills/jobindex-search/cli/src/commands/detail.ts @@ -1,6 +1,6 @@ import { defineCommand, option } from "@bunli/core" import { z } from "zod" -import { htmlFetch, writeError } from "../helpers.js" +import { htmlFetch, writeError, extractDivContent } from "../helpers.js" const BASE_URL = "https://www.jobindex.dk" @@ -180,9 +180,9 @@ function parseDetailPage(html: string, url: string, id: string): DetailResult { let description: string | null = null // Try job-text class first - const jobTextMatch = html.match(/class="job-text"[^>]*>([\s\S]*?)<\/div>\s*(?:)/i) - if (jobTextMatch) { - description = decodeHtmlEntities(stripTags(jobTextMatch[1])).replace(/\s+/g, " ").trim() || null + const jobTextHtml = extractDivContent(html, "job-text") + if (jobTextHtml) { + description = decodeHtmlEntities(stripTags(jobTextHtml)).replace(/\s+/g, " ").trim() || null } // Fallback: try og:description meta tag for a brief description diff --git a/.agents/skills/jobindex-search/cli/src/helpers.ts b/.agents/skills/jobindex-search/cli/src/helpers.ts index 48d940f..3852921 100644 --- a/.agents/skills/jobindex-search/cli/src/helpers.ts +++ b/.agents/skills/jobindex-search/cli/src/helpers.ts @@ -307,6 +307,33 @@ export function parseJobCards(html: string): JobCard[] { return results } +export function extractDivContent(html: string, className: string): string | null { + const escaped = className.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + const openRe = new RegExp(`]*class="[^"]*${escaped}[^"]*"[^>]*>`, 'i') + const open = openRe.exec(html) + if (!open) return null + + let i = open.index + open[0].length + let depth = 1 + + while (depth > 0 && i < html.length) { + const nextOpen = html.indexOf('', i) + + if (nextClose === -1) return null + + if (nextOpen !== -1 && nextOpen < nextClose) { + depth++ + i = nextOpen + 4 + } else { + depth-- + i = nextClose + 6 + } + } + + return html.slice(open.index + open[0].length, i - 6) +} + export function parseHitCount(html: string): number { const match = html.match(/af ([\d.]+)<\/strong>/) if (!match) return 0 diff --git a/.agents/skills/jobindex-search/cli/tests/parsing.test.ts b/.agents/skills/jobindex-search/cli/tests/parsing.test.ts index dde9a1a..240227e 100644 --- a/.agents/skills/jobindex-search/cli/tests/parsing.test.ts +++ b/.agents/skills/jobindex-search/cli/tests/parsing.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect } from "bun:test"; -import { parseJobCards } from "../src/helpers"; +import { parseJobCards, extractDivContent } from "../src/helpers"; // Minimal jobad-wrapper markup: parseJobCards splits on `jobad-wrapper-` // and reads the title from the

…

and the company from the @@ -44,3 +44,55 @@ describe("decodeHtmlEntities (via parseJobCards)", () => { expect(c.company).toBe("Nørrebro ApS"); }); }); + +describe("extractDivContent", () => { + test("extracts content from simple div", () => { + const html = '
Simple text
'; + expect(extractDivContent(html, "job-text")).toBe("Simple text"); + }); + + test("extracts content with nested divs — the regression case", () => { + const html = `
+
First section
+
Second section with important info
+
`; + expect(extractDivContent(html, "job-text")).toBe( + '\n
First section
\n
Second section with important info
\n ', + ); + }); + + test("returns null when class not found", () => { + expect(extractDivContent("
no class
", "nonexistent")).toBeNull(); + }); + + test("works with extra attributes on the div", () => { + const html = '
Content
'; + expect(extractDivContent(html, "job-text")).toBe("Content"); + }); + + test("handles deeply nested divs (3 levels)", () => { + const html = `
+
+
Deep content
+
+
`; + expect(extractDivContent(html, "job-text")).toBe( + '\n
\n
Deep content
\n
\n ', + ); + }); + + test("handles empty content", () => { + const html = '
'; + expect(extractDivContent(html, "job-text")).toBe(""); + }); + + test("handles br and other non-div tags", () => { + const html = '
Line1
Line2
Line3
'; + expect(extractDivContent(html, "job-text")).toBe("Line1
Line2
Line3"); + }); + + test("escapes special regex characters in class name", () => { + const html = '
Content
'; + expect(extractDivContent(html, "job-text (special)")).toBe("Content"); + }); +}); diff --git a/.agents/skills/linkedin-search/cli/src/helpers.ts b/.agents/skills/linkedin-search/cli/src/helpers.ts index 32d7bf8..8d2046a 100644 --- a/.agents/skills/linkedin-search/cli/src/helpers.ts +++ b/.agents/skills/linkedin-search/cli/src/helpers.ts @@ -68,6 +68,37 @@ export interface JobDetail extends JobCard { applyUrl: string | null } +/** + * Extract the inner HTML of a
identified by a CSS class name, correctly + * handling nested
elements by tracking tag depth. + */ +export function extractDivContent(html: string, className: string): string | null { + const escaped = className.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + const openRe = new RegExp(`]*class="[^"]*${escaped}[^"]*"[^>]*>`, 'i') + const open = openRe.exec(html) + if (!open) return null + + let i = open.index + open[0].length + let depth = 1 + + while (depth > 0 && i < html.length) { + const nextOpen = html.indexOf('', i) + + if (nextClose === -1) return null + + if (nextOpen !== -1 && nextOpen < nextClose) { + depth++ + i = nextOpen + 4 + } else { + depth-- + i = nextClose + 6 + } + } + + return html.slice(open.index + open[0].length, i - 6) +} + /** * Convert a Unicode code point to a string. Uses `fromCodePoint` (not * `fromCharCode`) so supplementary-plane code points (e.g. emoji, U+1F600) @@ -180,11 +211,11 @@ export function parseJobDetail(html: string, id: string): JobDetail { // Rich description block. Keep paragraph/line breaks as newlines. let description: string | null = null - const desc = html.match( - /class="(?:show-more-less-html__markup|description__text[^"]*)"[^>]*>([\s\S]*?)<\/div>/i, - ) - if (desc) { - const withBreaks = desc[1] + const descHtml = + extractDivContent(html, "show-more-less-html__markup") ?? + extractDivContent(html, "description__text") + if (descHtml) { + const withBreaks = descHtml .replace(/<\s*br\s*\/?>/gi, "\n") .replace(/<\/(p|li|ul|ol|div|h\d)>/gi, "\n") description = decodeHtmlEntities(stripTags(withBreaks)).replace(/\n{3,}/g, "\n\n").trim() || null diff --git a/.agents/skills/linkedin-search/cli/tests/parsing.test.ts b/.agents/skills/linkedin-search/cli/tests/parsing.test.ts index 49d3115..792659c 100644 --- a/.agents/skills/linkedin-search/cli/tests/parsing.test.ts +++ b/.agents/skills/linkedin-search/cli/tests/parsing.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect } from "bun:test"; -import { parseJobCards, parseJobDetail } from "../src/helpers"; +import { parseJobCards, parseJobDetail, extractDivContent } from "../src/helpers"; // Minimal search-card markup: parseJobCards splits on the job-posting URN and // needs an id, a base-search-card__title, and a full-link. Everything else is @@ -53,3 +53,61 @@ describe("decodeHtmlEntities (via parseJobDetail)", () => { expect(job.title).toBe("Señor Engineer"); }); }); + +describe("extractDivContent", () => { + test("extracts content from simple div", () => { + const html = '
Simple text
'; + expect(extractDivContent(html, "description__text")).toBe("Simple text"); + }); + + test("extracts content with nested divs — the regression case", () => { + const html = `
+
Requirements:
+
  • Skill A
+
About Us:
+

We are...

+
`; + expect(extractDivContent(html, "description__text")).toBe( + '\n
Requirements:
\n
  • Skill A
\n
About Us:
\n

We are...

\n ', + ); + }); + + test("returns null when class not found", () => { + expect(extractDivContent("
no class
", "nonexistent")).toBeNull(); + }); + + test("works with show-more-less-html__markup class", () => { + const html = '
LinkedIn content
'; + expect(extractDivContent(html, "show-more-less-html__markup")).toBe("LinkedIn content"); + }); + + test("handles deeply nested divs (3 levels)", () => { + const html = `
+
+
Deep content
+
+
`; + expect(extractDivContent(html, "description__text")).toBe( + '\n
\n
Deep content
\n
\n ', + ); + }); + + test("handles empty content", () => { + const html = '
'; + expect(extractDivContent(html, "description__text")).toBe(""); + }); + + test("parseJobDetail uses extractDivContent and preserves full description", () => { + const html = `
+
Requirements:
+
  • 5 years Python
+
About Us:
+

We are hiring!

+
`; + const job = parseJobDetail(html, "999"); + expect(job.description).toContain("Requirements:"); + expect(job.description).toContain("5 years Python"); + expect(job.description).toContain("About Us:"); + expect(job.description).toContain("We are hiring!"); + }); +});