diff --git a/.agents/skills/freehire-search/cli/src/commands/detail.ts b/.agents/skills/freehire-search/cli/src/commands/detail.ts index 405e0f99..1d87cd89 100644 --- a/.agents/skills/freehire-search/cli/src/commands/detail.ts +++ b/.agents/skills/freehire-search/cli/src/commands/detail.ts @@ -1,4 +1,4 @@ -import { apiGet, normalizeSlug, toDetail, writeError, type FreehireJob, type JobDetailResult } from "../helpers.js" +import { apiGet, normalizeSlug, stripInvisible, toDetail, writeError, type FreehireJob, type JobDetailResult } from "../helpers.js" export interface DetailOpts { id: string // a freehire public slug or a /jobs/ URL @@ -35,7 +35,7 @@ export async function runDetail(opts: DetailOpts): Promise { writeError("job not found", "NOT_FOUND") return 1 } - const job = toDetail(env.data) + const job = stripInvisible(toDetail(env.data)) if (opts.format === "plain") { const lines = [ diff --git a/.agents/skills/freehire-search/cli/src/commands/search.ts b/.agents/skills/freehire-search/cli/src/commands/search.ts index ac8a5e4a..33ec57f2 100644 --- a/.agents/skills/freehire-search/cli/src/commands/search.ts +++ b/.agents/skills/freehire-search/cli/src/commands/search.ts @@ -1,4 +1,4 @@ -import { apiGet, toResult, writeError, type FreehireJob, type JobResult } from "../helpers.js" +import { apiGet, stripInvisible, toResult, writeError, type FreehireJob, type JobResult } from "../helpers.js" // The agent variant of the job search: the same query, ranking, and facets as the // web's /jobs/search, but each hit carries the posting's full description instead @@ -175,7 +175,7 @@ export async function runSearch(opts: SearchOpts): Promise { ) return 1 } - let rows = (env.data ?? []).map(toResult) + let rows = stripInvisible((env.data ?? []).map(toResult)) // The web search still carries each hit's truncated preview, and the cost // this flag exists to avoid is the share of CLI output descriptions occupy // in agent context - so the lean mode drops them client-side. diff --git a/.agents/skills/freehire-search/cli/src/helpers.ts b/.agents/skills/freehire-search/cli/src/helpers.ts index 943b208c..206d7530 100644 --- a/.agents/skills/freehire-search/cli/src/helpers.ts +++ b/.agents/skills/freehire-search/cli/src/helpers.ts @@ -215,7 +215,7 @@ export function toResult(j: FreehireJob): JobResult { regions: j.regions, countries: j.countries, skills: j.skills, - description: j.description || null, + description: j.description ? visibleHtml(j.description) : null, } } @@ -263,13 +263,97 @@ function decodeHtmlEntities(text: string): string { .replace(/ /g, " ") } +/** + * Characters a reader of the posting never sees but a model reading this + * output does: C0/C1 controls other than tab, newline and carriage return, the + * soft hyphen, bidi marks/embeddings/overrides/isolates, zero-width characters + * and invisible operators, the BOM, variation selectors, and Unicode tag + * characters (U+E0000-E007F), which can spell out a whole hidden instruction. + * Listing text is third-party data, and these arrive raw in a portal's JSON or + * markup as easily as through a numeric entity. + */ +const INVISIBLE_CHARS = + /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F­؜᠎​-‏‪-‮⁠-︀-️\u{E0000}-\u{E007F}\u{E0100}-\u{E01EF}]/gu + +/** Strip INVISIBLE_CHARS from every string in a result, recursively: the CLI's output boundary. */ +export function stripInvisible(value: T): T { + if (typeof value === "string") return value.replace(INVISIBLE_CHARS, "") as T + if (Array.isArray(value)) return value.map(stripInvisible) as T + if (value !== null && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype) { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, stripInvisible(v)])) as T + } + return value +} + +const VOID_TAGS = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"]) + +/** Whether an opening tag's attributes keep it from rendering: `hidden`, or an inline style that hides it. */ +function isHiddenTag(attrs: string): boolean { + for (const m of attrs.matchAll(/([^\s"'=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g)) { + const name = m[1].toLowerCase() + const value = m[2] ?? m[3] ?? m[4] ?? "" + if (name === "hidden") return true + if (name === "style" && /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden|mso-hide\s*:\s*all)\b/i.test(value)) { + return true + } + } + return false +} + +/** + * Drop markup a browser never renders - comments (including Word's + * conditional ones), ', + '
HIDDEN styled
HIDDEN nested
HIDDEN after nesting
', + "", + '

HIDDEN mso

', + '', +].join(""); +const LEAKED = /HIDDEN|WordDocument|MsoNormal|alert/; +const INVISIBLE_CHAR = /[\u{E0000}-\u{E007F}‮​⁦\u001B]/u; + +// A freehire job as the API serves it, with hostile text where the source ATS +// posting controls it. +const hostileJob = { + public_slug: "backend-engineer-acme-ab12cd34", + source: "greenhouse", + external_id: "acme:1", + url: "https://boards.greenhouse.io/acme/jobs/1", + title: `Backend Engineer${ENTITY_TAGGED}`, + company: `Acme${INVISIBLE}`, + company_slug: "acme", + location: "Berlin, Germany", + description: `

Build things.${INVISIBLE}

${HIDDEN_MARKUP}

Ship them.

`, + skills: ["go"], + work_mode: "remote", + regions: ["eu"], + countries: ["de"], + cities: ["Berlin"], + posted_at: "2026-07-06T00:00:00Z", + created_at: "2026-07-06T15:00:00Z", + enrichment: {}, +}; + +// The documented CLI in a fresh process, pointed at a local freehire through +// FREEHIRE_API_URL, the self-hosting override. +async function cli(args: string[]) { + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + const path = new URL(request.url).pathname; + if (path.endsWith("/jobs/search")) return Response.json({ data: [hostileJob], meta: { total: 1 } }); + return Response.json({ data: hostileJob }); + }, + }); + try { + const result = await runCLI(args, { FREEHIRE_API_URL: server.url.toString() }); + expect(result.exitCode).toBe(0); + return result.stdout; + } finally { + server.stop(true); + } +} + +describe("hidden text from a listing never reaches the output", () => { + test("detail JSON: hidden markup dropped, invisible and entity-encoded characters stripped", async () => { + const stdout = await cli(["detail", "backend-engineer-acme-ab12cd34"]); + expect(stdout).not.toMatch(INVISIBLE_CHAR); + const job = JSON.parse(stdout); + expect(job.title).toBe("Backend Engineer"); + expect(job.company).toBe("Acme"); + expect(job.description).not.toMatch(LEAKED); + expect(job.description).toContain("Build things."); + expect(job.description).toContain("Visible despite the words."); + expect(job.description).toContain("Ship them."); + }); + + test("detail plain output is cleaned the same way", async () => { + const stdout = await cli(["detail", "backend-engineer-acme-ab12cd34", "--format", "plain"]); + expect(stdout).not.toMatch(INVISIBLE_CHAR); + expect(stdout).not.toMatch(LEAKED); + expect(stdout).toContain("Visible despite the words."); + }); + + test("search results carry no invisible characters or hidden markup", async () => { + const stdout = await cli(["search", "--query", "backend", "--description-format", "html"]); + expect(stdout).not.toMatch(INVISIBLE_CHAR); + const [row] = JSON.parse(stdout).results; + expect(row.title).toBe("Backend Engineer"); + expect(row.description).not.toMatch(LEAKED); + expect(row.description).toContain("Ship them."); + }); +}); diff --git a/.agents/skills/jobbank-search/cli/src/commands/search.ts b/.agents/skills/jobbank-search/cli/src/commands/search.ts index 1b1eb89f..72f97a8d 100644 --- a/.agents/skills/jobbank-search/cli/src/commands/search.ts +++ b/.agents/skills/jobbank-search/cli/src/commands/search.ts @@ -1,6 +1,6 @@ import { defineCommand, option } from "@bunli/core" import { z } from "zod" -import { rssFetch, fetchWithUA, writeError, parseRssDescription, extractJobIdFromUrl, BASE_URL, type RssItem } from "../helpers.js" +import { rssFetch, fetchWithUA, writeError, parseRssDescription, extractJobIdFromUrl, stripInvisible, BASE_URL, type RssItem } from "../helpers.js" export function normalizeSearchItem(item: RssItem): Record { const parsed = parseRssDescription(item.description) @@ -12,7 +12,7 @@ export function normalizeSearchItem(item: RssItem): Record { // as an absent one: posted "", date null. const parsedDate = item.pubDate ? new Date(item.pubDate) : null const posted = parsedDate && !Number.isNaN(parsedDate.getTime()) ? parsedDate.toISOString() : "" - return { + return stripInvisible({ id, title: item.title, company: parsed.company, @@ -23,7 +23,7 @@ export function normalizeSearchItem(item: RssItem): Record { posted, date: posted ? posted.slice(0, 10) : null, deadline: parsed.deadline, - } + }) } export const search = defineCommand({ diff --git a/.agents/skills/jobbank-search/cli/src/helpers.ts b/.agents/skills/jobbank-search/cli/src/helpers.ts index aafd0f39..45ef5607 100644 --- a/.agents/skills/jobbank-search/cli/src/helpers.ts +++ b/.agents/skills/jobbank-search/cli/src/helpers.ts @@ -182,6 +182,90 @@ export function extractJobIdFromUrl(url: string): string { return normalizeJobId(url) ?? "" } +/** + * Characters a reader of the posting never sees but a model reading this + * output does: C0/C1 controls other than tab, newline and carriage return, the + * soft hyphen, bidi marks/embeddings/overrides/isolates, zero-width characters + * and invisible operators, the BOM, variation selectors, and Unicode tag + * characters (U+E0000-E007F), which can spell out a whole hidden instruction. + * Listing text is third-party data, and these arrive raw in a portal's JSON or + * markup as easily as through a numeric entity. + */ +const INVISIBLE_CHARS = + /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F­؜᠎​-‏‪-‮⁠-︀-️\u{E0000}-\u{E007F}\u{E0100}-\u{E01EF}]/gu + +/** Strip INVISIBLE_CHARS from every string in a result, recursively: the CLI's output boundary. */ +export function stripInvisible(value: T): T { + if (typeof value === "string") return value.replace(INVISIBLE_CHARS, "") as T + if (Array.isArray(value)) return value.map(stripInvisible) as T + if (value !== null && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype) { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, stripInvisible(v)])) as T + } + return value +} + +const VOID_TAGS = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"]) + +/** Whether an opening tag's attributes keep it from rendering: `hidden`, or an inline style that hides it. */ +function isHiddenTag(attrs: string): boolean { + for (const m of attrs.matchAll(/([^\s"'=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g)) { + const name = m[1].toLowerCase() + const value = m[2] ?? m[3] ?? m[4] ?? "" + if (name === "hidden") return true + if (name === "style" && /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden|mso-hide\s*:\s*all)\b/i.test(value)) { + return true + } + } + return false +} + +/** + * Drop markup a browser never renders - comments (including Word's + * conditional ones), ', + '
HIDDEN styled
HIDDEN nested
HIDDEN after nesting
', + "", + '

HIDDEN mso

', + '', +].join("") +const LEAKED = /HIDDEN|WordDocument|MsoNormal|alert/ +const INVISIBLE_CHAR = /[\u{E0000}-\u{E007F}‮​⁦\u001B]/u + +// A job page's JobPosting JSON-LD with hostile text where the employer controls +// it; "` + +const originalFetch = globalThis.fetch +const originalLog = console.log +const originalExit = process.exit + +afterEach(() => { + globalThis.fetch = originalFetch + console.log = originalLog + process.exit = originalExit +}) + +// The real command handler with a stubbed fetch; process.exit throws so a +// parse failure fails the test, not the run. +async function runDetail(format: "json" | "plain"): Promise { + globalThis.fetch = (async () => new Response(PAGE, { status: 200 })) as unknown as typeof fetch + process.exit = ((code?: number) => { + throw new Error(`process.exit(${code})`) + }) as unknown as typeof process.exit + const stdout: string[] = [] + console.log = ((...args: unknown[]) => stdout.push(args.join(" "))) as typeof console.log + const handler = (detail as unknown as { handler: (ctx: unknown) => Promise }).handler + await handler({ flags: { format }, positional: ["123456"], signal: new AbortController().signal }) + return stdout.join("\n") +} + +describe("hidden text from a posting never reaches the output", () => { + test("detail JSON: hidden markup dropped from the description, invisible characters stripped", async () => { + const stdout = await runDetail("json") + expect(stdout).not.toMatch(INVISIBLE_CHAR) + const job = JSON.parse(stdout) + expect(job.title).toBe("Data Engineer") + expect(job.company.name).toBe("Acme") + expect(job.description).not.toMatch(LEAKED) + expect(job.description).toContain("

Byg datapipelines.

") + expect(job.description).toContain("Visible despite the words.") + expect(job.description).toContain("

Søg i dag.

") + }) + + test("detail plain output is cleaned the same way", async () => { + const stdout = await runDetail("plain") + expect(stdout).not.toMatch(INVISIBLE_CHAR) + expect(stdout).not.toMatch(LEAKED) + expect(stdout).toContain("Visible despite the words.") + }) + + test("search results strip invisible characters from feed fields", () => { + const result = normalizeSearchItem({ + title: `Data Engineer${INVISIBLE}`, + description: `Fuldtidsjob hos Acme${INVISIBLE}, København (Ansøgningsfrist: 01.10.2026)`, + link: "https://jobbank.dk/job/123456/acme/data-engineer", + pubDate: "Mon, 01 Sep 2026 08:00:00 GMT", + }) + expect(JSON.stringify(result)).not.toMatch(INVISIBLE_CHAR) + expect(result.title).toBe("Data Engineer") + expect(result.company).toBe("Acme") + }) +}) diff --git a/.agents/skills/jobdanmark-search/cli/src/commands/detail.ts b/.agents/skills/jobdanmark-search/cli/src/commands/detail.ts index 406b4bbc..6b593f2b 100644 --- a/.agents/skills/jobdanmark-search/cli/src/commands/detail.ts +++ b/.agents/skills/jobdanmark-search/cli/src/commands/detail.ts @@ -1,7 +1,7 @@ import { defineCommand, option } from "@bunli/core" import { z } from "zod" import { parse } from "node-html-parser" -import { BASE_URL, htmlFetch, normalizeSlug, writeError } from "../helpers.js" +import { BASE_URL, htmlFetch, normalizeSlug, stripInvisible, visibleHtml, writeError } from "../helpers.js" import { extractCity, toContractDate } from "./search.js" interface JsonLdJobPosting { @@ -74,7 +74,7 @@ function cleanText(text: string): string { */ function descriptionToText(html: string | undefined): string { if (!html) return "" - return parse(html) + return parse(visibleHtml(html)) .structuredText.split("\n") .map(cleanText) .filter(Boolean) @@ -235,9 +235,12 @@ function fromRenderedHtml(root: ReturnType, slug: string, url: str } export function parseJobPostingFromHtml(html: string, slug: string, url: string): DetailResult { - const root = parse(html) - const jobPosting = findJobPostingJsonLd(root) - return jobPosting ? fromJsonLd(jobPosting, slug, url) : fromRenderedHtml(root, slug, url) + const jobPosting = findJobPostingJsonLd(parse(html)) + // The rendered branch reads text nodes, so it parses the page without the + // markup a browser never shows; the JSON-LD lookup needs the scripts. + return stripInvisible( + jobPosting ? fromJsonLd(jobPosting, slug, url) : fromRenderedHtml(parse(visibleHtml(html)), slug, url), + ) } export const detail = defineCommand({ diff --git a/.agents/skills/jobdanmark-search/cli/src/commands/search.ts b/.agents/skills/jobdanmark-search/cli/src/commands/search.ts index 838880a8..b484d273 100644 --- a/.agents/skills/jobdanmark-search/cli/src/commands/search.ts +++ b/.agents/skills/jobdanmark-search/cli/src/commands/search.ts @@ -1,6 +1,6 @@ import { defineCommand, option } from "@bunli/core" import { z } from "zod" -import { apiPost, writeError, BASE_URL } from "../helpers.js" +import { apiPost, writeError, stripInvisible, BASE_URL } from "../helpers.js" export interface ApiSearchItem { title: string @@ -64,7 +64,7 @@ export function normalizeItem(item: ApiSearchItem): Record { // payload and the /scrape agent can never use an image or overlay colour. // The #340 compatibility duplicates (companyName, publishedDate, // applicationDeadline) stay. - return { + return stripInvisible({ title: item.title, companyName: item.companyName, companyAddress: item.companyAddress, @@ -82,7 +82,7 @@ export function normalizeItem(item: ApiSearchItem): Record { location: extractCity(item.companyAddress), date: toContractDate(item.publishedDate), deadline: toContractDate(item.applicationDeadline), - } + }) } export const search = defineCommand({ diff --git a/.agents/skills/jobdanmark-search/cli/src/helpers.ts b/.agents/skills/jobdanmark-search/cli/src/helpers.ts index 45c4480b..b43e7962 100644 --- a/.agents/skills/jobdanmark-search/cli/src/helpers.ts +++ b/.agents/skills/jobdanmark-search/cli/src/helpers.ts @@ -107,8 +107,92 @@ export function writeError(error: string, code: string): void { process.stderr.write(JSON.stringify({ error, code }) + "\n") } +/** + * Characters a reader of the posting never sees but a model reading this + * output does: C0/C1 controls other than tab, newline and carriage return, the + * soft hyphen, bidi marks/embeddings/overrides/isolates, zero-width characters + * and invisible operators, the BOM, variation selectors, and Unicode tag + * characters (U+E0000-E007F), which can spell out a whole hidden instruction. + * Listing text is third-party data, and these arrive raw in a portal's JSON or + * markup as easily as through a numeric entity. + */ +const INVISIBLE_CHARS = + /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F­؜᠎​-‏‪-‮⁠-︀-️\u{E0000}-\u{E007F}\u{E0100}-\u{E01EF}]/gu + +/** Strip INVISIBLE_CHARS from every string in a result, recursively: the CLI's output boundary. */ +export function stripInvisible(value: T): T { + if (typeof value === "string") return value.replace(INVISIBLE_CHARS, "") as T + if (Array.isArray(value)) return value.map(stripInvisible) as T + if (value !== null && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype) { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, stripInvisible(v)])) as T + } + return value +} + +const VOID_TAGS = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"]) + +/** Whether an opening tag's attributes keep it from rendering: `hidden`, or an inline style that hides it. */ +function isHiddenTag(attrs: string): boolean { + for (const m of attrs.matchAll(/([^\s"'=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g)) { + const name = m[1].toLowerCase() + const value = m[2] ?? m[3] ?? m[4] ?? "" + if (name === "hidden") return true + if (name === "style" && /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden|mso-hide\s*:\s*all)\b/i.test(value)) { + return true + } + } + return false +} + +/** + * Drop markup a browser never renders - comments (including Word's + * conditional ones), ', + '

HIDDEN styled

HIDDEN nested

HIDDEN after nesting

', + "", + '

HIDDEN mso

', + '', +].join(""); +const LEAKED = /HIDDEN|WordDocument|MsoNormal|alert/; +const INVISIBLE_CHAR = /[\u{E0000}-\u{E007F}‮​⁦\u001B]/u; + +// The two page shapes `detail` parses, with hostile text where the employer +// controls it. JSON-LD carries the description as HTML inside a JSON string, +// with " +`; + +const RENDERED_PAGE = ` + + Journalistisk udvikler søges | jobdanmark + +
+
+

Journalistisk udvikler søges${INVISIBLE}

+ JFM +
+

Hvad nu hvis du med få klik kunne undersøge dit lokalområde?${INVISIBLE}

+ ${HIDDEN_MARKUP} +
  • identificere relevante datasæt og muligheder
+
+
    +
  • Udgivet: 03-07-2026
    • +
  • Jobtype: Fuldtid
    • +
+ +`; + +const originalFetch = globalThis.fetch; +const originalLog = console.log; +const originalExit = process.exit; + +afterEach(() => { + globalThis.fetch = originalFetch; + console.log = originalLog; + process.exit = originalExit; +}); + +// The real command handler with a stubbed fetch, as detail-backoff.test.ts +// drives it; process.exit throws so a parse failure fails the test, not the run. +async function runDetail(page: string): Promise { + globalThis.fetch = (async () => new Response(page, { status: 200 })) as unknown as typeof fetch; + process.exit = ((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as unknown as typeof process.exit; + const stdout: string[] = []; + console.log = ((...args: unknown[]) => stdout.push(args.join(" "))) as typeof console.log; + const handler = (detail as unknown as { handler: (ctx: unknown) => Promise }).handler; + await handler({ flags: { format: "json" }, positional: ["some-slug"], signal: new AbortController().signal }); + return stdout.join("\n"); +} + +describe("hidden text from a posting never reaches the output", () => { + test("JSON-LD page: hidden markup dropped from the description, invisible characters stripped", async () => { + const stdout = await runDetail(JSON_LD_PAGE); + expect(stdout).not.toMatch(INVISIBLE_CHAR); + const job = JSON.parse(stdout); + expect(job.title).toBe("Data Engineer"); + expect(job.hiringOrganization.name).toBe("Acme"); + expect(job.description).not.toMatch(LEAKED); + expect(job.description).toBe("Build pipelines.\nVisible despite the words.\nShip them."); + }); + + test("rendered page: hidden paragraphs never reach the description", async () => { + const stdout = await runDetail(RENDERED_PAGE); + expect(stdout).not.toMatch(INVISIBLE_CHAR); + const job = JSON.parse(stdout); + expect(job.title).toBe("Journalistisk udvikler søges"); + expect(job.description).not.toMatch(LEAKED); + expect(job.description).toContain("Visible despite the words."); + expect(job.description).toContain("identificere relevante datasæt og muligheder"); + }); + + test("search results strip invisible characters from listing fields", () => { + const result = normalizeItem({ + title: `Softwareudvikler${INVISIBLE}`, + companyName: `Statens It${INVISIBLE}`, + companyLogo: null, + companyLogoSvgMarkup: null, + overlayColor: null, + companyAddress: "Lautruphøj 2, 2750 Ballerup", + jobTypes: ["fuldtid"], + boostJob: false, + publishedDate: "27-07-2026", + applicationDeadline: "17-08-2026", + url: "/job/softwareudvikler-til-statens-it", + coverImage: null, + silhouetteLogo: false, + }); + expect(JSON.stringify(result)).not.toMatch(INVISIBLE_CHAR); + expect(result.title).toBe("Softwareudvikler"); + expect(result.company).toBe("Statens It"); + }); +}); diff --git a/.agents/skills/jobindex-search/cli/src/commands/detail.ts b/.agents/skills/jobindex-search/cli/src/commands/detail.ts index 5f8be673..d277d56e 100644 --- a/.agents/skills/jobindex-search/cli/src/commands/detail.ts +++ b/.agents/skills/jobindex-search/cli/src/commands/detail.ts @@ -1,6 +1,6 @@ import { defineCommand, option } from "@bunli/core" import { z } from "zod" -import { htmlFetch, writeError } from "../helpers.js" +import { htmlFetch, stripInvisible, visibleHtml, writeError } from "../helpers.js" const BASE_URL = "https://www.jobindex.dk" @@ -135,17 +135,13 @@ function jdBlockValue(html: string, cls: string): string | null { return m ? decodeHtmlEntities(stripTags(m[1])).replace(/\s+/g, " ").trim() || null : null } -/** Drop head/script/style content so text scans never read CSS or JS. */ -function visibleHtml(html: string): string { - return html - .replace(//gi, "") - .replace(//gi, "") - .replace(//gi, "") - .replace(//g, "") +/** Drop head content and unrendered markup so text scans never read CSS, JS or hidden text. */ +function visibleBody(html: string): string { + return visibleHtml(html.replace(//gi, "")) } function bodyText(html: string): string | null { - const text = decodeHtmlEntities(stripTags(visibleHtml(html).replace(/<(br|\/p|\/div|\/li|\/h[1-6])[^>]*>/gi, "\n"))) + const text = decodeHtmlEntities(stripTags(visibleBody(html).replace(/<(br|\/p|\/div|\/li|\/h[1-6])[^>]*>/gi, "\n"))) .split("\n") .map((line) => line.replace(/\s+/g, " ").trim()) .filter(Boolean) @@ -204,13 +200,13 @@ export function parseDetailPage(html: string, url: string, id: string): DetailRe deadline = toIsoDate(jdBlockValue(html, "jd-deadline")) employmentType = jdBlockValue(html, "jd-type") hours = jdBlockValue(html, "jd-workhours") - const desc = html.match(/class="jd-description"[^>]*>([\s\S]*?)<\/div>/i) + const desc = visibleHtml(html).match(/class="jd-description"[^>]*>([\s\S]*?)<\/div>/i) description = desc ? decodeHtmlEntities(stripTags(desc[1])).replace(/\s+/g, " ").trim() || null : null } else { // hr-manager-style widget: a rowheader label followed by the value span. - const workplace = visibleHtml(html).match( + const workplace = visibleBody(html).match( /class="workplace[^"]*"[\s\S]*?([\s\S]*?)<\/span>/i, ) location = workplace @@ -220,7 +216,7 @@ export function parseDetailPage(html: string, url: string, id: string): DetailRe // Deadline: label + a real date within range, scanned only over visible // markup - the label also appears inside a CSS comment on these pages, // which the previous parser captured verbatim as the deadline. - const due = visibleHtml(html).match( + const due = visibleBody(html).match( /(?:Ansøgningsfrist|Application\s*due|Frist)[\s\S]{0,300}?(\d{2}-\d{2}-\d{4}|\d{4}-\d{2}-\d{2}|\d{1,2}\.?\s+[a-zæøå]+\s+\d{4})/i, ) deadline = due ? toIsoDate(due[1]) : null @@ -244,7 +240,7 @@ export function parseDetailPage(html: string, url: string, id: string): DetailRe const timeMatch = html.match(/]+datetime="([^"]+)"/) - return { + return stripInvisible({ id, title, company, @@ -257,7 +253,7 @@ export function parseDetailPage(html: string, url: string, id: string): DetailRe applyUrl, url, description, - } + }) } export const detail = defineCommand({ diff --git a/.agents/skills/jobindex-search/cli/src/helpers.ts b/.agents/skills/jobindex-search/cli/src/helpers.ts index e6db4aa1..5f5e663d 100644 --- a/.agents/skills/jobindex-search/cli/src/helpers.ts +++ b/.agents/skills/jobindex-search/cli/src/helpers.ts @@ -182,7 +182,7 @@ export function parseSearchPage(html: string): SearchPageResult { }) const total = typeof sr.hitcount === "number" ? sr.hitcount : results.length - return { total, results } + return stripInvisible({ total, results }) } /** @@ -211,11 +211,95 @@ function decodeHtmlEntities(text: string): string { .replace(/ /g, " ") } +/** + * Characters a reader of the posting never sees but a model reading this + * output does: C0/C1 controls other than tab, newline and carriage return, the + * soft hyphen, bidi marks/embeddings/overrides/isolates, zero-width characters + * and invisible operators, the BOM, variation selectors, and Unicode tag + * characters (U+E0000-E007F), which can spell out a whole hidden instruction. + * Listing text is third-party data, and these arrive raw in a portal's JSON or + * markup as easily as through a numeric entity. + */ +const INVISIBLE_CHARS = + /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F­؜᠎​-‏‪-‮⁠-︀-️\u{E0000}-\u{E007F}\u{E0100}-\u{E01EF}]/gu + +/** Strip INVISIBLE_CHARS from every string in a result, recursively: the CLI's output boundary. */ +export function stripInvisible(value: T): T { + if (typeof value === "string") return value.replace(INVISIBLE_CHARS, "") as T + if (Array.isArray(value)) return value.map(stripInvisible) as T + if (value !== null && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype) { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, stripInvisible(v)])) as T + } + return value +} + +const VOID_TAGS = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"]) + +/** Whether an opening tag's attributes keep it from rendering: `hidden`, or an inline style that hides it. */ +function isHiddenTag(attrs: string): boolean { + for (const m of attrs.matchAll(/([^\s"'=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g)) { + const name = m[1].toLowerCase() + const value = m[2] ?? m[3] ?? m[4] ?? "" + if (name === "hidden") return true + if (name === "style" && /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden|mso-hide\s*:\s*all)\b/i.test(value)) { + return true + } + } + return false +} + +/** + * Drop markup a browser never renders - comments (including Word's + * conditional ones), ', + '
HIDDEN styled
HIDDEN nested
HIDDEN after nesting
', + "", + '

HIDDEN mso

', + '', +].join(""); +const LEAKED = /HIDDEN|WordDocument|MsoNormal|alert/; +const INVISIBLE_CHAR = /[\u{E0000}-\u{E007F}‮​⁦\u001B]/u; + +// Both live page shapes from detail-parsing.test.ts, trimmed, with hostile text +// where the employer controls it. +const NATIVE_PAGE = ` + + + VELLIV - Udvikler til Camunda/AWS + + + +
+
+

Hos Velliv får du mulighed for at arbejde med Camunda.${INVISIBLE}

+ ${HIDDEN_MARKUP} +

Vi glæder os til at høre fra dig.

+
+
+

Arbejdssted:

Ballerup

+ +`; + +const EXTERNAL_PAGE = ` + + + Talentech - C#-udvikler + + + +

C#-udvikler til elnettet

+

Vil du være med til at udvikle og drifte de systemer, der understøtter udbygningen af Danmarks kommende elnet?${INVISIBLE}

+ ${HIDDEN_MARKUP} +

Vi arbejder i krydsfeltet mellem IT og energi, og vi søger en kollega med erfaring i C#.

+ +`; + +describe("hidden text from a posting never reaches the output", () => { + test("native page: hidden markup dropped from the description, invisible characters stripped", () => { + const job = parseDetailPage(NATIVE_PAGE, "https://www.jobindex.dk/jobannonce/h1690934", "h1690934"); + expect(JSON.stringify(job)).not.toMatch(INVISIBLE_CHAR); + expect(job.title).toBe("Udvikler til Camunda/AWS"); + expect(job.description).not.toMatch(LEAKED); + expect(job.description).toContain("Visible despite the words."); + expect(job.description).toContain("Vi glæder os til at høre fra dig."); + }); + + test("external ATS page: hidden markup never reaches the body text", () => { + const job = parseDetailPage(EXTERNAL_PAGE, "https://www.jobindex.dk/vis-job/r123", "r123"); + expect(JSON.stringify(job)).not.toMatch(INVISIBLE_CHAR); + expect(job.description).not.toMatch(LEAKED); + expect(job.description).toContain("Visible despite the words."); + expect(job.description).toContain("Vi arbejder i krydsfeltet mellem IT og energi"); + }); + + test("search results strip invisible characters from Stash fields", () => { + const stash = { + jobsearch: { + result_app: { + storeData: { + searchResponse: { + hitcount: 1, + results: [ + { + tid: "h1689961", + headline: `Softwareudvikler${INVISIBLE}`, + company: { name: `Acme A/S${INVISIBLE}` }, + area: "Aarhus", + firstdate: "2026-08-10", + }, + ], + }, + }, + }, + }, + }; + const page = ``; + const [job] = parseSearchPage(page).results; + expect(JSON.stringify(job)).not.toMatch(INVISIBLE_CHAR); + expect(job.title).toBe("Softwareudvikler"); + expect(job.company).toBe("Acme A/S"); + }); +}); diff --git a/.agents/skills/jobnet-search/cli/src/commands/detail.ts b/.agents/skills/jobnet-search/cli/src/commands/detail.ts index 2d2d5da1..f8e988ba 100644 --- a/.agents/skills/jobnet-search/cli/src/commands/detail.ts +++ b/.agents/skills/jobnet-search/cli/src/commands/detail.ts @@ -1,6 +1,6 @@ import { defineCommand, option } from "@bunli/core" import { z } from "zod" -import { apiFetch, normalizeJobId, writeError, stripHtml } from "../helpers.js" +import { apiFetch, normalizeJobId, writeError, stripHtml, stripInvisible, visibleHtml } from "../helpers.js" import type { JobAdRaw, SearchApiResponse } from "./search.js" export interface DetailApiResponse { @@ -132,13 +132,18 @@ export function mapSearchAdToDetail(raw: JobAdRaw & { jobAdUrl?: string | null; * The search command already maps that sentinel to null; detail must agree, * or an undisclosed deadline reads as 126 years expired and /rank's expiry * sweep retires the job the moment it is stored. + * + * The body is the employer's HTML: markup a browser never renders is dropped, + * and invisible characters are stripped from every field (see visibleHtml and + * stripInvisible). */ export function prepareDetail(data: DetailApiResponse): DetailApiResponse { const deadline = data.application.deadlineDate if (deadline && deadline.startsWith("1900-01-01")) { data.application.deadlineDate = null } - return data + if (typeof data.body === "string") data.body = visibleHtml(data.body) + return stripInvisible(data) } export const detail = defineCommand({ diff --git a/.agents/skills/jobnet-search/cli/src/commands/search.ts b/.agents/skills/jobnet-search/cli/src/commands/search.ts index ca5b07f7..40a284da 100644 --- a/.agents/skills/jobnet-search/cli/src/commands/search.ts +++ b/.agents/skills/jobnet-search/cli/src/commands/search.ts @@ -1,6 +1,6 @@ import { defineCommand, option } from "@bunli/core" import { z } from "zod" -import { apiFetch, writeError } from "../helpers.js" +import { apiFetch, stripInvisible, writeError } from "../helpers.js" export interface SearchApiResponse { jobAds: JobAdRaw[] @@ -175,7 +175,7 @@ export function createSearchOutput(data: SearchApiResponse, flags: SearchFlags) searchString: data.searchString ?? null, } - return { meta, facets, results } + return stripInvisible({ meta, facets, results }) } export const search = defineCommand({ diff --git a/.agents/skills/jobnet-search/cli/src/helpers.ts b/.agents/skills/jobnet-search/cli/src/helpers.ts index f409dced..c58aa395 100644 --- a/.agents/skills/jobnet-search/cli/src/helpers.ts +++ b/.agents/skills/jobnet-search/cli/src/helpers.ts @@ -43,8 +43,92 @@ export function writeError(error: string, code: string): void { process.stderr.write(JSON.stringify({ error, code }) + "\n") } +/** + * Characters a reader of the posting never sees but a model reading this + * output does: C0/C1 controls other than tab, newline and carriage return, the + * soft hyphen, bidi marks/embeddings/overrides/isolates, zero-width characters + * and invisible operators, the BOM, variation selectors, and Unicode tag + * characters (U+E0000-E007F), which can spell out a whole hidden instruction. + * Listing text is third-party data, and these arrive raw in a portal's JSON or + * markup as easily as through a numeric entity. + */ +const INVISIBLE_CHARS = + /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F­؜᠎​-‏‪-‮⁠-︀-️\u{E0000}-\u{E007F}\u{E0100}-\u{E01EF}]/gu + +/** Strip INVISIBLE_CHARS from every string in a result, recursively: the CLI's output boundary. */ +export function stripInvisible(value: T): T { + if (typeof value === "string") return value.replace(INVISIBLE_CHARS, "") as T + if (Array.isArray(value)) return value.map(stripInvisible) as T + if (value !== null && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype) { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, stripInvisible(v)])) as T + } + return value +} + +const VOID_TAGS = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"]) + +/** Whether an opening tag's attributes keep it from rendering: `hidden`, or an inline style that hides it. */ +function isHiddenTag(attrs: string): boolean { + for (const m of attrs.matchAll(/([^\s"'=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g)) { + const name = m[1].toLowerCase() + const value = m[2] ?? m[3] ?? m[4] ?? "" + if (name === "hidden") return true + if (name === "style" && /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden|mso-hide\s*:\s*all)\b/i.test(value)) { + return true + } + } + return false +} + +/** + * Drop markup a browser never renders - comments (including Word's + * conditional ones), ', + '
HIDDEN styled
HIDDEN nested
HIDDEN after nesting
', + "", + '

HIDDEN mso

', + '', +].join("") +const LEAKED = /HIDDEN|WordDocument|MsoNormal|alert/ +const INVISIBLE_CHAR = /[\u{E0000}-\u{E007F}‮​⁦\u001B]/u + +const ID = "9ef43bce-d82b-4ea1-a098-7ff6520f99be" + +// The README's documented detail response, with hostile text where the +// employer controls it: the title and the HTML body. +async function hostileDetail(): Promise> { + const readme = (await Bun.file(join(import.meta.dir, "../README.md")).text()).replace(/\r\n/g, "\n") + const section = readme.split("## `detail`")[1].split("\n---")[0] + const documented = JSON.parse(section.match(/```json\n([\s\S]*?)```/)![1]) + return { + ...documented, + title: `${documented.title}${INVISIBLE}`, + body: `

Vi søger en kollega.${INVISIBLE}

${HIDDEN_MARKUP}

Søg senest 5. april.

`, + } +} + +async function detail(format: "json" | "plain"): Promise { + const fixture = await hostileDetail() + const server = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => Response.json(fixture) }) + const dir = mkdtempSync(join(tmpdir(), "jobnet-hidden-")) + const preload = join(dir, "fetch.ts") + // The documented CLI in a fresh process; only this posting is served. + writeFileSync(preload, ` + const originalFetch = globalThis.fetch; + globalThis.fetch = (input, init) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.protocol === "data:") return originalFetch(input, init); + if (url.origin !== "https://jobnet.dk" || url.pathname !== "/bff/FindJob/JobAdDetails/${ID}") { + throw new Error("Unexpected detail fixture request"); + } + return originalFetch(${JSON.stringify(server.url.origin)} + url.pathname + url.search, init); + }; + `) + try { + const proc = Bun.spawn([ + process.execPath, "run", "--preload", preload, + join(import.meta.dir, "../src/cli.ts"), "detail", ID, "--format", format, + ], { stdout: "pipe", stderr: "pipe" }) + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]) + expect(exitCode).toBe(0) + return stdout + } finally { + server.stop(true) + rmSync(dir, { recursive: true, force: true }) + } +} + +describe("hidden text from a job ad never reaches the output", () => { + test("detail JSON: hidden markup dropped from the body, invisible characters stripped", async () => { + const stdout = await detail("json") + expect(stdout).not.toMatch(INVISIBLE_CHAR) + const job = JSON.parse(stdout) + expect(job.title).toBe("Akademisk medarbejder med interesse for arbejdsmiljø og uddannelse") + expect(job.body).not.toMatch(LEAKED) + expect(job.body).toContain("

Vi søger en kollega.

") + expect(job.body).toContain("Visible despite the words.") + expect(job.body).toContain("

Søg senest 5. april.

") + }) + + test("detail plain output is cleaned the same way", async () => { + const stdout = await detail("plain") + expect(stdout).not.toMatch(INVISIBLE_CHAR) + expect(stdout).not.toMatch(LEAKED) + expect(stdout).toContain("Visible despite the words.") + }) + + test("search output strips invisible characters from employer-supplied fields", () => { + const data: SearchApiResponse = { + totalJobAdCount: 1, + searchString: "udvikler", + searchFacets: { regions: [], workHours: [], employmentDurations: [], occupationAreas: [], countries: [] }, + jobAds: [ + { + jobAdId: ID, + title: `Udvikler${INVISIBLE}`, + hiringOrgName: `Acme${INVISIBLE}`, + country: "Danmark", + publicationDate: "2026-03-13T00:00:00+01:00", + workHourPartTime: false, + isExternal: false, + hasLogo: false, + isSeen: false, + isFavorite: false, + } as SearchApiResponse["jobAds"][number], + ], + } + const output = createSearchOutput(data, { page: 1, "per-page": 10 } as SearchFlags) + expect(JSON.stringify(output)).not.toMatch(INVISIBLE_CHAR) + expect(output.results[0].title).toBe("Udvikler") + expect(output.results[0].company).toBe("Acme") + }) +}) diff --git a/.agents/skills/linkedin-search/cli/src/commands/detail.ts b/.agents/skills/linkedin-search/cli/src/commands/detail.ts index 5c800791..b3f9bde7 100644 --- a/.agents/skills/linkedin-search/cli/src/commands/detail.ts +++ b/.agents/skills/linkedin-search/cli/src/commands/detail.ts @@ -1,4 +1,4 @@ -import { DETAIL_URL, htmlFetch, parseJobDetail, writeError } from "../helpers.js" +import { DETAIL_URL, htmlFetch, parseJobDetail, stripInvisible, writeError } from "../helpers.js" export interface DetailOpts { id: string @@ -55,7 +55,7 @@ export async function runDetail(opts: DetailOpts): Promise { writeError("Job not found", "NOT_FOUND") return 1 } - const job = parseJobDetail(html, id) + const job = stripInvisible(parseJobDetail(html, id)) if (opts.format === "plain") { const lines = [ diff --git a/.agents/skills/linkedin-search/cli/src/commands/search.ts b/.agents/skills/linkedin-search/cli/src/commands/search.ts index 37eaac21..c8d1ac35 100644 --- a/.agents/skills/linkedin-search/cli/src/commands/search.ts +++ b/.agents/skills/linkedin-search/cli/src/commands/search.ts @@ -4,6 +4,7 @@ import { parseJobCards, jobageToTPR, minutesToTPR, + stripInvisible, workTypeFlag, writeError, type JobCard, @@ -56,7 +57,7 @@ function renderTable(cards: JobCard[]): string { export async function runSearch(opts: SearchOpts): Promise { try { const html = await htmlFetch(buildUrl(opts)) - let cards = parseJobCards(html) + let cards = stripInvisible(parseJobCards(html)) if (opts.limit !== undefined && opts.limit >= 0) cards = cards.slice(0, opts.limit) if (opts.format === "table") { diff --git a/.agents/skills/linkedin-search/cli/src/helpers.ts b/.agents/skills/linkedin-search/cli/src/helpers.ts index 4bce89ad..beb8269d 100644 --- a/.agents/skills/linkedin-search/cli/src/helpers.ts +++ b/.agents/skills/linkedin-search/cli/src/helpers.ts @@ -120,8 +120,92 @@ function decodeHtmlEntities(text: string): string { .replace(/ /g, " ") } +/** + * Characters a reader of the posting never sees but a model reading this + * output does: C0/C1 controls other than tab, newline and carriage return, the + * soft hyphen, bidi marks/embeddings/overrides/isolates, zero-width characters + * and invisible operators, the BOM, variation selectors, and Unicode tag + * characters (U+E0000-E007F), which can spell out a whole hidden instruction. + * Listing text is third-party data, and these arrive raw in a portal's JSON or + * markup as easily as through a numeric entity. + */ +const INVISIBLE_CHARS = + /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F­؜᠎​-‏‪-‮⁠-︀-️\u{E0000}-\u{E007F}\u{E0100}-\u{E01EF}]/gu + +/** Strip INVISIBLE_CHARS from every string in a result, recursively: the CLI's output boundary. */ +export function stripInvisible(value: T): T { + if (typeof value === "string") return value.replace(INVISIBLE_CHARS, "") as T + if (Array.isArray(value)) return value.map(stripInvisible) as T + if (value !== null && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype) { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, stripInvisible(v)])) as T + } + return value +} + +const VOID_TAGS = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"]) + +/** Whether an opening tag's attributes keep it from rendering: `hidden`, or an inline style that hides it. */ +function isHiddenTag(attrs: string): boolean { + for (const m of attrs.matchAll(/([^\s"'=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g)) { + const name = m[1].toLowerCase() + const value = m[2] ?? m[3] ?? m[4] ?? "" + if (name === "hidden") return true + if (name === "style" && /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden|mso-hide\s*:\s*all)\b/i.test(value)) { + return true + } + } + return false +} + +/** + * Drop markup a browser never renders - comments (including Word's + * conditional ones), ', + '
HIDDEN styled
HIDDEN nested
HIDDEN after nesting
', + "", + '

HIDDEN mso

', + '', +].join(""); +const LEAKED = /HIDDEN|WordDocument|MsoNormal|alert/; +const INVISIBLE_CHAR = /[\u{E0000}-\u{E007F}‮​⁦\u001B]/u; + +const endpoint = "https://www.linkedin.com/jobs-guest/jobs/api/jobPosting/4345841651"; +const cliPath = join(import.meta.dir, "../src/cli.ts"); +const fixturePath = join(import.meta.dir, "fixtures/detail-orbis-excerpt.html"); + +// The captured guest response with hostile text added where an employer +// controls it: the title and the description body. +async function hostilePage(): Promise { + const page = await Bun.file(fixturePath).text(); + return page + .replace("Agentic AI Engineer - Full remote - Europe", `Agentic AI Engineer - Full remote - Europe${INVISIBLE}`) + .replace( + "

Location : Fully remote (Europe)

", + `

Location : Fully remote (Europe)${INVISIBLE}

${HIDDEN_MARKUP}`, + ); +} + +async function detail(format: "json" | "plain"): Promise { + const page = await hostilePage(); + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: () => new Response(page, { headers: { "Content-Type": "text/html" } }), + }); + const dir = mkdtempSync(join(tmpdir(), "linkedin-hidden-")); + const preload = join(dir, "fetch.ts"); + // The documented CLI in a fresh process; only this posting is served. + writeFileSync(preload, ` + const originalFetch = globalThis.fetch; + globalThis.fetch = (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url !== ${JSON.stringify(endpoint)}) throw new Error("Unexpected fixture request"); + return originalFetch(${JSON.stringify(server.url.origin)} + new URL(url).pathname, init); + }; + `); + try { + const proc = Bun.spawn([ + process.execPath, "run", "--preload", preload, cliPath, + "detail", "4345841651", "--format", format, + ], { stdout: "pipe", stderr: "pipe" }); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + expect(exitCode).toBe(0); + return stdout; + } finally { + server.stop(true); + rmSync(dir, { recursive: true, force: true }); + } +} + +const originalFetch = globalThis.fetch; +const originalStdoutWrite = process.stdout.write; + +afterEach(() => { + globalThis.fetch = originalFetch; + process.stdout.write = originalStdoutWrite; +}); + +test("detail JSON drops hidden markup and invisible characters, keeping the visible text", async () => { + const stdout = await detail("json"); + expect(stdout).not.toMatch(INVISIBLE_CHAR); + const job = JSON.parse(stdout); + expect(job.title).toBe("Agentic AI Engineer - Full remote - Europe"); + expect(job.description).not.toMatch(LEAKED); + expect(job.description).toContain("Location : Fully remote (Europe)\n"); + expect(job.description).toContain("Visible despite the words."); + // The real description container's class list includes overflow-hidden. + expect(job.description).toContain("Designing and building agentic & multi-agent systems\nDeveloping"); +}); + +test("detail plain output is cleaned the same way", async () => { + const stdout = await detail("plain"); + expect(stdout).not.toMatch(INVISIBLE_CHAR); + expect(stdout).not.toMatch(LEAKED); + expect(stdout).toContain("Visible despite the words."); +}); + +test("search drops invisible characters from card text", async () => { + globalThis.fetch = (async () => + new Response(`
  • +
    + +

    Data Engineer${INVISIBLE}

    +
    +
    • `)) as typeof fetch; + let stdout = ""; + process.stdout.write = ((chunk: string | Uint8Array) => { + stdout += chunk.toString(); + return true; + }) as typeof process.stdout.write; + + const code = await runSearch({ location: "Copenhagen, Denmark", jobage: 9999, page: 1, format: "json" }); + + expect(code).toBe(0); + expect(stdout).not.toMatch(INVISIBLE_CHAR); + expect(JSON.parse(stdout).results[0].title).toBe("Data Engineer"); +}); diff --git a/.claude/commands/add-portal.md b/.claude/commands/add-portal.md index ac5d0bf6..05b865b2 100644 --- a/.claude/commands/add-portal.md +++ b/.claude/commands/add-portal.md @@ -83,6 +83,7 @@ These conventions are what make portal skills interchangeable for `/scrape` and - **Errors:** written to **stderr** as `{ "error": "...", "code": "..." }`, exit code `1`. Never write errors to stdout. - **Fetching:** an honest User-Agent that names the tool (`Mozilla/5.0 (compatible; -cli/1.0)`, the convention every shipped portal CLI follows) - never a full browser impersonation; if the portal refuses that UA, escalation to browser headers goes through the robots.txt gate in `.claude/skills/job-application-assistant/09-web-research.md`, not through the CLI's default. Exponential backoff with jitter on 429/5xx (max ~6 retries), `""`/`null` on 404 rather than a crash. - **HTML parsing:** split the response into per-result chunks and parse each independently, so one malformed card cannot break the rest (see `parseJobCards` in `linkedin-search/cli/src/helpers.ts`). +- **Output hygiene:** listing text is third-party data, so text a reader of the posting never sees must not reach the model. Run HTML through `visibleHtml` before stripping tags (comments, `