export const BASE_URL = "https://jobdanmark.dk" export const USER_AGENT = "Mozilla/5.0 (compatible; jobdanmark-cli/1.0)" export async function apiFetch(path: string, params?: Record): Promise { let url = `${BASE_URL}${path}` if (params && Object.keys(params).length > 0) { const qs = new URLSearchParams(params) url += `?${qs.toString()}` } const maxRetries = 6 let delay = 500 for (let attempt = 0; attempt <= maxRetries; attempt++) { const response = await fetch(url, { headers: { "User-Agent": USER_AGENT }, signal: AbortSignal.timeout(15000), }) if (response.status === 429 || response.status >= 500) { if (attempt === maxRetries) { throw new Error(`API request failed: ${response.status} ${response.statusText}`) } const jitter = Math.floor(Math.random() * 500) await new Promise((resolve) => setTimeout(resolve, delay + jitter)) delay = Math.min(delay * 2, 5000) continue } if (!response.ok) { throw new Error(`API request failed: ${response.status} ${response.statusText}`) } return response.json() as Promise } throw new Error("API request failed after max retries") } export async function apiPost(path: string, body: unknown): Promise { const url = `${BASE_URL}${path}` const maxRetries = 6 let delay = 500 for (let attempt = 0; attempt <= maxRetries; attempt++) { const response = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json", "User-Agent": USER_AGENT, }, body: JSON.stringify(body), signal: AbortSignal.timeout(15000), }) if (response.status === 429 || response.status >= 500) { if (attempt === maxRetries) { throw new Error(`API request failed: ${response.status} ${response.statusText}`) } const jitter = Math.floor(Math.random() * 500) await new Promise((resolve) => setTimeout(resolve, delay + jitter)) delay = Math.min(delay * 2, 5000) continue } if (!response.ok) { throw new Error(`API request failed: ${response.status} ${response.statusText}`) } return response.json() as Promise } throw new Error("API request failed after max retries") } /** * Fetch a rendered jobdanmark.dk page as text, with the same 429/5xx backoff, * request timeout, and User-Agent as apiFetch/apiPost. `detail` reads HTML * rather than the JSON API; it used to call fetch() directly with none of the * three, so a rate-limited detail page failed on the first 429 while every * other portal's detail command retried. Returns null on 404 so the caller * keeps its own NOT_FOUND contract. */ export async function htmlFetch(url: string): Promise { const maxRetries = 6 let delay = 500 for (let attempt = 0; attempt <= maxRetries; attempt++) { const response = await fetch(url, { headers: { "Accept": "text/html,application/xhtml+xml", "User-Agent": USER_AGENT, }, signal: AbortSignal.timeout(15000), }) if (response.status === 429 || response.status >= 500) { if (attempt === maxRetries) { throw new Error(`API request failed: ${response.status} ${response.statusText}`) } const jitter = Math.floor(Math.random() * 500) await new Promise((resolve) => setTimeout(resolve, delay + jitter)) delay = Math.min(delay * 2, 5000) continue } if (response.status === 404) { return null } if (!response.ok) { throw new Error(`API request failed: ${response.status} ${response.statusText}`) } return response.text() } throw new Error("API request failed after max retries") } export function writeError(error: string, code: string): void { process.stderr.write(JSON.stringify({ error, code }) + "\n") } /** * Characters a reader of the posting never sees but a model reading this * output does: C0/C1 controls other than tab, newline and carriage return, the * soft hyphen, bidi marks/embeddings/overrides/isolates, zero-width characters * and invisible operators, the BOM, variation selectors, and Unicode tag * characters (U+E0000-E007F), which can spell out a whole hidden instruction. * Listing text is third-party data, and these arrive raw in a portal's JSON or * markup as easily as through a numeric entity. */ const INVISIBLE_CHARS = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F­؜᠎​-‏‪-‮⁠-︀-️\u{E0000}-\u{E007F}\u{E0100}-\u{E01EF}]/gu /** Strip INVISIBLE_CHARS from every string in a result, recursively: the CLI's output boundary. */ export function stripInvisible(value: T): T { if (typeof value === "string") return value.replace(INVISIBLE_CHARS, "") as T if (Array.isArray(value)) return value.map(stripInvisible) as T if (value !== null && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype) { return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, stripInvisible(v)])) as T } return value } const VOID_TAGS = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"]) /** Whether an opening tag's attributes keep it from rendering: `hidden`, or an inline style that hides it. */ function isHiddenTag(attrs: string): boolean { for (const m of attrs.matchAll(/([^\s"'=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g)) { const name = m[1].toLowerCase() const value = m[2] ?? m[3] ?? m[4] ?? "" if (name === "hidden") return true if (name === "style" && /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden|mso-hide\s*:\s*all)\b/i.test(value)) { return true } } return false } /** * Drop markup a browser never renders - comments (including Word's * conditional ones),