mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-09-17 08:36:25 +00:00
fix(jobindex-search): decode hex HTML entities in CLI output (#56)
decodeHtmlEntities (duplicated in src/helpers.ts and src/commands/detail.ts) only handled decimal numeric character references (é); the equally valid hexadecimal form (é) fell through undecoded and surfaced as raw text in titles, companies, locations and descriptions. This bites Danish content especially (ae/o/aa often arrive as entities). It also used String.fromCharCode, which corrupts supplementary-plane code points (e.g. emoji, U+1F600). Add a hexadecimal numeric-entity rule and route both decimal and hex through a fromCodePoint-based helper with a valid-range guard, in both copies. Add network-free unit tests via the exported parseJobCards.
This commit is contained in:
@@ -19,6 +19,15 @@ interface DetailResult {
|
||||
description: string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Unicode code point to a string. Uses `fromCodePoint` (not
|
||||
* `fromCharCode`) so supplementary-plane code points (e.g. emoji, U+1F600)
|
||||
* decode correctly, and drops out-of-range values instead of throwing.
|
||||
*/
|
||||
function numericEntity(cp: number): string {
|
||||
return cp >= 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : ""
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode HTML entities in text
|
||||
*/
|
||||
@@ -30,7 +39,9 @@ function decodeHtmlEntities(text: string): string {
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&#(\d+);/g, (_, code) => String.fromCharCode(parseInt(code, 10)))
|
||||
// Numeric character references: decimal (é) and hexadecimal (é).
|
||||
.replace(/&#(\d+);/g, (_, dec) => numericEntity(parseInt(dec, 10)))
|
||||
.replace(/&#[xX]([0-9a-fA-F]+);/g, (_, hex) => numericEntity(parseInt(hex, 16)))
|
||||
.replace(/ /g, " ")
|
||||
}
|
||||
|
||||
|
||||
@@ -180,6 +180,15 @@ export function parseSearchPage(html: string): SearchPageResult {
|
||||
return { total, results }
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Unicode code point to a string. Uses `fromCodePoint` (not
|
||||
* `fromCharCode`) so supplementary-plane code points (e.g. emoji, U+1F600)
|
||||
* decode correctly, and drops out-of-range values instead of throwing.
|
||||
*/
|
||||
function numericEntity(cp: number): string {
|
||||
return cp >= 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : ""
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode HTML entities in text
|
||||
*/
|
||||
@@ -191,7 +200,9 @@ function decodeHtmlEntities(text: string): string {
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&#(\d+);/g, (_, code) => String.fromCharCode(parseInt(code, 10)))
|
||||
// Numeric character references: decimal (é) and hexadecimal (é).
|
||||
.replace(/&#(\d+);/g, (_, dec) => numericEntity(parseInt(dec, 10)))
|
||||
.replace(/&#[xX]([0-9a-fA-F]+);/g, (_, hex) => numericEntity(parseInt(hex, 16)))
|
||||
.replace(/ /g, " ")
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, test, expect } from "bun:test";
|
||||
import { parseJobCards } from "../src/helpers";
|
||||
|
||||
// Minimal jobad-wrapper markup: parseJobCards splits on `jobad-wrapper-<id>`
|
||||
// and reads the title from the <h4><a href>…</a></h4> and the company from the
|
||||
// jix-toolbar-top__company link. We inject HTML entities to exercise decoding.
|
||||
function card(id: string, title: string, company = "Acme A/S"): string {
|
||||
return `<div id="jobad-wrapper-${id}" class="PaidJob">
|
||||
<h4><a href="https://www.jobindex.dk/jobannonce/${id}">${title}</a></h4>
|
||||
<div class="jix-toolbar-top__company">
|
||||
<a href="https://www.jobindex.dk/virksomhed/acme">${company}</a>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
describe("decodeHtmlEntities (via parseJobCards)", () => {
|
||||
test("decodes hexadecimal numeric entities (ø -> ø)", () => {
|
||||
const [c] = parseJobCards(card("h1", "Smørrebro Chef"));
|
||||
expect(c.title).toBe("Smørrebro Chef");
|
||||
});
|
||||
|
||||
test("decodes uppercase-X hexadecimal entities (æ -> æ)", () => {
|
||||
const [c] = parseJobCards(card("h2", "Kære Kollega"));
|
||||
expect(c.title).toBe("Kære Kollega");
|
||||
});
|
||||
|
||||
test("still decodes decimal numeric entities (å -> å) — regression", () => {
|
||||
const [c] = parseJobCards(card("h3", "århus Lead"));
|
||||
expect(c.title).toBe("århus Lead");
|
||||
});
|
||||
|
||||
test("decodes supplementary-plane code points with fromCodePoint (😀)", () => {
|
||||
const [c] = parseJobCards(card("h4", "Growth 😀"));
|
||||
expect(c.title).toBe("Growth 😀");
|
||||
});
|
||||
|
||||
test("decodes hex supplementary-plane code points (😀)", () => {
|
||||
const [c] = parseJobCards(card("h5", "Growth 😀"));
|
||||
expect(c.title).toBe("Growth 😀");
|
||||
});
|
||||
|
||||
test("decodes hex entities in the company name too", () => {
|
||||
const [c] = parseJobCards(card("h6", "Engineer", "Nørrebro ApS"));
|
||||
expect(c.company).toBe("Nørrebro ApS");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user