mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-09-17 00:26:26 +00:00
feat(cli): identify jobnet and jobdanmark API requests with an honest User-Agent (#283)
* fix(cli): send User-Agent on jobnet and jobdanmark API requests apiFetch/apiPost hit the portals' APIs without a User-Agent header, while every other Danish-portal CLI sends one on purpose (jobbank exports USER_AGENT and its tests assert it; jobindex sets it on htmlFetch). Requests without one are rejected by the portals' bot filters. * fix(cli): satisfy strict typecheck in user-agent regression test * refactor(cli): reframe user-agent tests as honest self-identification * docs(changelog): entry for #283 user-agent self-identification
This commit is contained in:
@@ -1,4 +1,5 @@
|
|||||||
export const BASE_URL = "https://jobdanmark.dk"
|
export const BASE_URL = "https://jobdanmark.dk"
|
||||||
|
export const USER_AGENT = "Mozilla/5.0 (compatible; jobdanmark-cli/1.0)"
|
||||||
|
|
||||||
export async function apiFetch<T>(path: string, params?: Record<string, string>): Promise<T> {
|
export async function apiFetch<T>(path: string, params?: Record<string, string>): Promise<T> {
|
||||||
let url = `${BASE_URL}${path}`
|
let url = `${BASE_URL}${path}`
|
||||||
@@ -10,7 +11,10 @@ export async function apiFetch<T>(path: string, params?: Record<string, string>)
|
|||||||
const maxRetries = 6
|
const maxRetries = 6
|
||||||
let delay = 500
|
let delay = 500
|
||||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||||
const response = await fetch(url, { signal: AbortSignal.timeout(15000) })
|
const response = await fetch(url, {
|
||||||
|
headers: { "User-Agent": USER_AGENT },
|
||||||
|
signal: AbortSignal.timeout(15000),
|
||||||
|
})
|
||||||
if (response.status === 429 || response.status >= 500) {
|
if (response.status === 429 || response.status >= 500) {
|
||||||
if (attempt === maxRetries) {
|
if (attempt === maxRetries) {
|
||||||
throw new Error(`API request failed: ${response.status} ${response.statusText}`)
|
throw new Error(`API request failed: ${response.status} ${response.statusText}`)
|
||||||
@@ -38,6 +42,7 @@ export async function apiPost<T>(path: string, body: unknown): Promise<T> {
|
|||||||
method: "POST",
|
method: "POST",
|
||||||
headers: {
|
headers: {
|
||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
|
"User-Agent": USER_AGENT,
|
||||||
},
|
},
|
||||||
body: JSON.stringify(body),
|
body: JSON.stringify(body),
|
||||||
signal: AbortSignal.timeout(15000),
|
signal: AbortSignal.timeout(15000),
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
import { afterEach, describe, expect, test } from "bun:test";
|
||||||
|
import { apiFetch, apiPost, USER_AGENT } from "../src/helpers";
|
||||||
|
|
||||||
|
// Bun's fetch injects an anonymous default User-Agent (Bun/1.3.10) when code
|
||||||
|
// sets none. This CLI should say who is asking, in the honest style jobindex
|
||||||
|
// already uses on htmlFetch ("Mozilla/5.0 (compatible; jobindex-cli/1.0)").
|
||||||
|
// Assert the header is present on every request. Fails on the pre-change code.
|
||||||
|
const originalFetch = globalThis.fetch;
|
||||||
|
afterEach(() => {
|
||||||
|
globalThis.fetch = originalFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
function headerValue(headers: RequestInit["headers"], name: string): string | null {
|
||||||
|
if (headers instanceof Headers) return headers.get(name);
|
||||||
|
if (Array.isArray(headers)) {
|
||||||
|
const found = headers.find(([k]) => k === name);
|
||||||
|
return found ? String(found[1]) : null;
|
||||||
|
}
|
||||||
|
const value = headers?.[name];
|
||||||
|
return typeof value === "string" ? value : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("apiFetch user agent", () => {
|
||||||
|
test("sends a User-Agent header", async () => {
|
||||||
|
let init: RequestInit | undefined;
|
||||||
|
globalThis.fetch = (async (_url: string | URL | Request, i?: RequestInit) => {
|
||||||
|
init = i;
|
||||||
|
return new Response("{}", { status: 200 });
|
||||||
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
|
await apiFetch("/api/search/autocomplete", { q: "it" });
|
||||||
|
expect(headerValue(init?.headers, "User-Agent")).toBe(USER_AGENT);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("apiPost user agent", () => {
|
||||||
|
test("sends a User-Agent header alongside Content-Type", async () => {
|
||||||
|
let init: RequestInit | undefined;
|
||||||
|
globalThis.fetch = (async (_url: string | URL | Request, i?: RequestInit) => {
|
||||||
|
init = i;
|
||||||
|
return new Response("{}", { status: 200 });
|
||||||
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
|
await apiPost("/api/jobsearch/search/1", { q: "it" });
|
||||||
|
expect(headerValue(init?.headers, "User-Agent")).toBe(USER_AGENT);
|
||||||
|
expect(headerValue(init?.headers, "Content-Type")).toBe("application/json");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
export const BASE_URL = "https://jobnet.dk/bff"
|
export const BASE_URL = "https://jobnet.dk/bff"
|
||||||
|
export const USER_AGENT = "Mozilla/5.0 (compatible; jobnet-cli/1.0)"
|
||||||
|
|
||||||
export async function apiFetch<T>(path: string, params?: Record<string, string>): Promise<T> {
|
export async function apiFetch<T>(path: string, params?: Record<string, string>): Promise<T> {
|
||||||
let url = `${BASE_URL}${path}`
|
let url = `${BASE_URL}${path}`
|
||||||
@@ -12,6 +13,7 @@ export async function apiFetch<T>(path: string, params?: Record<string, string>)
|
|||||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||||
const response = await fetch(url, {
|
const response = await fetch(url, {
|
||||||
headers: {
|
headers: {
|
||||||
|
"User-Agent": USER_AGENT,
|
||||||
"x-csrf": "1",
|
"x-csrf": "1",
|
||||||
},
|
},
|
||||||
signal: AbortSignal.timeout(15000),
|
signal: AbortSignal.timeout(15000),
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
import { afterEach, describe, expect, test } from "bun:test";
|
||||||
|
import { apiFetch, USER_AGENT } from "../src/helpers";
|
||||||
|
|
||||||
|
// Bun's fetch injects an anonymous default User-Agent (Bun/1.3.10) when code
|
||||||
|
// sets none. This CLI should say who is asking, in the honest style jobindex
|
||||||
|
// already uses on htmlFetch ("Mozilla/5.0 (compatible; jobindex-cli/1.0)").
|
||||||
|
// Assert the header is present on every request. Fails on the pre-change code.
|
||||||
|
const originalFetch = globalThis.fetch;
|
||||||
|
afterEach(() => {
|
||||||
|
globalThis.fetch = originalFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("apiFetch user agent", () => {
|
||||||
|
test("sends a User-Agent header", async () => {
|
||||||
|
let init: RequestInit | undefined;
|
||||||
|
globalThis.fetch = (async (_url: string | URL | Request, i?: RequestInit) => {
|
||||||
|
init = i;
|
||||||
|
return new Response("{}", { status: 200 });
|
||||||
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
|
await apiFetch("/search");
|
||||||
|
const headers = init?.headers as Record<string, string> | Headers | undefined;
|
||||||
|
const value =
|
||||||
|
headers instanceof Headers ? headers.get("User-Agent") : headers?.["User-Agent"];
|
||||||
|
expect(value).toBe(USER_AGENT);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -43,6 +43,13 @@ per-file diff commands.
|
|||||||
during #275 (vetoes reported in console output but `language_gate: null` on every persisted
|
during #275 (vetoes reported in console output but `language_gate: null` on every persisted
|
||||||
entry). Mirrors the existing `gaps`/`strengths` pinning pattern. No behavior change.
|
entry). Mirrors the existing `gaps`/`strengths` pinning pattern. No behavior change.
|
||||||
|
|
||||||
|
- **The jobnet and jobdanmark CLIs identify themselves on every API request** (#283) - their
|
||||||
|
`apiFetch`/`apiPost` wrappers now send an explicit `User-Agent` (`jobnet-cli/1.0`,
|
||||||
|
`jobdanmark-cli/1.0`) instead of Bun's anonymous default token, matching the honest
|
||||||
|
self-identification jobindex already uses on `htmlFetch`. The new `user-agent.test.ts`
|
||||||
|
suites assert the header on every request wrapper. No response behavior observed to
|
||||||
|
change.
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
- **Negative and fractional filter flags are rejected in the Danish portal CLIs** (#281) -
|
- **Negative and fractional filter flags are rejected in the Danish portal CLIs** (#281) -
|
||||||
|
|||||||
Reference in New Issue
Block a user