mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-09-17 08:36:25 +00:00
feat(freehire-search): add --no-description for cheap discovery passes
A default search hydrates full bodies - ~73% of the payload, ~20k tokens per query fed into agent context - while /scrape's own Step 2 says to pre-filter by title before reading bodies. The flag keeps every other field and drops the bodies (live 10-result search: ~58k -> ~10k chars); hydration stays the default per the documented trade-off. The API returns bodies regardless of include_description=false (verified live), so the lean guarantee is enforced client-side. Review opportunity O1 (2026-08-19), approved as an enhancement. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
c85640e30a
commit
dd02c82485
@@ -95,7 +95,10 @@ posting's complete text, so a search of 20 roles is 1 request rather than 1 + 20
|
||||
Do **not** loop `detail` over search hits to read their descriptions — reach for
|
||||
`detail` only to look one posting up by slug (e.g. from the tracker, or a posting
|
||||
already closed and therefore absent from search). Full descriptions are verbose:
|
||||
keep `--limit` modest, and pre-filter on title/company before reading bodies.
|
||||
keep `--limit` modest, and pre-filter on title/company before reading bodies -
|
||||
or pass `--no-description` for a cheap discovery pass that keeps every other
|
||||
field and drops the bodies entirely (fetch a shortlisted job's body with
|
||||
`detail`, or re-run the search without the flag).
|
||||
|
||||
Facet filters (values come from freehire's controlled vocabularies; comma-separate for OR within a facet):
|
||||
- `--region <codes>` — macro-region, e.g. `global`, `eu`, `us`, `apac`, `latam`, `cis`. `--region eu,us`. Use `none` to match jobs whose region could **not** be resolved (see "Partial data" below).
|
||||
|
||||
@@ -81,6 +81,8 @@ SEARCH FLAGS
|
||||
--page <n> 1-indexed page. Default 1.
|
||||
--limit, -n <n> Results per page (API limit). Default 25.
|
||||
--format <fmt> json (default) | table | plain.
|
||||
--no-description Skip description hydration for a cheap discovery pass
|
||||
(results keep every other field; detail fetches the body).
|
||||
--description-format markdown (default) | text | html — how each result's
|
||||
full description is rendered (json output only).
|
||||
|
||||
@@ -124,7 +126,7 @@ function parseIntFlag(name: string, raw: string | boolean | string[]): number |
|
||||
const KNOWN_FLAGS: Record<string, Set<string>> = {
|
||||
search: new Set([
|
||||
"query", "category", "city", "company", "country", "facet", "format", "jobage", "limit",
|
||||
"page", "region", "remote", "seniority", "skill", "description-format", "help", "h",
|
||||
"page", "region", "remote", "seniority", "skill", "description-format", "no-description", "help", "h",
|
||||
]),
|
||||
detail: new Set(["format", "description-format", "help", "h"]),
|
||||
}
|
||||
@@ -202,6 +204,7 @@ async function main(): Promise<number> {
|
||||
limit: flags.limit ? Math.max(1, parseInt(flags.limit as string, 10)) : 25,
|
||||
format: (["json", "table", "plain"].includes(fmt) ? fmt : "json") as SearchOpts["format"],
|
||||
descriptionFormat: descFmt as DescriptionFormat,
|
||||
includeDescription: flags["no-description"] === undefined,
|
||||
regions: commaList(flags.region),
|
||||
countries: commaList(flags.country),
|
||||
cities: commaList(flags.city),
|
||||
|
||||
@@ -18,6 +18,10 @@ export interface SearchOpts {
|
||||
limit: number
|
||||
format: "json" | "table" | "plain"
|
||||
descriptionFormat: DescriptionFormat
|
||||
// Hydrate full description bodies (the documented default). False keeps a
|
||||
// discovery pass cheap: bodies are ~73% of a default search payload, and
|
||||
// /scrape pre-filters by title before reading bodies anyway.
|
||||
includeDescription?: boolean
|
||||
// Facet filters (already parsed into value lists; empty means unset).
|
||||
regions: string[]
|
||||
countries: string[]
|
||||
@@ -38,9 +42,11 @@ function buildQuery(opts: SearchOpts): URLSearchParams {
|
||||
p.set("offset", String((opts.page - 1) * opts.limit))
|
||||
p.set("semantic_ratio", "0") // keyword search; the semantic index is opt-in
|
||||
// The agent endpoint serves the index's truncated preview unless asked to
|
||||
// rehydrate each hit from the database, so both params travel together.
|
||||
p.set("include_description", "true")
|
||||
p.set("description_format", opts.descriptionFormat)
|
||||
// rehydrate each hit from the database, so both params travel together -
|
||||
// unless the caller opted out of hydration entirely (--no-description).
|
||||
const hydrate = opts.includeDescription !== false
|
||||
p.set("include_description", hydrate ? "true" : "false")
|
||||
if (hydrate) p.set("description_format", opts.descriptionFormat)
|
||||
if (opts.jobage > 0 && opts.jobage < 9999) p.set("posted_within_days", String(opts.jobage))
|
||||
if (opts.workMode) p.set("work_mode", opts.workMode)
|
||||
if (opts.company) p.set("company_slug", opts.company)
|
||||
@@ -117,7 +123,14 @@ export async function runSearch(opts: SearchOpts): Promise<number> {
|
||||
)
|
||||
return 1
|
||||
}
|
||||
const rows = (env.data ?? []).map(toResult)
|
||||
let rows = (env.data ?? []).map(toResult)
|
||||
// The API currently returns description bodies regardless of
|
||||
// include_description=false (verified live 2026-08-19), and the cost this
|
||||
// flag exists to avoid is the ~73% of CLI output the bodies occupy in
|
||||
// agent context - so the lean mode strips them client-side either way.
|
||||
if (opts.includeDescription === false) {
|
||||
rows = rows.map((r) => ({ ...r, description: null }))
|
||||
}
|
||||
const total = env.meta?.total ?? rows.length
|
||||
|
||||
if (opts.format === "table") {
|
||||
|
||||
@@ -112,6 +112,24 @@ describe("runSearch (mocked fetch)", () => {
|
||||
expect(requestedParams(mock).get("description_format")).toBe("markdown");
|
||||
});
|
||||
|
||||
test("skips description hydration when includeDescription is false", async () => {
|
||||
// A default search hydrates ~20k tokens of description bodies per query,
|
||||
// while /scrape's Step 2 says to pre-filter by title/snippet before
|
||||
// reading bodies. --no-description keeps the discovery pass cheap;
|
||||
// hydration stays the default (review opportunity O1, 2026-08-19).
|
||||
const mock = mockFetch(200, { data: [job()], meta: { total: 1 } });
|
||||
|
||||
const out = captureStdout();
|
||||
await runSearch({ ...searchOpts, query: "backend", includeDescription: false });
|
||||
|
||||
expect(requestedParams(mock).get("include_description")).toBe("false");
|
||||
expect(requestedParams(mock).get("description_format")).toBeNull();
|
||||
// The live API ignores include_description=false and sends bodies anyway
|
||||
// (verified 2026-08-19), so the lean guarantee is enforced client-side.
|
||||
const parsed = JSON.parse(out.get());
|
||||
expect(parsed.results[0].description).toBeNull();
|
||||
});
|
||||
|
||||
test("asks for the requested description format", async () => {
|
||||
const mock = mockFetch(200, { data: [job()], meta: { total: 1 } });
|
||||
captureStdout();
|
||||
|
||||
@@ -15,6 +15,13 @@ per-file diff commands.
|
||||
|
||||
### Added
|
||||
|
||||
- **`freehire-search` gains `--no-description` for cheap discovery passes** - a default
|
||||
search hydrates full description bodies (~73% of the payload, ~20k tokens per query)
|
||||
while `/scrape` is told to pre-filter by title before reading bodies. The new flag
|
||||
drops the bodies (a live 10-result search shrinks from ~58k to ~10k chars) while
|
||||
keeping every other field; hydration stays the default. The API currently returns
|
||||
bodies regardless of `include_description=false`, so the lean guarantee is enforced
|
||||
client-side. Pinned in `tests/commands.test.ts`.
|
||||
- **Fixture coverage for linkedin's date/location and jobindex's `parseSearchPage`** -
|
||||
linkedin's search-card fixture carried no `<time>` or location element, so deleting
|
||||
the `date` extraction (a `/scrape` contract field on a default-ON portal) left every
|
||||
|
||||
Reference in New Issue
Block a user