mirror of
https://github.com/prdlk/jobyssey.git
synced 2026-11-08 07:57:39 +00:00
feat: add skills
This commit is contained in:
Prad Nukala committed
1 parent
6984212b17
commit
c50aa92862
150 files changed
+12702
No files matched your search
+9
@@ -0,0 +1,9 @@
|
||||
# wellfound-cli
|
||||
|
||||
Zero-dependency Bun CLI over Wellfound's public role landing pages. See `../SKILL.md` for usage and `../url-reference.md` for parsing anchors.
|
||||
|
||||
```bash
|
||||
cd .agents/skills/wellfound-search/cli && bun install # dev types only
|
||||
bun run typecheck && bun run test
|
||||
bun run src/cli.ts search -q "senior software engineer" --location new-york --format table
|
||||
```
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "wellfound-cli",
|
||||
"version": "1.0.0",
|
||||
"description": "CLI for searching public Wellfound (wellfound.com) startup job listings by role and location. Zero runtime dependencies. Personal use only.",
|
||||
"type": "module",
|
||||
"main": "src/cli.ts",
|
||||
"bin": { "wellfound-search": "src/cli.ts" },
|
||||
"scripts": { "start": "bun run src/cli.ts", "test": "bun test --timeout 30000", "typecheck": "tsc --noEmit" },
|
||||
"dependencies": {},
|
||||
"devDependencies": { "typescript": "^5.4.0", "@types/bun": "1.3.14" }
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bun
|
||||
// wellfound-cli: search public Wellfound role pages. Zero runtime dependencies.
|
||||
import { runSearch, type SearchOpts } from "./commands/search.js"
|
||||
import { runDetail } from "./commands/detail.js"
|
||||
import { writeError } from "./helpers.js"
|
||||
|
||||
const HELP = `wellfound-cli - search Wellfound (wellfound.com) startup jobs by role and location
|
||||
|
||||
Usage:
|
||||
cli.ts search -q "<role>" [--location "<city>"] [--remote] [--jobage <days>] [--limit <n>] [--format json|table|plain] [--no-description]
|
||||
cli.ts detail <id-slug|url> [--format json|plain]
|
||||
|
||||
Search flags:
|
||||
-q, --query <role> Role title, slugified into Wellfound's /role/ path (e.g. "senior software engineer", "backend engineer", "blockchain engineer"). Required.
|
||||
-l, --location <city> City slug/name (e.g. "new-york", "San Francisco"). Omit or pass --remote for remote roles.
|
||||
--remote Use the remote landing page (/role/r/<role>).
|
||||
--jobage <days> Keep only jobs posted within N days (client-side, from liveStartAt).
|
||||
--page <n> Only page 1 is available server-side; any other value errors.
|
||||
-n, --limit <n> Cap results (client-side).
|
||||
--format json|table|plain Default json.
|
||||
--no-description Drop the description bodies from JSON output (cheap discovery pass).
|
||||
--all-listings Keep every highlighted listing of matching startups, not just titles sharing a token with the query.
|
||||
`
|
||||
|
||||
type Flags = Record<string, string | boolean>
|
||||
function parse(argv: string[]) {
|
||||
const [cmd = "", ...rest] = argv; const pos: string[] = []; const flags: Flags = {}
|
||||
for (let i = 0; i < rest.length; i++) {
|
||||
const a = rest[i]
|
||||
if (a === "-q") { flags.query = rest[++i] ?? ""; continue }
|
||||
if (a === "-l") { flags.location = rest[++i] ?? ""; continue }
|
||||
if (a === "-n") { flags.limit = rest[++i] ?? ""; continue }
|
||||
if (a.startsWith("--")) {
|
||||
const [k, inline] = a.slice(2).split("=", 2)
|
||||
if (inline !== undefined) { flags[k] = inline; continue }
|
||||
const next = rest[i + 1]
|
||||
if (k === "remote" || k === "no-description" || k === "all-listings" || next === undefined || next.startsWith("-")) flags[k] = true
|
||||
else flags[k] = rest[++i]
|
||||
continue
|
||||
}
|
||||
pos.push(a)
|
||||
}
|
||||
return { cmd, pos, flags }
|
||||
}
|
||||
const { cmd, pos, flags } = parse(process.argv.slice(2))
|
||||
const KNOWN = new Set(["query", "location", "remote", "jobage", "page", "limit", "format", "no-description", "all-listings"])
|
||||
for (const k of Object.keys(flags)) if (!KNOWN.has(k)) { writeError(`Unknown flag --${k}`, "BAD_ARG"); process.exit(1) }
|
||||
function num(name: string, dflt?: number): number | undefined {
|
||||
const raw = flags[name]; if (raw === undefined) return dflt
|
||||
const n = Number(raw); if (!Number.isFinite(n) || n < 0) { writeError(`--${name} must be a non-negative number, got "${raw}"`, "BAD_ARG"); process.exit(1) }
|
||||
return n
|
||||
}
|
||||
const str = (v: string | boolean | undefined) => (typeof v === "string" && v ? v : undefined)
|
||||
|
||||
if (!cmd || cmd === "help" || cmd === "--help" || cmd === "-h") { process.stdout.write(HELP); process.exit(0) }
|
||||
if (cmd === "search") {
|
||||
const format = (str(flags.format) ?? "json") as SearchOpts["format"]
|
||||
if (!["json", "table", "plain"].includes(format)) { writeError("--format must be json, table or plain", "BAD_ARG"); process.exit(1) }
|
||||
await runSearch({ query: str(flags.query) ?? "", location: str(flags.location), remote: flags.remote === true, jobage: num("jobage"), page: num("page", 1)!, limit: num("limit"), format, noDescription: flags["no-description"] === true, allListings: flags["all-listings"] === true })
|
||||
} else if (cmd === "detail") {
|
||||
if (!pos[0]) { writeError("detail requires an <id|url>", "NO_ID"); process.exit(1) }
|
||||
const format = (str(flags.format) ?? "json") as "json" | "plain"
|
||||
await runDetail(pos[0], format)
|
||||
} else { writeError(`Unknown command "${cmd}"`, "BAD_CMD"); process.exit(1) }
|
||||
@@ -0,0 +1,19 @@
|
||||
import { BASE, htmlFetch, parseDetail, writeError } from "../helpers.js"
|
||||
|
||||
export async function runDetail(idOrUrl: string, format: "json" | "plain"): Promise<void> {
|
||||
// Wellfound requires the exact slug: /jobs/<id> and /jobs/<id>-wrong-slug both 404.
|
||||
const m = /(\d{5,})-([a-z0-9-]+)/i.exec(idOrUrl)
|
||||
if (!m) { writeError("detail requires <id>-<slug> or the full wellfound.com/jobs/<id>-<slug> URL (bare ids 404 on Wellfound); copy the url field from a search result", "BAD_ARG"); process.exit(1) }
|
||||
const id = m[1]
|
||||
const url = idOrUrl.startsWith("http") ? idOrUrl : `${BASE}/jobs/${m[1]}-${m[2]}`
|
||||
let html: string
|
||||
try { html = await htmlFetch(url) } catch (e) { writeError(`could not reach ${url} (${(e as Error).message})`, "DETAIL_FAILED"); process.exit(1) }
|
||||
if (!html) { writeError(`job ${id} not found (404)`, "NOT_FOUND"); process.exit(1) }
|
||||
const d = parseDetail(html, id, url)
|
||||
if (!d) { writeError(`no JobPosting data in ${url} (markup changed? see url-reference.md)`, "PARSE_FAILED"); process.exit(1) }
|
||||
if (format === "plain") {
|
||||
process.stdout.write([`${d.title} - ${d.company}`, `${d.location ?? "-"} | ${d.employmentType ?? "-"} | posted ${d.date ?? "-"}${d.salary ? ` | ${d.salary}` : ""}`, `URL: ${d.url}`, "", d.description ?? ""].join("\n") + "\n")
|
||||
return
|
||||
}
|
||||
process.stdout.write(JSON.stringify(d, null, 2) + "\n")
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { BASE, htmlFetch, extractApollo, parseJobCards, slugify, writeError, type JobCard } from "../helpers.js"
|
||||
|
||||
export interface SearchOpts {
|
||||
query: string
|
||||
location?: string
|
||||
remote: boolean
|
||||
jobage?: number
|
||||
page: number
|
||||
limit?: number
|
||||
format: "json" | "table" | "plain"
|
||||
noDescription: boolean
|
||||
allListings: boolean
|
||||
}
|
||||
|
||||
export function buildUrl(opts: SearchOpts): string {
|
||||
const role = slugify(opts.query)
|
||||
if (opts.remote || !opts.location) return `${BASE}/role/r/${role}`
|
||||
return `${BASE}/role/l/${role}/${slugify(opts.location)}`
|
||||
}
|
||||
|
||||
function renderTable(cards: JobCard[]): string {
|
||||
if (cards.length === 0) return "No results."
|
||||
return cards
|
||||
.map((c) => [
|
||||
(c.title || "").slice(0, 44).padEnd(44),
|
||||
(c.company || "-").slice(0, 22).padEnd(22),
|
||||
(c.location || "-").slice(0, 20).padEnd(20),
|
||||
(c.compensation || "-").slice(0, 14).padEnd(14),
|
||||
c.date ?? "-".padEnd(10),
|
||||
c.url,
|
||||
].join(" "))
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
export async function runSearch(opts: SearchOpts): Promise<void> {
|
||||
if (!opts.query) { writeError("search requires --query/-q (a role, e.g. \"senior software engineer\")", "NO_QUERY"); process.exit(1) }
|
||||
if (opts.page !== 1) { writeError("Wellfound role pages expose only page 1 server-side; --page > 1 is not supported", "BAD_ARG"); process.exit(1) }
|
||||
const url = buildUrl(opts)
|
||||
let html: string
|
||||
try { html = await htmlFetch(url) } catch (e) { writeError(`could not reach ${url} (${(e as Error).message})`, "SEARCH_FAILED"); process.exit(1) }
|
||||
if (!html) {
|
||||
// 404: Wellfound has no landing page for this role/location combination
|
||||
emit([], opts, url); return
|
||||
}
|
||||
const apollo = extractApollo(html)
|
||||
if (!apollo) { writeError(`no job data found in ${url} (markup changed? see url-reference.md)`, "PARSE_FAILED"); process.exit(1) }
|
||||
let cards = parseJobCards(apollo)
|
||||
if (!opts.allListings) {
|
||||
// Role pages list each matching startup's *other* highlighted jobs too; keep titles that share a
|
||||
// meaningful token with the query (drops "Product Designer" from a "senior software engineer" page).
|
||||
const stop = new Set(["senior", "sr", "staff", "lead", "principal", "junior", "engineer", "developer", "of", "and", "the"])
|
||||
const tokens = opts.query.toLowerCase().split(/[^a-z0-9+#]+/).filter((t) => t && !stop.has(t))
|
||||
if (tokens.length) cards = cards.filter((c) => { const t = c.title.toLowerCase(); return tokens.some((k) => t.includes(k)) })
|
||||
}
|
||||
if (opts.jobage !== undefined) {
|
||||
const cutoff = Date.now() - opts.jobage * 86_400_000
|
||||
cards = cards.filter((c) => c.date && Date.parse(c.date) >= cutoff)
|
||||
}
|
||||
cards.sort((a, b) => (b.date ?? "").localeCompare(a.date ?? ""))
|
||||
if (opts.limit) cards = cards.slice(0, opts.limit)
|
||||
emit(cards, opts, url)
|
||||
}
|
||||
|
||||
function emit(cards: JobCard[], opts: SearchOpts, source: string): void {
|
||||
if (opts.format === "table") { process.stdout.write(renderTable(cards) + "\n"); return }
|
||||
if (opts.format === "plain") { process.stdout.write(cards.map((c) => `${c.title} | ${c.company ?? "-"} | ${c.location ?? "-"} | ${c.date ?? "-"} | ${c.url}`).join("\n") + "\n"); return }
|
||||
const results = opts.noDescription ? cards.map((c) => ({ ...c, description: null })) : cards
|
||||
process.stdout.write(JSON.stringify({ meta: { count: results.length, page: opts.page, source }, results }, null, 2) + "\n")
|
||||
}
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
// Data source: Wellfound's public SEO landing pages /role/l/<role>/<location> and
|
||||
// /role/r/<role> (remote). Each page embeds a Next.js __NEXT_DATA__ blob whose Apollo
|
||||
// cache holds StartupResult and JobListingSearchResult records, including full
|
||||
// descriptions. Detail pages /jobs/<id>-<slug> carry a schema.org JobPosting JSON-LD
|
||||
// block. /search is disallowed by robots.txt, so free-text queries are slugified
|
||||
// into the role path instead.
|
||||
|
||||
export const BASE = "https://wellfound.com"
|
||||
const UA = "Mozilla/5.0 (compatible; wellfound-cli/1.0)"
|
||||
|
||||
export function writeError(error: string, code: string): void {
|
||||
process.stderr.write(JSON.stringify({ error, code }) + "\n")
|
||||
}
|
||||
|
||||
/** Fetch HTML with exponential backoff on 429/5xx. Returns "" on a 404. */
|
||||
export async function htmlFetch(url: string): Promise<string> {
|
||||
const maxRetries = 6
|
||||
let delay = 500
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
const response = await fetch(url, {
|
||||
headers: { "User-Agent": UA, Accept: "text/html,application/xhtml+xml", "Accept-Language": "en-US,en;q=0.9" },
|
||||
redirect: "follow",
|
||||
signal: AbortSignal.timeout(20000),
|
||||
})
|
||||
if (response.status === 429 || response.status >= 500) {
|
||||
if (attempt === maxRetries) throw new Error(`Request failed: ${response.status} ${response.statusText}`)
|
||||
const jitter = Math.floor(Math.random() * 500)
|
||||
await new Promise((r) => setTimeout(r, delay + jitter))
|
||||
delay = Math.min(delay * 2, 8000)
|
||||
continue
|
||||
}
|
||||
if (response.status === 404) return ""
|
||||
if (!response.ok) throw new Error(`Request failed: ${response.status} ${response.statusText}`)
|
||||
return response.text()
|
||||
}
|
||||
throw new Error("Request failed after max retries")
|
||||
}
|
||||
|
||||
export function slugify(s: string): string {
|
||||
return s.toLowerCase().trim().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "")
|
||||
}
|
||||
|
||||
export interface JobCard {
|
||||
id: string
|
||||
title: string
|
||||
company: string | null
|
||||
location: string | null
|
||||
date: string | null
|
||||
url: string
|
||||
description: string | null
|
||||
compensation: string | null
|
||||
remote: boolean | null
|
||||
jobType: string | null
|
||||
companySlug: string | null
|
||||
companySize: string | null
|
||||
ats: string | null
|
||||
}
|
||||
|
||||
type ApolloRecord = Record<string, unknown>
|
||||
|
||||
/** Pull the Apollo cache out of a role page. Returns null when the page has no data blob. */
|
||||
export function extractApollo(html: string): Record<string, ApolloRecord> | null {
|
||||
const m = /<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/.exec(html)
|
||||
if (!m) return null
|
||||
try {
|
||||
const nd = JSON.parse(m[1])
|
||||
const data = nd?.props?.pageProps?.apolloState?.data
|
||||
return data && typeof data === "object" ? (data as Record<string, ApolloRecord>) : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v ? v : null)
|
||||
|
||||
/** Walk StartupResult -> highlightedJobListings -> JobListingSearchResult. Each record is parsed
|
||||
* independently so one malformed entry cannot break the rest. */
|
||||
export function parseJobCards(apollo: Record<string, ApolloRecord>): JobCard[] {
|
||||
const cards: JobCard[] = []
|
||||
for (const [key, startup] of Object.entries(apollo)) {
|
||||
if (!key.startsWith("StartupResult:")) continue
|
||||
const refs = (startup.highlightedJobListings as { __ref?: string }[] | undefined) ?? []
|
||||
for (const ref of refs) {
|
||||
try {
|
||||
const job = ref?.__ref ? apollo[ref.__ref] : undefined
|
||||
if (!job || !str(job.id)) continue
|
||||
const id = String(job.id)
|
||||
const slug = str(job.slug) ?? ""
|
||||
const live = Number(job.liveStartAt)
|
||||
const locs = Array.isArray(job.locationNames) ? (job.locationNames as unknown[]).filter((x) => typeof x === "string").join(", ") : null
|
||||
cards.push({
|
||||
id,
|
||||
title: str(job.title) ?? "",
|
||||
company: str(startup.name),
|
||||
location: locs || (job.remote ? "Remote" : null),
|
||||
date: Number.isFinite(live) && live > 0 ? new Date(live * 1000).toISOString().slice(0, 10) : null,
|
||||
url: `${BASE}/jobs/${id}-${slug}`,
|
||||
description: str(job.description),
|
||||
compensation: str(job.compensation),
|
||||
remote: typeof job.remote === "boolean" ? job.remote : null,
|
||||
jobType: str(job.jobType),
|
||||
companySlug: str(startup.slug),
|
||||
companySize: str(startup.companySize),
|
||||
ats: str(job.atsSource),
|
||||
})
|
||||
} catch {
|
||||
// skip malformed record
|
||||
}
|
||||
}
|
||||
}
|
||||
return cards
|
||||
}
|
||||
|
||||
export function decodeEntities(s: string): string {
|
||||
return s
|
||||
.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'|'/g, "'").replace(/ /g, " ")
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n)))
|
||||
}
|
||||
|
||||
export function stripHtml(s: string): string {
|
||||
return decodeEntities(
|
||||
s.replace(/<br\s*\/?>/gi, "\n").replace(/<\/(p|div|li|h\d|ul|ol)>/gi, "\n").replace(/<li[^>]*>/gi, "- ").replace(/<[^>]+>/g, ""),
|
||||
).replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim()
|
||||
}
|
||||
|
||||
export interface JobDetail {
|
||||
id: string
|
||||
title: string | null
|
||||
company: string | null
|
||||
location: string | null
|
||||
date: string | null
|
||||
url: string
|
||||
description: string | null
|
||||
employmentType: string | null
|
||||
salary: string | null
|
||||
applyUrl: string | null
|
||||
}
|
||||
|
||||
/** Parse the schema.org JobPosting JSON-LD on a detail page. */
|
||||
export function parseDetail(html: string, id: string, url: string): JobDetail | null {
|
||||
const blocks = [...html.matchAll(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/g)]
|
||||
for (const b of blocks) {
|
||||
try {
|
||||
const j = JSON.parse(b[1])
|
||||
if (j?.["@type"] !== "JobPosting") continue
|
||||
const loc = j.jobLocation
|
||||
const addr = Array.isArray(loc) ? loc[0]?.address : loc?.address
|
||||
const locStr = addr ? [addr.addressLocality, addr.addressRegion, addr.addressCountry].filter(Boolean).join(", ") : null
|
||||
const sal = j.baseSalary?.value
|
||||
const salary = sal ? [sal.minValue, sal.maxValue].filter((x: unknown) => x != null).join("-") + (j.baseSalary?.currency ? ` ${j.baseSalary.currency}` : "") : null
|
||||
return {
|
||||
id,
|
||||
title: str(j.title),
|
||||
company: str(j.hiringOrganization?.name),
|
||||
location: locStr || null,
|
||||
date: str(j.datePosted)?.slice(0, 10) ?? null,
|
||||
url,
|
||||
description: typeof j.description === "string" ? stripHtml(j.description) : null,
|
||||
employmentType: Array.isArray(j.employmentType) ? j.employmentType.join(", ") : str(j.employmentType),
|
||||
salary: salary || null,
|
||||
applyUrl: str(j.url) ?? url,
|
||||
}
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
import { join } from "path";
|
||||
|
||||
const CLI_PATH = join(import.meta.dir, "../src/cli.ts");
|
||||
|
||||
export interface CLIResult {
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
exitCode: number;
|
||||
}
|
||||
|
||||
export async function runCLI(args: string[]): Promise<CLIResult> {
|
||||
const proc = Bun.spawn(["bun", "run", CLI_PATH, ...args], {
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
|
||||
const [stdout, stderr, exitCode] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text(),
|
||||
proc.exited,
|
||||
]);
|
||||
|
||||
return { stdout: stdout.trim(), stderr: stderr.trim(), exitCode };
|
||||
}
|
||||
|
||||
export function parseJSON<T = unknown>(result: CLIResult): T {
|
||||
if (result.exitCode !== 0) {
|
||||
throw new Error(
|
||||
`CLI exited with code ${result.exitCode}. stderr: ${result.stderr}`
|
||||
);
|
||||
}
|
||||
try {
|
||||
return JSON.parse(result.stdout) as T;
|
||||
} catch {
|
||||
throw new Error(
|
||||
`Failed to parse JSON. stdout: ${result.stdout}\nstderr: ${result.stderr}`
|
||||
);
|
||||
}
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { runCLI, parseJSON } from "./helpers"
|
||||
|
||||
describe("wellfound-cli", () => {
|
||||
test("search returns real results for a common role", async () => {
|
||||
const res = await runCLI(["search", "-q", "senior software engineer", "--location", "new-york", "--limit", "5", "--no-description"])
|
||||
expect(res.exitCode).toBe(0)
|
||||
const json = parseJSON<{ meta: { count: number }; results: { id: string; title: string; company: string | null; location: string | null; date: string | null; url: string }[] }>(res)
|
||||
expect(json.results.length).toBeGreaterThan(0)
|
||||
for (const r of json.results) {
|
||||
expect(r.id).toBeTruthy(); expect(r.title).toBeTruthy(); expect(r.url).toContain("wellfound.com/jobs/")
|
||||
expect(r).toHaveProperty("company"); expect(r).toHaveProperty("location"); expect(r).toHaveProperty("date")
|
||||
}
|
||||
})
|
||||
test("bogus flag exits 1 with JSON error on stderr", async () => {
|
||||
const res = await runCLI(["search", "-q", "x", "--bogus"])
|
||||
expect(res.exitCode).toBe(1); expect(JSON.parse(res.stderr)).toHaveProperty("error")
|
||||
})
|
||||
test("missing query exits 1 with JSON error", async () => {
|
||||
const res = await runCLI(["search"])
|
||||
expect(res.exitCode).toBe(1); expect(JSON.parse(res.stderr).code).toBe("NO_QUERY")
|
||||
})
|
||||
})
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"types": ["bun"],
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"skipLibCheck": true,
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
Reference in new issue
Block a user