jobyssey/.agents/skills/wellfound-search/cli/src/helpers.ts

// Data source: Wellfound's public SEO landing pages /role/l/<role>/<location> and
// /role/r/<role> (remote). Each page embeds a Next.js __NEXT_DATA__ blob whose Apollo
// cache holds StartupResult and JobListingSearchResult records, including full
// descriptions. Detail pages /jobs/<id>-<slug> carry a schema.org JobPosting JSON-LD
// block. /search is disallowed by robots.txt, so free-text queries are slugified
// into the role path instead.

export const BASE = "https://wellfound.com"
const UA = "Mozilla/5.0 (compatible; wellfound-cli/1.0)"

export function writeError(error: string, code: string): void {
  process.stderr.write(JSON.stringify({ error, code }) + "\n")
}

/** Fetch HTML with exponential backoff on 429/5xx. Returns "" on a 404. */
export async function htmlFetch(url: string): Promise<string> {
  const maxRetries = 6
  let delay = 500
  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    const response = await fetch(url, {
      headers: { "User-Agent": UA, Accept: "text/html,application/xhtml+xml", "Accept-Language": "en-US,en;q=0.9" },
      redirect: "follow",
      signal: AbortSignal.timeout(20000),
    })
    if (response.status === 429 || response.status >= 500) {
      if (attempt === maxRetries) throw new Error(`Request failed: ${response.status} ${response.statusText}`)
      const jitter = Math.floor(Math.random() * 500)
      await new Promise((r) => setTimeout(r, delay + jitter))
      delay = Math.min(delay * 2, 8000)
      continue
    }
    if (response.status === 404) return ""
    if (!response.ok) throw new Error(`Request failed: ${response.status} ${response.statusText}`)
    return response.text()
  }
  throw new Error("Request failed after max retries")
}

export function slugify(s: string): string {
  return s.toLowerCase().trim().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "")
}

export interface JobCard {
  id: string
  title: string
  company: string | null
  location: string | null
  date: string | null
  url: string
  description: string | null
  compensation: string | null
  remote: boolean | null
  jobType: string | null
  companySlug: string | null
  companySize: string | null
  ats: string | null
}

type ApolloRecord = Record<string, unknown>

/** Pull the Apollo cache out of a role page. Returns null when the page has no data blob. */
export function extractApollo(html: string): Record<string, ApolloRecord> | null {
  const m = /<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/.exec(html)
  if (!m) return null
  try {
    const nd = JSON.parse(m[1])
    const data = nd?.props?.pageProps?.apolloState?.data
    return data && typeof data === "object" ? (data as Record<string, ApolloRecord>) : null
  } catch {
    return null
  }
}

const str = (v: unknown): string | null => (typeof v === "string" && v ? v : null)

/** Walk StartupResult -> highlightedJobListings -> JobListingSearchResult. Each record is parsed
 * independently so one malformed entry cannot break the rest. */
export function parseJobCards(apollo: Record<string, ApolloRecord>): JobCard[] {
  const cards: JobCard[] = []
  for (const [key, startup] of Object.entries(apollo)) {
    if (!key.startsWith("StartupResult:")) continue
    const refs = (startup.highlightedJobListings as { __ref?: string }[] | undefined) ?? []
    for (const ref of refs) {
      try {
        const job = ref?.__ref ? apollo[ref.__ref] : undefined
        if (!job || !str(job.id)) continue
        const id = String(job.id)
        const slug = str(job.slug) ?? ""
        const live = Number(job.liveStartAt)
        const locs = Array.isArray(job.locationNames) ? (job.locationNames as unknown[]).filter((x) => typeof x === "string").join(", ") : null
        cards.push({
          id,
          title: str(job.title) ?? "",
          company: str(startup.name),
          location: locs || (job.remote ? "Remote" : null),
          date: Number.isFinite(live) && live > 0 ? new Date(live * 1000).toISOString().slice(0, 10) : null,
          url: `${BASE}/jobs/${id}-${slug}`,
          description: str(job.description),
          compensation: str(job.compensation),
          remote: typeof job.remote === "boolean" ? job.remote : null,
          jobType: str(job.jobType),
          companySlug: str(startup.slug),
          companySize: str(startup.companySize),
          ats: str(job.atsSource),
        })
      } catch {
        // skip malformed record
      }
    }
  }
  return cards
}

export function decodeEntities(s: string): string {
  return s
    .replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;|&apos;/g, "'").replace(/&nbsp;/g, " ")
    .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n)))
}

export function stripHtml(s: string): string {
  return decodeEntities(
    s.replace(/<br\s*\/?>/gi, "\n").replace(/<\/(p|div|li|h\d|ul|ol)>/gi, "\n").replace(/<li[^>]*>/gi, "- ").replace(/<[^>]+>/g, ""),
  ).replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim()
}

export interface JobDetail {
  id: string
  title: string | null
  company: string | null
  location: string | null
  date: string | null
  url: string
  description: string | null
  employmentType: string | null
  salary: string | null
  applyUrl: string | null
}

/** Parse the schema.org JobPosting JSON-LD on a detail page. */
export function parseDetail(html: string, id: string, url: string): JobDetail | null {
  const blocks = [...html.matchAll(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/g)]
  for (const b of blocks) {
    try {
      const j = JSON.parse(b[1])
      if (j?.["@type"] !== "JobPosting") continue
      const loc = j.jobLocation
      const addr = Array.isArray(loc) ? loc[0]?.address : loc?.address
      const locStr = addr ? [addr.addressLocality, addr.addressRegion, addr.addressCountry].filter(Boolean).join(", ") : null
      const sal = j.baseSalary?.value
      const salary = sal ? [sal.minValue, sal.maxValue].filter((x: unknown) => x != null).join("-") + (j.baseSalary?.currency ? ` ${j.baseSalary.currency}` : "") : null
      return {
        id,
        title: str(j.title),
        company: str(j.hiringOrganization?.name),
        location: locStr || null,
        date: str(j.datePosted)?.slice(0, 10) ?? null,
        url,
        description: typeof j.description === "string" ? stripHtml(j.description) : null,
        employmentType: Array.isArray(j.employmentType) ? j.employmentType.join(", ") : str(j.employmentType),
        salary: salary || null,
        applyUrl: str(j.url) ?? url,
      }
    } catch {
      continue
    }
  }
  return null
}