diff --git a/.claude/settings.json b/.claude/settings.json index cc6f37b9..9fa1618d 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -14,6 +14,8 @@ "Bash(python3 tools/rank_state.py:*)", "Bash(python tools/job_key.py:*)", "Bash(python3 tools/job_key.py:*)", + "Bash(python tools/scrape_state.py:*)", + "Bash(python3 tools/scrape_state.py:*)", "Bash(python tools/verify_pdf.py:*)", "Bash(python3 tools/verify_pdf.py:*)", "Bash(python tools/verify_layout.py:*)", diff --git a/.claude/skills/job-scraper/SKILL.md b/.claude/skills/job-scraper/SKILL.md index 937c8b88..68a35878 100644 --- a/.claude/skills/job-scraper/SKILL.md +++ b/.claude/skills/job-scraper/SKILL.md @@ -5,7 +5,7 @@ description: > (LinkedIn, local job boards, and any skills added with /add-portal). Deduplicates across runs. Triggers on: job scrape, find jobs, search jobs, new jobs, job search, scrape jobs, /scrape -allowed-tools: Read, Write, Edit, Glob, Grep, Bash(bun --version), Bash(python tools/job_key.py:*), Bash(python3 tools/job_key.py:*), WebFetch, WebSearch, Agent, AskUserQuestion +allowed-tools: Read, Write, Edit, Glob, Grep, Bash(bun --version), Bash(python tools/job_key.py:*), Bash(python3 tools/job_key.py:*), Bash(python tools/scrape_state.py:*), Bash(python3 tools/scrape_state.py:*), WebFetch, WebSearch, Agent, AskUserQuestion --- # Job Scraper @@ -36,11 +36,11 @@ Optional arguments: ## Execution Steps -### Step 0: Load State +### Step 0: Load Search Strategy -1. Read `job_scraper/seen_jobs.json` (create if missing - start with `{"seen": {}}`) -2. Read `job_search_tracker.csv` to extract already-applied companies+roles -3. Read `search-queries.md` (this directory) for the search strategy +Read `search-queries.md` (this directory) for the search strategy. `tools/scrape_state.py` reads `job_scraper/seen_jobs.json` and `job_search_tracker.csv` for deduplication; do not load either growing file into the conversation. + +> **Windows:** run these tools as `python`, not `python3`. A python.org install ships no `python3`, so that name reaches the Microsoft Store alias, which either exits 49 with no output (#669) or starts a separate Store Python without the packages `pip` installed (such as pypdf). `.claude/settings.json` allows both spellings. ### Step 1: Search @@ -87,7 +87,15 @@ Tag each fallback result as WebSearch-sourced, keeping the portal tag when the f ### Step 2: Fetch & Parse -For each promising result from Step 1: +Before detail fetches, combine the promising search hits into a JSON array in a temporary file outside the repo. Include each hit's source portal and `source` (`cli` or `websearch`), then run: + +```bash +python3 tools/scrape_state.py candidates --results "" +``` + +Only fetch details for the returned `selected` hits. The helper checks prior URLs (including legacy-key entries), company+title pairs, the application tracker, and duplicates within this run. This avoids repeat detail requests on every scrape. If `invalid` lists malformed hits, report them as portal data errors and continue with the valid hits; never invent a missing company or URL. Keep the returned hits for Step 4; never paste the history file into the agent context. + +For each selected result from Step 1: **From CLI results:** Search output already includes title, company, location, date, and URL. For jobs worth a deeper look, fetch full detail with that portal's `detail` @@ -116,7 +124,7 @@ that entry. When WebSearch only yields a listing page, search the employer's own site for the role and store that URL instead, or drop the candidate rather than saving a fragment link. -For every candidate: +The `candidates` command applies these rules before a detail request: - Skip if the URL matches any existing `job_scraper/seen_jobs.json` entry, regardless of that entry's key. This preserves dedup continuity for postings stored under the pre-helper key rule while new entries use the canonical key from Step 4. @@ -141,9 +149,7 @@ For each new job, do a rapid fit check (NOT the full evaluation from `04-job-eva ### Step 4: Deduplicate & Store -1. Derive each entry's key with the helper, never by slugifying in the moment: - -> **Windows:** run these tools as `python`, not `python3`. A python.org install ships no `python3`, so that name reaches the Microsoft Store alias, which either exits 49 with no output (#669) or starts a separate Store Python without the packages `pip` installed (such as pypdf). `.claude/settings.json` allows both spellings. +1. Keep the canonical key rule; `scrape_state.py store` calls `tools/job_key.py`'s `make_key` for every inserted entry, so a separate key subprocess per job is unnecessary: ```bash python3 tools/job_key.py --company "" --title "" --url "<url>" @@ -151,7 +157,7 @@ python3 tools/job_key.py --company "<company>" --title "<title>" --url "<url>" It prints one line: the canonical key for that posting. The key must be a pure function of the posting, because two runs that slugify differently store the same job twice and defeat the dedup this step exists to provide. The helper also length-caps long titles and disambiguates the cap with a hash of the full slug, so a truncated title is stable across runs and two different long titles never collide. `python3 tools/job_key.py --audit` reports entries in an existing state file that predate this rule; it only reports, and never rewrites keys, since a rewritten key breaks the tracker's own company+role matching. -2. Add ALL fetched jobs (new and skipped) to `job_scraper/seen_jobs.json` with structure: +2. Add ALL fetched jobs (new and skipped) to `job_scraper/seen_jobs.json` through the tool below. Each stored entry has this structure: ```json { "seen": { @@ -171,6 +177,14 @@ It prints one line: the canonical key for that posting. The key must be a pure f } ``` +Pass each entry object (without the outer `seen` map or key) to the tool; include the search hit's `location` and any other fetched fields already used by the workflow. Then run: + +```bash +python3 tools/scrape_state.py store --results "<temporary fetched jobs.json>" +``` + +The tool rechecks deduplication against the current state and tracker, computes keys, and atomically adds only new entries to `job_scraper/seen_jobs.json`. Use its `added` keys to present only newly stored jobs. It leaves every older entry and its `/rank` fields untouched. Do not read or rewrite the full state file in the conversation. + The `portal` field records which CLI skill produced the job (results are already tagged per portal in Step 1b - persist that tag here). Entries written before this field existed lack it; the health check (Step 4.75) attributes those by matching the URL's domain against each portal's base URL, so do not backfill. The `source` field records which mechanism produced the entry: `cli` for Step 1b portal-CLI output, `websearch` for the Step 1c fallback. This is what keeps a ghost-job report diagnosable after the run's summary is gone: a stored entry whose URL later resolves to nothing (or to a different job) reads very differently depending on whether it came from live CLI output or from a search index that can be weeks stale - and a presented job with no entry here at all points at fabrication, which Rule 1 forbids. Entries written before this field existed lack it; never backfill it - the mechanism was not recorded. @@ -215,7 +229,7 @@ Scraper-based portal CLIs rot silently: when a portal changes its markup, the pa **Free pass (no extra requests).** For each enabled portal that ran in Step 1b: - **Degraded scan:** inspect the results it returned this run. Flags: `company` null or empty on every result, empty titles, undecoded entities (`&`) or HTML fragments in titles, URLs that do not point at the portal. Any of these means the parser is half-working and `/scrape` is silently collecting junk. -- **Yield history:** if the portal returned zero results across all of this run's queries, check whether `job_scraper/seen_jobs.json` holds prior entries from it (via the `portal` field, or by matching URL domains for entries predating the field). A portal that produced jobs on earlier runs and produces nothing now is suspect - the same queries worked before. +- **Yield history:** if the portal returned zero results across all of this run's queries, run `python3 tools/scrape_state.py history --portal <skill-name> --domain <portal-host>` (the host from that portal's documented base URL). It counts prior `job_scraper/seen_jobs.json` entries by `portal`, or by URL host for entries predating that field, without loading the history into context. A portal that produced jobs on earlier runs and produces nothing now is suspect - the same queries worked before. **Escalation (bounded, on suspicion only).** A suspect portal gets **one** sentinel probe: run its documented `search` with the example query from its own SKILL.md (that query provably worked when the skill was registered), the portal's limit flag capped at 3, `--format json`. If that returns nothing, retry **once** with a single common word. Only then is the verdict **broken**. A 429 or block page is **never** evidence of breakage - record the portal as **inconclusive (rate-limited)**, back off, and do not retry. diff --git a/CHANGELOG.md b/CHANGELOG.md index 45cbd1a8..15936051 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,18 @@ per-file diff commands. or `-q "golang rust"`, which returns fewer jobs than `golang` alone. Measured live: `golang` 3,550, `golang OR rust` 6,805. The CLI already passes `-q` through unchanged. +- **`/scrape` drops already-seen hits before fetching details and keeps its history out of the conversation** + (#710, `tools/scrape_state.py`, `tests/test_scrape_state.py`, `.claude/skills/job-scraper/SKILL.md`, + `.claude/settings.json`, `tools/security_guards.py`) - previously `/scrape` read the whole + `job_scraper/seen_jobs.json` and the application tracker into the conversation on every run, fetched + details for every promising hit before checking whether it had been seen, and rewrote the file at the end; + a history of a few hundred jobs is already about 100 KB. A new helper does the deduplication outside the + conversation: `candidates` drops hits seen before (by URL, including entries stored under older keys, or by + company and title), already in the tracker, or repeated within the run, before any detail fetch; `store` + rechecks, keys each new entry with `job_key.make_key` and adds it atomically without touching existing + entries; `history` gives the portal health check its prior-entry count. The helper gets path-scoped allow + entries in `.claude/settings.json` and the guard allowlist, like `rank_state.py`. + ### Fixed - **`freehire-search detail` no longer asks the API for a slug cut out of a URL on another diff --git a/tests/test_scrape_state.py b/tests/test_scrape_state.py new file mode 100644 index 00000000..d2dbe2c0 --- /dev/null +++ b/tests/test_scrape_state.py @@ -0,0 +1,100 @@ +"""The scrape helper filters before detail fetches and never rewrites old entries.""" + +import argparse +import contextlib +import io +import json +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "tools")) +import scrape_state # noqa: E402 + + +class ScrapeStateTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.state = self.root / "job_scraper" / "seen_jobs.json" + self.tracker = self.root / "tracker.csv" + self.results = self.root / "results.json" + + def run_tool(self, command, jobs): + self.results.write_text(json.dumps(jobs), encoding="utf-8") + args = argparse.Namespace(command=command, results=self.results, state=self.state, tracker=self.tracker) + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.assertEqual(scrape_state.run(args), 0) + return json.loads(output.getvalue()) + + def test_candidates_excludes_history_tracker_and_repeated_hits_before_detail(self): + old = {f"old_{i}": {"title": f"Old {i}", "company": "Archive", "url": f"https://jobs.test/{i}"} for i in range(1000)} + old["legacy_key"] = {"title": "Engineer", "company": "Acme", "url": "https://jobs.test/old"} + self.state.parent.mkdir() + self.state.write_text(json.dumps({"seen": old}), encoding="utf-8") + self.tracker.write_text("date,company,role\n2026-10-21,Tracked,Analyst\n", encoding="utf-8") + jobs = [ + {"title": "Engineer", "company": "Acme", "url": "https://jobs.test/old"}, + {"title": "Analyst", "company": "Tracked", "url": "https://jobs.test/tracked"}, + {"title": "Builder", "company": "Fresh", "url": "https://jobs.test/fresh"}, + {"title": "Builder", "company": "Fresh", "url": "https://jobs.test/fresh"}, + ] + out = self.run_tool("candidates", jobs) + self.assertEqual(out["selected"], [jobs[2]]) + self.assertEqual((out["already_seen"], out["already_applied"], out["repeated_in_batch"]), (1, 1, 1)) + self.assertNotIn("Old 999", json.dumps(out)) + + def test_one_malformed_portal_hit_does_not_stop_other_candidates(self): + valid = {"title": "Builder", "company": "Fresh", "url": "https://jobs.test/fresh"} + out = self.run_tool("candidates", [valid, {"title": "Unknown", "company": None, "url": "https://jobs.test/bad"}, None]) + self.assertEqual(out["selected"], [valid]) + self.assertEqual([item["index"] for item in out["invalid"]], [1, 2]) + + def test_store_adds_only_new_jobs_and_preserves_ranked_history(self): + old = {"legacy_key": {"title": "Engineer", "company": "Acme", "url": "https://jobs.test/old", "status": "ranked", "rank_score": 91}} + self.state.parent.mkdir() + self.state.write_text(json.dumps({"seen": old}), encoding="utf-8") + fresh = { + "title": "Builder", "company": "Fresh", "url": "https://jobs.test/fresh", + "first_seen": "2026-10-21", "posted_date": None, "deadline": None, + "fit": "high", "status": "new", "portal": "example-search", "source": "cli", + "location": "Remote", + } + out = self.run_tool("store", [fresh, {**fresh, "url": "https://jobs.test/another"}]) + self.assertEqual(out["added"], ["fresh_builder"]) + stored = json.loads(self.state.read_text(encoding="utf-8"))["seen"] + self.assertEqual(stored["legacy_key"], old["legacy_key"]) + self.assertEqual(stored["fresh_builder"], fresh) + + def test_store_creates_missing_history_and_rejects_partial_entries_without_writing(self): + good = { + "title": "Builder", "company": "Fresh", "url": "https://jobs.test/fresh", + "first_seen": "2026-10-21", "posted_date": None, "deadline": None, + "fit": "high", "status": "new", "portal": "example-search", "source": "cli", + } + with self.assertRaisesRegex(ValueError, "invalid portal or source"): + self.run_tool("store", [good, {**good, "title": "Analyst", "url": "https://jobs.test/other", "source": None}]) + self.assertFalse(self.state.exists()) + self.assertEqual(self.run_tool("store", [good])["added"], ["fresh_builder"]) + self.assertIn("fresh_builder", json.loads(self.state.read_text(encoding="utf-8"))["seen"]) + + def test_history_counts_portal_and_legacy_host_without_emitting_entries(self): + self.state.parent.mkdir() + self.state.write_text(json.dumps({"seen": { + "one": {"portal": "example-search", "url": "https://jobs.test/1"}, + "two": {"url": "https://careers.jobs.test/2"}, + "three": {"url": "https://other.test/3"}, + }}), encoding="utf-8") + args = argparse.Namespace(command="history", state=self.state, portal="example-search", domain="jobs.test") + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.assertEqual(scrape_state.run(args), 0) + self.assertEqual(json.loads(output.getvalue()), {"portal": "example-search", "prior_entries": 2}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/scrape_state.py b/tools/scrape_state.py new file mode 100644 index 00000000..814703ea --- /dev/null +++ b/tools/scrape_state.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Keep /scrape's growing history out of the agent context. + +Pass a JSON array of search hits to `candidates` before fetching details, then +pass the enriched entries to `store`. Both commands deduplicate against the +current history and tracker; `store` writes the history atomically. +""" + +import argparse +import json +import sys +from pathlib import Path +from urllib.parse import urlsplit + +from job_key import make_key +from rank_state import STATE, TRACKER, load_state, norm, save_state, tracker_pairs + + +def read_jobs(path: Path) -> list: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"cannot read {path}: {exc}") from exc + if isinstance(data, dict): + data = data.get("results") + if not isinstance(data, list): + raise ValueError("results must be a JSON array of jobs") + return data + + +def identity(job: dict) -> tuple[str, tuple[str, str]]: + for field in ("title", "company", "url"): + if not isinstance(job.get(field), str) or not job[field].strip(): + raise ValueError(f"job needs a non-empty {field}") + url = job["url"].strip() + parsed = urlsplit(url) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + raise ValueError(f"job has an invalid posting URL: {url}") + return url.rstrip("/"), (norm(job["company"]), norm(job["title"])) + + +def select(jobs: list, seen: dict, tracker: set, *, strict: bool = False) -> tuple[list[dict], dict]: + urls, pairs = set(), set() + for entry in seen.values(): + if not isinstance(entry, dict): + raise ValueError("job_scraper/seen_jobs.json contains a non-object entry") + if isinstance(entry.get("url"), str): + urls.add(entry["url"].rstrip("/")) + pairs.add((norm(entry.get("company")), norm(entry.get("title")))) + + selected = [] + counts = {"already_seen": 0, "already_applied": 0, "repeated_in_batch": 0, "invalid": []} + batch_urls, batch_pairs = set(), set() + for index, job in enumerate(jobs): + try: + if not isinstance(job, dict): + raise ValueError("job is not an object") + url, pair = identity(job) + except ValueError as exc: + if strict: + raise ValueError(f"job {index + 1}: {exc}") from exc + counts["invalid"].append({"index": index, "error": str(exc)}) + continue + if url in urls or pair in pairs: + counts["already_seen"] += 1 + elif pair in tracker: + counts["already_applied"] += 1 + elif url in batch_urls or pair in batch_pairs: + counts["repeated_in_batch"] += 1 + else: + selected.append(job) + batch_urls.add(url) + batch_pairs.add(pair) + return selected, counts + + +def run(args) -> int: + if args.state.exists(): + doc, seen = load_state(args.state) + else: + doc = {"seen": {}} + seen = doc["seen"] + if args.command == "history": + domain = (args.domain or "").lower().strip(".") + if not args.portal or not domain: + raise ValueError("history needs --portal and --domain") + count = 0 + for entry in seen.values(): + if not isinstance(entry, dict): + continue + if entry.get("portal") == args.portal: + count += 1 + elif not entry.get("portal"): + url = entry.get("url") + try: + host = (urlsplit(url if isinstance(url, str) else "").hostname or "").lower() + except ValueError: + continue + if host == domain or host.endswith("." + domain): + count += 1 + print(json.dumps({"portal": args.portal, "prior_entries": count})) + return 0 + + jobs = read_jobs(args.results) + selected, counts = select(jobs, seen, tracker_pairs(args.tracker), strict=args.command == "store") + if args.command == "candidates": + print(json.dumps({"selected": selected, **counts}, ensure_ascii=False)) + return 0 + + for job in selected: + for field in ("first_seen", "posted_date", "deadline", "fit", "status", "portal", "source"): + if field not in job: + raise ValueError(f"job needs {field} before storage") + if job["fit"] not in ("high", "medium", "low") or job["status"] not in ("new", "skipped", "expired"): + raise ValueError("job has an invalid fit or status") + if job["source"] not in ("cli", "websearch") or not isinstance(job["portal"], str): + raise ValueError("job has an invalid portal or source") + key = make_key(job["company"], job["title"], job["url"]) + if key in seen: + raise ValueError(f"canonical key already exists for another posting: {key}") + seen[key] = job + if selected: + args.state.parent.mkdir(parents=True, exist_ok=True) + save_state(args.state, doc) + print(json.dumps({"added": [make_key(j["company"], j["title"], j["url"]) for j in selected], **counts}, ensure_ascii=False)) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("command", choices=("candidates", "store", "history")) + parser.add_argument("--results", type=Path) + parser.add_argument("--portal") + parser.add_argument("--domain") + args = parser.parse_args() + args.state, args.tracker = STATE, TRACKER + try: + if args.command != "history" and args.results is None: + parser.error("candidates and store need --results") + return run(args) + except ValueError as exc: + print(f"scrape_state: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/security_guards.py b/tools/security_guards.py index c6c35a4c..0ff89aa9 100644 --- a/tools/security_guards.py +++ b/tools/security_guards.py @@ -71,6 +71,8 @@ ALLOWED_PERMISSIONS = { "Bash(python3 tools/rank_state.py:*)", "Bash(python tools/job_key.py:*)", "Bash(python3 tools/job_key.py:*)", + "Bash(python tools/scrape_state.py:*)", + "Bash(python3 tools/scrape_state.py:*)", "Bash(python tools/verify_pdf.py:*)", "Bash(python3 tools/verify_pdf.py:*)", "Bash(python tools/verify_layout.py:*)", @@ -242,6 +244,8 @@ ALLOWED_SKILL_TOOLS = { "Bash(bun --version)", "Bash(python tools/job_key.py:*)", "Bash(python3 tools/job_key.py:*)", + "Bash(python tools/scrape_state.py:*)", + "Bash(python3 tools/scrape_state.py:*)", } # The tools a shipped .claude/agents/*.md may list. These agents read untrusted