diff --git a/.claude/skills/job-scraper/SKILL.md b/.claude/skills/job-scraper/SKILL.md index 9036894..8af8719 100644 --- a/.claude/skills/job-scraper/SKILL.md +++ b/.claude/skills/job-scraper/SKILL.md @@ -147,6 +147,7 @@ For each new job, do a rapid fit check (NOT the full evaluation from `04-job-eva "company": "...", "url": "...", "first_seen": "YYYY-MM-DD", + "posted_date": "YYYY-MM-DD" | null, "deadline": "YYYY-MM-DD" | null, "fit": "high/medium/low", "status": "new/skipped/ranked/expired", @@ -165,6 +166,8 @@ The `source` field records which mechanism produced the entry: `cli` for Step 1b `deadline` is a base field rather than a `/rank` extension: Step 2's detail fetch already extracts the application deadline, so it is written when the job is first seen and refreshed by `/rank` Step 4 when a scoring agent returns a different value. `null` means the posting states no deadline; a missing key means the entry predates this field - **never infer a deadline** from either, and never backfill by guessing. +`posted_date` is the posting's own publication date, taken from the `date` field Step 2's contract already guarantees on every portal CLI's search output. Step 1b uses that date to scope the run to the last 14 days and then drops it, so nothing downstream can distinguish a posting published yesterday from one published two years ago - `first_seen` is when this scraper first saw the entry, not when the employer posted it. Persisting it makes Step 1b's window auditable after the run and gives `/rank` a freshness signal to weigh, instead of rediscovering the date and recording it in prose that nothing reads. That gap landed for real: a freehire-search posting dated 2024-05-13 was scraped and ranked Strong Fit at position 1 of 133, its own scoring note observing the listing "may be long stale" with nothing able to act on it. `null` means the portal returned no date for that result (the CLIs emit `date: null` when a listing omits it); a missing key means the entry predates this field - **never infer a posting date** from either, and never backfill by guessing. + 2. Only present jobs NOT already in the seen list or tracker. ### Step 4.5: Generate Referral Contact Links (High & Medium Fit Only) diff --git a/CHANGELOG.md b/CHANGELOG.md index feb7c64..5c893ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,23 @@ prefer updating to a tagged release over pulling raw `master` (see files a release touched; `python3 tools/check_upstream_updates.py` lists them with per-file diff commands. +## [Unreleased] + +### Fixed + +- **`/scrape` now persists each posting's publication date** (#390) - Step 2's contract guarantees a + `date` on every portal CLI's search output (CI enforces it in `test_scrape_contract.py`) and + Step 1b uses that date to scope a run to the last 14 days, but Step 4's `seen_jobs.json` schema + stored no posting date at all: `first_seen` is when the scraper saw an entry, not when the + employer posted it. The freshness window was therefore unauditable the moment a run ended, and + `/rank` - which reads the stored entry, not the run - had no age signal to weigh. A + `freehire-search` posting dated 2024-05-13 was scraped 27 months later and ranked Strong Fit at + position 1 of 133; the scoring note recorded that the listing "may be long stale" in prose + nothing reads, and an `/apply` run drafted a tailored CV and cover letter against it. The schema + gains `posted_date` (`null` when the portal returned no date, never inferred or backfilled). + Pinned by three new cases in `test_scrape_contract.py`, each verified to fail on the unfixed + spec. Reported and diagnosed from a real run by @sandunwijerathne. + ## [1.7.0] - 2026-08-29 ### Fixed diff --git a/tests/test_scrape_contract.py b/tests/test_scrape_contract.py index f93371d..9aec685 100644 --- a/tests/test_scrape_contract.py +++ b/tests/test_scrape_contract.py @@ -74,5 +74,67 @@ class ScrapeSearchOutputContractTests(unittest.TestCase): self.assertEqual([], failures, "; ".join(failures) or "no portal CLIs checked") + +# Step 4's storage schema, derived the same way as the Step 2 contract above: +# the field list lives in the spec, never duplicated here, so a schema change +# fails this test instead of silently agreeing with a stale copy. +_STEP4_SCHEMA_BLOCK = re.compile(r"Add ALL fetched jobs.*?```json(.*?)```", re.DOTALL) + + +def derive_stored_fields() -> frozenset[str]: + text = SCRAPER_SKILL.read_text(encoding="utf-8") + match = _STEP4_SCHEMA_BLOCK.search(text) + if match is None: + raise AssertionError("Step 4 seen_jobs.json schema block not found in job-scraper/SKILL.md") + return frozenset(re.findall(r'"([a-z_]+)":', match.group(1))) + + +class SeenJobsPostingDateTests(unittest.TestCase): + """The posting date Step 2 guarantees must survive into Step 4's storage. + + Step 2's contract promises a `date` on every portal CLI's search output and + the test above keeps every CLI honest about emitting it. Step 1b then uses + that date to scope the run to the last 14 days - and Step 4's schema drops + it. `first_seen` records when this scraper first saw an entry, not when the + employer posted it, so once the run ends nothing can tell a posting + published yesterday from one published two years ago: the Step 1b window is + unauditable and /rank has no freshness signal to weigh. + + That failure landed for real: a freehire-search posting dated 2024-05-13 was + scraped and ranked Strong Fit at position 1 of 133, its own scoring note + observing the listing "may be long stale" with nothing able to act on it. + """ + + def test_step4_schema_persists_a_posting_date(self): + stored = derive_stored_fields() + self.assertIn( + "posted_date", + stored, + "Step 4's seen_jobs.json schema stores no posting-date field, so a " + "posting's age is unrecoverable after the run that scraped it", + ) + + def test_the_step2_date_field_survives_into_storage(self): + contract = derive_contract_fields() + self.assertIn("date", contract, "Step 2 no longer guarantees a posting date") + stored = derive_stored_fields() + self.assertIn( + "posted_date", + stored, + "Step 2 guarantees a posting `date` and CI enforces every CLI emits it, " + "but Step 4 discards it at write time", + ) + + def test_posted_date_semantics_are_documented(self): + """A stored field the spec never explains gets backfilled by guessing.""" + text = SCRAPER_SKILL.read_text(encoding="utf-8") + self.assertIn("`posted_date`", text, "posted_date is in the schema but never documented") + self.assertRegex( + text, + r"never infer a posting date", + "posted_date must carry the same never-backfill rule as `deadline`", + ) + + if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main()