From 9ab697de6412806963ce0c9d7f93f2c0356d3d61 Mon Sep 17 00:00:00 2001 From: Mads Lorentzen Date: Wed, 19 Aug 2026 19:52:21 +0200 Subject: [PATCH] fix(evaluation): update stale Language Gate preamble to reflect tracking 04-job-evaluation.md still said the gate result "is not a field /scrape or /rank track" - true when the gate was introduced, false since /rank began persisting language_gate/language_note as shortlist veto fields and /scrape began surfacing the flag. The authoritative framework file taught agents the opposite of rank.md's own persistence rule. New coupling test pins that the section names the tracked fields and never reverts to the untracked claim. framework_version 1.2.3 -> 1.2.4. Review finding F24 (2026-08-19). Co-Authored-By: Claude Opus 5 (1M context) --- .../04-job-evaluation.md | 4 +-- CHANGELOG.md | 8 +++++ tests/test_rank_command.py | 30 +++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/.claude/skills/job-application-assistant/04-job-evaluation.md b/.claude/skills/job-application-assistant/04-job-evaluation.md index b11eb07..376ced9 100644 --- a/.claude/skills/job-application-assistant/04-job-evaluation.md +++ b/.claude/skills/job-application-assistant/04-job-evaluation.md @@ -1,5 +1,5 @@ --- -framework_version: 1.2.3 +framework_version: 1.2.4 --- # Job Evaluation Framework @@ -32,7 +32,7 @@ A role that fails this gate is not scored and not drafted. Everything below appl ## Language Gate — run before scoring -No dimension or gate anywhere in this framework currently checks a posting's language requirements against what the candidate actually speaks - it is not one of the five Scoring Dimensions below, not a field `/scrape` or `/rank` track, and not something `/apply`'s language detection (Step 1, which already extracts a posting's required language generically) has anywhere to report to. This gate adds that check, structured the same way as the Eligibility Gate above: read the posting, classify against profile data, and treat a hard mismatch as FAIL before scoring. +This gate checks a posting's language requirements against what the candidate actually speaks. It is not one of the five Scoring Dimensions below - it runs before them, structured the same way as the Eligibility Gate above: read the posting, classify against profile data, and treat a hard mismatch as FAIL before scoring. Its verdict is tracked downstream: `/rank` records the result as `language_gate` (PASS/FAIL/FLAG) with a supporting `language_note`, persists both into `seen_jobs.json`, and treats a FAIL as a shortlist veto; `/scrape` surfaces the flag in its results table and carries a language-override rule for postings whose ad language differs from the role's working language. `/apply`'s language detection (Step 1, which extracts a posting's required language generically) feeds this same check. Read the posting's language requirements as stated for **the role itself** — not the language the ad happens to be written in. A posting written in a language you don't work in, for a role that only needs languages you do work in on the job, passes fine; only an explicit job-condition requirement ("fluent X required," "must communicate with the Y team in Z") triggers this check. For each language the posting requires as a job condition, compare it against your Languages table in CLAUDE.md / `01-candidate-profile.md`: diff --git a/CHANGELOG.md b/CHANGELOG.md index 13e077b..acf4217 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,14 @@ per-file diff commands. ### Fixed +- **Language Gate preamble no longer claims the gate is untracked** (`framework_version` + 1.2.3 -> 1.2.4 in `04-job-evaluation.md`) - the paragraph still said the result "is not + a field `/scrape` or `/rank` track", written before the gate was wired into both + consumers. An agent reading the authoritative framework file learned the opposite of + what `rank.md` itself insists on ("These veto fields are as important to persist as + the score itself"). The preamble now names `language_gate`/`language_note` and how each + consumer uses them; a coupling test in `tests/test_rank_command.py` keeps the framework + text honest about the tracking. - **`/reset documents` now clears `documents/postings/`** - the drop folder for hand-pasted job posting text was absent from the preview, the delete block, and the user-facing scope description, after which the command told the user "The `documents/` diff --git a/tests/test_rank_command.py b/tests/test_rank_command.py index bc20976..dbb8d06 100644 --- a/tests/test_rank_command.py +++ b/tests/test_rank_command.py @@ -20,6 +20,9 @@ except ImportError: REPO = Path(__file__).resolve().parent.parent COMMAND = REPO / ".claude" / "commands" / "rank.md" SCRAPER_SKILL = REPO / ".claude" / "skills" / "job-scraper" / "SKILL.md" +EVALUATION = ( + REPO / ".claude" / "skills" / "job-application-assistant" / "04-job-evaluation.md" +) def _sections(text: str) -> dict[str, str]: @@ -102,6 +105,33 @@ class RankCommandSpec(unittest.TestCase): "not only when /rank re-scores it", ) + def test_evaluation_framework_acknowledges_language_gate_tracking(self): + """04-job-evaluation.md is the authoritative file /rank tells its agents + to read. Its Language Gate preamble once said the gate result "is not a + field /scrape or /rank track" - written before the gate was wired into + both consumers, and never updated. An agent reading that learns the + opposite of what rank.md itself insists on ("These veto fields are as + important to persist as the score itself"). The framework text must name + the tracked fields and must not claim they are untracked.""" + text = EVALUATION.read_text(encoding="utf-8") + gate = text.partition("## Language Gate")[2].partition("\n## ")[0] + self.assertTrue(gate, "04-job-evaluation.md has no Language Gate section") + self.assertIn( + "language_gate", + gate, + "the Language Gate section must name the language_gate field /rank persists", + ) + self.assertIn( + "language_note", + gate, + "the Language Gate section must name the language_note field /rank persists", + ) + self.assertNotIn( + "not a field", + gate, + "stale claim: the gate result IS tracked by /scrape and /rank now", + ) + def test_step2_schema_includes_language_gate_fields(self): sections = _sections(COMMAND.read_text(encoding="utf-8")) step2 = sections.get("Step 2: Batch-Fetch and Score", "")