diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f4416e..99aa0fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,13 @@ per-file diff commands. ### Added +- **Discriminating tests for `robots_check`'s tie-break and browser-UA fallback** - the + existing tie test put Disallow first, the one ordering that cannot detect deletion of + the tie-break clause; and the browser-readback recovery that `09-web-research.md` + claims is covered had no test at all. Three new tests in `tests/test_robots_check.py` + pin the Allow-first tie, the 403-to-honest/200-to-browser recovery, and that a + browser-fetched policy is still obeyed strictly. Each was mutation-verified: deleting + the tie-break clause or the UA fallback now fails the suite. - **LaTeX special-character guidance for CVs** (`framework_version` 1.4.1 -> 1.4.2 in `05-cv-templates.md`, 1.0.1 -> 1.0.2 in `06-cover-letter-templates.md`) - `05` gains a "LaTeX Special Characters" section and `06`'s existing one is completed beyond `\_`/`\&`. diff --git a/tests/test_robots_check.py b/tests/test_robots_check.py index 08c7860..beb9286 100644 --- a/tests/test_robots_check.py +++ b/tests/test_robots_check.py @@ -52,6 +52,13 @@ class TestPathRules(unittest.TestCase): """Cautious tie-break: Google resolves ties to Allow, we do not.""" self.assertFalse(allowed("User-agent: *\nDisallow: /a\nAllow: /a\n", "*", "/a")) + def test_equal_specificity_tie_goes_to_disallow_when_allow_listed_first(self): + """The only ordering that exercises the tie-break clause: with Allow + first, deleting the clause makes the first rule at a given length win + and Allow would leak through. The Disallow-first sibling above cannot + detect that mutation (review finding F21, 2026-08-19).""" + self.assertFalse(allowed("User-agent: *\nAllow: /a\nDisallow: /a\n", "*", "/a")) + def test_api_block_and_sibling_path(self): self.assertFalse(allowed(JOBUP, "*", "/api/v1/public/search")) self.assertTrue(allowed(JOBUP, "*", "/en/jobs/")) @@ -130,6 +137,49 @@ class TestSoftTwoHundred(unittest.TestCase): self.assertEqual(rc, 1) self.assertIn("not a robots.txt", msg) + def test_gate_reads_policy_as_browser_when_honest_request_is_refused(self): + """09-web-research.md's Barclays-class recovery: the policy file itself + returns 403 to Claude-User and 200 to a browser, and the checker must + then read it as a browser and obey it strictly. This is gate()'s UA + fallback loop, previously untested despite the doc's coverage claim + (review finding F30, 2026-08-19).""" + import robots_check + + original = robots_check._fetch + + def waf(url, ua): + if ua == robots_check.BROWSER: + return ("User-agent: *\nAllow: /\n", 200) + return ("403 Forbidden", 403) + + robots_check._fetch = waf + try: + rc, msg = robots_check.gate("https://waf.example/jobs") + finally: + robots_check._fetch = original + self.assertEqual(rc, 0) + self.assertIn("ALLOWED", msg) + + def test_gate_obeys_a_browser_fetched_policy_strictly(self): + """The fallback must not fail open: a policy readable only as a browser + still disallows what it disallows.""" + import robots_check + + original = robots_check._fetch + + def waf(url, ua): + if ua == robots_check.BROWSER: + return ("User-agent: *\nDisallow: /jobs\n", 200) + return ("403 Forbidden", 403) + + robots_check._fetch = waf + try: + rc, msg = robots_check.gate("https://waf.example/jobs") + finally: + robots_check._fetch = original + self.assertEqual(rc, 1) + self.assertIn("DISALLOWED", msg) + def test_a_genuinely_empty_robots_is_still_allow_all(self): """RFC 9309: an empty file permits everything. Do not over-correct.""" self.assertTrue(is_robots_body(""))