py/ai-job-search
1
0
Fork 0
mirror of https://github.com/MadsLorentzen/ai-job-search.git synced 2026-09-17 00:26:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(robots-check): pin the tie-break clause and the browser-UA fallback

Browse Source 
The tie-break test listed Disallow first - the one ordering where
deleting the clause changes nothing - and gate()'s read-the-policy-as-a-
browser recovery (the Barclays-class case 09-web-research.md documents
as covered) had no test. Both gaps are guard code whose breakage is
silent by construction. Mutation-verified: the tie-break deletion and
the UA-loop reduction each now fail exactly the new tests. Review
findings F21 and F30 (2026-08-19).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Mads Lorentzen
2026-08-19 19:57:46 +02:00
co-authored by Claude Opus 5
parent 4ed5fee221
commit c20458d768
2 changed files with 57 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_robots_check.py
+50
View File
@@ -52,6 +52,13 @@ class TestPathRules(unittest.TestCase):
"""Cautious tie-break: Google resolves ties to Allow, we do not."""
self.assertFalse(allowed("User-agent: *\nDisallow: /a\nAllow: /a\n", "*", "/a"))
def test_equal_specificity_tie_goes_to_disallow_when_allow_listed_first(self):
"""The only ordering that exercises the tie-break clause: with Allow
first, deleting the clause makes the first rule at a given length win
and Allow would leak through. The Disallow-first sibling above cannot
detect that mutation (review finding F21, 2026-08-19)."""
self.assertFalse(allowed("User-agent: *\nAllow: /a\nDisallow: /a\n", "*", "/a"))
def test_api_block_and_sibling_path(self):
self.assertFalse(allowed(JOBUP, "*", "/api/v1/public/search"))
self.assertTrue(allowed(JOBUP, "*", "/en/jobs/"))
@@ -130,6 +137,49 @@ class TestSoftTwoHundred(unittest.TestCase):
self.assertEqual(rc, 1)
self.assertIn("not a robots.txt", msg)
def test_gate_reads_policy_as_browser_when_honest_request_is_refused(self):
"""09-web-research.md's Barclays-class recovery: the policy file itself
returns 403 to Claude-User and 200 to a browser, and the checker must
then read it as a browser and obey it strictly. This is gate()'s UA
fallback loop, previously untested despite the doc's coverage claim
(review finding F30, 2026-08-19)."""
import robots_check
original = robots_check._fetch
def waf(url, ua):
if ua == robots_check.BROWSER:
return ("User-agent: *\nAllow: /\n", 200)
return ("<html>403 Forbidden</html>", 403)
robots_check._fetch = waf
try:
rc, msg = robots_check.gate("https://waf.example/jobs")
finally:
robots_check._fetch = original
self.assertEqual(rc, 0)
self.assertIn("ALLOWED", msg)
def test_gate_obeys_a_browser_fetched_policy_strictly(self):
"""The fallback must not fail open: a policy readable only as a browser
still disallows what it disallows."""
import robots_check
original = robots_check._fetch
def waf(url, ua):
if ua == robots_check.BROWSER:
return ("User-agent: *\nDisallow: /jobs\n", 200)
return ("<html>403 Forbidden</html>", 403)
robots_check._fetch = waf
try:
rc, msg = robots_check.gate("https://waf.example/jobs")
finally:
robots_check._fetch = original
self.assertEqual(rc, 1)
self.assertIn("DISALLOWED", msg)
def test_a_genuinely_empty_robots_is_still_allow_all(self):
"""RFC 9309: an empty file permits everything. Do not over-correct."""
self.assertTrue(is_robots_body(""))