mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-09-17 00:26:26 +00:00
fcefb8150f
* fix(web-research): stop treating a WebFetch 403 as a dead posting WebFetch sends a bot user agent, and many bank and corporate sites answer with HTTP 403 while serving the same page to a browser normally. Every command treated that as "page unavailable" and degraded silently rather than failing loudly: - /rank marked live postings `expired` - /apply fell back to search snippets, or to vague cover-letter prose - /scrape stored listing-page `#fragment` URLs, which fetch fine and return unrelated jobs, so every later /rank and /apply run on that entry failed Adds 09-web-research.md as the single reference: the trust boundary, a curl browser-header retry with a tag-stripping extractor, a four-step escalation order, the login-wall case, why the employer's own careers posting beats an aggregator listing (the requisition ID and the grade survive there), and the rule that a search-result snippet is a lead rather than a source. Wires it into /apply, /rank, /interview, /outcome, /notion-sync, the job-scraper skill, and writing-style rule 5. Bumps 03-writing-style.md to 1.2.0; 09-web-research.md starts at 1.0.0. Aggregator examples are given generically (LinkedIn, Indeed, national job boards) so the guidance holds in any market. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> * fix(web-research): gate the browser-header retry on robots.txt Addresses review feedback on #277. WebFetch identifies itself as Claude-User and honors robots.txt, so a 403 has two very different causes and they must not be treated the same: a WAF default on a site whose published policy allows access, or a site that has actually declined. Retrying with browser headers in the second case circumvents the very opt-out mechanism site owners are told they can rely on, and the core framework cannot hold a looser standard than it asks of community forks. The escalation now runs tools/robots_check.py before the retry. A disallow for "*" or for "Claude-User" skips the retry entirely and goes to step 3 (find the employer's own posting). The rule is stated plainly in 09-web-research.md so later edits do not erode it: the retry exists to get past bot-filtering firewalls on sites whose robots.txt permits access; it is never used to override a site that has said no. Two findings from testing the gate against live sites, both pinned by tests/test_robots_check.py (15 offline cases): - The WAF usually blocks robots.txt too. privatebank.barclays.com returns 403 on the policy file to Claude-User and 200 to a browser, so a naive gate would block the retry on exactly the sites the retry is for. The checker reads the policy as a browser when the honest request is refused, then obeys it strictly - a policy you are prevented from reading cannot be honored, and robots.txt is not the protected resource. - urllib.robotparser cannot be used. It ends a record at a blank line and matches rules in file order, so Barclays' real file (blank lines between "User-agent: *" and its rules, "Allow: /" before "Disallow: /cs/") reads as everything-allowed. That fails open, in the one direction that matters. The checker implements RFC 9309 longest-match instead, with ties resolved to Disallow rather than Allow. Verified live: barclays /careers/ allowed and /cs/ blocked, ubs.com allowed, jobup.ch /api/ blocked while /en/jobs/ stays allowed. 09-web-research.md 1.0.0 to 1.1.0. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --------- Co-authored-by: kgb <kevingblackman@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
108 lines
4.3 KiB
Python
108 lines
4.3 KiB
Python
"""Offline tests for tools/robots_check.py.
|
|
|
|
No network: every case exercises the parser against literal robots.txt bodies,
|
|
matching the repo's CI policy of making no live portal requests.
|
|
|
|
The cases marked FAIL-OPEN REGRESSION are the ones Python's own
|
|
urllib.robotparser gets wrong. They are pinned here because getting them wrong
|
|
means the browser-header retry runs against a site that said no, which is the
|
|
exact boundary this tool exists to hold.
|
|
"""
|
|
|
|
import subprocess
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO_ROOT / "tools"))
|
|
|
|
from robots_check import allowed # noqa: E402
|
|
|
|
|
|
# Real body served by privatebank.barclays.com: blank lines sit between the
|
|
# User-agent line and its rules. Python's robotparser treats those as record
|
|
# separators and drops every rule, so /cs/ reads as allowed.
|
|
BARCLAYS = "User-agent: *\n\n\nAllow: /\n\nDisallow: /cs/\n\nSitemap: https://x/sitemap.xml\n"
|
|
|
|
# jobup.ch: the case a community fork was asked to ship opt-in.
|
|
JOBUP = "User-agent: *\nDisallow: /api/\n"
|
|
|
|
|
|
class TestPathRules(unittest.TestCase):
|
|
def test_blank_lines_inside_record_do_not_end_it(self):
|
|
"""FAIL-OPEN REGRESSION: /cs/ is disallowed despite the blank lines."""
|
|
self.assertFalse(allowed(BARCLAYS, "*", "/cs/"))
|
|
|
|
def test_allowed_path_on_same_site_still_allowed(self):
|
|
self.assertTrue(allowed(BARCLAYS, "*", "/careers/"))
|
|
|
|
def test_longest_match_wins_over_rule_order(self):
|
|
"""FAIL-OPEN REGRESSION: 'Allow: /' precedes 'Disallow: /cs/' in the
|
|
file; specificity must win, not position."""
|
|
body = "User-agent: *\nAllow: /\nDisallow: /cs/\n"
|
|
self.assertFalse(allowed(body, "*", "/cs/deep/page"))
|
|
|
|
def test_longest_match_can_unblock(self):
|
|
body = "User-agent: *\nDisallow: /\nAllow: /jobs/\n"
|
|
self.assertTrue(allowed(body, "*", "/jobs/x"))
|
|
self.assertFalse(allowed(body, "*", "/other"))
|
|
|
|
def test_equal_specificity_tie_goes_to_disallow(self):
|
|
"""Cautious tie-break: Google resolves ties to Allow, we do not."""
|
|
self.assertFalse(allowed("User-agent: *\nDisallow: /a\nAllow: /a\n", "*", "/a"))
|
|
|
|
def test_api_block_and_sibling_path(self):
|
|
self.assertFalse(allowed(JOBUP, "*", "/api/v1/public/search"))
|
|
self.assertTrue(allowed(JOBUP, "*", "/en/jobs/"))
|
|
|
|
def test_wildcard_and_end_anchor(self):
|
|
body = "User-agent: *\nDisallow: /*.pdf$\n"
|
|
self.assertFalse(allowed(body, "*", "/files/cv.pdf"))
|
|
self.assertTrue(allowed(body, "*", "/files/cv.pdf.html"))
|
|
|
|
def test_empty_disallow_means_allow_everything(self):
|
|
self.assertTrue(allowed("User-agent: *\nDisallow:\n", "*", "/anything"))
|
|
|
|
def test_empty_or_ruleless_robots_allows(self):
|
|
self.assertTrue(allowed("", "*", "/x"))
|
|
self.assertTrue(allowed("# just a comment\n", "*", "/x"))
|
|
|
|
def test_comments_are_stripped(self):
|
|
self.assertFalse(allowed("User-agent: *\nDisallow: /x # nope\n", "*", "/x"))
|
|
|
|
|
|
class TestAgentSelection(unittest.TestCase):
|
|
def test_named_claude_user_opt_out_is_honored(self):
|
|
body = "User-agent: Claude-User\nDisallow: /\n\nUser-agent: *\nAllow: /\n"
|
|
self.assertFalse(allowed(body, "Claude-User", "/a"))
|
|
self.assertTrue(allowed(body, "*", "/a"))
|
|
|
|
def test_agent_match_is_case_insensitive(self):
|
|
body = "User-agent: CLAUDE-USER\nDisallow: /x\n"
|
|
self.assertFalse(allowed(body, "claude-user", "/x"))
|
|
|
|
def test_falls_back_to_star_when_agent_absent(self):
|
|
self.assertFalse(allowed(JOBUP, "Claude-User", "/api/v1"))
|
|
|
|
def test_multiple_agents_share_one_ruleset(self):
|
|
body = "User-agent: A\nUser-agent: Claude-User\nDisallow: /z\n"
|
|
self.assertFalse(allowed(body, "Claude-User", "/z"))
|
|
self.assertFalse(allowed(body, "A", "/z"))
|
|
|
|
|
|
class TestCli(unittest.TestCase):
|
|
def test_module_is_importable_and_cli_exists(self):
|
|
"""The doc calls this by path; make sure that entry point stays valid."""
|
|
script = REPO_ROOT / "tools" / "robots_check.py"
|
|
self.assertTrue(script.is_file())
|
|
out = subprocess.run(
|
|
[sys.executable, str(script)], capture_output=True, text=True, timeout=30
|
|
)
|
|
# No URL argument: must fail loudly rather than defaulting to "allowed".
|
|
self.assertNotEqual(out.returncode, 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|