fix(robots_check): match wildcard rules in linear time and hand curl only http(s) URLs (#779)

Two findings on tools/robots_check.py, reported privately by Martin Vlach
(@Huge, BudgetScan).

GHSA-xhqf-8q8x-4jwm: _match turned every '*' into a regex '.*', so a rule
with repeated wildcard segments that does not match the path
("Disallow: /*a*a*a...*b$") sent Python's backtracking engine into
exponential time - minutes at 16 segments - in the main process, where
no curl timeout applies, on a file the site being checked controls. The
matcher is now the reporter's linear table walk in O(len(pattern) *
len(path)) with no regex and no backtracking; consecutive '*' match as
one. A 200,000-case differential fuzz against the old regex matcher
found no change in answers, and three tests pin the prefix, anchored and
slash-spanning semantics.

GHSA-h8vc-753p-wpmh: gate() rebuilt the target as scheme://host/robots.txt
without checking the scheme, so a direct call with file:, gopher: or ftp:
passed that scheme to curl -L, which could also follow a redirect to any
protocol it was built with. gate() now fails closed with UNCONFIRMED on
any scheme but http(s) and on a URL without a host, before curl runs, and
curl gets --proto =http,https --proto-redir =http,https.

Seven new tests, all failing before this change.

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

tests/test_robots_check.py

@@ -73,6 +73,47 @@ class TestPathRules(unittest.TestCase):
         self.assertFalse(allowed(body, "*", "/tag/$var"))
         self.assertFalse(allowed(body, "*", "/tag/$var/sub"))
 
+    def test_repeated_wildcards_do_not_catastrophically_backtrack(self):
+        # GHSA-xhqf-8q8x-4jwm: every '*' became a regex '.*', so a rule with
+        # repeated wildcard segments that does not match took exponential time
+        # in Python's backtracking engine (16 segments: minutes). The matcher
+        # must run in O(len(pattern) * len(path)) whatever the rule says.
+        import time
+
+        pat = "/" + "*a" * 13 + "*b$"
+        body = f"User-agent: *\nDisallow: {pat}\n"
+        path = "/" + "a" * 28 + "X"
+        start = time.perf_counter()
+        res = allowed(body, "*", path)
+        elapsed = time.perf_counter() - start
+        self.assertTrue(res)
+        self.assertLess(elapsed, 0.5, f"_match took {elapsed:.2f}s (catastrophic backtracking)")
+
+    def test_consecutive_wildcards_match_identically_to_single_wildcard(self):
+        body = "User-agent: *\nDisallow: /docs/*******.pdf$\n"
+        self.assertFalse(allowed(body, "*", "/docs/cv.pdf"))
+        self.assertFalse(allowed(body, "*", "/docs/sub/folder/cv.pdf"))
+        self.assertTrue(allowed(body, "*", "/docs/sub/folder/cv.html"))
+
+    def test_wildcard_semantics_survive_the_matcher_change(self):
+        # Pinned against the regex matcher's answers: prefix match without '$',
+        # full match with it, a wildcard spanning slashes and the empty string,
+        # and a trailing wildcard before the anchor.
+        body = (
+            "User-agent: *\n"
+            "Disallow: /a*b\n"
+            "Disallow: /x/*/y$\n"
+            "Disallow: /end*$\n"
+        )
+        self.assertFalse(allowed(body, "*", "/ab"))
+        self.assertFalse(allowed(body, "*", "/a/long/path/b/and-more"))
+        self.assertTrue(allowed(body, "*", "/a"))
+        self.assertFalse(allowed(body, "*", "/x/anything/at/all/y"))
+        self.assertTrue(allowed(body, "*", "/x/anything/y/more"))
+        self.assertFalse(allowed(body, "*", "/end"))
+        self.assertFalse(allowed(body, "*", "/ending/here"))
+        self.assertTrue(allowed(body, "*", "/en"))
+
     def test_percent_encoded_query_string_matches_rule_in_gate(self):
         import robots_check
 
@@ -462,3 +503,78 @@ class TestNonAsciiCurlOutput(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class TestSchemeHardening(unittest.TestCase):
+    """GHSA-h8vc-753p-wpmh: only http(s) reaches curl, on the first hop and on
+    every redirect.
+
+    gate() rebuilt the target as scheme://host/robots.txt but never checked the
+    scheme, so a direct call with file:, gopher: or ftp: handed that scheme to
+    curl, and curl -L was free to follow a redirect to any protocol it supports.
+    """
+
+    def _gate_without_curl(self, url):
+        # Record rather than fail inside the spy: gate() swallows every
+        # exception from _fetch into an UNCONFIRMED verdict, which would pass.
+        import robots_check
+
+        fetched = []
+        original = robots_check._fetch
+        robots_check._fetch = lambda url, ua: fetched.append(url) or ("", 404)
+        try:
+            result = robots_check.gate(url)
+        finally:
+            robots_check._fetch = original
+        self.assertEqual(fetched, [], f"curl was asked for {fetched}")
+        return result
+
+    def test_non_http_scheme_is_unconfirmed_without_a_fetch(self):
+        for url in ("file:///etc/passwd", "gopher://127.0.0.1:6379/_", "ftp://x.example/robots.txt", "x.example/path"):
+            with self.subTest(url=url):
+                rc, msg = self._gate_without_curl(url)
+                self.assertEqual(rc, 1)
+                self.assertTrue(msg.startswith("UNCONFIRMED"), msg)
+                self.assertIn("scheme", msg)
+
+    def test_missing_host_is_unconfirmed_without_a_fetch(self):
+        rc, msg = self._gate_without_curl("http:///robots.txt")
+        self.assertEqual(rc, 1)
+        self.assertTrue(msg.startswith("UNCONFIRMED"), msg)
+
+    def test_curl_is_pinned_to_http_and_https_on_every_hop(self):
+        import robots_check
+
+        seen = []
+
+        class Done:
+            returncode, stdout, stderr = 0, "\n404", ""
+
+        def fake_run(argv, **kwargs):
+            seen.append(argv)
+            return Done()
+
+        original = robots_check.subprocess.run
+        robots_check.subprocess.run = fake_run
+        try:
+            body, code = robots_check._fetch("https://x.example/robots.txt", "Claude-User")
+        finally:
+            robots_check.subprocess.run = original
+        self.assertEqual(code, 404)
+        argv = seen[0]
+        self.assertIn("--proto", argv)
+        self.assertEqual(argv[argv.index("--proto") + 1], "=http,https")
+        self.assertIn("--proto-redir", argv)
+        self.assertEqual(argv[argv.index("--proto-redir") + 1], "=http,https")
+        self.assertLess(argv.index("--proto-redir"), argv.index("--"))
+
+    def test_real_curl_refuses_a_file_url_through_fetch(self):
+        import shutil
+
+        import robots_check
+
+        if not shutil.which("curl"):
+            self.skipTest("curl is not installed")
+        target = (REPO_ROOT / "tools" / "robots_check.py").resolve().as_uri()
+        with self.assertRaises(RuntimeError):
+            robots_check._fetch(target, "Claude-User")