fix(robots-check): unquote query strings and restrict '$' anchor to pattern end (#543) (#546)

CHANGELOG.md

@@ -29,6 +29,17 @@ per-file diff commands.
 
 ### Fixed
 
+- **`robots_check` unquotes incoming query strings and restricts `$` anchors to pattern end**
+  (#543, `tools/robots_check.py`, `tests/test_robots_check.py`) - two path-matching edge cases in
+  the robots.txt gate failed open. First, `gate(url)` called `unquote(parts.path)` but concatenated
+  `parts.query` raw, while `_match()` decodes the pattern: rules targeting encoded query parameters
+  (such as `Disallow: /search?q=machine%20learning`) never matched the incoming URL, evaluating
+  disallowed scraping targets as allowed. Query parameters are now decoded when assembling `path`.
+  Second, `_match()` converted every `$` character unconditionally into a regular-expression
+  end-of-string anchor, causing rules with an internal `$` (e.g. `/tag/$var`) to never match; under
+  RFC 9309 section 2.2.3, `$` is an end anchor only at the end of the pattern, and internal `$`
+  characters are now matched literally.
+
 - **CV template guidance aligns geometry scale with master template** (#553,
   `.claude/skills/job-application-assistant/05-cv-templates.md`, `tests/test_latex_guidance.py`) -
   the preamble snippet in `05-cv-templates.md` specified `\usepackage[scale=0.77]{geometry}`,

tests/test_robots_check.py

@@ -68,6 +68,24 @@ class TestPathRules(unittest.TestCase):
         self.assertFalse(allowed(body, "*", "/files/cv.pdf"))
         self.assertTrue(allowed(body, "*", "/files/cv.pdf.html"))
 
+    def test_dollar_sign_mid_pattern_is_literal_not_end_anchor(self):
+        body = "User-agent: *\nDisallow: /tag/$var\n"
+        self.assertFalse(allowed(body, "*", "/tag/$var"))
+        self.assertFalse(allowed(body, "*", "/tag/$var/sub"))
+
+    def test_percent_encoded_query_string_matches_rule_in_gate(self):
+        import robots_check
+
+        body = "User-agent: *\nDisallow: /search?q=machine%20learning\n"
+        original = robots_check._fetch
+        robots_check._fetch = lambda url, ua: (body, 200)
+        try:
+            rc, msg = robots_check.gate("https://example.com/search?q=machine%20learning")
+        finally:
+            robots_check._fetch = original
+        self.assertEqual(rc, 1, msg)
+        self.assertIn("DISALLOWED", msg)
+
     def test_empty_disallow_means_allow_everything(self):
         self.assertTrue(allowed("User-agent: *\nDisallow:\n", "*", "/anything"))
 

tools/robots_check.py

@@ -122,11 +122,15 @@ def _match(pattern, path):
     The pattern is percent-decoded to match the already-decoded path. Without
     this, "Disallow: /foo%20bar" never matched "/foo bar" and the rule was
     silently skipped - a fail-open on any site that encodes its own rules.
+    Under RFC 9309 section 2.2.2, '$' anchors the end of the URL only when
+    at the end of the pattern; mid-pattern '$' is matched literally.
     """
     if pattern == '':
         return -1
     pattern = unquote(pattern)
-    rx = '^' + ''.join('.*' if c == '*' else ('$' if c == '$' else re.escape(c)) for c in pattern)
+    end_anchor = pattern.endswith('$')
+    pat = pattern[:-1] if end_anchor else pattern
+    rx = '^' + ''.join('.*' if c == '*' else re.escape(c) for c in pat) + ('$' if end_anchor else '')
     return len(pattern) if re.match(rx, path) else -1
 
 def allowed(text, agent, path):
@@ -143,7 +147,7 @@ def gate(url):
     parts = urlsplit(url)
     path = unquote(parts.path) or '/'
     if parts.query:
-        path += '?' + parts.query
+        path += '?' + unquote(parts.query)
     robots = f'{parts.scheme}://{parts.netloc}/robots.txt'
     body, last = None, 'no attempt'
     for ua in ('Claude-User', BROWSER):