mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-11-12 18:07:41 +00:00
fix(scrape): send one plain term per CLI call and make portals state their query syntax (#536) (#653)
* fix(scrape): send one plain term per CLI call and make portals state their query syntax (#536) search-queries.md is written in Google syntax, and Step 1b said to translate its terms into portal flags without saying that OR and quotes do not survive the translation. - job-scraper Step 1b: one plain term per CLI call, never OR or a combined string; results merge across calls (Step 2 dedups on URL). Quoting follows the portal's own SKILL.md. - Portal SKILL.md files carry a one-line "Query syntax:" note; /add-portal asks the generator to write it for new portals. - search-queries.md: site: blocks are labelled WebSearch-only and each priority gains a plain-terms block for the CLIs. - tests/test_scrape_contract.py: every installed portal SKILL.md must carry the Query syntax line; the portal list is derived from the skills tree. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * fix(scrape): fall back to site: terms for placeholder blocks; update freehire query syntax Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: drop the CHANGELOG hunk, frozen by #800 (release notes come from the PR title) Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * fix(scrape): dedup the pool by URL before Step 2 and give each portal's literal quote form Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * fix(add-portal): compare the portal's reported total, not capped rows, when probing query syntax Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2b3f613fda
commit
7432228340
11 files changed
+88
-6
No files matched your search
@@ -152,5 +152,36 @@ class SeenJobsDedupContinuityTests(unittest.TestCase):
|
||||
self.assertRegex(text, r"matched by URL or\s+company\+title")
|
||||
|
||||
|
||||
# Step 1b sends one plain term per CLI call and takes quoting from the portal's
|
||||
# own SKILL.md. The rule is data each portal carries, so the test derives the
|
||||
# portal list from the skills tree and names no portal.
|
||||
_QUERY_SYNTAX_LINE = re.compile(r"^Query syntax:\s*\S", re.MULTILINE)
|
||||
|
||||
|
||||
class PortalQuerySyntaxTests(unittest.TestCase):
|
||||
"""Every installed portal SKILL.md must state its query syntax."""
|
||||
|
||||
def test_step1b_points_at_the_query_syntax_line(self):
|
||||
text = SCRAPER_SKILL.read_text(encoding="utf-8")
|
||||
step1b = re.search(r"#### 1b\..*?(?=#### 1c\.)", text, re.DOTALL)
|
||||
self.assertIsNotNone(step1b, "Step 1b section not found")
|
||||
self.assertIn("`Query syntax:`", step1b.group(0), "Step 1b no longer defers quoting to the portal's Query syntax line")
|
||||
self.assertIn("inside the argument the CLI receives", step1b.group(0))
|
||||
self.assertIn("deduplicate the pool by URL before Step 2", step1b.group(0))
|
||||
|
||||
def test_every_installed_portal_skill_has_a_query_syntax_line(self):
|
||||
self.assertTrue(PORTAL_CLIS, "no installed portal skills found")
|
||||
missing = [
|
||||
portal.name
|
||||
for portal in PORTAL_CLIS
|
||||
if not _QUERY_SYNTAX_LINE.search((portal / "SKILL.md").read_text(encoding="utf-8"))
|
||||
]
|
||||
self.assertEqual(
|
||||
[],
|
||||
missing,
|
||||
"portal SKILL.md files without a `Query syntax:` line: " + ", ".join(missing),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user