diff --git a/.agents/skills/freehire-search/SKILL.md b/.agents/skills/freehire-search/SKILL.md index 76bf4990..6b223173 100644 --- a/.agents/skills/freehire-search/SKILL.md +++ b/.agents/skills/freehire-search/SKILL.md @@ -79,6 +79,8 @@ at the hosted API. bun run .agents/skills/freehire-search/cli/src/cli.ts search [-q ""] [facet flags] ``` +Query syntax: quotes require every quoted word (any order, not a contiguous phrase); `-word` excludes; uppercase `OR` between terms matches any alternative (max 5, merged and deduplicated server-side); `title:` at the start of an alternative matches it in the title only, e.g. `title:"product owner" -junior OR title:produktchef`. + Key flags: - `--query ` / `-q ` — keyword search (title, skill, role). Full-text; optional. Supports `OR` and `title:` (see "Query operators" below). diff --git a/.agents/skills/jobbank-search/SKILL.md b/.agents/skills/jobbank-search/SKILL.md index f747c2df..4f93197e 100644 --- a/.agents/skills/jobbank-search/SKILL.md +++ b/.agents/skills/jobbank-search/SKILL.md @@ -47,6 +47,8 @@ Invoke this skill when the user wants to: bun run .agents/skills/jobbank-search/cli/src/cli.ts search [flags] ``` +Query syntax: double quotes inside the argument make an exact phrase, word order included (e.g. `--key '"product owner"'`); single quotes are ignored; no OR. + Key flags: - `--key ` — keyword search (title, company, keyword) - `--exclude ` — exclude keywords from results diff --git a/.agents/skills/jobdanmark-search/SKILL.md b/.agents/skills/jobdanmark-search/SKILL.md index 9dde4caf..3b01bc00 100644 --- a/.agents/skills/jobdanmark-search/SKILL.md +++ b/.agents/skills/jobdanmark-search/SKILL.md @@ -46,6 +46,8 @@ Invoke this skill when the user wants to: bun run .agents/skills/jobdanmark-search/cli/src/cli.ts search [flags] ``` +Query syntax: double quotes inside the argument make an exact phrase, word order included (e.g. `--text '"product owner"'`); single quotes are searched literally and return nothing; no OR. + Key flags: - `--text ` — free-text search, e.g. `elektriker`, `sygeplejerske`, `softwareudvikler` - `--category ` — category ID (see Categories table below) diff --git a/.agents/skills/jobindex-search/SKILL.md b/.agents/skills/jobindex-search/SKILL.md index 6169da1b..d1f149f8 100644 --- a/.agents/skills/jobindex-search/SKILL.md +++ b/.agents/skills/jobindex-search/SKILL.md @@ -44,6 +44,8 @@ Invoke this skill when the user wants to: bun run .agents/skills/jobindex-search/cli/src/cli.ts search --query "" [flags] ``` +Query syntax: double quotes inside the argument make an exact phrase, word order included (e.g. `--query '"product owner"'`); single quotes are ignored; no OR. + Key flags: - `--query ` / `-q ` — keyword search (job title, skill, company, city). **Required.** Searches without a query exit with `MISSING_REQUIRED`. - `--jobage ` — filter by posting age: `1` (today), `7`, `14`, `30`, or `9999` (all, default) diff --git a/.agents/skills/jobindex-search/url-reference.md b/.agents/skills/jobindex-search/url-reference.md index ac39c650..b6e10d67 100644 --- a/.agents/skills/jobindex-search/url-reference.md +++ b/.agents/skills/jobindex-search/url-reference.md @@ -59,7 +59,7 @@ Category uses a two-part path: `{group}/{slug}`: | Parameter | Description | Examples | |-----------|-------------|----------| -| `q` | Search query | `q=data+engineer` (broad), `q=%27data+engineer%27` (exact match) | +| `q` | Search query | `q=data+engineer` (broad), `q=%22data+engineer%22` (exact match) | | `jobage` | Max age in days | `jobage=1` (today), `jobage=3`, `jobage=7`, `jobage=30` | | `page` | Page number (1-indexed) | `page=1`, `page=2` | @@ -69,8 +69,8 @@ Category uses a two-part path: `{group}/{slug}`: # Basic keyword search playwright-cli goto "https://www.jobindex.dk/jobsoegning?q=python+developer" -# Exact match search (single quotes around query) -playwright-cli goto "https://www.jobindex.dk/jobsoegning?q=%27python+developer%27" +# Exact match search (double quotes around query) +playwright-cli goto "https://www.jobindex.dk/jobsoegning?q=%22python+developer%22" # Search in Storkøbenhavn area playwright-cli goto "https://www.jobindex.dk/jobsoegning/storkoebenhavn?q=python+developer" diff --git a/.agents/skills/jobnet-search/SKILL.md b/.agents/skills/jobnet-search/SKILL.md index f177d6b9..cb2b602c 100644 --- a/.agents/skills/jobnet-search/SKILL.md +++ b/.agents/skills/jobnet-search/SKILL.md @@ -49,6 +49,8 @@ Invoke this skill when the user wants to: bun run .agents/skills/jobnet-search/cli/src/cli.ts search [flags] ``` +Query syntax: phrase quoting not honoured; no OR. + Key flags: - `--search-string ` — keyword search, e.g. `sygeplejerske`, `ingeniør`, `pædagog` - `--region ` — `HovedstadenOgBornholm`, `Midtjylland`, `Syddanmark`, `OevrigeSjaelland`, `Nordjylland` diff --git a/.agents/skills/linkedin-search/SKILL.md b/.agents/skills/linkedin-search/SKILL.md index f533f1f0..689bb398 100644 --- a/.agents/skills/linkedin-search/SKILL.md +++ b/.agents/skills/linkedin-search/SKILL.md @@ -46,6 +46,8 @@ Run it on your own responsibility. bun run .agents/skills/linkedin-search/cli/src/cli.ts search --location "" [flags] ``` +Query syntax: phrase quoting not honoured; OR not verified. + Key flags: - `--location ` / `-l ` — **required.** A LinkedIn place string, e.g. `"Mumbai, Maharashtra, India"`, `"Berlin, Germany"`, `"London, United Kingdom"`, or `"Remote"`. - `--query ` / `-q ` — keyword search (title, skill, role). Recommended. diff --git a/.claude/commands/add-portal.md b/.claude/commands/add-portal.md index 05b865b2..5e41430f 100644 --- a/.claude/commands/add-portal.md +++ b/.claude/commands/add-portal.md @@ -90,7 +90,7 @@ These conventions are what make portal skills interchangeable for `/scrape` and ### File specifics - **`SKILL.md` frontmatter:** `name`, `version: 1.0.0`, a `description` written for skill triggering - it must name the portal, the market, and include trigger phrases in English **and** the market's language; `context: fork`; `allowed-tools: Bash(bun run .agents/skills//cli/src/cli.ts *)`. -- **`SKILL.md` body:** what the skill searches, the personal-use warning if Step 2 found terms restrictions, command reference with flags, 4-6 usage examples using the user's market (real cities, realistic roles), output-format table, and a Notes section recording portal quirks found in Step 2. If Step 2.5 found the portal needs a credential, add a **Setup** section naming the service, the exact environment variable to export, and the fact that every call is billed - stated where the user reads it before running the skill, not after. +- **`SKILL.md` body:** what the skill searches, the personal-use warning if Step 2 found terms restrictions, command reference with flags, 4-6 usage examples using the user's market (real cities, realistic roles), output-format table, and a Notes section recording portal quirks found in Step 2. If Step 2.5 found the portal needs a credential, add a **Setup** section naming the service, the exact environment variable to export, and the fact that every call is billed - stated where the user reads it before running the skill, not after. Under the search command, write one line `Query syntax: ...` stating whether the portal honours phrase quoting and which operators, if any, it supports (e.g. `Query syntax: double quotes inside the argument make an exact phrase, word order included; no OR.`). `/scrape` sends one plain term per call and follows this line for quoting, so the rule lives in the portal skill rather than in a list upstream maintains. Determine it in Step 4 by running a two-word test query bare, inside double quotes, inside single quotes, reversed inside double quotes, and with `OR` between two terms, each with `--format json` and no `--limit`, then comparing the total the portal itself reports (not the rows returned, which `--limit` and the portal's page size cap): a quoted total below the bare one means the quotes narrow the match, and a reversed total below the quoted one means word order counts. If the portal reports no total, write the behaviour as not verified - never assume it from another portal. - **`url-reference.md`:** the endpoints, parameters table, and response-structure notes from Step 2 - this is the file a future maintainer needs when the portal changes its markup. - **`package.json`:** name `-cli`, `"type": "module"`, scripts `start`, `test` (`bun test --timeout 30000`), and `typecheck` (`tsc --noEmit`); dev-only dependencies in the zero-dependency default. - **`tests/`:** copy `runCLI`/`parseJSON` from `jobindex-search/cli/tests/helpers.ts`, then add a small live smoke-test file: `search` with the test query returns exit code 0 and ≥1 result with non-null `id`/`title`/`url`; a bogus flag or missing required arg exits 1 with a JSON error on stderr. diff --git a/.claude/skills/job-scraper/SKILL.md b/.claude/skills/job-scraper/SKILL.md index 5a079435..68b2e86d 100644 --- a/.claude/skills/job-scraper/SKILL.md +++ b/.claude/skills/job-scraper/SKILL.md @@ -67,12 +67,12 @@ Discover all installed portal CLI skills by reading every `SKILL.md` found under For each **enabled** portal skill: 1. Read its `SKILL.md` to find the correct `bun run …` invocation and supported flags. -2. Translate the query terms from `search-queries.md` into that portal's flag format (e.g. `--key`, `--search-string`, `--query`, filter codes — whatever the portal's SKILL.md specifies). +2. Take the plain terms from `search-queries.md`'s **Plain terms** blocks and translate each into that portal's flag format (e.g. `--key`, `--search-string`, `--query`, filter codes — whatever the portal's SKILL.md specifies). When a priority has no Plain terms block, or its terms still carry a `[YOUR_...]` placeholder, take the terms from that priority's `site:` lines instead (drop `site:`, `OR` and the location words; each quoted phrase is one term). **One plain term per CLI call — never `OR`, never a combined string.** Portal search boxes do not share Google's syntax, so a combined query returns a different result set than its terms would (on some portals it is silently reduced to one of them). Quoting follows the portal's own SKILL.md `Query syntax:` line: quote a multi-word term when that line says quotes narrow the match (as a phrase or as a requirement that every word appear), otherwise pass the words bare. The quote characters go inside the argument the CLI receives (`--query '"product owner"'`); shell quotes alone are stripped before the CLI sees them. The `site:` blocks in `search-queries.md` are WebSearch-only and are never passed to a CLI. 3. Scope to the last 14 days using the portal's supported recency **filter** flag (`--jobage`, `--since `, etc. — as documented per portal). A portal with **no recency flag** (jobdanmark offers none) still gets scoped: every portal's search output carries a `date` field, so filter client-side — drop results whose `date` is older than 14 days after the call returns, and never invent a flag the portal's SKILL.md does not document (the CLIs reject unknown flags). `--order PublicationDate` is a sort, and a sort is not a filter — pairing it with a `--limit` is a defensible approximation on a portal that offers nothing better (jobnet), but apply the client-side date filter on top all the same. 4. Cap results to ~20 per call using the portal's limit flag. 5. Use `--format json` for machine-readable output. -Run all portal CLI calls in parallel where possible using the Agent tool. Collect all `results` arrays into a single pool for Step 2, keeping each result tagged with its source portal skill (for Step 2 `detail` lookups). +Run all portal CLI calls in parallel where possible using the Agent tool. Collect all `results` arrays — across terms as well as portals — into a single pool for Step 2, keeping each result tagged with its source portal skill (for Step 2 `detail` lookups). A posting that several terms return appears once per term, and Step 2 only skips URLs already in `job_scraper/seen_jobs.json`, so deduplicate the pool by URL before Step 2. If a CLI tool exits with a non-zero code, log the error message and continue — do not abort the whole search. diff --git a/.claude/skills/job-scraper/search-queries.md b/.claude/skills/job-scraper/search-queries.md index 4bf2660c..0a71054e 100644 --- a/.claude/skills/job-scraper/search-queries.md +++ b/.claude/skills/job-scraper/search-queries.md @@ -25,12 +25,24 @@ Secondary (company career pages via Google): Queries are grouped by priority. Write **each category in every language from your Languages table** (see Language scope above). Combine each query with your location terms (e.g. your city, region, or metro area) where the site supports it. +Each priority has two kinds of block. **Plain terms** blocks are what the portal CLIs consume: one term per line, one CLI call per term, no `OR`, no `site:` prefix, and no location (the CLI's location flag, or the city in `--query` where that portal's SKILL.md says so, carries it). Quote a multi-word phrase only if the portal's `Query syntax:` line says quoting is honoured. **WebSearch-only** blocks are Google syntax (`site:`, quotes, `OR`) for the fallback path and are never passed to a CLI. + **Organize by function, not job title.** The same underlying work carries different titles across companies and markets (a "Data Scientist" role at one employer may be posted as "Insights Analyst" or "Data Consultant" at another). Name each priority category after the function it covers, and list several plausible job titles as query variants within that category rather than betting an entire priority tier on one exact title string. ### Priority 1: [YOUR_PRIMARY_ROLE_TYPE] These match your strongest and most desired career direction. +#### Plain terms (portal CLIs) + +``` +[YOUR_PRIMARY_JOB_TITLE_1] +[YOUR_PRIMARY_JOB_TITLE_2] +[YOUR_KEY_SKILL] +``` + +#### WebSearch-only (`site:` fallback) + ``` site:[YOUR_JOB_BOARD] "[YOUR_PRIMARY_JOB_TITLE_1]" [YOUR_CITY] site:[YOUR_JOB_BOARD] "[YOUR_PRIMARY_JOB_TITLE_2]" [YOUR_CITY] @@ -42,6 +54,15 @@ site:linkedin.com/jobs "[YOUR_PRIMARY_JOB_TITLE_1]" [YOUR_COUNTRY] These match your domain expertise. +#### Plain terms (portal CLIs) + +``` +[YOUR_DOMAIN_KEYWORD_1] +[YOUR_DOMAIN_KEYWORD_2] +``` + +#### WebSearch-only (`site:` fallback) + ``` site:[YOUR_JOB_BOARD] [YOUR_DOMAIN_KEYWORD_1] [YOUR_CITY] OR [YOUR_REGION] site:[YOUR_JOB_BOARD] [YOUR_DOMAIN_KEYWORD_2] [YOUR_COUNTRY] @@ -52,6 +73,15 @@ site:linkedin.com/jobs [YOUR_DOMAIN_KEYWORD_1] [YOUR_CITY] [YOUR_COUNTRY] Adjacent roles you could pivot into. +#### Plain terms (portal CLIs) + +``` +[YOUR_ADJACENT_TITLE_1] +[YOUR_ADJACENT_TITLE_2] +``` + +#### WebSearch-only (`site:` fallback) + ``` site:[YOUR_JOB_BOARD] "[YOUR_ADJACENT_TITLE_1]" [YOUR_KEY_SKILL] [YOUR_CITY] site:[YOUR_JOB_BOARD] "[YOUR_ADJACENT_TITLE_2]" [YOUR_KEY_SKILL] [YOUR_CITY] @@ -61,6 +91,15 @@ site:[YOUR_JOB_BOARD] "[YOUR_ADJACENT_TITLE_2]" [YOUR_KEY_SKILL] [YOUR_CITY] Wider net for general technical roles. +#### Plain terms (portal CLIs) + +``` +[YOUR_KEY_SKILL] developer +technical consultant +``` + +#### WebSearch-only (`site:` fallback) + ``` site:[YOUR_JOB_BOARD] [YOUR_KEY_SKILL] developer [YOUR_CITY] site:linkedin.com/jobs "[YOUR_KEY_SKILL] developer" [YOUR_CITY] diff --git a/tests/test_scrape_contract.py b/tests/test_scrape_contract.py index 185e289e..18b5c542 100644 --- a/tests/test_scrape_contract.py +++ b/tests/test_scrape_contract.py @@ -152,5 +152,36 @@ class SeenJobsDedupContinuityTests(unittest.TestCase): self.assertRegex(text, r"matched by URL or\s+company\+title") +# Step 1b sends one plain term per CLI call and takes quoting from the portal's +# own SKILL.md. The rule is data each portal carries, so the test derives the +# portal list from the skills tree and names no portal. +_QUERY_SYNTAX_LINE = re.compile(r"^Query syntax:\s*\S", re.MULTILINE) + + +class PortalQuerySyntaxTests(unittest.TestCase): + """Every installed portal SKILL.md must state its query syntax.""" + + def test_step1b_points_at_the_query_syntax_line(self): + text = SCRAPER_SKILL.read_text(encoding="utf-8") + step1b = re.search(r"#### 1b\..*?(?=#### 1c\.)", text, re.DOTALL) + self.assertIsNotNone(step1b, "Step 1b section not found") + self.assertIn("`Query syntax:`", step1b.group(0), "Step 1b no longer defers quoting to the portal's Query syntax line") + self.assertIn("inside the argument the CLI receives", step1b.group(0)) + self.assertIn("deduplicate the pool by URL before Step 2", step1b.group(0)) + + def test_every_installed_portal_skill_has_a_query_syntax_line(self): + self.assertTrue(PORTAL_CLIS, "no installed portal skills found") + missing = [ + portal.name + for portal in PORTAL_CLIS + if not _QUERY_SYNTAX_LINE.search((portal / "SKILL.md").read_text(encoding="utf-8")) + ] + self.assertEqual( + [], + missing, + "portal SKILL.md files without a `Query syntax:` line: " + ", ".join(missing), + ) + + if __name__ == "__main__": unittest.main()