From 8dbc941642f51c05cd401300a99357a2868e0c79 Mon Sep 17 00:00:00 2001 From: Ayobami Adegoke Date: Thu, 8 Oct 2026 20:21:28 +0100 Subject: [PATCH] test(apply): derive the host-check's installed portals from the skills tree (#528) SHIPPED_PORTAL_HOSTS was six names typed into the test and nothing tied it to anything: a portal added by /add-portal fetches from a host the copy never heard of, so its postings reached /apply classified "unverified" while the suite stayed green, and a dropped portal left a stale name behind. apply.md rule 1 defines an installed portal as "any configured job portal in .agents/skills/", so that tree is the source: the host literals in each *-search/cli/src/**/*.ts (one BASE_URL per shipped CLI, www. dropped so apex and subdomain both match). Discovery is by installation, not by the enabled: flag - an installed-but-disabled board is still a legitimate source for a pasted posting. classify_posting_host's default is the discovered set, and a new class asserts every installed portal has a discoverable host that classifies as installed_portal at the apex, under www. and under a subdomain. The ATS apex list stays the fixed set it is. Reported by Holo-Eter in #497; shape per the maintainer's note there. --- CHANGELOG.md | 16 ++++ tests/test_apply_host_check.py | 132 ++++++++++++++++++++++++++++++--- 2 files changed, 136 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2523daa4..832f077b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,22 @@ per-file diff commands. ### Fixed +- **The `/apply` host-check test derives its installed-portal hosts from the skills tree + instead of a hand-copied set** (#497, `tests/test_apply_host_check.py`) - `SHIPPED_PORTAL_HOSTS` + was six names typed into the test and nothing tied it to anything: a portal added by + `/add-portal` fetches from a host the copy never heard of, so its postings reached `/apply` + classified "unverified" while the suite stayed green, and a dropped portal left a stale + name behind. Rule 1 in `apply.md` defines an installed portal as "any configured job portal + in `.agents/skills/`", so that tree is now the source: the host literals in each + `*-search/cli/src/**/*.ts` (one `BASE_URL` per shipped CLI, `www.` dropped so apex and + subdomain both match), discovery by installation rather than by the `enabled:` flag + because an installed-but-disabled board is still a legitimate source for a pasted posting. + `classify_posting_host`'s default is that discovered set, and a new test class asserts that + every installed portal has a discoverable host and that each host classifies as + `installed_portal` at the apex, under `www.` and under a subdomain. The ATS apex list stays + the fixed set it is; parsing `apply.md`'s example list was deliberately not done, since a + parser keyed on prose wording is the next thing to go stale. Reported by Holo-Eter. + - **`/expand` and `/setup` now append behavioral findings to sections `02-behavioral-profile.md` actually has** (`.claude/commands/expand.md` Step 5, `.claude/commands/setup.md` Step A7, `tests/test_expand_command.py`, `tests/test_setup_command.py`) - both told the model to add diff --git a/tests/test_apply_host_check.py b/tests/test_apply_host_check.py index a54f8467..7d922d6c 100644 --- a/tests/test_apply_host_check.py +++ b/tests/test_apply_host_check.py @@ -6,6 +6,14 @@ Pins the invariants from the maintainer design in issue #431: myworkdayjobs.com (or workday.com), ashbyhq.com, smartrecruiters.com, workable.com. - Look-alike attacks (prefixes, suffixes, userinfo tricks) must fail closed. - Any other host must be named plainly in the output as unverified. + +The installed-portal half is derived, not copied (#497). Rule 1 in apply.md +defines an installed portal as "any configured job portal in `.agents/skills/`", +so the source of truth is the skills tree: the host literals in each +`*-search/cli/src/**/*.ts`. A hand-copied set in this file went stale the moment +a portal was added or dropped, and a new portal CLI fetching from a host the +copy lacked had its postings classified "unverified" with no test failing. +The ATS apex list stays the fixed set it is. """ import re @@ -15,6 +23,59 @@ from urllib.parse import urlparse REPO = Path(__file__).resolve().parent.parent APPLY_COMMAND_FILE = REPO / ".claude" / "commands" / "apply.md" +SKILLS_DIR = REPO / ".agents" / "skills" + +# A host literal in a CLI source file. Portal CLIs build every request URL from +# a BASE_URL / SEARCH_URL constant, so this is where the board's host lives. +HOST_LITERAL = re.compile(r"https?://([A-Za-z0-9.-]+\.[A-Za-z]{2,})") +# Hosts a source file may name that are not the portal: schema vocabularies, +# RFC example domains, loopback. Extend if a CLI ever embeds another one. +NON_PORTAL_HOSTS = {"schema.org", "www.w3.org", "example.com", "localhost"} + + +def installed_portal_skills() -> list[Path]: + """Every `*-search` skill with a CLI source tree, enabled or not. + + Rule 1 says "configured job portal in `.agents/skills/`" - a portal that is + installed but `enabled: false` for /scrape is still a legitimate board for + a posting the user pastes into /apply, so discovery is by installation. + """ + return sorted( + d for d in SKILLS_DIR.glob("*-search") if (d / "cli" / "src").is_dir() + ) + + +def portal_enabled(skill_dir: Path) -> bool: + """The `enabled:` frontmatter flag /scrape honours; a missing key means enabled.""" + skill_md = skill_dir / "SKILL.md" + if not skill_md.is_file(): + return True + match = re.search(r"^enabled:\s*(true|false)\b", skill_md.read_text(encoding="utf-8"), re.M) + return match is None or match.group(1) == "true" + + +def portal_hosts(skill_dir: Path) -> set[str]: + """The board hosts a portal CLI fetches from, as registrable names. + + A leading `www.` is dropped so the classifier's exact-or-subdomain rule + accepts both `jobindex.dk` and `www.jobindex.dk` (the CLI builds its URLs + with the latter; postings link to both). + """ + hosts: set[str] = set() + for source in (skill_dir / "cli" / "src").rglob("*.ts"): + for host in HOST_LITERAL.findall(source.read_text(encoding="utf-8")): + host = host.lower() + if host in NON_PORTAL_HOSTS: + continue + hosts.add(host[4:] if host.startswith("www.") else host) + return hosts + + +def discover_installed_portal_hosts() -> set[str]: + return set().union(*(portal_hosts(d) for d in installed_portal_skills())) + + +INSTALLED_PORTAL_HOSTS = discover_installed_portal_hosts() KNOWN_ATS_APEXES = { "greenhouse.io", @@ -26,24 +87,21 @@ KNOWN_ATS_APEXES = { "workable.com", } -SHIPPED_PORTAL_HOSTS = { - "jobindex.dk", - "linkedin.com", - "jobnet.dk", - "jobbank.dk", - "jobdanmark.dk", - "freehire.me", -} -def classify_posting_host(url_str: str, installed_portals: set[str] = SHIPPED_PORTAL_HOSTS) -> tuple[str, str]: +def classify_posting_host(url_str: str, installed_portals: set[str] | None = None) -> tuple[str, str]: """Reference implementation of the host provenance rule in /apply Step 1. + `installed_portals` defaults to the set discovered from the skills tree, so + the classifier can never lag a portal that /add-portal installed. + Returns (tier, host), where tier is one of: - 'installed_portal' - 'official_ats' - 'unverified' """ + if installed_portals is None: + installed_portals = INSTALLED_PORTAL_HOSTS try: parsed = urlparse(url_str) host = (parsed.hostname or "").lower().strip() @@ -102,17 +160,67 @@ class ApplyHostVerificationSpecTests(unittest.TestCase): self.assertEqual(tier, "official_ats", f"{url} should classify as official_ats, got {tier}") def test_classifier_identifies_installed_portal_hosts(self): + # A regional subdomain and the bare apex must both pass for every portal; + # the derived set is what makes this hold for a portal added by /add-portal. urls = [ "https://www.jobindex.dk/jobannonce/12345", - "https://www.linkedin.com/jobs/view/999999", + "https://dk.linkedin.com/jobs/view/999999", "https://jobnet.dk/find-job/8888", - "https://jobbank.dk/job/7777", - "https://freehire.me/job/6666", + "https://freehire.me/jobs/golang-zensar-2bxu6dxm", ] for url in urls: tier, host = classify_posting_host(url) self.assertEqual(tier, "installed_portal", f"{url} should classify as installed_portal, got {tier}") + def test_classifier_default_is_the_discovered_set_not_a_copy(self): + self.assertEqual(classify_posting_host.__defaults__, (None,)) + self.assertTrue(INSTALLED_PORTAL_HOSTS, "discovery found no portal hosts under .agents/skills/") + # An explicit set still overrides, which is how a fork's test can probe a candidate portal. + self.assertEqual( + classify_posting_host("https://jobs.newboard.test/x", {"newboard.test"})[0], "installed_portal" + ) + self.assertEqual(classify_posting_host("https://jobs.newboard.test/x")[0], "unverified") + + +class InstalledPortalsAreVerifiedTests(unittest.TestCase): + """Every installed portal CLI's board host must classify as installed_portal. + + This is the assertion the hand-copied set could not make: a portal added + by /add-portal fetches from a host the copy never heard of, and its + postings reached /apply flagged "unverified" while the suite stayed green. + """ + + def test_every_installed_portal_has_a_discoverable_host(self): + skills = installed_portal_skills() + self.assertTrue(skills, "no *-search skill with a cli/src tree under .agents/skills/") + for skill in skills: + with self.subTest(portal=skill.name): + self.assertTrue( + portal_hosts(skill), + f"{skill.name}/cli/src has no host literal - the classifier cannot verify its postings", + ) + + def test_every_installed_portals_host_classifies_as_installed_portal(self): + for skill in installed_portal_skills(): + for host in sorted(portal_hosts(skill)): + for url in (f"https://{host}/job/1", f"https://www.{host}/job/1", f"https://sub.{host}/job/1"): + with self.subTest(portal=skill.name, url=url, enabled=portal_enabled(skill)): + tier, _ = classify_posting_host(url) + self.assertEqual(tier, "installed_portal", f"{url} classified as {tier}") + + def test_portal_hosts_are_registrable_names_without_www(self): + for skill in installed_portal_skills(): + for host in portal_hosts(skill): + with self.subTest(portal=skill.name, host=host): + self.assertFalse(host.startswith("www."), host) + self.assertRegex(host, r"^[a-z0-9.-]+\.[a-z]{2,}$") + + def test_enabled_flag_reads_like_scrape(self): + # /scrape: "a missing key means enabled". Both shipped states occur in the tree. + states = {skill.name: portal_enabled(skill) for skill in installed_portal_skills()} + self.assertTrue(any(states.values()), f"no enabled portal among {states}") + self.assertIsInstance(portal_enabled(SKILLS_DIR / "does-not-exist-search"), bool) + def test_classifier_fails_closed_on_look_alikes_and_unverified_hosts(self): suspicious = [ "https://evil-greenhouse.io/job/1",