mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-09-17 08:36:25 +00:00
test(framework-version): cover the CI gate that had zero tests
check_framework_version.py guards fork-rebase safety (Gate E) and could be neutralised by a one-line change that reads as a refactor, with nothing in the repo noticing - a broken guard is silent by construction. Four new tests run the real script inside an isolated git repo: clean tree passes, unbumped edit fails, bumped edit passes, missing marker fails. Mutation-verified against the exact return-False disable the review demonstrated. Review finding F22 (2026-08-19). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
9a074b262d
commit
65fbe8b8a4
@@ -15,6 +15,12 @@ per-file diff commands.
|
|||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
|
- **Tests for `check_framework_version.py`** - the CI gate that stops a framework file
|
||||||
|
from being edited without a `framework_version` bump had zero tests, so the one-line
|
||||||
|
mutation `return meaningful_changes > 0` -> `return False` disabled it while the suite
|
||||||
|
stayed green. Four cases in the new `tests/test_check_framework_version.py` (clean
|
||||||
|
tree, unbumped edit, bumped edit, missing marker), each running the real script inside
|
||||||
|
an isolated git repo. Mutation-verified against that exact disable.
|
||||||
- **Tests for `lint_skills.py`'s skill and command checks** - only `check_settings()`
|
- **Tests for `lint_skills.py`'s skill and command checks** - only `check_settings()`
|
||||||
had coverage; the linter's main job (frontmatter keys, `allowed-tools` targets
|
had coverage; the linter's main job (frontmatter keys, `allowed-tools` targets
|
||||||
existing, the `# /<name>` command title rule) was unasserted, so deleting the
|
existing, the `# /<name>` command title rule) was unasserted, so deleting the
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
"""Guards for tools/check_framework_version.py - the CI gate itself.
|
||||||
|
|
||||||
|
This gate is what stops a PR from editing a profile-bearing framework
|
||||||
|
file without bumping `framework_version` (the fork-rebase safety marker).
|
||||||
|
It ran in CI with zero tests, so a one-line mutation
|
||||||
|
(`return meaningful_changes > 0` -> `return False`) disabled it while
|
||||||
|
the whole suite stayed green (review finding F22, 2026-08-19). A broken
|
||||||
|
guard is silent by construction: nothing fails, it just stops catching.
|
||||||
|
|
||||||
|
Each test builds an isolated git repo with the script copied inside it
|
||||||
|
(the script resolves ROOT from __file__), so the real repo is never read
|
||||||
|
or written.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
SCRIPT = REPO_ROOT / "tools" / "check_framework_version.py"
|
||||||
|
|
||||||
|
FRONTMATTER = "---\nframework_version: 1.0.0\n---\n"
|
||||||
|
BODY = "# Test framework file\n\nOriginal guidance sentence.\n"
|
||||||
|
|
||||||
|
|
||||||
|
class CheckerRepoFixture(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.root = Path(tempfile.mkdtemp())
|
||||||
|
self.addCleanup(shutil.rmtree, self.root, ignore_errors=True)
|
||||||
|
|
||||||
|
tools = self.root / "tools"
|
||||||
|
tools.mkdir()
|
||||||
|
shutil.copy(SCRIPT, tools / "check_framework_version.py")
|
||||||
|
|
||||||
|
self.skill_dir = self.root / ".claude" / "skills" / "job-application-assistant"
|
||||||
|
self.skill_dir.mkdir(parents=True)
|
||||||
|
self.framework_file = self.skill_dir / "01-test-profile.md"
|
||||||
|
self.framework_file.write_text(FRONTMATTER + BODY, encoding="utf-8")
|
||||||
|
|
||||||
|
self.git("init", "-q")
|
||||||
|
self.git("add", "-A")
|
||||||
|
self.git("commit", "-q", "-m", "base")
|
||||||
|
|
||||||
|
def git(self, *args):
|
||||||
|
subprocess.run(
|
||||||
|
["git", "-c", "user.name=test", "-c", "user.email=test@example.com", *args],
|
||||||
|
cwd=self.root,
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def run_checker(self):
|
||||||
|
# Strip GitHub Actions variables so get_base_commit() takes the
|
||||||
|
# local path (uncommitted changes vs HEAD) regardless of where the
|
||||||
|
# test suite itself runs.
|
||||||
|
env = {k: v for k, v in os.environ.items() if not k.startswith("GITHUB_")}
|
||||||
|
return subprocess.run(
|
||||||
|
[sys.executable, str(self.root / "tools" / "check_framework_version.py")],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FrameworkVersionGateTests(CheckerRepoFixture):
|
||||||
|
def test_clean_tree_passes(self):
|
||||||
|
result = self.run_checker()
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||||
|
self.assertIn("Framework Version Check: OK", result.stdout)
|
||||||
|
|
||||||
|
def test_unbumped_edit_fails(self):
|
||||||
|
self.framework_file.write_text(
|
||||||
|
FRONTMATTER + BODY + "\nA new sentence without a version bump.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = self.run_checker()
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
|
||||||
|
self.assertIn("modified without bumping 'framework_version'", result.stdout)
|
||||||
|
|
||||||
|
def test_bumped_edit_passes(self):
|
||||||
|
bumped = FRONTMATTER.replace("1.0.0", "1.0.1")
|
||||||
|
self.framework_file.write_text(
|
||||||
|
bumped + BODY + "\nA new sentence with a version bump.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = self.run_checker()
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||||
|
|
||||||
|
def test_file_without_version_marker_fails(self):
|
||||||
|
(self.skill_dir / "02-unmarked.md").write_text(
|
||||||
|
"# No frontmatter at all\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
result = self.run_checker()
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
|
||||||
|
self.assertIn("missing 'framework_version'", result.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user