diff --git a/tools/README_SALARY_TOOL.md b/tools/README_SALARY_TOOL.md new file mode 100755 index 0000000..6882638 --- /dev/null +++ b/tools/README_SALARY_TOOL.md @@ -0,0 +1,124 @@ +# Salary Benchmark Tool + +## What is this? + +The salary lookup tool (`salary_lookup.py`) lets you benchmark company salaries against a baseline from your own data. It's used during the `/apply` workflow to show how a company's compensation compares to market rates. + +**This tool is optional.** If you don't have salary data, the salary step is simply skipped during `/apply`. + +## How it works + +The tool reads a `salary_data.json` file in the repo root containing company salary benchmarks. It uses fuzzy matching to find companies by name, handling Danish/Nordic characters, legal suffixes (A/S, ApS), and common spelling variations. + +The data format supports any index-based or absolute salary data. For example: +- Index 100 = median salary, higher is better +- Absolute salary values in your currency +- Any custom metric you want to track + +## Data format + +The tool expects `salary_data.json` with this structure: + +```json +{ + "metadata": { + "source": "My Union Statistics 2025", + "index_baseline": 100, + "index_label": "Index", + "baseline_description": "Index 100 = median salary for private sector" + }, + "companies": [ + { + "company": "Novo Nordisk A/S", + "city": "Bagsværd", + "categories": { + "all_employees": { "count": 500, "index": 108.5 }, + "engineering": { "count": 120, "index": 112.3 } + } + }, + { + "company": "Ørsted A/S", + "city": "Fredericia", + "categories": { + "all_employees": { "count": 200, "index": 105.2 } + } + } + ] +} +``` + +### Fields + +- **metadata.source**: Where the data comes from (for reference) +- **metadata.index_baseline**: The baseline value (e.g., 100 for index-based data) +- **metadata.index_label**: Label for the index column in output +- **metadata.baseline_description**: Human-readable explanation of the baseline +- **companies[].company**: Company name (required) +- **companies[].city**: City/location (optional, used for filtering) +- **companies[].categories**: Named salary categories, each with `count` and/or `index` + +## Setup options + +### Option A: Create salary_data.json manually + +Create the file by hand with data from any source: union statistics, Glassdoor, salary surveys, networking, or personal research. + +### Option B: Convert from Excel + +If you have salary data in an Excel file: + +```bash +pip install openpyxl +python3 tools/convert_salary_excel.py path/to/salary-data.xlsx \ + --source "My Salary Data 2025" \ + --baseline 100 \ + --baseline-desc "Index 100 = median salary" +``` + +On Windows, use `py` if that is how Python is exposed on your PATH. If your system uses `python` instead of `python3`, substitute that in the examples. + +The converter auto-detects the Excel layout: +- Looks for a "Company"/"Firma" column and an optional "City"/"By" column +- Treats remaining columns as salary data (auto-pairs count/index columns) + +### Option C: Build from research + +Start with an empty template and add companies as you research them: + +```json +{ + "metadata": { + "source": "Personal research", + "index_baseline": 0, + "index_label": "Monthly salary (DKK)", + "baseline_description": "Approximate monthly salary before tax" + }, + "companies": [ + { + "company": "Example Corp", + "city": "Copenhagen", + "categories": { + "entry_level": { "index": 42000 }, + "senior": { "index": 55000 } + } + } + ] +} +``` + +## Usage + +```bash +python3 salary_lookup.py "Novo Nordisk" +python3 salary_lookup.py "Ørsted" --city "Fredericia" +python3 salary_lookup.py "COWI" --json +python3 salary_lookup.py --list-all +python3 salary_lookup.py --validate # pre-flight check your salary_data.json +``` + +## Important notes + +- The data file (`salary_data.json`) is **excluded from git** (see `.gitignore`). Your salary data may be proprietary or confidential. +- If the data file is missing, `salary_lookup.py` exits with a helpful error message and the `/apply` workflow skips the salary benchmark step. +- The fuzzy matcher handles Danish company name variations: legal suffixes, Nordic characters, anglicized spellings, and partial matches. +- `--validate` checks your data file for malformed category values and duplicate company names and prints a report, without performing a lookup. diff --git a/tools/check_framework_version.py b/tools/check_framework_version.py new file mode 100755 index 0000000..cfdea06 --- /dev/null +++ b/tools/check_framework_version.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""CI check: ensure that modified framework files have updated version markers. + +Fails if any markdown file under .claude/skills/job-application-assistant/ is +modified in git without a change/bump to its 'framework_version' frontmatter key. +Also ensures all framework files have a valid 'framework_version' frontmatter key. +""" + +from __future__ import annotations +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SKILL_DIR = ROOT / ".claude/skills/job-application-assistant" +FRAMEWORK_FILES = sorted(SKILL_DIR.glob("*.md")) + +# Add root AGENTS.md if it exists +root_agents = ROOT / "AGENTS.md" +if root_agents.exists(): + FRAMEWORK_FILES.append(root_agents) + +def run_git(args: list[str]) -> tuple[int, str, str]: + res = subprocess.run(["git"] + args, cwd=str(ROOT), capture_output=True, text=True) + return res.returncode, res.stdout, res.stderr + +def get_base_commit() -> str | None: + # If in GitHub Actions PR, use the target branch's base ref + base_ref = os.environ.get("GITHUB_BASE_REF") + if base_ref: + # Check if origin/base_ref is fetched + rc, _, _ = run_git(["rev-parse", "--verify", f"origin/{base_ref}"]) + if rc == 0: + return f"origin/{base_ref}" + # Try without origin/ + rc, _, _ = run_git(["rev-parse", "--verify", base_ref]) + if rc == 0: + return base_ref + + # If running in GitHub Actions but not a PR (e.g. push to master) + if os.environ.get("GITHUB_ACTIONS"): + rc, _, _ = run_git(["rev-parse", "--verify", "HEAD~1"]) + if rc == 0: + return "HEAD~1" + + # Otherwise (running locally), check uncommitted changes against HEAD + rc, _, _ = run_git(["rev-parse", "--verify", "HEAD"]) + if rc == 0: + return "HEAD" + + return None + +def parse_frontmatter(path: Path) -> dict: + if not path.exists(): + return {} + text = path.read_text(encoding="utf-8") + if not text.startswith("---\n"): + return {} + end = text.find("\n---", 4) + if end == -1: + return {} + + # Simple parser for YAML/frontmatter + data = {} + for line in text[4:end].splitlines(): + if ":" in line: + k, v = line.split(":", 1) + data[k.strip()] = v.strip().strip('"').strip("'") + return data + +def has_non_trivial_changes(file_path: Path, base_commit: str) -> bool: + # Get diff of the file from base_commit to HEAD + rel_path = str(file_path.relative_to(ROOT)) + rc, stdout, stderr = run_git(["diff", "-U0", base_commit, "--", rel_path]) + if rc != 0: + # If diff fails (e.g. file is new/untracked), it's a change + return True + + # Parse diff lines + # We want to count lines added/removed that: + # - do not match framework_version line + # - are not empty/whitespace only + meaningful_changes = 0 + version_changed = False + + for line in stdout.splitlines(): + if line.startswith("+++") or line.startswith("---") or line.startswith("@@"): + continue + if line.startswith("+") or line.startswith("-"): + content = line[1:].strip() + if not content: + continue + if re.match(r"^framework_version\s*:", content): + version_changed = True + continue + # Check if it's just frontmatter syntax (e.g. ---) + if content == "---": + continue + meaningful_changes += 1 + + # If the version key itself was modified, we don't fail, regardless of other changes + if version_changed: + return False + + # If there are meaningful changes but the version was not changed + return meaningful_changes > 0 + +def main() -> int: + errors = [] + + # 1. Lint: Check that all framework files have framework_version in frontmatter + for path in FRAMEWORK_FILES: + rel_path = str(path.relative_to(ROOT)) + fm = parse_frontmatter(path) + if "framework_version" not in fm: + errors.append(f"{rel_path}: missing 'framework_version' in frontmatter") + + # 2. Check for missing version bumps in modified files + base_commit = get_base_commit() + if base_commit: + print(f"Comparing HEAD against base commit: {base_commit}") + for path in FRAMEWORK_FILES: + rel_path = str(path.relative_to(ROOT)) + if "framework_version" not in parse_frontmatter(path): + # Skip checking changes if it doesn't even have frontmatter (already reported above) + continue + if has_non_trivial_changes(path, base_commit): + errors.append( + f"{rel_path}: modified without bumping 'framework_version'. " + f"Please update the version in the frontmatter." + ) + else: + print("No base commit found (e.g. initial commit or shallow clone without base branch). Skipping diff checks.") + + if errors: + print("Framework Version Check Failed:") + for err in errors: + print(f" - {err}") + return 1 + + print("Framework Version Check: OK") + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/check_upstream_updates.py b/tools/check_upstream_updates.py new file mode 100755 index 0000000..f1595b9 --- /dev/null +++ b/tools/check_upstream_updates.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Check for framework updates in the upstream repository. + +Usage: python tools/check_upstream_updates.py [--remote ] [--branch ] + +This script: +1. Identifies the upstream remote (defaults to 'upstream', falls back to 'origin'). +2. Fetches the latest commits from the upstream remote. +3. Compares the 'framework_version' in your local files under + .claude/skills/job-application-assistant/ with those in the upstream remote. +4. Alerts you if a file has been updated upstream with a newer version. +""" + +from __future__ import annotations +import argparse +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +FRAMEWORK_FILES = [ + ".claude/skills/job-application-assistant/01-candidate-profile.md", + ".claude/skills/job-application-assistant/02-behavioral-profile.md", + ".claude/skills/job-application-assistant/03-writing-style.md", + ".claude/skills/job-application-assistant/04-job-evaluation.md", + ".claude/skills/job-application-assistant/05-cv-templates.md", + ".claude/skills/job-application-assistant/06-cover-letter-templates.md", + ".claude/skills/job-application-assistant/07-interview-prep.md", + ".claude/skills/job-application-assistant/08-application-forms.md", + ".claude/skills/job-application-assistant/09-web-research.md", + ".claude/skills/job-application-assistant/SKILL.md", + "AGENTS.md", +] + +UPSTREAM_REPO_SLUG = "MadsLorentzen/ai-job-search" + +def run_git(args: list[str]) -> tuple[int, str, str]: + res = subprocess.run(["git"] + args, cwd=str(ROOT), capture_output=True, text=True) + return res.returncode, res.stdout, res.stderr + +def get_remote_url(remote_name: str) -> str: + rc, stdout, _ = run_git(["remote", "get-url", remote_name]) + return stdout.strip() if rc == 0 else "" + +def get_framework_version_from_text(text: str) -> str | None: + if not text.startswith("---\n"): + return None + end = text.find("\n---", 4) + if end == -1: + return None + for line in text[4:end].splitlines(): + if ":" in line: + k, v = line.split(":", 1) + if k.strip() == "framework_version": + return v.strip().strip('"').strip("'") + return None + +def parse_semver(version_str: str) -> tuple[int, int, int]: + # Clean version string (e.g. remove 'v' prefix) + match = re.match(r"^v?(\d+)\.(\d+)\.(\d+)", version_str) + if not match: + return (0, 0, 0) + return tuple(int(x) for x in match.groups()) + +def main() -> int: + parser = argparse.ArgumentParser(description="Check for framework updates upstream.") + parser.add_argument("--remote", default="upstream", help="Name of the git remote for upstream (default: upstream)") + parser.add_argument("--branch", default="master", help="Branch name of the upstream repo (default: master)") + parser.add_argument("--no-fetch", action="store_true", help="Skip fetching from remote") + args = parser.parse_args() + + # Verify remote exists + rc, stdout, _ = run_git(["remote"]) + remotes = stdout.splitlines() + remote = args.remote + if remote not in remotes: + if "origin" in remotes: + print(f"Warning: Remote '{remote}' not found. Falling back to 'origin'.") + remote = "origin" + else: + print("Error: No git remotes found.") + return 1 + + # A fork's own 'origin' can never reveal upstream updates: warn so the + # user is not misled by the final '[OK]' line below. (Direct clones of + # the template repo have origin == the upstream repo, so no warning.) + # GitHub serves repo paths case-insensitively, so compare lowercased. + if remote != args.remote and UPSTREAM_REPO_SLUG.lower() not in get_remote_url(remote).lower(): + print( + f"Warning: Remote '{remote}' does not point to the ai-job-search " + f"template repo ({UPSTREAM_REPO_SLUG}), so this check compares your " + f"fork against itself and will never report upstream updates. " + f"Add the template repo as a remote to track upstream changes, e.g.:\n" + f" git remote add upstream https://github.com/{UPSTREAM_REPO_SLUG}.git" + ) + + if not args.no_fetch: + print(f"Fetching latest from remote '{remote}'...") + rc, _, stderr = run_git(["fetch", remote]) + if rc != 0: + print(f"Warning: Failed to fetch from remote '{remote}': {stderr.strip()}") + print("Proceeding with cached remote tracking branches.") + + ref = f"{remote}/{args.branch}" + # Verify ref exists + rc, _, _ = run_git(["rev-parse", "--verify", ref]) + if rc != 0: + print(f"Error: Ref '{ref}' does not exist. Make sure you fetched and specified the correct branch.") + return 1 + + print(f"Comparing local files against upstream '{ref}'...\n") + + updates_available = [] + errors = [] + missing_upstream = [] + + for rel_path in FRAMEWORK_FILES: + local_path = ROOT / rel_path + if not local_path.exists(): + print(f"Local file missing: {rel_path}") + continue + + # Get local version + local_text = local_path.read_text(encoding="utf-8") + local_ver = get_framework_version_from_text(local_text) + + # Get upstream version + rc, upstream_text, git_err = run_git(["show", f"{ref}:{rel_path}"]) + if rc != 0: + # A file present locally but missing from the upstream ref means + # it was renamed or deleted upstream; any other git failure means + # the comparison is incomplete. Either way, never report a clean + # '[OK]' while silently skipping the file. + if "does not exist" in git_err or "exists on disk, but not in" in git_err: + missing_upstream.append(rel_path) + else: + errors.append(f"Failed to read upstream version of {rel_path}: {git_err.strip()}") + continue + + upstream_ver = get_framework_version_from_text(upstream_text) + + if not local_ver: + errors.append(f"Local file {rel_path} is missing 'framework_version' in frontmatter.") + continue + if not upstream_ver: + continue + + if parse_semver(upstream_ver) > parse_semver(local_ver): + updates_available.append({ + "filename": Path(rel_path).name, + "local": local_ver, + "upstream": upstream_ver, + "path": rel_path + }) + + + if errors: + print("Configuration errors:") + for err in errors: + print(f" - {err}") + print() + + if missing_upstream: + print("Files present locally but missing from the upstream ref (possibly renamed or deleted upstream):") + for path in missing_upstream: + print(f" - {path}") + print() + + if updates_available: + print("[UPDATE] Upstream updates available for framework methodology files:") + for up in updates_available: + print(f" - {up['filename']}: local {up['local']} < upstream {up['upstream']}") + print(f" Diff command: git diff {ref} -- {up['path']}") + print() + print("Review these changes to see if they fit your personalized fork!") + else: + if errors or missing_upstream: + print( + f"[WARNING] Framework check incomplete against {ref}: " + f"{len(errors)} configuration error(s), {len(missing_upstream)} file(s) missing upstream. " + "Review the messages above before assuming you are up to date." + ) + else: + print(f"[OK] All framework files are up to date with {ref}!") + # Version stamps answer "which of my files changed"; commit-level triage + # answers "which upstream commits deserve review". Point at the companion. + print( + f"\nFor commit-level triage of upstream commits, run: " + f"python3 tools/upstream_triage.py --remote {remote}" + ) + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/convert_salary_excel.py b/tools/convert_salary_excel.py new file mode 100755 index 0000000..2e8b6bb --- /dev/null +++ b/tools/convert_salary_excel.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +""" +Convert salary data from Excel to JSON format. + +This script converts an Excel file containing company salary data +into the JSON format expected by salary_lookup.py. + +Prerequisites: + pip install openpyxl + +Usage: + python tools/convert_salary_excel.py + python tools/convert_salary_excel.py --source "My Union Stats 2025" + python tools/convert_salary_excel.py --baseline 100 --baseline-desc "Index 100 = median salary" + +The output file (salary_data.json) will be written to the repository root. + +Expected Excel format: + - A header row with column names + - A "Company" or "Firma" column (required) + - An optional "City" or "By" column + - Any number of numeric data columns (salary index, count, etc.) + +The script auto-detects the header row and column layout. For Excel files +with paired count/index columns per category, it groups them automatically. +""" + +import json +import sys +import argparse +import re +from pathlib import Path + +try: + import openpyxl +except ImportError: + openpyxl = None + + +# Column name patterns for auto-detection +COMPANY_PATTERNS = {"firma", "company", "virksomhed", "employer", "arbejdsgiver"} +CITY_PATTERNS = {"by", "city", "kommune", "location", "lokation", "sted"} +COUNT_PATTERNS = {"antal", "count", "number", "n", "employees", "medarbejdere"} +INDEX_PATTERNS = {"indeks", "index", "idx", "salary", "løn", "median", "average", "gennemsnit"} +# "Compound" tokens: pattern words allowed to match as a substring of a larger +# header token, for languages that glue words together (e.g. Danish "lønindeks" +# -> løn + indeks). Languages that write headers as separate words need none. +# Ships populated for this repo's Danish demonstration data; a fork targeting +# another locale edits this constant. +COMPOUND_PATTERNS = {"antal", "indeks", "løn", "gennemsnit", "medarbejdere"} +# Identifier columns (employee id, Danish "personnummer", etc.) are never salary +# data. They are dropped at classification so they are not mistaken for a salary +# category. Matched as whole tokens only, like other pattern sets. +ID_PATTERNS = {"id", "personnummer"} + + +def parse_numeric_cell(value): + """Parse numeric Excel values, including localized string cells.""" + if isinstance(value, (int, float)): + return float(value) + if not isinstance(value, str): + raise ValueError("not numeric") + + text = value.strip().replace("\u00a0", " ").replace(" ", "") + if not text: + raise ValueError("not numeric") + if "," in text and "." in text: + # The separator that appears last is the decimal separator: European + # "1.234,56" and US "1,234.56" are both unambiguous here, unlike the + # single-separator cases below. + if text.rfind(",") > text.rfind("."): + text = text.replace(".", "").replace(",", ".") + else: + text = text.replace(",", "") + elif "," in text: + if re.fullmatch(r"[+-]?\d+,\d{3}", text): + raise ValueError("ambiguous comma separator") + text = text.replace(",", ".") + elif "." in text: + if re.fullmatch(r"[+-]?\d+\.\d{3}", text): + raise ValueError("ambiguous dot separator") + return float(text) + + +def header_matches(header, patterns): + """Return True when a header contains a meaningful pattern match. + + Patterns match whole tokens; any pattern also listed in + ``COMPOUND_PATTERNS`` may additionally match as a substring, to handle + languages that form compound words. + """ + h = header.lower().strip() + tokens = set(re.findall(r"[a-zæøåöäü0-9]+", h)) + + for p in patterns: + if p in tokens: + return True + if p in COMPOUND_PATTERNS and p in h: + return True + return False + + +def strip_type_patterns(header, patterns): + """Remove count/index words from a header to derive a category name. + + Mirrors ``header_matches``: patterns strip as whole tokens, and any + pattern also listed in ``COMPOUND_PATTERNS`` additionally strips as a + substring - otherwise a compound header like "Lønindeks alle" keeps the + type word in its category name and can never pair with "Antal alle". + """ + name = header.lower() + for p in patterns: + name = re.sub(rf"(?= len(row) or not row[company_col]: + continue + + company_name = str(row[company_col]).strip() + if city_col is not None and city_col < len(row) and row[city_col]: + city_name = str(row[city_col]).strip() + else: + city_name = "" + + entry = { + "company": company_name, + "city": city_name, + "categories": {}, + } + + for cat in categories: + cat_name = cat["name"] + if "count_col" in cat and "index_col" in cat: + count_val = None + index_val = None + if cat["count_col"] < len(row) and row[cat["count_col"]] is not None: + try: + count_val = int(parse_numeric_cell(row[cat["count_col"]])) + except (ValueError, TypeError): + pass + if cat["index_col"] < len(row) and row[cat["index_col"]] is not None: + try: + index_val = parse_numeric_cell(row[cat["index_col"]]) + except (ValueError, TypeError): + pass + # A count/index pair that is entirely empty for this row carries + # no salary information, so skip it rather than emit nulls. + if count_val is None and index_val is None: + continue + entry["categories"][cat_name] = {"count": count_val, "index": index_val} + elif "value_col" in cat: + if cat["value_col"] < len(row) and row[cat["value_col"]] is not None: + val = row[cat["value_col"]] + try: + val = parse_numeric_cell(val) + except (ValueError, TypeError): + # Non-numeric standalone value (e.g. a free-text "Notes" + # column) is not salary data; skip it for this row. + continue + field = cat.get("field", "index") + entry["categories"][cat_name] = {field: int(val) if field == "count" else val} + + companies.append(entry) + + return companies + + +def main(): + parser = argparse.ArgumentParser( + description="Convert salary Excel data to JSON" + ) + parser.add_argument("excel_file", help="Path to the Excel file with salary data") + parser.add_argument( + "--output", default=None, + help="Output JSON file path (default: salary_data.json in repo root)", + ) + parser.add_argument( + "--source", default=None, + help="Name of the data source (e.g., 'Union Statistics 2025')", + ) + parser.add_argument( + "--baseline", type=float, default=100, + help="Baseline value for index comparison (default: 100)", + ) + parser.add_argument( + "--baseline-desc", default=None, + help="Description of what the baseline means (e.g., 'Index 100 = median salary')", + ) + args = parser.parse_args() + + excel_path = Path(args.excel_file) + if not excel_path.exists(): + print(f"Error: File not found: {excel_path}", file=sys.stderr) + sys.exit(1) + + if openpyxl is None: + print("Error: openpyxl is required. Install it with: pip install openpyxl", file=sys.stderr) + sys.exit(1) + + output_path = Path(args.output) if args.output else Path(__file__).parent.parent / "salary_data.json" + + print(f"Reading: {excel_path}") + wb = openpyxl.load_workbook(excel_path, read_only=True, data_only=True) + + all_companies = [] + for sheet_name in wb.sheetnames: + print(f" Parsing sheet: {sheet_name}") + ws = wb[sheet_name] + companies = parse_sheet(ws, sheet_label=sheet_name) + all_companies.extend(companies) + + wb.close() + + if not all_companies: + print("Error: No data could be parsed from the Excel file.", file=sys.stderr) + print("Make sure the Excel file has a header row with a 'Company'/'Firma' column.", file=sys.stderr) + sys.exit(1) + + # Build output + output = { + "metadata": { + "source": args.source or excel_path.stem, + "index_baseline": args.baseline, + "index_label": "Index", + "baseline_description": args.baseline_desc or f"Index {args.baseline} = baseline", + }, + "companies": all_companies, + } + + with open(output_path, "w", encoding="utf-8") as f: + json.dump(output, f, ensure_ascii=False, indent=2) + + print(f"\nDone! Wrote {len(all_companies)} company entries to {output_path}") + + +if __name__ == "__main__": + main() diff --git a/tools/lint_skills.py b/tools/lint_skills.py new file mode 100755 index 0000000..cf4a6c0 --- /dev/null +++ b/tools/lint_skills.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Lint the repo's skill, command, and settings files. + +Run from anywhere: python tools/lint_skills.py + +Checks: +- Every SKILL.md (.claude/skills/*, .agents/skills/*) has YAML frontmatter that + parses, with non-empty `name` and `description` keys +- `allowed-tools` entries of the form `Bash(bun run *)` point at files + that exist (skill paths resolve relative to the repo root and to .agents/) +- Every .claude/commands/*.md starts with a `# /` title +- .claude/settings.json is valid JSON with a permissions.allow list + +Exit code 0 on success, 1 with a failure list otherwise. +""" + +import json +import re +import sys +from pathlib import Path + +try: + import yaml +except ImportError: + sys.exit("lint_skills.py requires PyYAML: pip install pyyaml") + +ROOT = Path(__file__).resolve().parent.parent +errors: list[str] = [] + + +def rel(path: Path) -> str: + return str(path.relative_to(ROOT)) + + +def check_skill(path: Path) -> None: + text = path.read_text(encoding="utf-8") + if not text.startswith("---\n"): + errors.append(f"{rel(path)}: missing YAML frontmatter (file must start with ---)") + return + end = text.find("\n---", 4) + if end == -1: + errors.append(f"{rel(path)}: unterminated YAML frontmatter") + return + try: + data = yaml.safe_load(text[4:end]) + except yaml.YAMLError as exc: + errors.append(f"{rel(path)}: frontmatter is not valid YAML: {exc}") + return + if not isinstance(data, dict): + errors.append(f"{rel(path)}: frontmatter did not parse to a mapping") + return + for key in ("name", "description"): + if not data.get(key): + errors.append(f"{rel(path)}: frontmatter missing required key '{key}'") + + allowed = data.get("allowed-tools", "") + if isinstance(allowed, str): + for match in re.finditer(r"bun run ([^\s)]+)", allowed): + target = match.group(1).rstrip("*") + if not target or target.endswith("/"): + continue + # Targets may contain globs (e.g. .agents/skills/*/cli/src/cli.ts); + # require at least one existing file to match. + if "*" in target: + if not list(ROOT.glob(target)) and not list((ROOT / ".agents").glob(target)): + errors.append(f"{rel(path)}: allowed-tools glob matches no files: {target}") + else: + candidates = [ROOT / target, ROOT / ".agents" / target] + if not any(c.is_file() for c in candidates): + errors.append(f"{rel(path)}: allowed-tools references a missing file: {target}") + + +def check_command(path: Path) -> None: + lines = path.read_text(encoding="utf-8").lstrip().splitlines() + first = lines[0] if lines else "" + if not first.startswith("# /"): + errors.append(f"{rel(path)}: command file must start with a '# /' title (found: {first[:50]!r})") + + +def check_settings() -> None: + path = ROOT / ".claude" / "settings.json" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + errors.append(f".claude/settings.json: {exc}") + return + if not isinstance(data, dict): + errors.append(".claude/settings.json: expected top-level JSON value to be an object") + return + permissions = data.get("permissions", {}) + if not isinstance(permissions, dict): + errors.append(".claude/settings.json: expected permissions to be an object") + return + if not isinstance(permissions.get("allow"), list): + errors.append(".claude/settings.json: expected permissions.allow to be a list") + + +def main() -> int: + skills = sorted(ROOT.glob(".claude/skills/*/SKILL.md")) + sorted(ROOT.glob(".agents/skills/*/SKILL.md")) + commands = sorted((ROOT / ".claude" / "commands").glob("*.md")) + if not skills: + errors.append("no SKILL.md files found - glob roots are wrong or the tree moved") + if not commands: + errors.append("no command files found under .claude/commands/") + + for skill in skills: + check_skill(skill) + for command in commands: + check_command(command) + check_settings() + + if errors: + print(f"lint_skills: {len(errors)} failure(s)") + for err in errors: + print(f" - {err}") + return 1 + print(f"lint_skills: OK ({len(skills)} skills, {len(commands)} commands, settings.json)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/robots_check.py b/tools/robots_check.py new file mode 100755 index 0000000..b59c972 --- /dev/null +++ b/tools/robots_check.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Decide whether the browser-header curl retry in 09-web-research.md may run. + +The retry exists to get past bot-filtering firewalls on sites whose robots.txt +permits access. It is never used to override a site that has said no. + +WebFetch identifies itself as Claude-User and honors robots.txt, so a 403 has +two very different causes: a WAF default on a site whose published policy +allows access, or a site that has actually declined. This tells them apart. + +Rules implemented (RFC 9309), deliberately on the cautious side: + * longest-match wins; on equal specificity Disallow wins + * a Disallow for either "*" or "Claude-User" blocks the retry + * blank lines inside a record do not end it (Python's robotparser drops + rules in that case, which fails open - see tests) + * 404 means no published policy, which is permission + * any other failure to read robots.txt leaves permission unconfirmed, + and the retry does not happen + +Usage: python3 tools/robots_check.py +Exit 0 = the retry may proceed. Exit 1 = do not retry; go to escalation step 3. +""" + +import re, subprocess, sys +from urllib.parse import urlsplit, unquote + +BROWSER = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36') + +def _fetch(url, ua): + """curl, not urllib: some hosts (jobup.ch) hang urllib indefinitely while + answering curl in under a second, and --max-time is a hard ceiling.""" + # "--" terminates option parsing, so a URL beginning with a dash can never + # be read by curl as a flag. gate() rebuilds the target as + # scheme://host/robots.txt before calling here, so this is hardening for + # direct callers rather than a hole in the gate path itself. + r = subprocess.run( + ['curl', '-sS', '-L', '--max-redirs', '5', '--max-time', '12', '-A', ua, + '-H', 'Accept: text/plain,*/*', '-w', '\n%{http_code}', '--', url], + capture_output=True, text=True, timeout=20) + if r.returncode != 0: + raise RuntimeError('curl exit %d' % r.returncode) + body, _, code = r.stdout.rpartition('\n') + return body, int(code or 0) + + +def is_robots_body(text): + """Does this actually look like a robots.txt? + + A misconfigured host can answer /robots.txt with 200 and an HTML error page. + That body parses to zero rules, and zero rules read as "allowed" - so a + soft-200 granted permission that was never given. An empty or whitespace-only + body IS a valid allow-all under RFC 9309 and stays allowed; a non-empty body + with no recognised directive is treated as unreadable. + """ + if not text.strip(): + return True + for raw in text.splitlines(): + line = raw.split('#', 1)[0].strip().lower() + if ':' in line and line.split(':', 1)[0].strip() in ( + 'user-agent', 'allow', 'disallow', 'sitemap', 'crawl-delay', 'host', + ): + return True + return False + +def _groups(text): + """user-agent -> [(is_allow, pattern)], tolerating blank lines inside a record.""" + out, agents, expect = {}, [], True + for raw in text.splitlines(): + line = raw.split('#', 1)[0].strip() + if not line or ':' not in line: + continue + field, _, value = line.partition(':') + field, value = field.strip().lower(), value.strip() + if field == 'user-agent': + if not expect: + agents, expect = [], True + agents.append(value.lower()) + out.setdefault(value.lower(), []) + elif field in ('allow', 'disallow') and agents: + expect = False + for a in agents: + out[a].append((field == 'allow', value)) + return out + +def _match(pattern, path): + """RFC 9309 wildcard match; returns match length or -1. + + The pattern is percent-decoded to match the already-decoded path. Without + this, "Disallow: /foo%20bar" never matched "/foo bar" and the rule was + silently skipped - a fail-open on any site that encodes its own rules. + """ + if pattern == '': + return -1 + pattern = unquote(pattern) + rx = '^' + ''.join('.*' if c == '*' else ('$' if c == '$' else re.escape(c)) for c in pattern) + return len(pattern) if re.match(rx, path) else -1 + +def allowed(text, agent, path): + g = _groups(text) + rules = g.get(agent.lower()) or g.get('*') or [] + best_len, best_allow = -1, True + for is_allow, pat in rules: + n = _match(pat, path) + if n > best_len or (n == best_len and n >= 0 and not is_allow): + best_len, best_allow = n, is_allow # ties -> Disallow wins (cautious) + return True if best_len < 0 else best_allow + +def gate(url): + parts = urlsplit(url) + path = unquote(parts.path) or '/' + if parts.query: + path += '?' + parts.query + robots = f'{parts.scheme}://{parts.netloc}/robots.txt' + body, last = None, 'no attempt' + for ua in ('Claude-User', BROWSER): + try: + text, code = _fetch(robots, ua) + except Exception as e: + last = type(e).__name__; continue + if code == 404: + return 0, 'ALLOWED - no robots.txt published' + if code == 200: + if not is_robots_body(text): + last = 'HTTP 200 but the body is not a robots.txt' + continue + body = text; break + last = 'HTTP %d' % code + if body is None: + return 1, 'UNCONFIRMED (%s) - do not retry, go to step 3' % last + for a in ('Claude-User', '*'): + if not allowed(body, a, path): + return 1, f'DISALLOWED for {a} - do not retry, go to step 3' + return 0, 'ALLOWED - robots.txt permits this path' + +if __name__ == '__main__': + if len(sys.argv) != 2: + print('usage: python3 tools/robots_check.py ', file=sys.stderr) + sys.exit(2) + rc, msg = gate(sys.argv[1]) + print(msg) + sys.exit(rc) diff --git a/tools/security_guards.py b/tools/security_guards.py new file mode 100755 index 0000000..2111781 --- /dev/null +++ b/tools/security_guards.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +"""Supply-chain guards for the template's riskiest surfaces. + +Run from anywhere: python tools/security_guards.py + +This repo ships pre-approved Claude Code permissions and CLI code that every +fork user executes. These guards make the dangerous changes LOUD, not +impossible: a PR that intentionally needs one of them must update the +allowlists in this file in the same diff, so the change is explicit and +reviewable rather than buried. + +Checks: +1. .claude/settings.json — every permissions.allow entry must be in the exact + allowlist below. Catches permission widening (e.g. Bash(*), Bash(curl:*)), + which would auto-approve commands on every fork. The same file's `hooks` + key is held to an allowlist too: a hook runs automatically when its event + fires, with no prompt, so it is strictly more dangerous than a pre-approved + permission. +2. .gitignore — the personal-data ignore rules must all still be present, + and no un-allowlisted negation (!pattern) may re-include them. Catches + weakening that would make future users silently commit their tracker, + profile exports, or application archives. +3. .agents/**/package.json — no npm/bun lifecycle scripts (preinstall, + install, postinstall, prepare, prepack) and no trustedDependencies. + Catches code execution smuggled into `bun install`. + +Stdlib only. Exit 0 on success, 1 with a failure list otherwise. +""" + +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +errors: list[str] = [] + +# The exact permission entries the template ships. A PR that adds or changes +# an entry must add it here too - that is the point: the diff shows both. +ALLOWED_PERMISSIONS = { + "Skill(job-application-assistant)", + "Bash(bun run:*)", + "Bash(python salary_lookup.py:*)", + "Bash(python3 salary_lookup.py:*)", + "Bash(pdftotext:*)", +} + +# Personal-data ignore rules that must never disappear from .gitignore. +REQUIRED_IGNORE_RULES = [ + "salary_data.json", + # Depth-independent: the job-scraper skill resolves `job_scraper/` relative + # to its own directory, so the state file lands under .claude/skills/... and + # a repo-rooted rule silently fails to match it. + "**/job_scraper/seen_jobs.json", + "**/job_scraper/notion_sync.json", + "**/job_scraper/*.md", + "*_BehavioralReport.pdf", + "linkedin_Profile.pdf", + "cv/main_*.*", + "!cv/main_example.tex", + # ATS text extractions (/apply step 5d) carry the CV's full text. + "cv/*.txt", + "cover_letters/cover_*.*", + # /apply also recognizes the uppercase Cover_* naming variant. + "cover_letters/Cover_*.*", + "documents/cv/**", + "documents/linkedin/**", + "documents/diplomas/**", + "documents/references/**", + "documents/applications/**", + "documents/postings/**", + "documents/interview/**", + "job_search_tracker.csv", + "gmail_sync/", + "reports/", + "upskill/*.md", + # Depth-independent twin of the rule above. The upskill *skill* resolves + # `upskill/` relative to its own directory - the same observed behavior + # the **/job_scraper rules exist for - so reports can land at + # .claude/skills/upskill/upskill/*.md where the rooted rule cannot see + # them. `**/upskill/*.md` would also ignore the skill's own SKILL.md + # (the directory shares the name), so the report-file prefix is pinned. + "**/upskill/report-*.md", + # Not personal data but the same failure mode: /add-portal can generate a + # skill for a portal that only returns usable content through a paid + # fetching service, and that skill reads an API token from the environment. + ".env", + ".env.*", +] + +# Negation (re-include) rules the template legitimately ships. .gitignore is +# order-sensitive: a later `!pattern` re-includes a path an earlier rule +# excluded, so a rule can be physically present in REQUIRED_IGNORE_RULES yet +# no longer ignored (e.g. adding `!salary_data.json`). Set membership on the +# required rules cannot see that. Any negation outside this allowlist is a +# failure - add an intentional one here in the same PR, exactly as with +# ALLOWED_PERMISSIONS, so the widening is explicit and reviewable. +ALLOWED_IGNORE_NEGATIONS = { + "!cover_letters/OpenFonts/fonts/**", + "!cv/main_example.tex", + "!cover_letters/cover_example.tex", + "!documents/**/.gitkeep", +} + +# Hook commands the template legitimately ships, as ":" strings. +# Empty by design - the template ships no hooks at all. +# +# A hook is strictly more dangerous than a permissions.allow entry. A permission +# pre-approves something Claude may choose to do; a hook runs unconditionally when +# its event fires, with no prompt and no model decision in between. Cloning a repo +# and opening it is enough. This is the vector the Shai-Hulud worm used in its +# August 2026 wave, planting a SessionStart hook in .claude/settings.json that +# executed on session start: +# https://research.jfrog.com/post/shai-hulud-is-back-august/ +ALLOWED_HOOKS: set[str] = set() + +FORBIDDEN_SCRIPTS = {"preinstall", "install", "postinstall", "prepare", "prepack"} + + +def _hook_commands(event: str, entries: object): + """Yield ":" for every command a hook event would run. + + Fails closed: any shape this does not recognise yields a marker that cannot + be in the allowlist, so an unfamiliar hook layout is rejected rather than + silently skipped. + """ + unrecognised = f"{event}:" + if not isinstance(entries, list): + yield unrecognised + return + for entry in entries: + if not isinstance(entry, dict): + yield unrecognised + continue + inner = entry.get("hooks") + if not isinstance(inner, list): + yield unrecognised + continue + for hook in inner: + command = hook.get("command") if isinstance(hook, dict) else None + yield f"{event}:{command}" if isinstance(command, str) else unrecognised + + +def check_permissions() -> None: + path = ROOT / ".claude" / "settings.json" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + errors.append(f".claude/settings.json: unreadable or invalid JSON: {exc}") + return + if not isinstance(data, dict): + errors.append(".claude/settings.json: top-level JSON value must be an object") + return + + # Checked before the permissions shape guards below, so a file that pairs a + # malformed permissions block with a hook cannot return early and skip this. + hooks = data.get("hooks", {}) + if hooks: + if not isinstance(hooks, dict): + errors.append(".claude/settings.json: hooks must be an object") + else: + for event, entries in hooks.items(): + for command in _hook_commands(str(event), entries): + if command not in ALLOWED_HOOKS: + errors.append( + f".claude/settings.json: hook not in the reviewed allowlist: " + f"{command!r}. A hook runs automatically when its event fires - it " + "is never gated by the permissions prompt, so it executes on every " + "fork without the user agreeing to anything. If this hook is " + "intentional, add it to ALLOWED_HOOKS in tools/security_guards.py " + "in the same PR so the addition is explicit and reviewable." + ) + + permissions = data.get("permissions", {}) + if not isinstance(permissions, dict): + errors.append(".claude/settings.json: permissions must be an object") + return + allow = permissions.get("allow", []) + if not isinstance(allow, list) or not all(isinstance(entry, str) for entry in allow): + errors.append(".claude/settings.json: permissions.allow must be a list of strings") + return + for entry in allow: + if entry not in ALLOWED_PERMISSIONS: + errors.append( + f".claude/settings.json: permission not in the reviewed allowlist: {entry!r}. " + "Pre-approved permissions run without prompting on every fork. If this entry is " + "intentional, add it to ALLOWED_PERMISSIONS in tools/security_guards.py in the " + "same PR so the widening is explicit and reviewable." + ) + for entry in ALLOWED_PERMISSIONS - set(allow): + # Not an error: settings may legitimately drop an entry. But an + # allowlist entry that no longer exists should be pruned. + print(f"note: allowlisted permission not present in settings.json: {entry!r}") + + +def check_gitignore() -> None: + path = ROOT / ".gitignore" + try: + lines = [line.strip() for line in path.read_text(encoding="utf-8").splitlines()] + except OSError as exc: + errors.append(f".gitignore: unreadable: {exc}") + return + rules = set(lines) + for rule in REQUIRED_IGNORE_RULES: + if rule not in rules: + errors.append( + f".gitignore: required personal-data rule missing: {rule!r}. " + "These rules keep fork users from committing personal data. If the rule moved " + "or was renamed intentionally, update REQUIRED_IGNORE_RULES in " + "tools/security_guards.py in the same PR." + ) + for line in lines: + if line.startswith("!") and line not in ALLOWED_IGNORE_NEGATIONS: + errors.append( + f".gitignore: negation rule not in the reviewed allowlist: {line!r}. " + "A negation re-includes a path an earlier rule excluded and can silently " + "re-expose personal data (a required ignore rule stays present but stops " + "taking effect). If this negation is intentional, add it to " + "ALLOWED_IGNORE_NEGATIONS in tools/security_guards.py in the same PR." + ) + + +def check_package_manifests() -> None: + manifests = [ + p for p in ROOT.glob(".agents/**/package.json") if "node_modules" not in p.parts + ] + if not manifests: + errors.append(".agents: no package.json files found - glob roots are wrong or the tree moved") + for manifest in manifests: + relpath = manifest.relative_to(ROOT) + try: + data = json.loads(manifest.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + errors.append(f"{relpath}: unreadable or invalid JSON: {exc}") + continue + if not isinstance(data, dict): + errors.append(f"{relpath}: top-level JSON value must be an object") + continue + scripts = data.get("scripts", {}) + if not isinstance(scripts, dict): + errors.append(f"{relpath}: scripts must be an object") + continue + bad = FORBIDDEN_SCRIPTS & set(scripts) + if bad: + errors.append( + f"{relpath}: lifecycle script(s) {sorted(bad)} are forbidden - they execute " + "arbitrary code during `bun install` on every fork user's machine." + ) + if "trustedDependencies" in data: + errors.append( + f"{relpath}: trustedDependencies is forbidden - it re-enables dependency " + "lifecycle scripts that bun blocks by default." + ) + + +def main() -> int: + check_permissions() + check_gitignore() + check_package_manifests() + if errors: + print(f"security_guards: {len(errors)} failure(s)") + for err in errors: + print(f" - {err}") + return 1 + print( + "security_guards: OK (permissions allowlist, hooks allowlist, gitignore rules, " + "package manifests)" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/upstream_triage.py b/tools/upstream_triage.py new file mode 100755 index 0000000..74a2fb8 --- /dev/null +++ b/tools/upstream_triage.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Triage upstream commits this fork has not picked up yet. + +Emits a Markdown report that sorts the behind-list into "worth reviewing" vs +"probably skip", so a human decides what to merge/port. It never merges, +pushes, or edits anything - it only reads git history and prints. This is the +deliberate report/act boundary: on a fork "applies cleanly" is not "correct" - +a commit for portals the fork dropped can cherry-pick fine and still be wrong, +and that silent-wrong case is worse than a conflict. So the report stops at +ready-to-run cherry-pick lines; a human runs them. + +This is the commit-level companion to check_upstream_updates.py. That tool +answers "which of my personalized framework files changed" (version stamps); +this one answers "which upstream commits deserve my attention" (commit history). +Two tools, two questions - each cross-references the other in its output. + +Two signals drive the sort: + +1. Already applied? A cherry-pick lands with a NEW sha but the same patch, so a + raw sha comparison misreports it as missing. We compute git patch-ids for the + fork-only commits and treat any upstream commit whose patch-id (or exact + subject) matches as already applied. + +2. Relevant to this fork? A commit that only touches files this fork deleted + (e.g. removed demo portals) is almost certainly N/A. We check each commit's + touched paths against the working tree and flag accordingly. + +Usage: python tools/upstream_triage.py [--remote upstream] [--branch master] +Exits 0 always (a report, not a gate). Prints a note to stderr and exits 0 if +the upstream ref is unavailable, so a scheduled job degrades gracefully. +""" +from __future__ import annotations + +import argparse +import subprocess +import sys + + +def git(*args: str) -> str: + return subprocess.run( + ["git", *args], capture_output=True, text=True, check=True + ).stdout + + +def rev_list(range_spec: str) -> list[str]: + out = git("rev-list", "--no-merges", range_spec).strip() + return out.splitlines() if out else [] + + +def patch_id(sha: str) -> str | None: + """Stable patch-id for a commit, or None if it has no diff.""" + show = subprocess.run( + ["git", "show", sha], capture_output=True, text=True, check=True + ).stdout + r = subprocess.run( + ["git", "patch-id", "--stable"], input=show, capture_output=True, text=True + ) + line = r.stdout.strip() + return line.split()[0] if line else None + + +def subject(sha: str) -> str: + return git("show", "-s", "--format=%s", sha).strip() + + +def files_touched(sha: str) -> list[str]: + out = git("show", "--name-only", "--format=", sha).strip() + return [f for f in out.splitlines() if f] + + +def path_exists(path: str) -> bool: + # ls-tree against HEAD is authoritative for "does this fork still ship it". + r = subprocess.run( + ["git", "cat-file", "-e", f"HEAD:{path}"], capture_output=True + ) + return r.returncode == 0 + + +def remote_slug(remote: str) -> str | None: + """owner/repo for a GitHub remote, or None if it can't be parsed.""" + try: + url = git("remote", "get-url", remote).strip() + except subprocess.CalledProcessError: + return None + for sep in ("github.com/", "github.com:"): + if sep in url: + path = url.split(sep, 1)[1] + return path[:-4] if path.endswith(".git") else path + return None + + +def load_wontport(path: str) -> list[str]: + """SHA prefixes the fork has decided never to port; missing file -> [].""" + try: + with open(path, encoding="utf-8") as f: + raw = f.read() + except FileNotFoundError: + return [] + entries = [] + for line in raw.splitlines(): + line = line.split("#", 1)[0].strip() + if line: + entries.append(line) + return entries + + +def commit_cell(short: str, sha: str, slug: str | None) -> str: + if slug: + return f"[`{short}`](https://github.com/{slug}/commit/{sha})" + return f"`{short}`" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--remote", default="upstream") + ap.add_argument("--branch", default="master") + ap.add_argument("--wontport", default=".github/upstream-wontport.txt") + args = ap.parse_args() + ref = f"{args.remote}/{args.branch}" + slug = remote_slug(args.remote) + wontport = load_wontport(args.wontport) + + try: + git("rev-parse", "--verify", ref) + except subprocess.CalledProcessError: + print( + f"note: {ref} not available (add the remote and fetch it first); " + "nothing to triage.", + file=sys.stderr, + ) + print(f"_Upstream ref `{ref}` was not available when this ran._") + return 0 + + behind = rev_list(f"HEAD..{ref}") + if not behind: + print(f"Up to date with `{ref}`. Nothing to review. :white_check_mark:") + _print_crossref(ref) + return 0 + + fork_only = rev_list(f"{ref}..HEAD") + fork_patch_ids = {p for p in (patch_id(s) for s in fork_only) if p} + fork_subjects = {subject(s) for s in fork_only} + + review: list[tuple[str, str, str, list[str]]] = [] + skip: list[tuple[str, str, str, str]] = [] + + for sha in behind: + subj = subject(sha) + short = sha[:9] + if patch_id(sha) in fork_patch_ids or subj in fork_subjects: + skip.append((short, sha, subj, "already applied (cherry-picked)")) + continue + if any(sha.startswith(e) for e in wontport): + skip.append((short, sha, subj, "on the fork's won't-port list")) + continue + touched = files_touched(sha) + present = [f for f in touched if path_exists(f)] + # A commit whose only surviving footprint is the changelog is one whose + # real change lives in files this fork removed - the code doesn't apply, + # only a doc line would. Low signal; demote it. + substantive = [f for f in present if f != "CHANGELOG.md"] + if touched and not present: + skip.append((short, sha, subj, "touches only files not in this fork")) + elif present and not substantive: + skip.append((short, sha, subj, "changelog-only footprint in this fork")) + else: + review.append((short, sha, subj, substantive)) + + lines: list[str] = [] + lines.append(f"Upstream `{ref}` has **{len(behind)}** commit(s) this fork lacks: " + f"**{len(review)}** worth reviewing, **{len(skip)}** probably skippable.") + lines.append("") + lines.append("_This is a triage report. Nothing was merged - review and port by hand._") + lines.append("") + + lines.append("### Worth reviewing") + if review: + lines.append("") + lines.append("| Commit | Subject | Fork files it touches |") + lines.append("|---|---|---|") + for short, sha, subj, present in review: + shown = ", ".join(f"`{p}`" for p in present[:4]) or "_(new/shared paths)_" + if len(present) > 4: + shown += f" +{len(present) - 4} more" + lines.append(f"| {commit_cell(short, sha, slug)} | {subj} | {shown} |") + # Ready-to-run cherry-pick lines - still information, not action. The + # report stops here on purpose; a human runs (and verifies) these. + lines.append("") + lines.append("
Ready-to-run cherry-picks (review each before running)") + lines.append("") + lines.append("```bash") + for short, sha, subj, _ in review: + lines.append(f"git cherry-pick {sha} # {subj}") + lines.append("```") + lines.append("") + lines.append("
") + else: + lines.append("") + lines.append("_None._") + lines.append("") + + lines.append("### Probably skip") + if skip: + lines.append("") + lines.append("| Commit | Subject | Why |") + lines.append("|---|---|---|") + for short, sha, subj, why in skip: + lines.append(f"| {commit_cell(short, sha, slug)} | {subj} | {why} |") + else: + lines.append("") + lines.append("_None._") + + print("\n".join(lines)) + _print_crossref(ref) + return 0 + + +def _print_crossref(ref: str) -> None: + print() + print( + "_For personalized-file version stamps (which methodology files changed), " + f"run `python tools/check_upstream_updates.py --remote {ref.split('/')[0]}`._" + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/verify_pdf.py b/tools/verify_pdf.py new file mode 100755 index 0000000..79b4dfd --- /dev/null +++ b/tools/verify_pdf.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Verify that a generated PDF has the expected pages and extractable text.""" + +import argparse +import re +import subprocess +import sys +from pathlib import Path + + +class VerificationError(Exception): + """Raised when a generated PDF does not satisfy its checks.""" + + +def run_tool(command): + try: + return subprocess.run( + command, + check=True, + capture_output=True, + text=True, + ).stdout + except FileNotFoundError as exc: + raise VerificationError( + f"required command '{command[0]}' was not found. " + "Install poppler-utils (macOS: brew install poppler, " + "Debian/Ubuntu: apt install poppler-utils, Windows: choco install poppler)" + ) from exc + except subprocess.CalledProcessError as exc: + detail = (exc.stderr or "").strip() or (exc.stdout or "").strip() + detail = detail or "command failed" + raise VerificationError(f"{command[0]} could not read the PDF: {detail}") from exc + + +def parse_page_count(pdfinfo_output): + match = re.search(r"^Pages:\s+(\d+)\s*$", pdfinfo_output, re.MULTILINE) + if not match: + raise VerificationError("pdfinfo output did not contain a page count") + return int(match.group(1)) + + +def normalize_text(text): + return " ".join(text.split()) + + +def verify_pdf(pdf_path, expected_pages=None, min_chars=1, required_text=()): + pdf_path = Path(pdf_path) + if not pdf_path.is_file(): + raise VerificationError(f"PDF does not exist: {pdf_path}") + + if expected_pages is not None: + actual_pages = parse_page_count(run_tool(["pdfinfo", str(pdf_path)])) + if actual_pages != expected_pages: + raise VerificationError( + f"expected {expected_pages} page(s), found {actual_pages}" + ) + + extracted_text = normalize_text( + run_tool(["pdftotext", "-layout", str(pdf_path), "-"]) + ) + if len(extracted_text) < min_chars: + raise VerificationError( + f"text layer has {len(extracted_text)} character(s); expected at least {min_chars}" + ) + + for required in required_text: + if normalize_text(required) not in extracted_text: + raise VerificationError(f"text layer is missing required text: {required!r}") + + +def build_parser(): + parser = argparse.ArgumentParser( + description="Verify a PDF's page count and ATS-readable text layer." + ) + parser.add_argument("pdf", type=Path, help="PDF file to verify") + parser.add_argument("--pages", type=int, help="required exact page count") + parser.add_argument( + "--min-chars", + type=int, + default=1, + help="minimum non-whitespace text-layer characters (default: 1)", + ) + parser.add_argument( + "--contains", + action="append", + default=[], + help="text that must appear after whitespace normalization; repeatable", + ) + return parser + + +def main(argv=None): + args = build_parser().parse_args(argv) + try: + verify_pdf(args.pdf, args.pages, args.min_chars, args.contains) + except VerificationError as exc: + print(f"Error: {args.pdf}: {exc}", file=sys.stderr) + return 1 + print(f"Verified {args.pdf}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())