From dea8140db24a682b2ff4dc887206db1c9c82dda7 Mon Sep 17 00:00:00 2001 From: sdrarunvarshan <143191778+sdrarunvarshan@users.noreply.github.com> Date: Wed, 26 Aug 2026 23:37:03 +0530 Subject: [PATCH] feat(ats): extract PDF text with pypdf before Poppler (#369) * feat(ats): extract PDF text with pypdf before Poppler Lead the ATS text-layer check with pypdf (BSD, optional pip install). Fall back to pdftotext -layout -enc UTF-8. No cache directory, no installer, no AGPL pymupdf. Windows users without Poppler still get a mechanical parseability check; visual review remains the last resort. * Update verify_pdf.py * Update apply.md * Update verify_pdf.py * Update verify_pdf.py --- .claude/commands/apply.md | 14 ++- .claude/settings.json | 2 + .../05-cv-templates.md | 6 +- CHANGELOG.md | 1 + CLAUDE.md | 2 +- README.md | 2 +- SETUP.md | 8 +- tests/test_verify_pdf.py | 50 ++++++-- tools/security_guards.py | 2 + tools/verify_pdf.py | 111 ++++++++++++++---- 10 files changed, 157 insertions(+), 41 deletions(-) diff --git a/.claude/commands/apply.md b/.claude/commands/apply.md index a4ffbf1..846fac0 100644 --- a/.claude/commands/apply.md +++ b/.claude/commands/apply.md @@ -258,15 +258,19 @@ Do not proceed to Step 6 until both PDFs pass inspection. An ATS parser reads the PDF's embedded **text layer**, not the rendered page — a CV that passed visual inspection can still extract as garbage (icon glyphs where the contact details should be, scrambled reading order in multi-column layouts). This step verifies what a parser actually sees. It applies to the **CV only**; cover letters rarely go through keyword screening. -**Availability check:** run `pdftotext -v`. `pdftotext` (poppler) is an optional dependency, not part of TeX distributions. If it is missing, print a one-line warning that the mechanical parse check is skipped, do the keyword-coverage check (item 3 below) against your visual Read of the PDF instead, and note the degraded mode in the Step 6 report. Same graceful-skip pattern as the salary lookup. Keep the `-enc UTF-8` flag: Xpdf-based builds default to Latin-1 output, and without it a correct non-ASCII CV fails the replacement-character check below. +**Availability check:** extract with `python tools/verify_pdf.py` (tries **pypdf** first — BSD, `pip install pypdf` — then Poppler `pdftotext`). If both are missing, print a one-line warning that the mechanical parse check is skipped, do the keyword-coverage check (item 3 below) against your visual Read of the PDF instead, and note the degraded mode in the Step 6 report. Same graceful-skip pattern as the salary lookup. If a documented fallback still shells out to `pdftotext -layout`, keep the `-enc UTF-8` flag: Xpdf-based builds default to Latin-1 output, and without it a correct non-ASCII CV fails the replacement-character check below. **1. Extract the text layer:** ```bash -cd cv && pdftotext -layout -enc UTF-8 main__.pdf main__.txt +python tools/verify_pdf.py cv/main__.pdf --dump-text cv/main__.txt ``` -Read the `.txt` file. +The command prints `extractor: pypdf` or `extractor: pdftotext`. Record that name in the Step 6 report. Read the `.txt` file. If that tool is unavailable, the Poppler fallback is: + +```bash +cd cv && pdftotext -layout -enc UTF-8 main__.pdf main__.txt +``` **2. Parseability checks** on the extracted text: @@ -288,6 +292,10 @@ Failures here are template-level problems: fix them in the `` source (e. - **missing (have it)** — the profile shows the candidate genuinely has this skill but the CV never says it: add it where it fits naturally, preferring experience bullets (concrete evidence) over the profile statement, then re-run 5a–5c. - **missing (gap)** — a genuine gap: leave it missing. **Never stuff keywords.** This is the same honesty rule the reviewer follows — a gap gets acknowledged in the cover letter's framing, not hidden in the CV. + +> **Note:** A multi-word phrase reported missing may be a punctuation-spacing artifact between extractors (pypdf sometimes inserts spaces around punctuation that Poppler does not). Re-check against the other extractor before concluding the text is absent. + + **4. Clean up:** delete the extracted `.txt` file. ### 5e. Clean up build artifacts diff --git a/.claude/settings.json b/.claude/settings.json index 16b6aca..347018c 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -5,6 +5,8 @@ "Bash(bun run:*)", "Bash(python salary_lookup.py:*)", "Bash(python3 salary_lookup.py:*)", + "Bash(python tools/verify_pdf.py:*)", + "Bash(python3 tools/verify_pdf.py:*)", "Bash(pdftotext:*)" ] } diff --git a/.claude/skills/job-application-assistant/05-cv-templates.md b/.claude/skills/job-application-assistant/05-cv-templates.md index 220c6ca..749d49e 100644 --- a/.claude/skills/job-application-assistant/05-cv-templates.md +++ b/.claude/skills/job-application-assistant/05-cv-templates.md @@ -1,5 +1,5 @@ --- -framework_version: 1.4.2 +framework_version: 1.4.3 --- # CV Templates and Tailoring Guide @@ -267,10 +267,10 @@ Restore the highest-relevance item that was previously cut — a CV that ends mi Most employers run CVs through an ATS before a human sees them, and the ATS reads the PDF's embedded **text layer**, not the rendered page. A CV can pass visual inspection and still extract as garbage. After the layout passes the compile-and-inspect loop, verify the text layer: ```bash -cd cv && pdftotext -layout -enc UTF-8 main__.pdf main__.txt +python tools/verify_pdf.py cv/main__.pdf --dump-text cv/main__.txt ``` -`pdftotext` comes from [poppler](https://poppler.freedesktop.org/), not the TeX distribution - it is an **optional** dependency. The `-enc UTF-8` flag is not optional: Xpdf-based `pdftotext` builds default to Latin-1 output, which makes every non-ASCII character in a perfectly good CV read back as a replacement character and fail the parseability check below for no real reason. If it is not installed, skip the mechanical check with a warning and rely on the visual PDF read for keyword coverage. +Extraction tries **pypdf** first (`pip install pypdf`, BSD license), then Poppler `pdftotext`. If a fallback still uses `pdftotext -layout`, it must also pass `-enc UTF-8`: Xpdf-based builds default to Latin-1, which makes every non-ASCII character in a perfectly good CV read back as a replacement character. If neither extractor is available, skip the mechanical check with a warning and rely on the visual PDF read for keyword coverage. What to check in the extraction: diff --git a/CHANGELOG.md b/CHANGELOG.md index 71bd035..9483aee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ per-file diff commands. ### Added +- **pypdf ATS text-layer fallback** - `/apply` Step 5d and `tools/verify_pdf.py` extract the CV PDF text layer with **pypdf** first (BSD, `pip install pypdf`) so Windows machines without Poppler still get a mechanical parseability check. Poppler `pdftotext -layout -enc UTF-8` remains the fallback; if both are missing the check still degrades to a visual keyword review. No extra cache or installer. `05-cv-templates.md` `framework_version` 1.4.2 → 1.4.3. - **CI now tests the full documented Python range** (#370) - the Python tool tests job runs a 3.10-3.14 version matrix instead of pinning 3.12, so both the documented 3.10 minimum and the newest Python are continuously verified. Grew out of an independent diff --git a/CLAUDE.md b/CLAUDE.md index 608cd1c..d904d0e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -140,7 +140,7 @@ Both documents MUST be compiled and visually inspected via the Read tool on the - [ ] **Cover letter bullet font matches body font** - `\lettercontent{}` must not wrap `\begin{itemize}...\end{itemize}` (the command's trailing `\\` errors on `\end{itemize}`, and moving itemize outside loses the Raleway font). Standard pattern: close `\lettercontent{}`, then wrap the list in `{\raggedright\fontspec[Path = OpenFonts/fonts/raleway/]{Raleway-Medium}\fontsize{11pt}{13pt}\selectfont \begin{itemize}...\end{itemize}\par}` ### ATS & keyword verification (CV) -ATS parsers read the PDF's embedded text layer, not the rendered page. Extract it with `pdftotext -layout -enc UTF-8` and verify what a parser sees. `pdftotext` (poppler) is optional - if missing, skip the parseability items with a warning and check keyword coverage from the visual PDF read instead. +ATS parsers read the PDF's embedded text layer, not the rendered page. Extract it with `python tools/verify_pdf.py cv/main__.pdf --dump-text cv/main__.txt` (pypdf, then `pdftotext -layout -enc UTF-8`) and verify what a parser sees. If both extractors are missing, skip the parseability items with a warning and check keyword coverage from the visual PDF read instead. - [ ] CV text layer extracts cleanly - no `(cid:*)` markers, `�` replacement characters, or text visible in the PDF but absent from the extraction - [ ] Email and phone appear as **literal text** in the extraction (icon-glyph noise like `MOBILE-ALT`/`Envelope` is harmless, but a contact detail carried only by an icon or hyperlink is invisible to ATS) - [ ] Reading order of the extracted text matches the visual order (single-column stock template is safe; multi-column custom templates are where this breaks) diff --git a/README.md b/README.md index 7ae7c9c..770bdc7 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ The framework encodes career guidance best practices, including structured evalu - Python 3.10+ - [Bun](https://bun.sh) (for job search CLI tools) - LaTeX distribution with `lualatex` and `xelatex`: [TeX Live](https://tug.org/texlive/), [MacTeX](https://tug.org/mactex/), [TinyTeX](https://yihui.org/tinytex/), or [MiKTeX](https://miktex.org/). The CV compiles with `lualatex` (pdflatex often fails on modern MiKTeX installs with `fontawesome5` font-expansion errors); the cover letter compiles with `xelatex` because `cover.cls` requires `fontspec`. If using a minimal TeX install such as TinyTeX or BasicTeX, install the extra packages listed in [SETUP.md](SETUP.md#minimal-tex-install-tinytexbasictex). -- Optional: `pdftotext` from [poppler](https://poppler.freedesktop.org/) (macOS: `brew install poppler`, Debian/Ubuntu: `apt install poppler-utils`, Windows: `choco install poppler`) — used by `/apply`'s ATS parseability check on the compiled CV. If missing, the check degrades gracefully to a visual keyword review. +- Optional: `pip install pypdf` for `/apply`'s ATS parseability check (BSD; no Poppler required). Poppler `pdftotext` remains a fallback (macOS: `brew install poppler`, Debian/Ubuntu: `apt install poppler-utils`, Windows: `choco install poppler`). If both are missing, the check degrades to a visual keyword review. ## Quick start diff --git a/SETUP.md b/SETUP.md index a4d7c77..754f090 100644 --- a/SETUP.md +++ b/SETUP.md @@ -141,15 +141,17 @@ Copy-Item cover_letters\cover.cls, cover_letters\OpenFonts -Destination $SmokeDi Push-Location $SmokeDir; xelatex -interaction=nonstopmode -halt-on-error cover_smoke.tex; Pop-Location ``` -### Optional: pdftotext (for the ATS check) +### Optional: ATS text extraction (pypdf, then pdftotext) -`/apply` runs an ATS parseability check on the compiled CV: it extracts the PDF's text layer and verifies contact details, reading order, and keyword coverage the way an applicant-tracking system sees them. This uses `pdftotext` from [poppler](https://poppler.freedesktop.org/), which is not part of TeX distributions: +`/apply` runs an ATS parseability check on the compiled CV: it extracts the PDF's text layer and verifies contact details, reading order, and keyword coverage the way an applicant-tracking system sees them. + +The default extractor is **pypdf** (BSD, `pip install pypdf`). Poppler `pdftotext` remains an optional fallback: - **macOS:** `brew install poppler` - **Debian/Ubuntu:** `sudo apt install poppler-utils` - **Windows:** `choco install poppler` -If `pdftotext` is missing, `/apply` skips the mechanical check with a warning and falls back to a visual keyword review — everything else works normally. +If a command still uses `pdftotext -layout`, it must pass `-enc UTF-8` as well. If **neither** extractor is available, `/apply` skips the mechanical check with a warning and falls back to a visual keyword review — everything else works normally. ## 2. Fork and clone diff --git a/tests/test_verify_pdf.py b/tests/test_verify_pdf.py index c9a938c..46e772b 100644 --- a/tests/test_verify_pdf.py +++ b/tests/test_verify_pdf.py @@ -4,7 +4,13 @@ import unittest from pathlib import Path from unittest.mock import patch -from tools.verify_pdf import VerificationError, parse_page_count, run_tool, verify_pdf +from tools.verify_pdf import ( + VerificationError, + extract_text_layer, + parse_page_count, + run_tool, + verify_pdf, +) class ParsePageCountTests(unittest.TestCase): @@ -25,11 +31,12 @@ class VerifyPdfTests(unittest.TestCase): def tearDown(self): self.temp_dir.cleanup() + @patch("tools.verify_pdf._extract_pypdf", return_value=None) @patch("tools.verify_pdf.run_tool") - def test_accepts_expected_pages_and_text(self, mock_run_tool): + def test_accepts_expected_pages_and_text(self, mock_run_tool, _pypdf): mock_run_tool.side_effect = [ - "Pages: 2\n", "Professional\nExperience [your.email@example.com]\n", + "Pages: 2\n", ] verify_pdf( @@ -39,23 +46,29 @@ class VerifyPdfTests(unittest.TestCase): required_text=("Professional Experience", "[your.email@example.com]"), ) + @patch("tools.verify_pdf._extract_pypdf", return_value=None) @patch("tools.verify_pdf.run_tool") - def test_rejects_wrong_page_count(self, mock_run_tool): - mock_run_tool.return_value = "Pages: 3\n" + def test_rejects_wrong_page_count(self, mock_run_tool, _pypdf): + mock_run_tool.side_effect = ["ok", "Pages: 3\n"] with self.assertRaisesRegex(VerificationError, "expected 2 page.*found 3"): verify_pdf(self.pdf, expected_pages=2) + @patch("tools.verify_pdf._extract_pypdf", return_value=None) @patch("tools.verify_pdf.run_tool") - def test_rejects_too_little_extractable_text(self, mock_run_tool): - mock_run_tool.return_value = "short" + def test_rejects_too_little_extractable_text(self, mock_run_tool, _pypdf): + mock_run_tool.side_effect = ["short", "Pages: 1\n"] with self.assertRaisesRegex(VerificationError, "expected at least 20"): verify_pdf(self.pdf, min_chars=20) + @patch("tools.verify_pdf._extract_pypdf", return_value=None) @patch("tools.verify_pdf.run_tool") - def test_rejects_missing_required_text(self, mock_run_tool): - mock_run_tool.return_value = "Readable text, but not the expected section." + def test_rejects_missing_required_text(self, mock_run_tool, _pypdf): + mock_run_tool.side_effect = [ + "Readable text, but not the expected section.", + "Pages: 1\n", + ] with self.assertRaisesRegex(VerificationError, "Professional Experience"): verify_pdf(self.pdf, required_text=("Professional Experience",)) @@ -64,11 +77,28 @@ class VerifyPdfTests(unittest.TestCase): with self.assertRaisesRegex(VerificationError, "PDF does not exist"): verify_pdf(Path(self.temp_dir.name) / "missing.pdf") + @patch("tools.verify_pdf._extract_pypdf", return_value=("Hello ATS body", 1)) + def test_pypdf_is_preferred_over_poppler(self, _pypdf): + text, pages, extractor = extract_text_layer(self.pdf) + self.assertEqual(extractor, "pypdf") + self.assertEqual(text, "Hello ATS body") + self.assertEqual(pages, 1) + + @patch("tools.verify_pdf._extract_pypdf", return_value=None) + @patch("tools.verify_pdf.run_tool") + def test_falls_back_to_pdftotext(self, mock_run_tool, _pypdf): + mock_run_tool.side_effect = ["poppler text", "Pages: 2\n"] + text, pages, extractor = extract_text_layer(self.pdf) + self.assertEqual(extractor, "pdftotext") + self.assertEqual(text, "poppler text") + self.assertEqual(pages, 2) + self.assertEqual(mock_run_tool.call_args_list[0][0][0][:3], ["pdftotext", "-layout", "-enc"]) + class RunToolTests(unittest.TestCase): @patch("tools.verify_pdf.subprocess.run", side_effect=FileNotFoundError) def test_reports_missing_poppler_command(self, _mock_run): - with self.assertRaisesRegex(VerificationError, "install poppler-utils"): + with self.assertRaisesRegex(VerificationError, "pip install pypdf"): run_tool(["pdftotext", "example.pdf", "-"]) @patch("tools.verify_pdf.subprocess.run") diff --git a/tools/security_guards.py b/tools/security_guards.py index 0e894e5..0285367 100644 --- a/tools/security_guards.py +++ b/tools/security_guards.py @@ -41,6 +41,8 @@ ALLOWED_PERMISSIONS = { "Bash(bun run:*)", "Bash(python salary_lookup.py:*)", "Bash(python3 salary_lookup.py:*)", + "Bash(python tools/verify_pdf.py:*)", + "Bash(python3 tools/verify_pdf.py:*)", "Bash(pdftotext:*)", } diff --git a/tools/verify_pdf.py b/tools/verify_pdf.py index 79b4dfd..da83bd1 100644 --- a/tools/verify_pdf.py +++ b/tools/verify_pdf.py @@ -1,5 +1,10 @@ #!/usr/bin/env python3 -"""Verify that a generated PDF has the expected pages and extractable text.""" +"""Verify that a generated PDF has the expected pages and extractable text. + +Text-layer extraction tries pypdf (BSD, optional `pip install pypdf`) first, +then Poppler `pdftotext` if pypdf is missing, raises, or returns zero +extractable characters. Poppler remains the fallback. +""" import argparse import re @@ -19,12 +24,15 @@ def run_tool(command): check=True, capture_output=True, text=True, + encoding="utf-8", + errors="replace", ).stdout except FileNotFoundError as exc: raise VerificationError( f"required command '{command[0]}' was not found. " - "Install poppler-utils (macOS: brew install poppler, " - "Debian/Ubuntu: apt install poppler-utils, Windows: choco install poppler)" + "Install pypdf (`pip install pypdf`) or poppler-utils " + "(macOS: brew install poppler, Debian/Ubuntu: apt install poppler-utils, " + "Windows: choco install poppler)" ) from exc except subprocess.CalledProcessError as exc: detail = (exc.stderr or "").strip() or (exc.stdout or "").strip() @@ -43,29 +51,81 @@ def normalize_text(text): return " ".join(text.split()) -def verify_pdf(pdf_path, expected_pages=None, min_chars=1, required_text=()): +def _extract_pypdf(pdf_path): + """Return (text, pages) or None if pypdf is unavailable, raises, or yields no text.""" + try: + from pypdf import PdfReader + except ImportError: + return None + try: + reader = PdfReader(str(pdf_path)) + pages = len(reader.pages) + text = "\n".join((page.extract_text() or "") for page in reader.pages) + except Exception: + return None + # Harden: treat empty/degraded extraction as failure so we fall back + if len(normalize_text(text)) == 0: + return None + return text, pages + + +def _extract_pdftotext(pdf_path): + text = run_tool(["pdftotext", "-layout", "-enc", "UTF-8", str(pdf_path), "-"]) +# Always call pdfinfo here so the fallback path returns a page count + # even when the caller did not request --pages (same Poppler package). + pages = parse_page_count(run_tool(["pdfinfo", str(pdf_path)])) + return text, pages + + +def extract_text_layer(pdf_path): + """Extract ATS-readable text. Returns (text, pages, extractor_name).""" + pypdf_result = _extract_pypdf(pdf_path) + if pypdf_result is not None: + text, pages = pypdf_result + return text, pages, "pypdf" + text, pages = _extract_pdftotext(pdf_path) + return text, pages, "pdftotext" + + +def verify_pdf(pdf_path, expected_pages=None, min_chars=1, required_text=(), dump_text=None): pdf_path = Path(pdf_path) if not pdf_path.is_file(): raise VerificationError(f"PDF does not exist: {pdf_path}") - if expected_pages is not None: - actual_pages = parse_page_count(run_tool(["pdfinfo", str(pdf_path)])) - if actual_pages != expected_pages: - raise VerificationError( - f"expected {expected_pages} page(s), found {actual_pages}" - ) + extracted_text, actual_pages, extractor = extract_text_layer(pdf_path) - extracted_text = normalize_text( - run_tool(["pdftotext", "-layout", str(pdf_path), "-"]) - ) - if len(extracted_text) < min_chars: + # Write dump *before* the checks so a failed verification still leaves a .txt + if dump_text is not None: + dump_path = Path(dump_text) + try: + dump_path.parent.mkdir(parents=True, exist_ok=True) + dump_path.write_text( + extracted_text if extracted_text.endswith("\n") else extracted_text + "\n", + encoding="utf-8", + ) + except OSError as exc: + raise VerificationError( + f"could not write --dump-text to {dump_path}: {exc}" + ) from exc + + if expected_pages is not None and actual_pages != expected_pages: raise VerificationError( - f"text layer has {len(extracted_text)} character(s); expected at least {min_chars}" + f"expected {expected_pages} page(s), found {actual_pages} (extractor: {extractor})" + ) + + normalized = normalize_text(extracted_text) + if len(normalized) < min_chars: + raise VerificationError( + f"text layer has {len(normalized)} character(s); expected at least {min_chars} " + f"(extractor: {extractor})" ) for required in required_text: - if normalize_text(required) not in extracted_text: - raise VerificationError(f"text layer is missing required text: {required!r}") + if normalize_text(required) not in normalized: + raise VerificationError( + f"text layer is missing required text: {required!r} (extractor: {extractor})" + ) + return extractor, extracted_text, actual_pages def build_parser(): @@ -86,19 +146,30 @@ def build_parser(): default=[], help="text that must appear after whitespace normalization; repeatable", ) + parser.add_argument( + "--dump-text", + type=Path, + help="write the extracted text layer to this path (UTF-8)", + ) return parser def main(argv=None): args = build_parser().parse_args(argv) try: - verify_pdf(args.pdf, args.pages, args.min_chars, args.contains) + extractor, text, pages = verify_pdf( + args.pdf, + args.pages, + args.min_chars, + args.contains, + dump_text=args.dump_text, + ) except VerificationError as exc: print(f"Error: {args.pdf}: {exc}", file=sys.stderr) return 1 - print(f"Verified {args.pdf}") + print(f"Verified {args.pdf} (extractor: {extractor}, pages: {pages})") return 0 if __name__ == "__main__": - sys.exit(main()) + sys.exit(main()) \ No newline at end of file