fix(verify_pdf): fold LaTeX's typographic substitutions before --contains; guard T1 fontenc for pdflatex (#385, #384) (#458)

`normalize_text()` folded whitespace only, so `--contains` compared what a
user types against what LaTeX renders. The stock CV compiled with the
documented lualatex command turns `'` into U+2019 and `--` into U+2013, so
`--contains "Master's degree"` and `--contains "2016-2024"` both reported
the keyword missing from a document that plainly contains it, through both
extractors. The documented remedy for a missing keyword is to add it, which
is the one thing the ATS section forbids.

Fold both sides at comparison time: NFC, then curly apostrophes and quotes
to ASCII, en/em dashes to `-`, no-break space to space. `--dump-text` still
writes the raw layer - that is what an ATS parses, and the date-range rule
in 05-cv-templates.md needs the raw en-dash visible there.

Separately, pdflatex without T1 font encoding stores accents decomposed
(`e` + U+0300). NFC repairs the pdftotext side of that, but pypdf reads the
same layer as `Z¨ urich` with a spacing accent, which no fold recovers.
moderncv 2.5 loads T1 itself under pdflatex; the apt-packaged 2.3.1 does
not - reproduced by compiling the template against moderncv v2.3.1 with
pdflatex (before: U+0308/U+0300 in pdftotext, `Z¨ urich` in pypdf; after:
U+00FC/U+00E8 in both). The template and the guide's preamble gain
`\ifpdftex\usepackage[T1]{fontenc}\fi`; the lualatex text layer is
byte-identical before and after.

Tests: ten new cases in test_verify_pdf.py (the fold-through and
normalize_text ones fail on the whitespace-only code) and a
test_latex_guidance.py guard that the fontenc line exists and stays inside
the pdflatex branch. framework_version 1.4.3 -> 1.4.4 on 05-cv-templates.md.

Reported and diagnosed by 9scorp4 in Discussions #385 and #384.

tests/test_latex_guidance.py

@@ -136,5 +136,45 @@ class TestAtsExtractionEncoding(unittest.TestCase):
         self.assert_pdftotext_commands_pin_utf8(CV_TEMPLATES)
 
 
+class TestPdflatexFontEncodingGuard(unittest.TestCase):
+    """#384: the pdflatex fallback must load T1 fontenc, and only under pdflatex.
+
+    Without T1, pdflatex stores accented letters decomposed in the text layer
+    (`e` + U+0300), so an ATS keyword match on `Genève` fails while the PDF
+    looks right. moderncv 2.5 loads T1 itself; the apt-packaged 2.3.1 does not.
+    The line must be guarded so the documented lualatex path is untouched.
+    """
+
+    GUARDED_FONTENC = re.compile(r"\\ifpdftex\s*\\usepackage\[T1\]\{fontenc\}\s*\\fi")
+
+    def assert_has_guarded_fontenc(self, path):
+        text = path.read_text(encoding="utf-8")
+        self.assertRegex(
+            text,
+            self.GUARDED_FONTENC,
+            f"{path.name} must carry `\\ifpdftex\\usepackage[T1]{{fontenc}}\\fi` so a "
+            "pdflatex fallback keeps accents precomposed in the text layer",
+        )
+        unguarded = [
+            f"{path.name}:{lineno}: {line.strip()}"
+            for lineno, line in enumerate(text.splitlines(), 1)
+            if "fontenc" in line
+            and not line.lstrip().startswith("%")
+            and not self.GUARDED_FONTENC.search(line)
+        ]
+        self.assertEqual(
+            unguarded,
+            [],
+            "fontenc must stay inside the \\ifpdftex guard - lualatex output "
+            "must not change:\n" + "\n".join(unguarded),
+        )
+
+    def test_example_cv_guards_fontenc_for_pdflatex(self):
+        self.assert_has_guarded_fontenc(EXAMPLE_CV)
+
+    def test_cv_guide_preamble_guards_fontenc_for_pdflatex(self):
+        self.assert_has_guarded_fontenc(CV_TEMPLATES)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_verify_pdf.py

@@ -7,6 +7,7 @@ from unittest.mock import patch
 from tools.verify_pdf import (
     VerificationError,
     extract_text_layer,
+    normalize_text,
     parse_page_count,
     run_tool,
     verify_pdf,
@@ -22,6 +23,44 @@ class ParsePageCountTests(unittest.TestCase):
             parse_page_count("Title: Example\n")
 
 
+class NormalizeTextTests(unittest.TestCase):
+    """`--contains` must see through what LaTeX does to plain source text.
+
+    Measured on the stock CV compiled with the documented `lualatex` command
+    (#385): the apostrophe in `Master's` reaches the text layer as U+2019 and
+    the `--` in `2016--2024` as U+2013, so a whitespace-only fold reports both
+    keywords missing from a CV that plainly contains them. Under pdflatex
+    without T1 font encoding, accents arrive decomposed (`e` + U+0300) instead
+    of precomposed (#384).
+    """
+
+    def test_folds_curly_apostrophe_to_ascii(self):
+        self.assertEqual(normalize_text("Master\u2019s degree"), "Master's degree")
+        self.assertEqual(normalize_text("\u2018quoted\u2019"), "'quoted'")
+
+    def test_folds_curly_double_quotes_to_ascii(self):
+        self.assertEqual(normalize_text("\u201cSix Sigma\u201d"), '"Six Sigma"')
+
+    def test_folds_en_and_em_dashes_to_hyphen(self):
+        self.assertEqual(normalize_text("2016\u20132024"), "2016-2024")
+        self.assertEqual(normalize_text("role\u2014title"), "role-title")
+
+    def test_folds_no_break_space_to_space(self):
+        self.assertEqual(normalize_text("EUR\u00a0600k"), "EUR 600k")
+
+    def test_folds_decomposed_accents_to_nfc(self):
+        decomposed = "Gene\u0300ve Universite\u0301"
+        precomposed = "Gen\u00e8ve Universit\u00e9"
+        self.assertEqual(normalize_text(decomposed), precomposed)
+
+    def test_still_collapses_whitespace(self):
+        self.assertEqual(normalize_text("Professional\n   Experience "), "Professional Experience")
+
+    def test_fold_is_symmetric(self):
+        # A user who pastes the curly form from a posting must match an ASCII layer too.
+        self.assertEqual(normalize_text("Master\u2019s"), normalize_text("Master's"))
+
+
 class VerifyPdfTests(unittest.TestCase):
     def setUp(self):
         self.temp_dir = tempfile.TemporaryDirectory()
@@ -73,6 +112,41 @@ class VerifyPdfTests(unittest.TestCase):
         with self.assertRaisesRegex(VerificationError, "Professional Experience"):
             verify_pdf(self.pdf, required_text=("Professional Experience",))
 
+    @patch("tools.verify_pdf._extract_pypdf", return_value=None)
+    @patch("tools.verify_pdf.run_tool")
+    def test_required_text_matches_latex_typographic_substitutions(self, mock_run_tool, _pypdf):
+        # What the stock template's lualatex text layer actually contains for the
+        # source `Master's degree ... 2016--2024` (code points measured, see class
+        # docstring of NormalizeTextTests).
+        mock_run_tool.side_effect = [
+            "Master\u2019s degree in Statistics. Six Sigma Green Belt, 2016\u20132024.\n",
+            "Pages:          1\n",
+        ]
+
+        verify_pdf(self.pdf, required_text=("Master's degree", "2016-2024"))
+
+    @patch("tools.verify_pdf._extract_pypdf", return_value=None)
+    @patch("tools.verify_pdf.run_tool")
+    def test_required_text_matches_decomposed_pdflatex_accents(self, mock_run_tool, _pypdf):
+        mock_run_tool.side_effect = [
+            "Universite\u0301 de Gene\u0300ve\n",  # pdflatex without T1 fontenc
+            "Pages:          1\n",
+        ]
+
+        verify_pdf(self.pdf, required_text=("Universit\u00e9 de Gen\u00e8ve",))
+
+    @patch("tools.verify_pdf._extract_pypdf", return_value=None)
+    @patch("tools.verify_pdf.run_tool")
+    def test_dump_text_keeps_the_raw_layer_unfolded(self, mock_run_tool, _pypdf):
+        # The fold is comparison-time only: the ATS parser sees the raw layer, and
+        # the date-range rule in 05-cv-templates.md needs the en-dash visible here.
+        mock_run_tool.side_effect = ["2016\u20132024\n", "Pages:          1\n"]
+        dump = Path(self.temp_dir.name) / "dump.txt"
+
+        verify_pdf(self.pdf, required_text=("2016-2024",), dump_text=dump)
+
+        self.assertEqual(dump.read_text(encoding="utf-8"), "2016\u20132024\n")
+
     def test_rejects_missing_pdf(self):
         with self.assertRaisesRegex(VerificationError, "PDF does not exist"):
             verify_pdf(Path(self.temp_dir.name) / "missing.pdf")