diff --git a/CHANGELOG.md b/CHANGELOG.md index 2196391b..27feec64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,17 @@ per-file diff commands. ### Fixed +- **`verify_pdf.py --ascii-dates` no longer reads a numeric range with year-shaped ends as a + date** - #512 bounded the year to four standalone digits, but `EUR 1500–2000 per month` and + `2000–5000 requests per second` still failed `/apply` Step 5d: each has a four-digit number + next to the dash that happens to fall in 1900-2099. A range whose two ends are both bare + numbers (the whole whitespace-delimited token, outer punctuation stripped) now counts as a + date only when both are years, so the salary band and the throughput range pass, while + `2016–2024`, `1990–2010` and `2016 – Present` still fail as before and a numerically + formatted range (`03/2016–07/2024`, `01.2016 – 12.2024`, `2016-03–2024-07`, `2016/17–2018/19`) + is not a bare number on either side and is still caught. Three new `test_verify_pdf.py` + cases; the quantity one fails on master. + - **`upstream_triage` forces UTF-8 stdout and decodes git output as UTF-8** (#544, `tools/upstream_triage.py`, `tests/test_tools_utf8_output.py`) - like the other Python tools, the triage report now calls `_force_utf8_output()` on startup and decodes git subprocess diff --git a/tests/test_verify_pdf.py b/tests/test_verify_pdf.py index 8dc69740..3085cffc 100644 --- a/tests/test_verify_pdf.py +++ b/tests/test_verify_pdf.py @@ -106,6 +106,39 @@ class FindNonAsciiDateRangesTests(unittest.TestCase): with self.subTest(text=text): self.assertEqual(find_non_ascii_date_ranges(text), []) + def test_numeric_range_with_year_shaped_ends_is_a_quantity_not_a_date(self): + # Four-digit numbers that are not both years: #512 bounded the year but + # a salary band or a throughput range still read as a date range. + for text in ( + "Salary EUR 1500\u20132000 per month", + "Scaled the API to serve 2000\u20135000 requests per second", + "Grew users from 2000\u20135000", + "Budget 1.5\u20132.0M, headcount 20\u201325", + ): + with self.subTest(text=text): + self.assertEqual(find_non_ascii_date_ranges(text), []) + + def test_numerically_formatted_date_ranges_are_still_caught(self): + # A composite token (slashed, dotted, hyphenated) is not a bare number, + # so the quantity rule does not apply and the year rule still fires. + # These are the Workday-drops-the-date shape the flag exists for. + for text in ( + "03/2016\u201307/2024", + "01.2016 \u2013 12.2024", + "2016-03\u20132024-07", + "2016/17\u20132018/19", + "2016/03\u20132024/07", + "01.03.2016\u201331.07.2024", + "2016\u201307/2024", + ): + with self.subTest(text=text): + self.assertEqual(len(find_non_ascii_date_ranges(text)), 1) + + def test_two_years_joined_by_a_dash_are_still_a_date_range(self): + for text in ("2016\u20132024", "1990\u20132010 programme", "Roles 2000\u20132024"): + with self.subTest(text=text): + self.assertEqual(len(find_non_ascii_date_ranges(text)), 1) + def test_a_standalone_year_next_to_a_longer_number_is_still_caught(self): self.assertEqual(find_non_ascii_date_ranges("2016\u20132024 (120000 users)")[0][1], "\u2013") diff --git a/tools/verify_pdf.py b/tools/verify_pdf.py index b4abfef8..83782625 100644 --- a/tools/verify_pdf.py +++ b/tools/verify_pdf.py @@ -103,6 +103,45 @@ _YEAR = r"(? 1 and any(_BARE_YEAR.fullmatch(g) for g in groups): + return None + return core + + +def _quantity_range(line, dash_index): + """True when both sides of the dash are bare numbers and not both years.""" + before = _TOKEN_BEFORE.search(line[:dash_index]) + after = _TOKEN_AFTER.search(line[dash_index + 1 :]) + if not before or not after: + return False + left, right = _bare_number(before.group(1)), _bare_number(after.group(1)) + if left is None or right is None: + return False + return not (_BARE_YEAR.fullmatch(left) and _BARE_YEAR.fullmatch(right)) def find_non_ascii_date_ranges(text): @@ -112,15 +151,21 @@ def find_non_ascii_date_ranges(text): maps U+2013 back to `-` so `--contains "2016-2024"` can match what the template renders, which is exactly why `--contains` cannot see this defect. A year on either side of the dash is enough ("Mar 2016 - Jul 2016", - "2016 - Present"), so the check is locale-agnostic; a numeric range with - no year ("EUR 600k-1M") is not a date and is left alone. + "2016 - Present"), so the check is locale-agnostic. A range whose two ends + are both bare numbers is a date only when both are years: "EUR 600k-1M", + "EUR 1500-2000" and "2000-5000 requests/s" are quantities and are left + alone, while a numerically formatted date ("03/2016-07/2024", "2016-03-2024-07") + is not a bare number on either side and is still caught. """ hits = [] for match in NON_ASCII_DATE_RANGE.finditer(text): dash = next(c for c in match.group(0) if c in NON_ASCII_DASHES) line_start = text.rfind("\n", 0, match.start()) + 1 line_end = text.find("\n", match.end()) - line = text[line_start : line_end if line_end != -1 else len(text)] + line_end = line_end if line_end != -1 else len(text) + line = text[line_start:line_end] + if _quantity_range(line, match.start() + match.group(0).index(dash) - line_start): + continue hits.append((" ".join(line.split()), dash)) return hits