mirror of
https://github.com/MadsLorentzen/ai-job-search.git
synced 2026-11-02 13:12:50 +00:00
fix(rank): preserve Unicode identity in tracker matching (#486)
This commit is contained in:
1 parent
ba4f2397a3
commit
120f476a08
3 files changed
+54
-1
No files matched your search
+8
-1
@@ -42,6 +42,7 @@ import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import unicodedata
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
@@ -97,7 +98,13 @@ def parse_iso(value) -> date | None:
|
||||
|
||||
|
||||
def norm(text) -> str:
|
||||
return re.sub(r"[^a-z0-9]", "", str(text or "").lower())
|
||||
"""Ignore case and separators without discarding non-Latin identity."""
|
||||
text = unicodedata.normalize("NFC", str(text or "").casefold())
|
||||
# Combining marks can distinguish names even after NFC (e.g. Indic vowels).
|
||||
return "".join(
|
||||
char for char in text
|
||||
if char.isalnum() or unicodedata.category(char).startswith("M")
|
||||
)
|
||||
|
||||
|
||||
def tracker_pairs(path: Path) -> set[tuple[str, str]]:
|
||||
|
||||
Reference in new issue
Block a user