normalization optimized

This commit is contained in:
2026-08-03 21:51:54 +00:00
parent bfc6128353
commit bd23858537
8 changed files with 95 additions and 46 deletions
+36 -19
View File
@@ -4869,40 +4869,57 @@ def _software_job_filter_values(request: Request) -> dict[str, str]:
def _normalize_filter_text(value: Any) -> str:
"""Normalize user-facing text for accent- and German umlaut-aware search."""
"""Normalize text while preserving natural German vowel combinations."""
import re
import unicodedata
text = str(value or "").translate(
str.maketrans(
{
"Ä": "Ae",
"Ö": "Oe",
"Ü": "Ue",
"ä": "ae",
"ö": "oe",
"ü": "ue",
"": "SS",
"ß": "ss",
}
)
text = str(value or "").casefold()
protected = (
text.replace("ä", "\ue000")
.replace("ö", "\ue001")
.replace("ü", "\ue002")
.replace("ß", "\ue003")
)
return "".join(
text = "".join(
character
for character in unicodedata.normalize("NFKD", text.casefold())
for character in unicodedata.normalize("NFKD", protected)
if not unicodedata.combining(character)
)
text = (
text.replace("\ue000", "ä")
.replace("\ue001", "ö")
.replace("\ue002", "ü")
.replace("\ue003", "ß")
)
# Mark only plausible ASCII umlaut spellings. This prevents natural
# combinations in words such as "Bauer", "teuer" and "euer" from being
# treated as an umlaut while keeping Mueller/Müller and Hyaene/Hyäne equal.
text = re.sub(r"(^|[^aeiouäöü])([aou])e", r"\1\2~e", text)
return (
text.replace("ä", "a~e")
.replace("ö", "o~e")
.replace("ü", "u~e")
.replace("ß", "s~s")
)
def _sql_normalized_filter_text(column: Any):
"""Return the SQL equivalent of :func:`_normalize_filter_text`."""
"""Return the PostgreSQL equivalent of :func:`_normalize_filter_text`."""
expression = func.lower(func.coalesce(cast(column, String), ""))
for source, target in (("ä", "ae"), ("ö", "oe"), ("ü", "ue"), ("ß", "ss")):
expression = func.regexp_replace(
expression,
r"(^|[^aeiouäöü])([aou])e",
r"\1\2~e",
"g",
)
for source, target in (("ä", "a~e"), ("ö", "o~e"), ("ü", "u~e"), ("ß", "s~s")):
expression = func.replace(expression, source, target)
return expression
def _sql_text_contains(column: Any, value: str):
"""Literal substring search with umlaut and ASCII spelling equivalence."""
"""Literal substring search with conservative German umlaut equivalence."""
escaped = (
_normalize_filter_text(value)
.replace("\\", "\\\\")