normalization optimized
This commit is contained in:
+36
-19
@@ -4869,40 +4869,57 @@ def _software_job_filter_values(request: Request) -> dict[str, str]:
|
||||
|
||||
|
||||
def _normalize_filter_text(value: Any) -> str:
|
||||
"""Normalize user-facing text for accent- and German umlaut-aware search."""
|
||||
"""Normalize text while preserving natural German vowel combinations."""
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
text = str(value or "").translate(
|
||||
str.maketrans(
|
||||
{
|
||||
"Ä": "Ae",
|
||||
"Ö": "Oe",
|
||||
"Ü": "Ue",
|
||||
"ä": "ae",
|
||||
"ö": "oe",
|
||||
"ü": "ue",
|
||||
"ẞ": "SS",
|
||||
"ß": "ss",
|
||||
}
|
||||
)
|
||||
text = str(value or "").casefold()
|
||||
protected = (
|
||||
text.replace("ä", "\ue000")
|
||||
.replace("ö", "\ue001")
|
||||
.replace("ü", "\ue002")
|
||||
.replace("ß", "\ue003")
|
||||
)
|
||||
return "".join(
|
||||
text = "".join(
|
||||
character
|
||||
for character in unicodedata.normalize("NFKD", text.casefold())
|
||||
for character in unicodedata.normalize("NFKD", protected)
|
||||
if not unicodedata.combining(character)
|
||||
)
|
||||
text = (
|
||||
text.replace("\ue000", "ä")
|
||||
.replace("\ue001", "ö")
|
||||
.replace("\ue002", "ü")
|
||||
.replace("\ue003", "ß")
|
||||
)
|
||||
|
||||
# Mark only plausible ASCII umlaut spellings. This prevents natural
|
||||
# combinations in words such as "Bauer", "teuer" and "euer" from being
|
||||
# treated as an umlaut while keeping Mueller/Müller and Hyaene/Hyäne equal.
|
||||
text = re.sub(r"(^|[^aeiouäöü])([aou])e", r"\1\2~e", text)
|
||||
return (
|
||||
text.replace("ä", "a~e")
|
||||
.replace("ö", "o~e")
|
||||
.replace("ü", "u~e")
|
||||
.replace("ß", "s~s")
|
||||
)
|
||||
|
||||
|
||||
def _sql_normalized_filter_text(column: Any):
|
||||
"""Return the SQL equivalent of :func:`_normalize_filter_text`."""
|
||||
"""Return the PostgreSQL equivalent of :func:`_normalize_filter_text`."""
|
||||
expression = func.lower(func.coalesce(cast(column, String), ""))
|
||||
for source, target in (("ä", "ae"), ("ö", "oe"), ("ü", "ue"), ("ß", "ss")):
|
||||
expression = func.regexp_replace(
|
||||
expression,
|
||||
r"(^|[^aeiouäöü])([aou])e",
|
||||
r"\1\2~e",
|
||||
"g",
|
||||
)
|
||||
for source, target in (("ä", "a~e"), ("ö", "o~e"), ("ü", "u~e"), ("ß", "s~s")):
|
||||
expression = func.replace(expression, source, target)
|
||||
return expression
|
||||
|
||||
|
||||
def _sql_text_contains(column: Any, value: str):
|
||||
"""Literal substring search with umlaut and ASCII spelling equivalence."""
|
||||
"""Literal substring search with conservative German umlaut equivalence."""
|
||||
escaped = (
|
||||
_normalize_filter_text(value)
|
||||
.replace("\\", "\\\\")
|
||||
|
||||
Reference in New Issue
Block a user