From bd2385853700254ba0ebfd9b56ed683f0daa3acd Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 3 Aug 2026 21:51:54 +0000 Subject: [PATCH] normalization optimized --- README.md | 2 +- VERSION | 2 +- app/main.py | 55 +++++++++++++++-------- app/static/js/asset-duplicates.js | 6 +-- app/static/js/asset-inventory.js | 8 ++-- app/static/js/table-tools.js | 58 +++++++++++++++++-------- app/version.py | 2 +- docs/version-history/UPDATE-0.5.5.32.md | 8 ++++ 8 files changed, 95 insertions(+), 46 deletions(-) create mode 100644 docs/version-history/UPDATE-0.5.5.32.md diff --git a/README.md b/README.md index dc3f3fa..66ca244 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# AssetManager 0.5.5.31 +# AssetManager 0.5.5.32 AssetManager is a self-hosted web application for managing IT equipment and other organizational assets. The project is released under the **Apache License 2.0** and may be used, modified, and redistributed for private and commercial purposes. diff --git a/VERSION b/VERSION index f13c9f2..599ba9c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.5.31 +0.5.5.32 diff --git a/app/main.py b/app/main.py index 56b2bb6..45ea843 100644 --- a/app/main.py +++ b/app/main.py @@ -4869,40 +4869,57 @@ def _software_job_filter_values(request: Request) -> dict[str, str]: def _normalize_filter_text(value: Any) -> str: - """Normalize user-facing text for accent- and German umlaut-aware search.""" + """Normalize text while preserving natural German vowel combinations.""" + import re import unicodedata - text = str(value or "").translate( - str.maketrans( - { - "Ä": "Ae", - "Ö": "Oe", - "Ü": "Ue", - "ä": "ae", - "ö": "oe", - "ü": "ue", - "ẞ": "SS", - "ß": "ss", - } - ) + text = str(value or "").casefold() + protected = ( + text.replace("ä", "\ue000") + .replace("ö", "\ue001") + .replace("ü", "\ue002") + .replace("ß", "\ue003") ) - return "".join( + text = "".join( character - for character in unicodedata.normalize("NFKD", text.casefold()) + for character in unicodedata.normalize("NFKD", protected) if not unicodedata.combining(character) ) + text = ( + text.replace("\ue000", "ä") + .replace("\ue001", "ö") + .replace("\ue002", "ü") + .replace("\ue003", "ß") + ) + + # Mark only plausible ASCII umlaut spellings. This prevents natural + # combinations in words such as "Bauer", "teuer" and "euer" from being + # treated as an umlaut while keeping Mueller/Müller and Hyaene/Hyäne equal. + text = re.sub(r"(^|[^aeiouäöü])([aou])e", r"\1\2~e", text) + return ( + text.replace("ä", "a~e") + .replace("ö", "o~e") + .replace("ü", "u~e") + .replace("ß", "s~s") + ) def _sql_normalized_filter_text(column: Any): - """Return the SQL equivalent of :func:`_normalize_filter_text`.""" + """Return the PostgreSQL equivalent of :func:`_normalize_filter_text`.""" expression = func.lower(func.coalesce(cast(column, String), "")) - for source, target in (("ä", "ae"), ("ö", "oe"), ("ü", "ue"), ("ß", "ss")): + expression = func.regexp_replace( + expression, + r"(^|[^aeiouäöü])([aou])e", + r"\1\2~e", + "g", + ) + for source, target in (("ä", "a~e"), ("ö", "o~e"), ("ü", "u~e"), ("ß", "s~s")): expression = func.replace(expression, source, target) return expression def _sql_text_contains(column: Any, value: str): - """Literal substring search with umlaut and ASCII spelling equivalence.""" + """Literal substring search with conservative German umlaut equivalence.""" escaped = ( _normalize_filter_text(value) .replace("\\", "\\\\") diff --git a/app/static/js/asset-duplicates.js b/app/static/js/asset-duplicates.js index 44cdc78..76efe14 100644 --- a/app/static/js/asset-duplicates.js +++ b/app/static/js/asset-duplicates.js @@ -47,9 +47,9 @@ } function normalize(value) { - const sharedNormalize = window.AssetManagerNormalizeSearchText; - if (typeof sharedNormalize === 'function') { - return sharedNormalize(value, { + const sharedKey = window.AssetManagerGermanSearchKey; + if (typeof sharedKey === 'function') { + return sharedKey(value, { caseSensitive: !caseInsensitive.checked, collapseWhitespace: true }); diff --git a/app/static/js/asset-inventory.js b/app/static/js/asset-inventory.js index 39c7bbb..6badfb8 100644 --- a/app/static/js/asset-inventory.js +++ b/app/static/js/asset-inventory.js @@ -11,11 +11,11 @@ const table = document.getElementById('software-inventory-table'); if (filter && table) { filter.addEventListener('input', () => { - const normalize = window.AssetManagerNormalizeSearchText || - ((value) => String(value || '').toLocaleLowerCase()); - const needle = normalize(filter.value.trim()); + const matches = window.AssetManagerSearchTextMatches || + ((value, term) => String(value || '').toLocaleLowerCase().includes(String(term || '').toLocaleLowerCase())); + const needle = filter.value.trim(); table.querySelectorAll('tbody tr').forEach((row) => { - row.hidden = Boolean(needle) && !normalize(row.textContent).includes(needle); + row.hidden = Boolean(needle) && !matches(row.textContent, needle); }); }); } diff --git a/app/static/js/table-tools.js b/app/static/js/table-tools.js index 58799ae..73015bb 100644 --- a/app/static/js/table-tools.js +++ b/app/static/js/table-tools.js @@ -4,28 +4,52 @@ const collapseWhitespace = Boolean(options.collapseWhitespace); let text = String(value ?? ''); - // Preserve the common German transliterations before Unicode accent - // folding so searches treat umlauts and their ASCII spellings equally. - text = text - .replace(/Ä/g, 'Ae') - .replace(/Ö/g, 'Oe') - .replace(/Ü/g, 'Ue') - .replace(/ä/g, 'ae') - .replace(/ö/g, 'oe') - .replace(/ü/g, 'ue') - .replace(/ẞ/g, 'SS') - .replace(/ß/g, 'ss') - .normalize('NFKD') - .replace(/[\u0300-\u036f]/g, ''); - if (collapseWhitespace) text = text.trim().replace(/\s+/g, ' '); if (!caseSensitive) text = text.toLocaleLowerCase(); - return text; + + const protectedGerman = text + .replace(/ä/g, '\uE000') + .replace(/ö/g, '\uE001') + .replace(/ü/g, '\uE002') + .replace(/ß/g, '\uE003'); + + return protectedGerman + .normalize('NFKD') + .replace(/[\u0300-\u036f]/g, '') + .replace(/\uE000/g, 'ä') + .replace(/\uE001/g, 'ö') + .replace(/\uE002/g, 'ü') + .replace(/\uE003/g, 'ß'); + } + + function germanSearchKey(value, options = {}) { + let text = normalizeSearchText(value, options); + + // Mark only plausible ASCII umlaut spellings. The marker keeps a real or + // transliterated umlaut distinct from natural letter pairs in words such + // as "Bauer", "teuer" and "euer". + text = text.replace(/(^|[^aeiouäöü])([aou])e/g, '$1$2~e'); + + return text + .replace(/ä/g, 'a~e') + .replace(/ö/g, 'o~e') + .replace(/ü/g, 'u~e') + .replace(/ß/g, 's~s'); + } + + function searchTextMatches(value, term, options = {}) { + const directValue = normalizeSearchText(value, options); + const directTerm = normalizeSearchText(term, options); + if (!directTerm) return true; + if (directValue.includes(directTerm)) return true; + return germanSearchKey(value, options).includes(germanSearchKey(term, options)); } // Standalone filters and duplicate detection use the exact same comparison // rules as the generic table filters. window.AssetManagerNormalizeSearchText = normalizeSearchText; + window.AssetManagerGermanSearchKey = germanSearchKey; + window.AssetManagerSearchTextMatches = searchTextMatches; const collator = new Intl.Collator(document.documentElement.lang || 'de', { numeric: true, @@ -387,7 +411,7 @@ const active = filterDefinitions .map(definition => ({ definition, - term: normalizeSearchText(definition.input?.value.trim() || '') + term: definition.input?.value.trim() || '' })) .filter(item => item.term); @@ -415,7 +439,7 @@ if (definition.numericFilter) { return !numericFilterMatches(value, term); } - return !normalizeSearchText(value).includes(term); + return !searchTextMatches(value, term); }); row.hidden = hidden; diff --git a/app/version.py b/app/version.py index 1941dec..2adc87f 100644 --- a/app/version.py +++ b/app/version.py @@ -1,2 +1,2 @@ -APP_VERSION = "0.5.5.31" +APP_VERSION = "0.5.5.32" __version__ = APP_VERSION diff --git a/docs/version-history/UPDATE-0.5.5.32.md b/docs/version-history/UPDATE-0.5.5.32.md new file mode 100644 index 0000000..a552a6d --- /dev/null +++ b/docs/version-history/UPDATE-0.5.5.32.md @@ -0,0 +1,8 @@ +# Version 0.5.5.32 + +## German search normalization refinement + +- Distinguishes real or transliterated umlauts from natural vowel combinations. +- Keeps searches such as `Müller`/`Mueller` and `Hyäne`/`Hyaene` equivalent. +- Prevents short umlaut filters from matching words such as `Bauer`, `teuer` or `euer` merely because they contain `ue`. +- Applies the same comparison rules to generic table filters, the server-side job history filters, asset software filtering and duplicate detection.