Merge branch 'release/5.28.0'

Author: lukavdplas

CITATION.cff

@@ -35,5 +35,5 @@ keywords:
   - elasticsearch
   - natural language processing
 license: MIT
-version: 5.27.1
-date-released: '2026-02-02'
+version: 5.28.0
+date-released: '2026-02-20'

backend/addcorpus/es_mappings.py

@@ -1,5 +1,6 @@
 from typing import Dict
 from addcorpus.es_settings import add_language_string, stopwords_available, stemming_available
+from langcodes import standardize_tag
 
 def primary_mapping_type(es_mapping: Dict) -> str:
     return es_mapping.get('type', None)
@@ -26,16 +27,21 @@ def main_content_mapping(
                 "type":     "token_count",
                 "analyzer": "standard"
             }
-        if stopword_analysis and stopwords_available(language):
+
+        if not language:
+            return mapping
+        tag = standardize_tag(language, macro=True)
+
+        if stopword_analysis and stopwords_available(tag):
             multifields['clean'] = {
                 "type": "text",
-                "analyzer": add_language_string('clean', language),
+                "analyzer": add_language_string('clean', tag),
                 "term_vector": "with_positions_offsets" # include character positions for highlighting
             }
-        if stemming_analysis and stemming_available(language):
+        if stemming_analysis and stemming_available(tag):
             multifields['stemmed'] = {
                 "type": "text",
-                "analyzer": add_language_string('stemmed', language),
+                "analyzer": add_language_string('stemmed', tag),
                 "term_vector": "with_positions_offsets",
             }
         mapping['fields'] = multifields

backend/addcorpus/es_settings.py

@@ -1,5 +1,6 @@
 import os
 import warnings
+from typing import Dict
 
 from django.conf import settings
 from langcodes import Language, standardize_tag
@@ -85,27 +86,30 @@ def es_settings(languages=[], stopword_analysis=False, stemming_analysis=False):
     for language in languages:
         # do not attach language isocodes if there is just one language
 
+        tag = standardize_tag(language, macro=True)
+
         if stopword_analysis or stemming_analysis:
-            if not set_stopword_filter(settings, add_language_string(stopword_filter_name, language), language):
+            if not set_stopword_filter(settings, add_language_string(stopword_filter_name, tag), tag):
                 continue # skip languages for which we do not have a stopword list
 
             if stopword_analysis:
                 set_clean_analyzer(
                     settings,
-                    add_language_string(stopword_filter_name, language),
-                    add_language_string(clean_analyzer_name, language),
+                    tag,
+                    add_language_string(stopword_filter_name, tag),
+                    add_language_string(clean_analyzer_name, tag),
                 )
             if stemming_analysis:
-                if not stemming_available(language):
+                if not stemming_available(tag):
                     warnings.warn('You specified `stemming_analysis=True`, but \
                                       there is no stemmer available for this language')
                     continue
                 set_stemmed_analyzer(
                     settings,
-                    add_language_string(stopword_filter_name, language),
-                    add_language_string(stemmer_filter_name, language),
-                    add_language_string(stemmed_analyzer_name, language),
-                    language
+                    tag,
+                    add_language_string(stopword_filter_name, tag),
+                    add_language_string(stemmer_filter_name, tag),
+                    add_language_string(stemmed_analyzer_name, tag),
                 )
 
     return settings
@@ -127,12 +131,30 @@ def make_stopword_filter(language):
     except:
         return None
 
-def make_clean_analyzer(stopword_filter_name):
-    return {
-        "tokenizer": "standard",
-        "char_filter": ["number_filter"],
-        "filter": ["lowercase", stopword_filter_name]
-    }
+def _standard_analyzer(language: str):
+    '''
+    Basic analyzer for a language.
+    '''
+    if language in ['zh', 'ja', 'ko']:
+        return {
+            'tokenizer': 'standard',
+            'filter': [
+                'cjk_width',
+                'lowercase',
+            ]
+        }
+    else:
+        return {
+            'tokenizer': 'standard',
+            'char_filter': ['number_filter'],
+            'filter': ['lowercase']
+        }
+
+def make_clean_analyzer(language: str, stopword_filter_name: str) -> Dict:
+    analyzer = _standard_analyzer(language)
+    analyzer['filter'].append(stopword_filter_name)
+    return analyzer
+
 
 def make_stemmer_filter(language):
     stemmer_language = get_language_key(language)
@@ -141,12 +163,13 @@ def make_stemmer_filter(language):
         "language": stemmer_language
     }
 
-def make_stemmed_analyzer(stopword_filter_name, stemmer_filter_name):
-    return {
-        "tokenizer": "standard",
-        "char_filter": ["number_filter"],
-        "filter": ["lowercase", stopword_filter_name, stemmer_filter_name]
-    }
+def make_stemmed_analyzer(
+    language: str, stopword_filter_name: str, stemmer_filter_name: str
+) -> Dict:
+    analyzer = make_clean_analyzer(language, stopword_filter_name)
+    analyzer['filter'].append(stemmer_filter_name)
+    return analyzer
+
 
 def get_stopwords_from_settings(es_settings, analyzer):
     try:
@@ -158,12 +181,19 @@ def get_stopwords_from_settings(es_settings, analyzer):
     except:
         return []
 
-def set_stemmed_analyzer(settings, stopword_filter_name, stemmer_filter_name, stemmed_analyzer_name, language):
+def set_stemmed_analyzer(
+        settings: Dict,
+        language: str,
+        stopword_filter_name: str,
+        stemmer_filter_name: str,
+        stemmed_analyzer_name: str,
+) -> None:
     filters = settings['analysis'].get('filter', {})
     filters.update({stemmer_filter_name: make_stemmer_filter(language)})
     settings['analysis']['filter'] = filters
     analyzers = settings['analysis'].get('analyzer')
-    analyzers.update({stemmed_analyzer_name: make_stemmed_analyzer(stopword_filter_name, stemmer_filter_name)})
+    analyzers.update({stemmed_analyzer_name: make_stemmed_analyzer(
+        language, stopword_filter_name, stemmer_filter_name)})
     settings['analysis']['analyzer'] = analyzers
 
 def set_char_filter(settings):
@@ -182,8 +212,13 @@ def set_stopword_filter(settings, stopword_filter_name, language):
     settings['analysis']['filter'] = filters
     return True
 
-def set_clean_analyzer(settings, stopword_filter_name, clean_analyzer_name):
-    clean_analyzer = make_clean_analyzer(stopword_filter_name)
+def set_clean_analyzer(
+    settings: Dict,
+    language: str,
+    stopword_filter_name: str,
+    clean_analyzer_name: str,
+) -> None:
+    clean_analyzer = make_clean_analyzer(language, stopword_filter_name)
     analyzers = settings['analysis'].get('analyzer', {})
     analyzers.update({clean_analyzer_name: clean_analyzer})
     settings["analysis"]['analyzer'] = analyzers