Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading (elastic#134253)

* Added keyword multi field with ignore_above to match only text bwc tests

* Rework ignore_above

* Added unit tests

* Undo changed to match only text bwc test

* formatting

* Removed indexMode from field type

* Added another test case

* Fixed failing bwc tests

* Improved msg

* Added additional tests

* Added IgnoreAbove record, addressed index-level ignore above

* Fixed test typos

* Added IgnoreAboveTest

* Enforce at least one value or defaultValue to always be non-null when IgnoreAbove is initialized

* When initializing IgnoreAbove, dont use defaultValue from builder - this fixes failing BWC test

* Fixed typo

* Switched IgnoreAbove to constructor only, removed the ability to set default directly

* Update docs/changelog/134253.yaml

* Update 134253.yaml

* Move IgnoreAbove into Mapper and make it final, move everything new out of IndexSettings and into IgnoreAbove

* Fixed typo

* Added helpful constructor to IgnoreAbove

* Added helpful constructor to IgnoreAbove

(cherry picked from commit 2c3bee7)

# Conflicts:
#	modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
#	qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java
#	server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
#	server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
#	server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
#	server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java
#	server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java
#	x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java

Author: Kubik42

docs/changelog/134253.yaml

@@ -0,0 +1,5 @@
+pr: 134253
+summary: Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading
+area: Mapping
+type: bug
+issues: []

plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java

@@ -250,12 +250,10 @@ public static class Builder extends FieldMapper.Builder {
             false
         ).acceptsNull();
 
-        final Parameter<Integer> ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, Integer.MAX_VALUE)
-            .addValidator(v -> {
-                if (v < 0) {
-                    throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
-                }
-            });
+        final Parameter<Integer> ignoreAbove = Parameter.ignoreAboveParam(
+            m -> toType(m).ignoreAbove,
+            IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE
+        );
         final Parameter<String> nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull();
 
         public Builder(String name) {

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -790,20 +790,21 @@ public Iterator<Setting<?>> settings() {
 
     public static final Setting<Integer> IGNORE_ABOVE_SETTING = Setting.intSetting(
         "index.mapping.ignore_above",
-        IndexSettings::getIgnoreAboveDefaultValue,
+        settings -> String.valueOf(getIgnoreAboveDefaultValue(settings)),
         0,
         Integer.MAX_VALUE,
         Property.IndexScope,
         Property.ServerlessPublic
     );
 
-    private static String getIgnoreAboveDefaultValue(final Settings settings) {
-        if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB
-            && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)) {
-            return "8191";
-        } else {
-            return String.valueOf(Integer.MAX_VALUE);
+    private static int getIgnoreAboveDefaultValue(final Settings settings) {
+        if (settings == null) {
+            return Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE;
         }
+        return Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(
+            IndexSettings.MODE.get(settings),
+            IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings)
+        );
     }
 
     private final Index index;

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -1311,6 +1311,14 @@ public static Parameter<Boolean> docValuesParam(Function<FieldMapper, Boolean> i
             return Parameter.boolParam("doc_values", false, initializer, defaultValue);
         }
 
+        public static Parameter<Integer> ignoreAboveParam(Function<FieldMapper, Integer> initializer, int defaultValue) {
+            return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> {
+                if (v < 0) {
+                    throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
+                }
+            });
+        }
+
         /**
          * Defines a script parameter
          * @param initializer   retrieves the equivalent parameter from an existing FieldMapper for use in merges

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -38,6 +38,7 @@
 import org.elasticsearch.common.lucene.search.AutomatonQueries;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -78,6 +79,7 @@
 import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH;
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
+import static org.elasticsearch.index.mapper.Mapper.IgnoreAbove.getIgnoreAboveDefaultValue;
 
 /**
  * A field mapper for keywords. This mapper accepts strings and indexes them as-is.
@@ -155,6 +157,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
         private final Parameter<Integer> ignoreAbove;
         private final int ignoreAboveDefault;
 
+        private final IndexMode indexMode;
         private final Parameter<String> indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions);
         private final Parameter<Boolean> hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false);
         private final Parameter<SimilarityProvider> similarity = TextParams.similarity(
@@ -189,16 +192,34 @@ public Builder(final String name, final MappingParserContext mappingParserContex
                 mappingParserContext.getIndexAnalyzers(),
                 mappingParserContext.scriptCompiler(),
                 IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()),
-                mappingParserContext.getIndexSettings().getIndexVersionCreated()
+                mappingParserContext.getIndexSettings().getIndexVersionCreated(),
+                mappingParserContext.getIndexSettings().getMode()
             );
         }
 
         Builder(
             String name,
             IndexAnalyzers indexAnalyzers,
             ScriptCompiler scriptCompiler,
-            int ignoreAboveDefault,
             IndexVersion indexCreatedVersion
+        ) {
+            this(
+                name,
+                indexAnalyzers,
+                scriptCompiler,
+                getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexCreatedVersion),
+                indexCreatedVersion,
+                IndexMode.STANDARD
+            );
+        }
+
+        private Builder(
+            String name,
+            IndexAnalyzers indexAnalyzers,
+            ScriptCompiler scriptCompiler,
+            int ignoreAboveDefault,
+            IndexVersion indexCreatedVersion,
+            IndexMode indexMode
         ) {
             super(name);
             this.indexAnalyzers = indexAnalyzers;
@@ -227,16 +248,12 @@ public Builder(final String name, final MappingParserContext mappingParserContex
                 }
             }).precludesParameters(normalizer);
             this.ignoreAboveDefault = ignoreAboveDefault;
-            this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).fieldType().ignoreAbove(), ignoreAboveDefault)
-                .addValidator(v -> {
-                    if (v < 0) {
-                        throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
-                    }
-                });
+            this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), ignoreAboveDefault);
+            this.indexMode = indexMode;
         }
 
         public Builder(String name, IndexVersion indexCreatedVersion) {
-            this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion);
+            this(name, null, ScriptCompiler.NONE, indexCreatedVersion);
         }
 
         public Builder ignoreAbove(int ignoreAbove) {
@@ -385,13 +402,16 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
 
     public static final class KeywordFieldType extends StringFieldType {
 
-        private final int ignoreAbove;
+        private static final IgnoreAbove IGNORE_ABOVE_DEFAULT = new IgnoreAbove(null, IndexMode.STANDARD);
+
+        private final IgnoreAbove ignoreAbove;
         private final String nullValue;
         private final NamedAnalyzer normalizer;
         private final boolean eagerGlobalOrdinals;
         private final FieldValues<String> scriptValues;
         private final boolean isDimension;
         private final boolean isSyntheticSource;
+        private final String originalName;
 
         public KeywordFieldType(
             String name,
@@ -412,26 +432,28 @@ public KeywordFieldType(
             );
             this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue();
             this.normalizer = normalizer;
-            this.ignoreAbove = builder.ignoreAbove.getValue();
+            this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion);
             this.nullValue = builder.nullValue.getValue();
             this.scriptValues = builder.scriptValues();
             this.isDimension = builder.dimension.getValue();
             this.isSyntheticSource = isSyntheticSource;
+            this.originalName = isSyntheticSource ? name + "._original" : null;
+        }
+
+        public KeywordFieldType(String name) {
+            this(name, true, true, Collections.emptyMap());
         }
 
         public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map<String, String> meta) {
             super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
             this.normalizer = Lucene.KEYWORD_ANALYZER;
-            this.ignoreAbove = Integer.MAX_VALUE;
+            this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
             this.nullValue = null;
             this.eagerGlobalOrdinals = false;
             this.scriptValues = null;
             this.isDimension = false;
             this.isSyntheticSource = false;
-        }
-
-        public KeywordFieldType(String name) {
-            this(name, true, true, Collections.emptyMap());
+            this.originalName = null;
         }
 
         public KeywordFieldType(String name, FieldType fieldType) {
@@ -444,23 +466,25 @@ public KeywordFieldType(String name, FieldType fieldType) {
                 Collections.emptyMap()
             );
             this.normalizer = Lucene.KEYWORD_ANALYZER;
-            this.ignoreAbove = Integer.MAX_VALUE;
+            this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
             this.nullValue = null;
             this.eagerGlobalOrdinals = false;
             this.scriptValues = null;
             this.isDimension = false;
             this.isSyntheticSource = false;
+            this.originalName = null;
         }
 
         public KeywordFieldType(String name, NamedAnalyzer analyzer) {
             super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap());
             this.normalizer = Lucene.KEYWORD_ANALYZER;
-            this.ignoreAbove = Integer.MAX_VALUE;
+            this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
             this.nullValue = null;
             this.eagerGlobalOrdinals = false;
             this.scriptValues = null;
             this.isDimension = false;
             this.isSyntheticSource = false;
+            this.originalName = null;
         }
 
         @Override
@@ -710,15 +734,16 @@ private SourceValueFetcher sourceValueFetcher(Set<String> sourcePaths) {
                 @Override
                 protected String parseSourceValue(Object value) {
                     String keywordValue = value.toString();
-                    if (keywordValue.length() > ignoreAbove) {
-                        return null;
-                    }
-
-                    return normalizeValue(normalizer(), name(), keywordValue);
+                    return applyIgnoreAboveAndNormalizer(keywordValue);
                 }
             };
         }
 
+        private String applyIgnoreAboveAndNormalizer(String value) {
+            if (ignoreAbove.isIgnored(value)) return null;
+            return normalizeValue(normalizer(), name(), value);
+        }
+
         @Override
         public Object valueForDisplay(Object value) {
             if (value == null) {
@@ -834,7 +859,7 @@ public CollapseType collapseType() {
 
         /** Values that have more chars than the return value of this method will
          *  be skipped at parsing time. */
-        public int ignoreAbove() {
+        public IgnoreAbove ignoreAbove() {
             return ignoreAbove;
         }
 
@@ -851,6 +876,15 @@ public boolean hasScriptValues() {
         public boolean hasNormalizer() {
             return normalizer != Lucene.KEYWORD_ANALYZER;
         }
+
+        /**
+         * The name used to store "original" that have been ignored
+         * by {@link KeywordFieldType#ignoreAbove()} so that they can be rebuilt
+         * for synthetic source.
+         */
+        public String originalName() {
+            return originalName;
+        }
     }
 
     private final boolean indexed;
@@ -862,11 +896,11 @@ public boolean hasNormalizer() {
     private final Script script;
     private final ScriptCompiler scriptCompiler;
     private final IndexVersion indexCreatedVersion;
+    private final IndexMode indexMode;
     private final boolean isSyntheticSource;
 
     private final IndexAnalyzers indexAnalyzers;
     private final int ignoreAboveDefault;
-    private final int ignoreAbove;
 
     private KeywordFieldMapper(
         String simpleName,
@@ -888,9 +922,9 @@ private KeywordFieldMapper(
         this.indexAnalyzers = builder.indexAnalyzers;
         this.scriptCompiler = builder.scriptCompiler;
         this.indexCreatedVersion = builder.indexCreatedVersion;
+        this.indexMode = builder.indexMode;
         this.isSyntheticSource = isSyntheticSource;
         this.ignoreAboveDefault = builder.ignoreAboveDefault;
-        this.ignoreAbove = builder.ignoreAbove.getValue();
     }
 
     @Override
@@ -923,7 +957,7 @@ private void indexValue(DocumentParserContext context, String value) {
             return;
         }
 
-        if (value.length() > fieldType().ignoreAbove()) {
+        if (fieldType().ignoreAbove().isIgnored(value)) {
             context.addIgnoredField(fullPath());
             if (isSyntheticSource) {
                 // Save a copy of the field so synthetic source can load it
@@ -1008,7 +1042,7 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion).dimension(
+        return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, indexMode).dimension(
             fieldType().isDimension()
         ).init(this);
     }
@@ -1078,7 +1112,7 @@ protected BytesRef preserve(BytesRef value) {
             });
         }
 
-        if (fieldType().ignoreAbove != Integer.MAX_VALUE) {
+        if (fieldType().ignoreAbove.isSet()) {
             layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) {
                 @Override
                 protected void writeValue(Object value, XContentBuilder b) throws IOException {