Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading (#134253)

* Added keyword multi field with ignore_above to match only text bwc tests

* Rework ignore_above

* Added unit tests

* Undo changed to match only text bwc test

* formatting

* Removed indexMode from field type

* Added another test case

* Fixed failing bwc tests

* Improved msg

* Added additional tests

* Added IgnoreAbove record, addressed index-level ignore above

* Fixed test typos

* Added IgnoreAboveTest

* Enforce at least one value or defaultValue to always be non-null when IgnoreAbove is initialized

* When initializing IgnoreAbove, dont use defaultValue from builder - this fixes failing BWC test

* Fixed typo

* Switched IgnoreAbove to constructor only, removed the ability to set default directly

* Update docs/changelog/134253.yaml

* Update 134253.yaml

* Move IgnoreAbove into Mapper and make it final, move everything new out of IndexSettings and into IgnoreAbove

* Fixed typo

* Added helpful constructor to IgnoreAbove

* Added helpful constructor to IgnoreAbove

Author: Kubik42

docs/changelog/134253.yaml

@@ -0,0 +1,5 @@
+pr: 134253
+summary: Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading
+area: Mapping
+type: bug
+issues: []

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -253,8 +253,7 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
                 String parentField = searchExecutionContext.parentPath(name());
                 var parent = searchExecutionContext.lookup().fieldType(parentField);
 
-                if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent
-                    && keywordParent.ignoreAbove() != Integer.MAX_VALUE) {
+                if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent && keywordParent.ignoreAbove().isSet()) {
                     if (parent.isStored()) {
                         return storedFieldFetcher(parentField, keywordParent.originalName());
                     } else if (parent.hasDocValues()) {
@@ -278,7 +277,7 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
                 if (kwd != null) {
                     var fieldType = kwd.fieldType();
 
-                    if (fieldType.ignoreAbove() != Integer.MAX_VALUE) {
+                    if (fieldType.ignoreAbove().isSet()) {
                         if (fieldType.isStored()) {
                             return storedFieldFetcher(fieldType.name(), fieldType.originalName());
                         } else if (fieldType.hasDocValues()) {

plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java

@@ -250,12 +250,10 @@ public static class Builder extends FieldMapper.Builder {
             false
         ).acceptsNull();
 
-        final Parameter<Integer> ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, Integer.MAX_VALUE)
-            .addValidator(v -> {
-                if (v < 0) {
-                    throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
-                }
-            });
+        final Parameter<Integer> ignoreAbove = Parameter.ignoreAboveParam(
+            m -> toType(m).ignoreAbove,
+            IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE
+        );
         final Parameter<String> nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull();
 
         public Builder(String name) {

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -805,20 +805,21 @@ public Iterator<Setting<?>> settings() {
 
     public static final Setting<Integer> IGNORE_ABOVE_SETTING = Setting.intSetting(
         "index.mapping.ignore_above",
-        IndexSettings::getIgnoreAboveDefaultValue,
+        settings -> String.valueOf(getIgnoreAboveDefaultValue(settings)),
         0,
         Integer.MAX_VALUE,
         Property.IndexScope,
         Property.ServerlessPublic
     );
 
-    private static String getIgnoreAboveDefaultValue(final Settings settings) {
-        if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB
-            && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)) {
-            return "8191";
-        } else {
-            return String.valueOf(Integer.MAX_VALUE);
+    private static int getIgnoreAboveDefaultValue(final Settings settings) {
+        if (settings == null) {
+            return Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE;
         }
+        return Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(
+            IndexSettings.MODE.get(settings),
+            IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings)
+        );
     }
 
     public static final Setting<SeqNoFieldMapper.SeqNoIndexOptions> SEQ_NO_INDEX_OPTIONS_SETTING = Setting.enumSetting(

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -1337,6 +1337,14 @@ public static Parameter<Boolean> normsParam(Function<FieldMapper, Boolean> initi
                 .setMergeValidator((prev, curr, c) -> prev == curr || (prev && curr == false));
         }
 
+        public static Parameter<Integer> ignoreAboveParam(Function<FieldMapper, Integer> initializer, int defaultValue) {
+            return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> {
+                if (v < 0) {
+                    throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
+                }
+            });
+        }
+
         /**
          * Defines a script parameter
          * @param initializer   retrieves the equivalent parameter from an existing FieldMapper for use in merges

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -92,6 +92,7 @@
 import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
 import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER;
 import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
+import static org.elasticsearch.index.mapper.Mapper.IgnoreAbove.getIgnoreAboveDefaultValue;
 
 /**
  * A field mapper for keywords. This mapper accepts strings and indexes them as-is.
@@ -232,15 +233,14 @@ public Builder(final String name, final MappingParserContext mappingParserContex
             String name,
             IndexAnalyzers indexAnalyzers,
             ScriptCompiler scriptCompiler,
-            int ignoreAboveDefault,
             IndexVersion indexCreatedVersion,
             SourceKeepMode sourceKeepMode
         ) {
             this(
                 name,
                 indexAnalyzers,
                 scriptCompiler,
-                ignoreAboveDefault,
+                getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexCreatedVersion),
                 indexCreatedVersion,
                 IndexMode.STANDARD,
                 null,
@@ -289,12 +289,7 @@ private Builder(
                 }
             }).precludesParameters(normalizer);
             this.ignoreAboveDefault = ignoreAboveDefault;
-            this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).fieldType().ignoreAbove(), ignoreAboveDefault)
-                .addValidator(v -> {
-                    if (v < 0) {
-                        throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
-                    }
-                });
+            this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), ignoreAboveDefault);
             this.indexSortConfig = indexSortConfig;
             this.indexMode = indexMode;
             this.enableDocValuesSkipper = enableDocValuesSkipper;
@@ -303,7 +298,7 @@ private Builder(
         }
 
         public Builder(String name, IndexVersion indexCreatedVersion) {
-            this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE);
+            this(name, null, ScriptCompiler.NONE, indexCreatedVersion, SourceKeepMode.NONE);
         }
 
         public static Builder buildWithDocValuesSkipper(
@@ -316,7 +311,7 @@ public static Builder buildWithDocValuesSkipper(
                 name,
                 null,
                 ScriptCompiler.NONE,
-                Integer.MAX_VALUE,
+                getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion),
                 indexCreatedVersion,
                 indexMode,
                 // Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed.
@@ -537,14 +532,15 @@ private static boolean indexSortConfigByHostName(final IndexSortConfig indexSort
 
     public static final class KeywordFieldType extends StringFieldType {
 
-        private final int ignoreAbove;
+        private static final IgnoreAbove IGNORE_ABOVE_DEFAULT = new IgnoreAbove(null, IndexMode.STANDARD);
+
+        private final IgnoreAbove ignoreAbove;
         private final String nullValue;
         private final NamedAnalyzer normalizer;
         private final boolean eagerGlobalOrdinals;
         private final FieldValues<String> scriptValues;
         private final boolean isDimension;
         private final boolean isSyntheticSource;
-        private final IndexMode indexMode;
         private final IndexSortConfig indexSortConfig;
         private final boolean hasDocValuesSkipper;
         private final String originalName;
@@ -568,36 +564,34 @@ public KeywordFieldType(
             );
             this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue();
             this.normalizer = normalizer;
-            this.ignoreAbove = builder.ignoreAbove.getValue();
+            this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion);
             this.nullValue = builder.nullValue.getValue();
             this.scriptValues = builder.scriptValues();
             this.isDimension = builder.dimension.getValue();
             this.isSyntheticSource = isSyntheticSource;
-            this.indexMode = builder.indexMode;
             this.indexSortConfig = builder.indexSortConfig;
             this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false;
             this.originalName = isSyntheticSource ? name + "._original" : null;
         }
 
+        public KeywordFieldType(String name) {
+            this(name, true, true, Collections.emptyMap());
+        }
+
         public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map<String, String> meta) {
             super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
             this.normalizer = Lucene.KEYWORD_ANALYZER;
-            this.ignoreAbove = Integer.MAX_VALUE;
+            this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
             this.nullValue = null;
             this.eagerGlobalOrdinals = false;
             this.scriptValues = null;
             this.isDimension = false;
             this.isSyntheticSource = false;
-            this.indexMode = IndexMode.STANDARD;
             this.indexSortConfig = null;
             this.hasDocValuesSkipper = false;
             this.originalName = null;
         }
 
-        public KeywordFieldType(String name) {
-            this(name, true, true, Collections.emptyMap());
-        }
-
         public KeywordFieldType(String name, FieldType fieldType) {
             super(
                 name,
@@ -608,13 +602,12 @@ public KeywordFieldType(String name, FieldType fieldType) {
                 Collections.emptyMap()
             );
             this.normalizer = Lucene.KEYWORD_ANALYZER;
-            this.ignoreAbove = Integer.MAX_VALUE;
+            this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
             this.nullValue = null;
             this.eagerGlobalOrdinals = false;
             this.scriptValues = null;
             this.isDimension = false;
             this.isSyntheticSource = false;
-            this.indexMode = IndexMode.STANDARD;
             this.indexSortConfig = null;
             this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false;
             this.originalName = null;
@@ -623,13 +616,12 @@ public KeywordFieldType(String name, FieldType fieldType) {
         public KeywordFieldType(String name, NamedAnalyzer analyzer) {
             super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap());
             this.normalizer = Lucene.KEYWORD_ANALYZER;
-            this.ignoreAbove = Integer.MAX_VALUE;
+            this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
             this.nullValue = null;
             this.eagerGlobalOrdinals = false;
             this.scriptValues = null;
             this.isDimension = false;
             this.isSyntheticSource = false;
-            this.indexMode = IndexMode.STANDARD;
             this.indexSortConfig = null;
             this.hasDocValuesSkipper = false;
             this.originalName = null;
@@ -938,10 +930,7 @@ protected String parseSourceValue(Object value) {
         }
 
         private String applyIgnoreAboveAndNormalizer(String value) {
-            if (value.length() > ignoreAbove) {
-                return null;
-            }
-
+            if (ignoreAbove.isIgnored(value)) return null;
             return normalizeValue(normalizer(), name(), value);
         }
 
@@ -1060,7 +1049,7 @@ public CollapseType collapseType() {
 
         /** Values that have more chars than the return value of this method will
          *  be skipped at parsing time. */
-        public int ignoreAbove() {
+        public IgnoreAbove ignoreAbove() {
             return ignoreAbove;
         }
 
@@ -1078,10 +1067,6 @@ public boolean hasNormalizer() {
             return normalizer != Lucene.KEYWORD_ANALYZER;
         }
 
-        public IndexMode getIndexMode() {
-            return indexMode;
-        }
-
         public IndexSortConfig getIndexSortConfig() {
             return indexSortConfig;
         }
@@ -1216,7 +1201,7 @@ private boolean indexValue(DocumentParserContext context, XContentString value)
             return false;
         }
 
-        if (value.stringLength() > fieldType().ignoreAbove()) {
+        if (fieldType().ignoreAbove().isIgnored(value)) {
             context.addIgnoredField(fullPath());
             if (isSyntheticSource) {
                 // Save a copy of the field so synthetic source can load it
@@ -1385,7 +1370,7 @@ protected BytesRef preserve(BytesRef value) {
             }
         }
 
-        if (fieldType().ignoreAbove != Integer.MAX_VALUE) {
+        if (fieldType().ignoreAbove.isSet()) {
             layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName) {
                 @Override
                 protected void writeValue(Object value, XContentBuilder b) throws IOException {

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentString;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -131,6 +132,78 @@ default boolean supportsVersion(IndexVersion indexCreatedVersion) {
         }
     }
 
+    /**
+     * This class models the ignore_above parameter in indices.
+     */
+    public static final class IgnoreAbove {
+
+        public static final int IGNORE_ABOVE_DEFAULT_VALUE = Integer.MAX_VALUE;
+        public static final int IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES = 8191;
+
+        private final Integer value;
+        private final Integer defaultValue;
+
+        public IgnoreAbove(Integer value) {
+            this(Objects.requireNonNull(value), IndexMode.STANDARD, IndexVersion.current());
+        }
+
+        public IgnoreAbove(Integer value, IndexMode indexMode) {
+            this(value, indexMode, IndexVersion.current());
+        }
+
+        public IgnoreAbove(Integer value, IndexMode indexMode, IndexVersion indexCreatedVersion) {
+            if (value != null && value < 0) {
+                throw new IllegalArgumentException("[ignore_above] must be positive, got [" + value + "]");
+            }
+
+            this.value = value;
+            this.defaultValue = getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion);
+        }
+
+        public int get() {
+            return value != null ? value : defaultValue;
+        }
+
+        /**
+         * Returns whether ignore_above is set; at field or index level.
+         */
+        public boolean isSet() {
+            // if ignore_above equals default, its not considered to be set, even if it was explicitly set to the default value
+            return Integer.valueOf(get()).equals(defaultValue) == false;
+        }
+
+        /**
+         * Returns whether the given string will be ignored.
+         */
+        public boolean isIgnored(final String s) {
+            if (s == null) return false;
+            return lengthExceedsIgnoreAbove(s.length());
+        }
+
+        public boolean isIgnored(final XContentString s) {
+            if (s == null) return false;
+            return lengthExceedsIgnoreAbove(s.stringLength());
+        }
+
+        private boolean lengthExceedsIgnoreAbove(int strLength) {
+            return strLength > get();
+        }
+
+        public static int getIgnoreAboveDefaultValue(final IndexMode indexMode, final IndexVersion indexCreatedVersion) {
+            if (diffIgnoreAboveDefaultForLogs(indexMode, indexCreatedVersion)) {
+                return IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES;
+            } else {
+                return IGNORE_ABOVE_DEFAULT_VALUE;
+            }
+        }
+
+        private static boolean diffIgnoreAboveDefaultForLogs(final IndexMode indexMode, final IndexVersion indexCreatedVersion) {
+            return indexMode == IndexMode.LOGSDB
+                && (indexCreatedVersion != null && indexCreatedVersion.onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB));
+        }
+
+    }
+
     private final String leafName;
 
     @SuppressWarnings("this-escape")

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -1021,15 +1021,15 @@ public boolean isAggregatable() {
          * A delegate by definition must have doc_values or be stored so most of the time it can be used for loading.
          */
         public boolean canUseSyntheticSourceDelegateForLoading() {
-            return syntheticSourceDelegate != null && syntheticSourceDelegate.ignoreAbove() == Integer.MAX_VALUE;
+            return syntheticSourceDelegate != null && syntheticSourceDelegate.ignoreAbove().isSet() == false;
         }
 
         /**
          * Returns true if the delegate sub-field can be used for querying only (ie. isIndexed must be true)
          */
         public boolean canUseSyntheticSourceDelegateForQuerying() {
             return syntheticSourceDelegate != null
-                && syntheticSourceDelegate.ignoreAbove() == Integer.MAX_VALUE
+                && syntheticSourceDelegate.ignoreAbove().isSet() == false
                 && syntheticSourceDelegate.isIndexed();
         }
 
@@ -1045,7 +1045,7 @@ public boolean canUseSyntheticSourceDelegateForQueryingEquality(String str) {
                 return false;
             }
             // Can't push equality if the field we're checking for is so big we'd ignore it.
-            return str.length() <= syntheticSourceDelegate.ignoreAbove();
+            return syntheticSourceDelegate.ignoreAbove().isIgnored(str) == false;
         }
 
         @Override