Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading

docs/changelog/134253.yaml

@@ -0,0 +1,5 @@
+pr: 134253
+summary: Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading
+area: Mapping
+type: bug
+issues: []

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -253,8 +253,7 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
                 String parentField = searchExecutionContext.parentPath(name());
                 var parent = searchExecutionContext.lookup().fieldType(parentField);
 
-                if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent
-                    && keywordParent.ignoreAbove() != Integer.MAX_VALUE) {
+                if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent && keywordParent.ignoreAbove().isSet()) {
                     if (parent.isStored()) {
                         return storedFieldFetcher(parentField, keywordParent.originalName());
                     } else if (parent.hasDocValues()) {
@@ -278,7 +277,7 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
                 if (kwd != null) {
                     var fieldType = kwd.fieldType();
 
-                    if (fieldType.ignoreAbove() != Integer.MAX_VALUE) {
+                    if (fieldType.ignoreAbove().isSet()) {
                         if (fieldType.isStored()) {
                             return storedFieldFetcher(fieldType.name(), fieldType.originalName());
                         } else if (fieldType.hasDocValues()) {

plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -250,12 +251,10 @@ public static class Builder extends FieldMapper.Builder {
             false
         ).acceptsNull();
 
-        final Parameter<Integer> ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, Integer.MAX_VALUE)
-            .addValidator(v -> {
-                if (v < 0) {
-                    throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
-                }
-            });
+        final Parameter<Integer> ignoreAbove = Parameter.ignoreAboveParam(
+            m -> toType(m).ignoreAbove,
+            IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES
+        );
         final Parameter<String> nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull();
 
         public Builder(String name) {

server/src/main/java/org/elasticsearch/index/IgnoreAbove.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index;
+
+import org.elasticsearch.xcontent.XContentString;
+
+import java.util.Objects;
+
+/**
+ * This class models the ignore_above parameter in indices.
+ */
+public class IgnoreAbove {
+
+    public static final IgnoreAbove IGNORE_ABOVE_STANDARD_INDICES = new IgnoreAbove(null, IndexMode.STANDARD, IndexVersion.current());
+    public static final IgnoreAbove IGNORE_ABOVE_LOGSDB_INDICES = new IgnoreAbove(null, IndexMode.LOGSDB, IndexVersion.current());
+
+    private final Integer value;
+    private final Integer defaultValue;
+
+    public IgnoreAbove(Integer value) {
+        this(Objects.requireNonNull(value), null, null);
+    }
+
+    public IgnoreAbove(Integer value, IndexMode indexMode, IndexVersion indexCreatedVersion) {
+        if (value != null && value < 0) {
+            throw new IllegalArgumentException("[ignore_above] must be positive, got [" + value + "]");
+        }
+
+        this.value = value;
+        this.defaultValue = IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion);
+    }
+
+    public int get() {
+        return value != null ? value : defaultValue;
+    }
+
+    /**
+     * Returns whether ignore_above is set; at field or index level.
+     */
+    public boolean isSet() {
+        // if ignore_above equals default, its not considered to be set, even if it was explicitly set to the default value
+        return Integer.valueOf(get()).equals(defaultValue) == false;
+    }
+
+    /**
+     * Returns whether the given string will be ignored.
+     */
+    public boolean isIgnored(final String s) {
+        if (s == null) return false;
+        return lengthExceedsIgnoreAbove(s.length());
+    }
+
+    public boolean isIgnored(final XContentString s) {
+        if (s == null) return false;
+        return lengthExceedsIgnoreAbove(s.stringLength());
+    }
+
+    private boolean lengthExceedsIgnoreAbove(int strLength) {
+        return strLength > get();
+    }
+
+}

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -803,24 +803,38 @@ public Iterator<Setting<?>> settings() {
      * occupy at most 4 bytes.
      */
 
+    public static final int IGNORE_ABOVE_DEFAULT_STANDARD_INDICES = Integer.MAX_VALUE;
+    public static final int IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES = 8191;
+
     public static final Setting<Integer> IGNORE_ABOVE_SETTING = Setting.intSetting(
         "index.mapping.ignore_above",
-        IndexSettings::getIgnoreAboveDefaultValue,
+        settings -> String.valueOf(getIgnoreAboveDefaultValue(settings)),
         0,
         Integer.MAX_VALUE,
         Property.IndexScope,
         Property.ServerlessPublic
     );
 
-    private static String getIgnoreAboveDefaultValue(final Settings settings) {
-        if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB
-            && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)) {
-            return "8191";
+    private static int getIgnoreAboveDefaultValue(final Settings settings) {
+        if (settings == null) {
+            return IGNORE_ABOVE_DEFAULT_STANDARD_INDICES;
+        }
+        return getIgnoreAboveDefaultValue(IndexSettings.MODE.get(settings), IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings));
+    }
+
+    public static int getIgnoreAboveDefaultValue(final IndexMode indexMode, final IndexVersion indexCreatedVersion) {
+        if (diffIgnoreAboveDefaultForLogs(indexMode, indexCreatedVersion)) {
+            return IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES;
         } else {
-            return String.valueOf(Integer.MAX_VALUE);
+            return IGNORE_ABOVE_DEFAULT_STANDARD_INDICES;
         }
     }
 
+    private static boolean diffIgnoreAboveDefaultForLogs(final IndexMode indexMode, final IndexVersion indexCreatedVersion) {
+        return indexMode == IndexMode.LOGSDB
+            && (indexCreatedVersion != null && indexCreatedVersion.onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB));
+    }
+
     public static final Setting<SeqNoFieldMapper.SeqNoIndexOptions> SEQ_NO_INDEX_OPTIONS_SETTING = Setting.enumSetting(
         SeqNoFieldMapper.SeqNoIndexOptions.class,
         settings -> {

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -1337,6 +1337,14 @@ public static Parameter<Boolean> normsParam(Function<FieldMapper, Boolean> initi
                 .setMergeValidator((prev, curr, c) -> prev == curr || (prev && curr == false));
         }
 
+        public static Parameter<Integer> ignoreAboveParam(Function<FieldMapper, Integer> initializer, int defaultValue) {
+            return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> {
+                if (v < 0) {
+                    throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
+                }
+            });
+        }
+
         /**
          * Defines a script parameter
          * @param initializer   retrieves the equivalent parameter from an existing FieldMapper for use in merges

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -41,7 +41,9 @@
 import org.elasticsearch.common.lucene.search.AutomatonQueries;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IgnoreAbove;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexSortConfig;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
@@ -91,6 +93,7 @@
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
 import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER;
+import static org.elasticsearch.index.IndexSettings.getIgnoreAboveDefaultValue;
 import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
 
 /**
@@ -232,15 +235,14 @@ public Builder(final String name, final MappingParserContext mappingParserContex
             String name,
             IndexAnalyzers indexAnalyzers,
             ScriptCompiler scriptCompiler,
-            int ignoreAboveDefault,
             IndexVersion indexCreatedVersion,
             SourceKeepMode sourceKeepMode
         ) {
             this(
                 name,
                 indexAnalyzers,
                 scriptCompiler,
-                ignoreAboveDefault,
+                getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexCreatedVersion),
                 indexCreatedVersion,
                 IndexMode.STANDARD,
                 null,
@@ -289,12 +291,7 @@ private Builder(
                 }
             }).precludesParameters(normalizer);
             this.ignoreAboveDefault = ignoreAboveDefault;
-            this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).fieldType().ignoreAbove(), ignoreAboveDefault)
-                .addValidator(v -> {
-                    if (v < 0) {
-                        throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
-                    }
-                });
+            this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), ignoreAboveDefault);
             this.indexSortConfig = indexSortConfig;
             this.indexMode = indexMode;
             this.enableDocValuesSkipper = enableDocValuesSkipper;
@@ -303,7 +300,7 @@ private Builder(
         }
 
         public Builder(String name, IndexVersion indexCreatedVersion) {
-            this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE);
+            this(name, null, ScriptCompiler.NONE, indexCreatedVersion, SourceKeepMode.NONE);
         }
 
         public static Builder buildWithDocValuesSkipper(
@@ -316,7 +313,7 @@ public static Builder buildWithDocValuesSkipper(
                 name,
                 null,
                 ScriptCompiler.NONE,
-                Integer.MAX_VALUE,
+                IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion),
                 indexCreatedVersion,
                 indexMode,
                 // Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed.
@@ -537,14 +534,13 @@ private static boolean indexSortConfigByHostName(final IndexSortConfig indexSort
 
     public static final class KeywordFieldType extends StringFieldType {
 
-        private final int ignoreAbove;
+        private final IgnoreAbove ignoreAbove;
         private final String nullValue;
         private final NamedAnalyzer normalizer;
         private final boolean eagerGlobalOrdinals;
         private final FieldValues<String> scriptValues;
         private final boolean isDimension;
         private final boolean isSyntheticSource;
-        private final IndexMode indexMode;
         private final IndexSortConfig indexSortConfig;
         private final boolean hasDocValuesSkipper;
         private final String originalName;
@@ -568,36 +564,34 @@ public KeywordFieldType(
             );
             this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue();
             this.normalizer = normalizer;
-            this.ignoreAbove = builder.ignoreAbove.getValue();
+            this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion);
             this.nullValue = builder.nullValue.getValue();
             this.scriptValues = builder.scriptValues();
             this.isDimension = builder.dimension.getValue();
             this.isSyntheticSource = isSyntheticSource;
-            this.indexMode = builder.indexMode;
             this.indexSortConfig = builder.indexSortConfig;
             this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false;
             this.originalName = isSyntheticSource ? name + "._original" : null;
         }
 
+        public KeywordFieldType(String name) {
+            this(name, true, true, Collections.emptyMap());
+        }
+
         public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map<String, String> meta) {
             super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
             this.normalizer = Lucene.KEYWORD_ANALYZER;
-            this.ignoreAbove = Integer.MAX_VALUE;
+            this.ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES;
             this.nullValue = null;
             this.eagerGlobalOrdinals = false;
             this.scriptValues = null;
             this.isDimension = false;
             this.isSyntheticSource = false;
-            this.indexMode = IndexMode.STANDARD;
             this.indexSortConfig = null;
             this.hasDocValuesSkipper = false;
             this.originalName = null;
         }
 
-        public KeywordFieldType(String name) {
-            this(name, true, true, Collections.emptyMap());
-        }
-
         public KeywordFieldType(String name, FieldType fieldType) {
             super(
                 name,
@@ -608,13 +602,12 @@ public KeywordFieldType(String name, FieldType fieldType) {
                 Collections.emptyMap()
             );
             this.normalizer = Lucene.KEYWORD_ANALYZER;
-            this.ignoreAbove = Integer.MAX_VALUE;
+            this.ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES;
             this.nullValue = null;
             this.eagerGlobalOrdinals = false;
             this.scriptValues = null;
             this.isDimension = false;
             this.isSyntheticSource = false;
-            this.indexMode = IndexMode.STANDARD;
             this.indexSortConfig = null;
             this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false;
             this.originalName = null;
@@ -623,13 +616,12 @@ public KeywordFieldType(String name, FieldType fieldType) {
         public KeywordFieldType(String name, NamedAnalyzer analyzer) {
             super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap());
             this.normalizer = Lucene.KEYWORD_ANALYZER;
-            this.ignoreAbove = Integer.MAX_VALUE;
+            this.ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES;
             this.nullValue = null;
             this.eagerGlobalOrdinals = false;
             this.scriptValues = null;
             this.isDimension = false;
             this.isSyntheticSource = false;
-            this.indexMode = IndexMode.STANDARD;
             this.indexSortConfig = null;
             this.hasDocValuesSkipper = false;
             this.originalName = null;
@@ -938,10 +930,7 @@ protected String parseSourceValue(Object value) {
         }
 
         private String applyIgnoreAboveAndNormalizer(String value) {
-            if (value.length() > ignoreAbove) {
-                return null;
-            }
-
+            if (ignoreAbove.isIgnored(value)) return null;
             return normalizeValue(normalizer(), name(), value);
         }
 
@@ -1060,7 +1049,7 @@ public CollapseType collapseType() {
 
         /** Values that have more chars than the return value of this method will
          *  be skipped at parsing time. */
-        public int ignoreAbove() {
+        public IgnoreAbove ignoreAbove() {
             return ignoreAbove;
         }
 
@@ -1078,10 +1067,6 @@ public boolean hasNormalizer() {
             return normalizer != Lucene.KEYWORD_ANALYZER;
         }
 
-        public IndexMode getIndexMode() {
-            return indexMode;
-        }
-
         public IndexSortConfig getIndexSortConfig() {
             return indexSortConfig;
         }
@@ -1216,7 +1201,7 @@ private boolean indexValue(DocumentParserContext context, XContentString value)
             return false;
         }
 
-        if (value.stringLength() > fieldType().ignoreAbove()) {
+        if (fieldType().ignoreAbove().isIgnored(value)) {
             context.addIgnoredField(fullPath());
             if (isSyntheticSource) {
                 // Save a copy of the field so synthetic source can load it
@@ -1385,7 +1370,7 @@ protected BytesRef preserve(BytesRef value) {
             }
         }
 
-        if (fieldType().ignoreAbove != Integer.MAX_VALUE) {
+        if (fieldType().ignoreAbove.isSet()) {
             layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName) {
                 @Override
                 protected void writeValue(Object value, XContentBuilder b) throws IOException {