Skip to content

[9.0] Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading (#134253) #134518

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

[9.0] Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading (#134253) #134518

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6d3637f
Fixed a bug where text fields in LogsDB indices did not use their key…
Kubik42 Sep 10, 2025
cb4fb49
[CI] Auto commit changes from spotless
Sep 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/134253.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 134253
summary: Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading
area: Mapping
type: bug
issues: []
10 changes: 4 additions & 6 deletions ...u/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,12 +250,10 @@ public static class Builder extends FieldMapper.Builder {
false
).acceptsNull();

final Parameter<Integer> ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, Integer.MAX_VALUE)
.addValidator(v -> {
if (v < 0) {
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
}
});
final Parameter<Integer> ignoreAbove = Parameter.ignoreAboveParam(
m -> toType(m).ignoreAbove,
IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE
);
final Parameter<String> nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull();

public Builder(String name) {
Expand Down
15 changes: 8 additions & 7 deletions server/src/main/java/org/elasticsearch/index/IndexSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -790,20 +790,21 @@ public Iterator<Setting<?>> settings() {

public static final Setting<Integer> IGNORE_ABOVE_SETTING = Setting.intSetting(
"index.mapping.ignore_above",
IndexSettings::getIgnoreAboveDefaultValue,
settings -> String.valueOf(getIgnoreAboveDefaultValue(settings)),
0,
Integer.MAX_VALUE,
Property.IndexScope,
Property.ServerlessPublic
);

private static String getIgnoreAboveDefaultValue(final Settings settings) {
if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB
&& IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)) {
return "8191";
} else {
return String.valueOf(Integer.MAX_VALUE);
private static int getIgnoreAboveDefaultValue(final Settings settings) {
if (settings == null) {
return Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE;
}
return Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(
IndexSettings.MODE.get(settings),
IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings)
);
}

private final Index index;
Expand Down
8 changes: 8 additions & 0 deletions server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1311,6 +1311,14 @@ public static Parameter<Boolean> docValuesParam(Function<FieldMapper, Boolean> i
return Parameter.boolParam("doc_values", false, initializer, defaultValue);
}

public static Parameter<Integer> ignoreAboveParam(Function<FieldMapper, Integer> initializer, int defaultValue) {
return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> {
if (v < 0) {
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
}
});
}

/**
* Defines a script parameter
* @param initializer retrieves the equivalent parameter from an existing FieldMapper for use in merges
Expand Down
89 changes: 59 additions & 30 deletions server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.elasticsearch.common.lucene.search.AutomatonQueries;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
Expand Down Expand Up @@ -78,6 +79,7 @@
import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH;
import static org.elasticsearch.core.Strings.format;
import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
import static org.elasticsearch.index.mapper.Mapper.IgnoreAbove.getIgnoreAboveDefaultValue;

/**
* A field mapper for keywords. This mapper accepts strings and indexes them as-is.
Expand Down Expand Up @@ -155,6 +157,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
private final Parameter<Integer> ignoreAbove;
private final int ignoreAboveDefault;

private final IndexMode indexMode;
private final Parameter<String> indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions);
private final Parameter<Boolean> hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false);
private final Parameter<SimilarityProvider> similarity = TextParams.similarity(
Expand Down Expand Up @@ -189,16 +192,29 @@ public Builder(final String name, final MappingParserContext mappingParserContex
mappingParserContext.getIndexAnalyzers(),
mappingParserContext.scriptCompiler(),
IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()),
mappingParserContext.getIndexSettings().getIndexVersionCreated()
mappingParserContext.getIndexSettings().getIndexVersionCreated(),
mappingParserContext.getIndexSettings().getMode()
);
}

Builder(
Builder(String name, IndexAnalyzers indexAnalyzers, ScriptCompiler scriptCompiler, IndexVersion indexCreatedVersion) {
this(
name,
indexAnalyzers,
scriptCompiler,
getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexCreatedVersion),
indexCreatedVersion,
IndexMode.STANDARD
);
}

private Builder(
String name,
IndexAnalyzers indexAnalyzers,
ScriptCompiler scriptCompiler,
int ignoreAboveDefault,
IndexVersion indexCreatedVersion
IndexVersion indexCreatedVersion,
IndexMode indexMode
) {
super(name);
this.indexAnalyzers = indexAnalyzers;
Expand Down Expand Up @@ -227,16 +243,12 @@ public Builder(final String name, final MappingParserContext mappingParserContex
}
}).precludesParameters(normalizer);
this.ignoreAboveDefault = ignoreAboveDefault;
this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).fieldType().ignoreAbove(), ignoreAboveDefault)
.addValidator(v -> {
if (v < 0) {
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
}
});
this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), ignoreAboveDefault);
this.indexMode = indexMode;
}

public Builder(String name, IndexVersion indexCreatedVersion) {
this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion);
this(name, null, ScriptCompiler.NONE, indexCreatedVersion);
}

public Builder ignoreAbove(int ignoreAbove) {
Expand Down Expand Up @@ -385,13 +397,16 @@ public KeywordFieldMapper build(MapperBuilderContext context) {

public static final class KeywordFieldType extends StringFieldType {

private final int ignoreAbove;
private static final IgnoreAbove IGNORE_ABOVE_DEFAULT = new IgnoreAbove(null, IndexMode.STANDARD);

private final IgnoreAbove ignoreAbove;
private final String nullValue;
private final NamedAnalyzer normalizer;
private final boolean eagerGlobalOrdinals;
private final FieldValues<String> scriptValues;
private final boolean isDimension;
private final boolean isSyntheticSource;
private final String originalName;

public KeywordFieldType(
String name,
Expand All @@ -412,26 +427,28 @@ public KeywordFieldType(
);
this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue();
this.normalizer = normalizer;
this.ignoreAbove = builder.ignoreAbove.getValue();
this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion);
this.nullValue = builder.nullValue.getValue();
this.scriptValues = builder.scriptValues();
this.isDimension = builder.dimension.getValue();
this.isSyntheticSource = isSyntheticSource;
this.originalName = isSyntheticSource ? name + "._original" : null;
}

public KeywordFieldType(String name) {
this(name, true, true, Collections.emptyMap());
}

public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map<String, String> meta) {
super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
this.normalizer = Lucene.KEYWORD_ANALYZER;
this.ignoreAbove = Integer.MAX_VALUE;
this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
this.nullValue = null;
this.eagerGlobalOrdinals = false;
this.scriptValues = null;
this.isDimension = false;
this.isSyntheticSource = false;
}

public KeywordFieldType(String name) {
this(name, true, true, Collections.emptyMap());
this.originalName = null;
}

public KeywordFieldType(String name, FieldType fieldType) {
Expand All @@ -444,23 +461,25 @@ public KeywordFieldType(String name, FieldType fieldType) {
Collections.emptyMap()
);
this.normalizer = Lucene.KEYWORD_ANALYZER;
this.ignoreAbove = Integer.MAX_VALUE;
this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
this.nullValue = null;
this.eagerGlobalOrdinals = false;
this.scriptValues = null;
this.isDimension = false;
this.isSyntheticSource = false;
this.originalName = null;
}

public KeywordFieldType(String name, NamedAnalyzer analyzer) {
super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap());
this.normalizer = Lucene.KEYWORD_ANALYZER;
this.ignoreAbove = Integer.MAX_VALUE;
this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
this.nullValue = null;
this.eagerGlobalOrdinals = false;
this.scriptValues = null;
this.isDimension = false;
this.isSyntheticSource = false;
this.originalName = null;
}

@Override
Expand Down Expand Up @@ -710,15 +729,16 @@ private SourceValueFetcher sourceValueFetcher(Set<String> sourcePaths) {
@Override
protected String parseSourceValue(Object value) {
String keywordValue = value.toString();
if (keywordValue.length() > ignoreAbove) {
return null;
}

return normalizeValue(normalizer(), name(), keywordValue);
return applyIgnoreAboveAndNormalizer(keywordValue);
}
};
}

private String applyIgnoreAboveAndNormalizer(String value) {
if (ignoreAbove.isIgnored(value)) return null;
return normalizeValue(normalizer(), name(), value);
}

@Override
public Object valueForDisplay(Object value) {
if (value == null) {
Expand Down Expand Up @@ -834,7 +854,7 @@ public CollapseType collapseType() {

/** Values that have more chars than the return value of this method will
* be skipped at parsing time. */
public int ignoreAbove() {
public IgnoreAbove ignoreAbove() {
return ignoreAbove;
}

Expand All @@ -851,6 +871,15 @@ public boolean hasScriptValues() {
public boolean hasNormalizer() {
return normalizer != Lucene.KEYWORD_ANALYZER;
}

/**
* The name used to store "original" that have been ignored
* by {@link KeywordFieldType#ignoreAbove()} so that they can be rebuilt
* for synthetic source.
*/
public String originalName() {
return originalName;
}
}

private final boolean indexed;
Expand All @@ -862,11 +891,11 @@ public boolean hasNormalizer() {
private final Script script;
private final ScriptCompiler scriptCompiler;
private final IndexVersion indexCreatedVersion;
private final IndexMode indexMode;
private final boolean isSyntheticSource;

private final IndexAnalyzers indexAnalyzers;
private final int ignoreAboveDefault;
private final int ignoreAbove;

private KeywordFieldMapper(
String simpleName,
Expand All @@ -888,9 +917,9 @@ private KeywordFieldMapper(
this.indexAnalyzers = builder.indexAnalyzers;
this.scriptCompiler = builder.scriptCompiler;
this.indexCreatedVersion = builder.indexCreatedVersion;
this.indexMode = builder.indexMode;
this.isSyntheticSource = isSyntheticSource;
this.ignoreAboveDefault = builder.ignoreAboveDefault;
this.ignoreAbove = builder.ignoreAbove.getValue();
}

@Override
Expand Down Expand Up @@ -923,7 +952,7 @@ private void indexValue(DocumentParserContext context, String value) {
return;
}

if (value.length() > fieldType().ignoreAbove()) {
if (fieldType().ignoreAbove().isIgnored(value)) {
context.addIgnoredField(fullPath());
if (isSyntheticSource) {
// Save a copy of the field so synthetic source can load it
Expand Down Expand Up @@ -1008,7 +1037,7 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {

@Override
public FieldMapper.Builder getMergeBuilder() {
return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion).dimension(
return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, indexMode).dimension(
fieldType().isDimension()
).init(this);
}
Expand Down Expand Up @@ -1078,7 +1107,7 @@ protected BytesRef preserve(BytesRef value) {
});
}

if (fieldType().ignoreAbove != Integer.MAX_VALUE) {
if (fieldType().ignoreAbove.isSet()) {
layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) {
@Override
protected void writeValue(Object value, XContentBuilder b) throws IOException {
Expand Down
Loading