Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/134253.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 134253
summary: Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading
area: Mapping
type: bug
issues: []
Original file line number Diff line number Diff line change
Expand Up @@ -247,8 +247,7 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
String parentField = searchExecutionContext.parentPath(name());
var parent = searchExecutionContext.lookup().fieldType(parentField);

if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent
&& keywordParent.ignoreAbove() != Integer.MAX_VALUE) {
if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent && keywordParent.ignoreAbove().isSet()) {
if (parent.isStored()) {
return storedFieldFetcher(parentField, keywordParent.originalName());
} else if (parent.hasDocValues()) {
Expand All @@ -272,7 +271,7 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
if (kwd != null) {
var fieldType = kwd.fieldType();

if (fieldType.ignoreAbove() != Integer.MAX_VALUE) {
if (fieldType.ignoreAbove().isSet()) {
if (fieldType.isStored()) {
return storedFieldFetcher(fieldType.name(), fieldType.originalName());
} else if (fieldType.hasDocValues()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -250,12 +250,10 @@ public static class Builder extends FieldMapper.Builder {
false
).acceptsNull();

final Parameter<Integer> ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, Integer.MAX_VALUE)
.addValidator(v -> {
if (v < 0) {
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
}
});
final Parameter<Integer> ignoreAbove = Parameter.ignoreAboveParam(
m -> toType(m).ignoreAbove,
IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE
);
final Parameter<String> nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull();

public Builder(String name) {
Expand Down
15 changes: 8 additions & 7 deletions server/src/main/java/org/elasticsearch/index/IndexSettings.java
Original file line number Diff line number Diff line change
Expand Up @@ -812,20 +812,21 @@ public Iterator<Setting<?>> settings() {

public static final Setting<Integer> IGNORE_ABOVE_SETTING = Setting.intSetting(
"index.mapping.ignore_above",
IndexSettings::getIgnoreAboveDefaultValue,
settings -> String.valueOf(getIgnoreAboveDefaultValue(settings)),
0,
Integer.MAX_VALUE,
Property.IndexScope,
Property.ServerlessPublic
);

private static String getIgnoreAboveDefaultValue(final Settings settings) {
if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB
&& IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)) {
return "8191";
} else {
return String.valueOf(Integer.MAX_VALUE);
private static int getIgnoreAboveDefaultValue(final Settings settings) {
if (settings == null) {
return Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE;
}
return Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(
IndexSettings.MODE.get(settings),
IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings)
);
}

public static final NodeFeature IGNORE_ABOVE_INDEX_LEVEL_SETTING = new NodeFeature("mapper.ignore_above_index_level_setting", true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1311,6 +1311,14 @@ public static Parameter<Boolean> docValuesParam(Function<FieldMapper, Boolean> i
return Parameter.boolParam("doc_values", false, initializer, defaultValue);
}

public static Parameter<Integer> ignoreAboveParam(Function<FieldMapper, Integer> initializer, int defaultValue) {
return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> {
if (v < 0) {
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
}
});
}

/**
* Defines a script parameter
* @param initializer retrieves the equivalent parameter from an existing FieldMapper for use in merges
Expand Down Expand Up @@ -1681,6 +1689,12 @@ public TypeParser(
this(builderFunction, (n, c) -> contextValidator.forEach(v -> v.accept(n, c)), IndexVersions.MINIMUM_COMPATIBLE);
}

private static final IndexVersion MINIMUM_LEGACY_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);

public static TypeParser createTypeParserWithLegacySupport(BiFunction<String, MappingParserContext, Builder> builderFunction) {
return new TypeParser(builderFunction, MINIMUM_LEGACY_COMPATIBILITY_VERSION);
}

private TypeParser(
BiFunction<String, MappingParserContext, Builder> builderFunction,
BiConsumer<String, MappingParserContext> contextValidator,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.features.NodeFeature;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.analysis.IndexAnalyzers;
Expand Down Expand Up @@ -90,6 +91,7 @@
import static org.elasticsearch.core.Strings.format;
import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
import static org.elasticsearch.index.mapper.Mapper.IgnoreAbove.getIgnoreAboveDefaultValue;

/**
* A field mapper for keywords. This mapper accepts strings and indexes them as-is.
Expand Down Expand Up @@ -169,6 +171,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
);
private final Parameter<Integer> ignoreAbove;
private final int ignoreAboveDefault;
private final IndexMode indexMode;

private final Parameter<String> indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions);
private final Parameter<Boolean> hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false);
Expand Down Expand Up @@ -206,16 +209,36 @@ public Builder(final String name, final MappingParserContext mappingParserContex
mappingParserContext.scriptCompiler(),
IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()),
mappingParserContext.getIndexSettings().getIndexVersionCreated(),
mappingParserContext.getIndexSettings().getMode(),
mappingParserContext.getIndexSettings().sourceKeepMode()
);
}

Builder(
String name,
IndexAnalyzers indexAnalyzers,
ScriptCompiler scriptCompiler,
IndexVersion indexCreatedVersion,
SourceKeepMode sourceKeepMode
) {
this(
name,
indexAnalyzers,
scriptCompiler,
getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexCreatedVersion),
indexCreatedVersion,
IndexMode.STANDARD,
sourceKeepMode
);
}

private Builder(
String name,
IndexAnalyzers indexAnalyzers,
ScriptCompiler scriptCompiler,
int ignoreAboveDefault,
IndexVersion indexCreatedVersion,
IndexMode indexMode,
SourceKeepMode indexSourceKeepMode
) {
super(name);
Expand Down Expand Up @@ -245,17 +268,13 @@ public Builder(final String name, final MappingParserContext mappingParserContex
}
}).precludesParameters(normalizer);
this.ignoreAboveDefault = ignoreAboveDefault;
this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).fieldType().ignoreAbove(), ignoreAboveDefault)
.addValidator(v -> {
if (v < 0) {
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
}
});
this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), ignoreAboveDefault);
this.indexMode = indexMode;
this.indexSourceKeepMode = indexSourceKeepMode;
}

public Builder(String name, IndexVersion indexCreatedVersion) {
this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE);
this(name, null, ScriptCompiler.NONE, indexCreatedVersion, SourceKeepMode.NONE);
}

public Builder ignoreAbove(int ignoreAbove) {
Expand Down Expand Up @@ -418,7 +437,9 @@ public KeywordFieldMapper build(MapperBuilderContext context) {

public static final class KeywordFieldType extends StringFieldType {

private final int ignoreAbove;
private static final IgnoreAbove IGNORE_ABOVE_DEFAULT = new IgnoreAbove(null, IndexMode.STANDARD);

private final IgnoreAbove ignoreAbove;
private final String nullValue;
private final NamedAnalyzer normalizer;
private final boolean eagerGlobalOrdinals;
Expand Down Expand Up @@ -446,18 +467,22 @@ public KeywordFieldType(
);
this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue();
this.normalizer = normalizer;
this.ignoreAbove = builder.ignoreAbove.getValue();
this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion);
this.nullValue = builder.nullValue.getValue();
this.scriptValues = builder.scriptValues();
this.isDimension = builder.dimension.getValue();
this.isSyntheticSource = isSyntheticSource;
this.originalName = isSyntheticSource ? name + "._original" : null;
}

public KeywordFieldType(String name) {
this(name, true, true, Collections.emptyMap());
}

public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map<String, String> meta) {
super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
this.normalizer = Lucene.KEYWORD_ANALYZER;
this.ignoreAbove = Integer.MAX_VALUE;
this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
this.nullValue = null;
this.eagerGlobalOrdinals = false;
this.scriptValues = null;
Expand All @@ -466,10 +491,6 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma
this.originalName = null;
}

public KeywordFieldType(String name) {
this(name, true, true, Collections.emptyMap());
}

public KeywordFieldType(String name, FieldType fieldType) {
super(
name,
Expand All @@ -480,7 +501,7 @@ public KeywordFieldType(String name, FieldType fieldType) {
Collections.emptyMap()
);
this.normalizer = Lucene.KEYWORD_ANALYZER;
this.ignoreAbove = Integer.MAX_VALUE;
this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
this.nullValue = null;
this.eagerGlobalOrdinals = false;
this.scriptValues = null;
Expand All @@ -492,7 +513,7 @@ public KeywordFieldType(String name, FieldType fieldType) {
public KeywordFieldType(String name, NamedAnalyzer analyzer) {
super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap());
this.normalizer = Lucene.KEYWORD_ANALYZER;
this.ignoreAbove = Integer.MAX_VALUE;
this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
this.nullValue = null;
this.eagerGlobalOrdinals = false;
this.scriptValues = null;
Expand Down Expand Up @@ -800,10 +821,7 @@ protected String parseSourceValue(Object value) {
}

private String applyIgnoreAboveAndNormalizer(String value) {
if (value.length() > ignoreAbove) {
return null;
}

if (ignoreAbove.isIgnored(value)) return null;
return normalizeValue(normalizer(), name(), value);
}

Expand Down Expand Up @@ -922,7 +940,7 @@ public CollapseType collapseType() {

/** Values that have more chars than the return value of this method will
* be skipped at parsing time. */
public int ignoreAbove() {
public IgnoreAbove ignoreAbove() {
return ignoreAbove;
}

Expand Down Expand Up @@ -974,7 +992,7 @@ public String originalName() {

private final IndexAnalyzers indexAnalyzers;
private final int ignoreAboveDefault;
private final int ignoreAbove;
private final IndexMode indexMode;
private final String offsetsFieldName;
private final SourceKeepMode indexSourceKeepMode;
private final String originalName;
Expand Down Expand Up @@ -1003,7 +1021,7 @@ private KeywordFieldMapper(
this.indexCreatedVersion = builder.indexCreatedVersion;
this.isSyntheticSource = isSyntheticSource;
this.ignoreAboveDefault = builder.ignoreAboveDefault;
this.ignoreAbove = builder.ignoreAbove.getValue();
this.indexMode = builder.indexMode;
this.offsetsFieldName = offsetsFieldName;
this.indexSourceKeepMode = indexSourceKeepMode;
this.originalName = mappedFieldType.originalName();
Expand Down Expand Up @@ -1060,7 +1078,7 @@ private boolean indexValue(DocumentParserContext context, XContentString value)
return false;
}

if (value.stringLength() > fieldType().ignoreAbove()) {
if (fieldType().ignoreAbove().isIgnored(value)) {
context.addIgnoredField(fullPath());
if (isSyntheticSource) {
// Save a copy of the field so synthetic source can load it
Expand Down Expand Up @@ -1151,9 +1169,15 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {

@Override
public FieldMapper.Builder getMergeBuilder() {
return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, indexSourceKeepMode)
.dimension(fieldType().isDimension())
.init(this);
return new Builder(
leafName(),
indexAnalyzers,
scriptCompiler,
ignoreAboveDefault,
indexCreatedVersion,
indexMode,
indexSourceKeepMode
).dimension(fieldType().isDimension()).init(this);
}

@Override
Expand Down Expand Up @@ -1216,7 +1240,7 @@ protected BytesRef preserve(BytesRef value) {
}
}

if (fieldType().ignoreAbove != Integer.MAX_VALUE) {
if (fieldType().ignoreAbove.isSet()) {
layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName) {
@Override
protected void writeValue(Object value, XContentBuilder b) throws IOException {
Expand Down
Loading