diff --git a/docs/changelog/134253.yaml b/docs/changelog/134253.yaml new file mode 100644 index 0000000000000..fd12f4c6a83b2 --- /dev/null +++ b/docs/changelog/134253.yaml @@ -0,0 +1,5 @@ +pr: 134253 +summary: Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading +area: Mapping +type: bug +issues: [] diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index 6c4a4ee42ed00..6ab4a7ad39614 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -250,12 +250,10 @@ public static class Builder extends FieldMapper.Builder { false ).acceptsNull(); - final Parameter ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, Integer.MAX_VALUE) - .addValidator(v -> { - if (v < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); - } - }); + final Parameter ignoreAbove = Parameter.ignoreAboveParam( + m -> toType(m).ignoreAbove, + IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE + ); final Parameter nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull(); public Builder(String name) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 66deea71148f2..b53d0f456e288 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -790,20 +790,21 @@ public Iterator> settings() { public static final Setting IGNORE_ABOVE_SETTING = Setting.intSetting( "index.mapping.ignore_above", - IndexSettings::getIgnoreAboveDefaultValue, + settings -> String.valueOf(getIgnoreAboveDefaultValue(settings)), 0, Integer.MAX_VALUE, Property.IndexScope, Property.ServerlessPublic ); - private static String getIgnoreAboveDefaultValue(final Settings settings) { - if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB - && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)) { - return "8191"; - } else { - return String.valueOf(Integer.MAX_VALUE); + private static int getIgnoreAboveDefaultValue(final Settings settings) { + if (settings == null) { + return Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE; } + return Mapper.IgnoreAbove.getIgnoreAboveDefaultValue( + IndexSettings.MODE.get(settings), + IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings) + ); } private final Index index; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 7c1f3678a5dc9..4efc66a925b65 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1311,6 +1311,14 @@ public static Parameter docValuesParam(Function i return Parameter.boolParam("doc_values", false, initializer, defaultValue); } + public static Parameter ignoreAboveParam(Function initializer, int defaultValue) { + return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> { + if (v < 0) { + throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); + } + }); + } + /** * Defines a script parameter * @param initializer retrieves the equivalent parameter from an existing FieldMapper for use in merges diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 7b14739d36246..8c95cddd0f96e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -78,6 +79,7 @@ import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; +import static org.elasticsearch.index.mapper.Mapper.IgnoreAbove.getIgnoreAboveDefaultValue; /** * A field mapper for keywords. This mapper accepts strings and indexes them as-is. @@ -155,6 +157,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder { private final Parameter ignoreAbove; private final int ignoreAboveDefault; + private final IndexMode indexMode; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions); private final Parameter hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false); private final Parameter similarity = TextParams.similarity( @@ -189,16 +192,29 @@ public Builder(final String name, final MappingParserContext mappingParserContex mappingParserContext.getIndexAnalyzers(), mappingParserContext.scriptCompiler(), IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), - mappingParserContext.getIndexSettings().getIndexVersionCreated() + mappingParserContext.getIndexSettings().getIndexVersionCreated(), + mappingParserContext.getIndexSettings().getMode() ); } - Builder( + Builder(String name, IndexAnalyzers indexAnalyzers, ScriptCompiler scriptCompiler, IndexVersion indexCreatedVersion) { + this( + name, + indexAnalyzers, + scriptCompiler, + getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexCreatedVersion), + indexCreatedVersion, + IndexMode.STANDARD + ); + } + + private Builder( String name, IndexAnalyzers indexAnalyzers, ScriptCompiler scriptCompiler, int ignoreAboveDefault, - IndexVersion indexCreatedVersion + IndexVersion indexCreatedVersion, + IndexMode indexMode ) { super(name); this.indexAnalyzers = indexAnalyzers; @@ -227,16 +243,12 @@ public Builder(final String name, final MappingParserContext mappingParserContex } }).precludesParameters(normalizer); this.ignoreAboveDefault = ignoreAboveDefault; - this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).fieldType().ignoreAbove(), ignoreAboveDefault) - .addValidator(v -> { - if (v < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); - } - }); + this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), ignoreAboveDefault); + this.indexMode = indexMode; } public Builder(String name, IndexVersion indexCreatedVersion) { - this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion); + this(name, null, ScriptCompiler.NONE, indexCreatedVersion); } public Builder ignoreAbove(int ignoreAbove) { @@ -385,13 +397,16 @@ public KeywordFieldMapper build(MapperBuilderContext context) { public static final class KeywordFieldType extends StringFieldType { - private final int ignoreAbove; + private static final IgnoreAbove IGNORE_ABOVE_DEFAULT = new IgnoreAbove(null, IndexMode.STANDARD); + + private final IgnoreAbove ignoreAbove; private final String nullValue; private final NamedAnalyzer normalizer; private final boolean eagerGlobalOrdinals; private final FieldValues scriptValues; private final boolean isDimension; private final boolean isSyntheticSource; + private final String originalName; public KeywordFieldType( String name, @@ -412,26 +427,28 @@ public KeywordFieldType( ); this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue(); this.normalizer = normalizer; - this.ignoreAbove = builder.ignoreAbove.getValue(); + this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion); this.nullValue = builder.nullValue.getValue(); this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); this.isSyntheticSource = isSyntheticSource; + this.originalName = isSyntheticSource ? name + "._original" : null; + } + + public KeywordFieldType(String name) { + this(name, true, true, Collections.emptyMap()); } public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = Integer.MAX_VALUE; + this.ignoreAbove = IGNORE_ABOVE_DEFAULT; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - } - - public KeywordFieldType(String name) { - this(name, true, true, Collections.emptyMap()); + this.originalName = null; } public KeywordFieldType(String name, FieldType fieldType) { @@ -444,23 +461,25 @@ public KeywordFieldType(String name, FieldType fieldType) { Collections.emptyMap() ); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = Integer.MAX_VALUE; + this.ignoreAbove = IGNORE_ABOVE_DEFAULT; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; + this.originalName = null; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = Integer.MAX_VALUE; + this.ignoreAbove = IGNORE_ABOVE_DEFAULT; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; + this.originalName = null; } @Override @@ -710,15 +729,16 @@ private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { - return null; - } - - return normalizeValue(normalizer(), name(), keywordValue); + return applyIgnoreAboveAndNormalizer(keywordValue); } }; } + private String applyIgnoreAboveAndNormalizer(String value) { + if (ignoreAbove.isIgnored(value)) return null; + return normalizeValue(normalizer(), name(), value); + } + @Override public Object valueForDisplay(Object value) { if (value == null) { @@ -834,7 +854,7 @@ public CollapseType collapseType() { /** Values that have more chars than the return value of this method will * be skipped at parsing time. */ - public int ignoreAbove() { + public IgnoreAbove ignoreAbove() { return ignoreAbove; } @@ -851,6 +871,15 @@ public boolean hasScriptValues() { public boolean hasNormalizer() { return normalizer != Lucene.KEYWORD_ANALYZER; } + + /** + * The name used to store "original" that have been ignored + * by {@link KeywordFieldType#ignoreAbove()} so that they can be rebuilt + * for synthetic source. + */ + public String originalName() { + return originalName; + } } private final boolean indexed; @@ -862,11 +891,11 @@ public boolean hasNormalizer() { private final Script script; private final ScriptCompiler scriptCompiler; private final IndexVersion indexCreatedVersion; + private final IndexMode indexMode; private final boolean isSyntheticSource; private final IndexAnalyzers indexAnalyzers; private final int ignoreAboveDefault; - private final int ignoreAbove; private KeywordFieldMapper( String simpleName, @@ -888,9 +917,9 @@ private KeywordFieldMapper( this.indexAnalyzers = builder.indexAnalyzers; this.scriptCompiler = builder.scriptCompiler; this.indexCreatedVersion = builder.indexCreatedVersion; + this.indexMode = builder.indexMode; this.isSyntheticSource = isSyntheticSource; this.ignoreAboveDefault = builder.ignoreAboveDefault; - this.ignoreAbove = builder.ignoreAbove.getValue(); } @Override @@ -923,7 +952,7 @@ private void indexValue(DocumentParserContext context, String value) { return; } - if (value.length() > fieldType().ignoreAbove()) { + if (fieldType().ignoreAbove().isIgnored(value)) { context.addIgnoredField(fullPath()); if (isSyntheticSource) { // Save a copy of the field so synthetic source can load it @@ -1008,7 +1037,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion).dimension( + return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, indexMode).dimension( fieldType().isDimension() ).init(this); } @@ -1078,7 +1107,7 @@ protected BytesRef preserve(BytesRef value) { }); } - if (fieldType().ignoreAbove != Integer.MAX_VALUE) { + if (fieldType().ignoreAbove.isSet()) { layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) { @Override protected void writeValue(Object value, XContentBuilder b) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index bafa74b662f00..495117ad5b76b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.IndexVersions; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentString; import java.io.IOException; import java.util.Arrays; @@ -131,6 +132,78 @@ default boolean supportsVersion(IndexVersion indexCreatedVersion) { } } + /** + * This class models the ignore_above parameter in indices. + */ + public static final class IgnoreAbove { + + public static final int IGNORE_ABOVE_DEFAULT_VALUE = Integer.MAX_VALUE; + public static final int IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES = 8191; + + private final Integer value; + private final Integer defaultValue; + + public IgnoreAbove(Integer value) { + this(Objects.requireNonNull(value), IndexMode.STANDARD, IndexVersion.current()); + } + + public IgnoreAbove(Integer value, IndexMode indexMode) { + this(value, indexMode, IndexVersion.current()); + } + + public IgnoreAbove(Integer value, IndexMode indexMode, IndexVersion indexCreatedVersion) { + if (value != null && value < 0) { + throw new IllegalArgumentException("[ignore_above] must be positive, got [" + value + "]"); + } + + this.value = value; + this.defaultValue = getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion); + } + + public int get() { + return value != null ? value : defaultValue; + } + + /** + * Returns whether ignore_above is set; at field or index level. + */ + public boolean isSet() { + // if ignore_above equals default, its not considered to be set, even if it was explicitly set to the default value + return Integer.valueOf(get()).equals(defaultValue) == false; + } + + /** + * Returns whether the given string will be ignored. + */ + public boolean isIgnored(final String s) { + if (s == null) return false; + return lengthExceedsIgnoreAbove(s.length()); + } + + public boolean isIgnored(final XContentString s) { + if (s == null) return false; + return lengthExceedsIgnoreAbove(s.stringLength()); + } + + private boolean lengthExceedsIgnoreAbove(int strLength) { + return strLength > get(); + } + + public static int getIgnoreAboveDefaultValue(final IndexMode indexMode, final IndexVersion indexCreatedVersion) { + if (diffIgnoreAboveDefaultForLogs(indexMode, indexCreatedVersion)) { + return IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES; + } else { + return IGNORE_ABOVE_DEFAULT_VALUE; + } + } + + private static boolean diffIgnoreAboveDefaultForLogs(final IndexMode indexMode, final IndexVersion indexCreatedVersion) { + return indexMode == IndexMode.LOGSDB + && (indexCreatedVersion != null && indexCreatedVersion.onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)); + } + + } + private final String leafName; @SuppressWarnings("this-escape") diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 01b275e0a382e..4fba2e0234783 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -970,7 +970,7 @@ public boolean isAggregatable() { */ public boolean canUseSyntheticSourceDelegateForLoading() { return syntheticSourceDelegate != null - && syntheticSourceDelegate.ignoreAbove() == Integer.MAX_VALUE + && syntheticSourceDelegate.ignoreAbove().isSet() == false && (syntheticSourceDelegate.isIndexed() || syntheticSourceDelegate.isStored()); } @@ -979,7 +979,7 @@ public boolean canUseSyntheticSourceDelegateForLoading() { */ public boolean canUseSyntheticSourceDelegateForQuerying() { return syntheticSourceDelegate != null - && syntheticSourceDelegate.ignoreAbove() == Integer.MAX_VALUE + && syntheticSourceDelegate.ignoreAbove().isSet() == false && syntheticSourceDelegate.isIndexed(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index f70a47caa080c..9b818570ba543 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -38,6 +38,8 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -55,6 +57,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextParams; @@ -124,9 +127,6 @@ private static Builder builder(Mapper in) { return ((FlattenedFieldMapper) in).builder; } - private final int ignoreAboveDefault; - private final int ignoreAbove; - public static class Builder extends FieldMapper.Builder { final Parameter depthLimit = Parameter.intParam( @@ -180,23 +180,37 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); + private final IndexMode indexMode; + private final IndexVersion indexCreatedVersion; + public static FieldMapper.Parameter> dimensionsParam(Function> initializer) { return FieldMapper.Parameter.stringArrayParam(TIME_SERIES_DIMENSIONS_ARRAY_PARAM, false, initializer); } public Builder(final String name) { - this(name, Integer.MAX_VALUE); + this( + name, + IgnoreAbove.getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current()), + IndexMode.STANDARD, + IndexVersion.current() + ); } - private Builder(String name, int ignoreAboveDefault) { + private Builder(String name, MappingParserContext mappingParserContext) { + this( + name, + IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), + mappingParserContext.getIndexSettings().getMode(), + mappingParserContext.indexVersionCreated() + ); + } + + private Builder(String name, int ignoreAboveDefault, IndexMode indexMode, IndexVersion indexCreatedVersion) { super(name); this.ignoreAboveDefault = ignoreAboveDefault; - this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> builder(m).ignoreAbove.get(), ignoreAboveDefault) - .addValidator(v -> { - if (v < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); - } - }); + this.indexMode = indexMode; + this.indexCreatedVersion = indexCreatedVersion; + this.ignoreAbove = Parameter.ignoreAboveParam(m -> builder(m).ignoreAbove.get(), ignoreAboveDefault); this.dimensions.precludesParameters(ignoreAbove); } @@ -233,13 +247,13 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { splitQueriesOnWhitespace.get(), eagerGlobalOrdinals.get(), dimensions.get(), - ignoreAbove.getValue() + new IgnoreAbove(ignoreAbove.getValue(), indexMode, indexCreatedVersion) ); - return new FlattenedFieldMapper(leafName(), ft, builderParams(this, context), ignoreAboveDefault, this); + return new FlattenedFieldMapper(leafName(), ft, builderParams(this, context), this); } } - public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, IGNORE_ABOVE_SETTING.get(c.getSettings()))); + public static final TypeParser PARSER = createTypeParserWithLegacySupport(FlattenedFieldMapper.Builder::new); /** * A field type that represents the values under a particular JSON key, used @@ -665,7 +679,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme private final boolean eagerGlobalOrdinals; private final List dimensions; private final boolean isDimension; - private final int ignoreAbove; + private final IgnoreAbove ignoreAbove; RootFlattenedFieldType( String name, @@ -674,7 +688,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme Map meta, boolean splitQueriesOnWhitespace, boolean eagerGlobalOrdinals, - int ignoreAbove + IgnoreAbove ignoreAbove ) { this(name, indexed, hasDocValues, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList(), ignoreAbove); } @@ -687,7 +701,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme boolean splitQueriesOnWhitespace, boolean eagerGlobalOrdinals, List dimensions, - int ignoreAbove + IgnoreAbove ignoreAbove ) { super( name, @@ -741,6 +755,10 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); } + public IgnoreAbove ignoreAbove() { + return ignoreAbove; + } + private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { return new SourceValueFetcher(sourcePaths, null) { @Override @@ -750,7 +768,7 @@ protected Object parseSourceValue(Object value) { final Map result = filterIgnoredValues((Map) valueAsMap); return result.isEmpty() ? null : result; } - if (value instanceof String valueAsString && valueAsString.length() <= ignoreAbove) { + if (value instanceof String valueAsString && ignoreAbove.isIgnored(valueAsString) == false) { return valueAsString; } return null; @@ -772,7 +790,7 @@ private Object filterIgnoredValues(final Object entryValue) { final List validValues = new ArrayList<>(); for (Object value : valueAsList) { if (value instanceof String valueAsString) { - if (valueAsString.length() <= ignoreAbove) { + if (ignoreAbove.isIgnored(valueAsString) == false) { validValues.add(valueAsString); } } else { @@ -788,7 +806,7 @@ private Object filterIgnoredValues(final Object entryValue) { } return validValues; } else if (entryValue instanceof String valueAsString) { - if (valueAsString.length() <= ignoreAbove) { + if (ignoreAbove.isIgnored(valueAsString) == false) { return valueAsString; } return null; @@ -828,17 +846,9 @@ public void validateMatchedRoutingPath(final String routingPath) { private final FlattenedFieldParser fieldParser; private final Builder builder; - private FlattenedFieldMapper( - String leafName, - MappedFieldType mappedFieldType, - BuilderParams builderParams, - int ignoreAboveDefault, - Builder builder - ) { + private FlattenedFieldMapper(String leafName, MappedFieldType mappedFieldType, BuilderParams builderParams, Builder builder) { super(leafName, mappedFieldType, builderParams); - this.ignoreAboveDefault = ignoreAboveDefault; this.builder = builder; - this.ignoreAbove = builder.ignoreAbove.get(); this.fieldParser = new FlattenedFieldParser( mappedFieldType.name(), mappedFieldType.name() + KEYED_FIELD_SUFFIX, @@ -864,10 +874,6 @@ int depthLimit() { return builder.depthLimit.get(); } - public int ignoreAbove() { - return ignoreAbove; - } - @Override public RootFlattenedFieldType fieldType() { return (RootFlattenedFieldType) super.fieldType(); @@ -905,7 +911,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), ignoreAboveDefault).init(this); + return new Builder(leafName(), builder.ignoreAboveDefault, builder.indexMode, builder.indexCreatedVersion).init(this); } @Override @@ -915,7 +921,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() { () -> new FlattenedSortedSetDocValuesSyntheticFieldLoader( fullPath(), fullPath() + KEYED_FIELD_SUFFIX, - ignoreAbove() < Integer.MAX_VALUE ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null, + fieldType().ignoreAbove.isSet() ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null, leafName() ) ); diff --git a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java new file mode 100644 index 0000000000000..f43b0eb95683e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index; + +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.Text; + +public class IgnoreAboveTests extends ESTestCase { + + private static final Mapper.IgnoreAbove IGNORE_ABOVE_DEFAULT = new Mapper.IgnoreAbove(null, IndexMode.STANDARD); + private static final Mapper.IgnoreAbove IGNORE_ABOVE_DEFAULT_LOGS = new Mapper.IgnoreAbove(null, IndexMode.LOGSDB); + + public void test_ignore_above_with_value_and_index_mode_and_index_version() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(123, IndexMode.STANDARD); + + // when/then + assertEquals(123, ignoreAbove.get()); + assertTrue(ignoreAbove.isSet()); + } + + public void test_ignore_above_with_value_only() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(123); + + // when/then + assertEquals(123, ignoreAbove.get()); + assertTrue(ignoreAbove.isSet()); + } + + public void test_ignore_above_with_null_value_should_throw() { + assertThrows(NullPointerException.class, () -> new Mapper.IgnoreAbove(null)); + } + + public void test_ignore_above_with_negative_value_should_throw() { + assertThrows(IllegalArgumentException.class, () -> new Mapper.IgnoreAbove(-1)); + assertThrows(IllegalArgumentException.class, () -> new Mapper.IgnoreAbove(-1, IndexMode.STANDARD)); + } + + public void test_ignore_above_with_null_value() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, IndexMode.STANDARD); + + // when/then + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); + } + + public void test_ignore_above_with_null_value_and_logsdb_index_mode() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, IndexMode.LOGSDB); + + // when/then + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); + } + + public void test_ignore_above_with_null_everything() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, null, null); + + // when/then + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); + } + + public void test_ignore_above_default_for_standard_indices() { + // given + Mapper.IgnoreAbove ignoreAbove = IGNORE_ABOVE_DEFAULT; + + // when/then + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); + } + + public void test_ignore_above_default_for_logsdb_indices() { + // given + Mapper.IgnoreAbove ignoreAbove = IGNORE_ABOVE_DEFAULT_LOGS; + + // when/then + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); + } + + public void test_string_isIgnored() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(10); + + // when/then + assertFalse(ignoreAbove.isIgnored("potato")); + assertFalse(ignoreAbove.isIgnored("1234567890")); + assertTrue(ignoreAbove.isIgnored("12345678901")); + assertTrue(ignoreAbove.isIgnored("potato potato tomato tomato")); + } + + public void test_XContentString_isIgnored() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(10); + + // when/then + assertFalse(ignoreAbove.isIgnored(new Text("potato"))); + assertFalse(ignoreAbove.isIgnored(new Text("1234567890"))); + assertTrue(ignoreAbove.isIgnored(new Text("12345678901"))); + assertTrue(ignoreAbove.isIgnored(new Text("potato potato tomato tomato"))); + } + + public void test_default_value() { + assertEquals( + Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, + Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current()) + ); + assertEquals( + Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, + Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(IndexMode.LOGSDB, IndexVersion.current()) + ); + assertEquals( + Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, + Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(IndexMode.LOGSDB, IndexVersions.ENABLE_IGNORE_MALFORMED_LOGSDB) + ); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 052bf995bdd48..9c17cd9fe56c0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -179,7 +179,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerUpdateCheck(b -> b.field("eager_global_ordinals", true), m -> assertTrue(m.fieldType().eagerGlobalOrdinals())); checker.registerUpdateCheck( b -> b.field("ignore_above", 256), - m -> assertEquals(256, ((KeywordFieldMapper) m).fieldType().ignoreAbove()) + m -> assertEquals(256, ((KeywordFieldMapper) m).fieldType().ignoreAbove().get()) ); checker.registerUpdateCheck( b -> b.field("split_queries_on_whitespace", true), @@ -243,6 +243,108 @@ public void testIgnoreAbove() throws IOException { assertTrue(doc.rootDoc().getFields("_ignored").stream().anyMatch(field -> "field".equals(field.stringValue()))); } + public void test_ignore_above_index_level_setting() throws IOException { + // given + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(), + mapping(b -> { + b.startObject("field"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + // when + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + + // then + assertEquals(123, mapper.fieldType().ignoreAbove().get()); + assertTrue(mapper.fieldType().ignoreAbove().isSet()); + } + + public void test_ignore_above_index_level_setting_is_overridden_by_field_level_ignore_above_in_standard_indices() throws IOException { + // given + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(), + mapping(b -> { + b.startObject("potato"); + b.field("type", "keyword"); + b.field("ignore_above", 456); + b.endObject(); + + b.startObject("tomato"); + b.field("type", "keyword"); + b.field("ignore_above", 789); + b.endObject(); + + b.startObject("cheese"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + // when + final KeywordFieldMapper fieldMapper1 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("potato"); + final KeywordFieldMapper fieldMapper2 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("tomato"); + final KeywordFieldMapper fieldMapper3 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("cheese"); + + // then + assertEquals(456, fieldMapper1.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper1.fieldType().ignoreAbove().isSet()); + + assertEquals(789, fieldMapper2.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper2.fieldType().ignoreAbove().isSet()); + + assertEquals(123, fieldMapper3.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper3.fieldType().ignoreAbove().isSet()); + } + + public void test_ignore_above_index_level_setting_is_overridden_by_field_level_ignore_above_in_logsdb_indices() throws IOException { + // given + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(), + mapping(b -> { + b.startObject("potato"); + b.field("type", "keyword"); + b.field("ignore_above", 456); + b.endObject(); + + b.startObject("tomato"); + b.field("type", "keyword"); + b.field("ignore_above", 789); + b.endObject(); + + b.startObject("cheese"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + // when + final KeywordFieldMapper fieldMapper1 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("potato"); + final KeywordFieldMapper fieldMapper2 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("tomato"); + final KeywordFieldMapper fieldMapper3 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("cheese"); + + // then + assertEquals(456, fieldMapper1.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper1.fieldType().ignoreAbove().isSet()); + + assertEquals(789, fieldMapper2.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper2.fieldType().ignoreAbove().isSet()); + + assertEquals(123, fieldMapper3.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper3.fieldType().ignoreAbove().isSet()); + } + public void testNullValue() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); ParsedDocument doc = mapper.parse(source(b -> b.nullField("field"))); @@ -331,7 +433,7 @@ public void testDimensionAndIgnoreAbove() throws IOException { b.field("time_series_dimension", true).field("ignore_above", 2048); })); KeywordFieldMapper field = (KeywordFieldMapper) documentMapper.mappers().getMapper("field"); - assertEquals(2048, field.fieldType().ignoreAbove()); + assertEquals(2048, field.fieldType().ignoreAbove().get()); } public void testDimensionAndNormalizer() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index e3bdb3d45818f..e495e60bead94 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -36,9 +36,13 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.CharFilterFactory; @@ -58,6 +62,9 @@ import java.util.List; import java.util.Map; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + public class KeywordFieldTypeTests extends FieldTypeTestCase { public void testIsFieldWithinQuery() throws IOException { @@ -243,7 +250,6 @@ public void testFetchSourceValue() throws IOException { "field", createIndexAnalyzers(), ScriptCompiler.NONE, - Integer.MAX_VALUE, IndexVersion.current() ).normalizer("lowercase").build(MapperBuilderContext.root(false, false)).fieldType(); assertEquals(List.of("value"), fetchSourceValue(normalizerMapper, "VALUE")); @@ -288,6 +294,243 @@ public void testGetTerms() throws IOException { } } + public void test_ignore_above_index_level_setting() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertTrue(fieldType.ignoreAbove().isSet()); + assertEquals(123, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_true_when_ignore_above_is_given() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + builder.ignoreAbove(123); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertTrue(fieldType.ignoreAbove().isSet()); + assertEquals(123, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_false_when_ignore_above_is_not_given() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertFalse(fieldType.ignoreAbove().isSet()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + builder.ignoreAbove(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertFalse(fieldType.ignoreAbove().isSet()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default_for_logsdb_indices() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + builder.ignoreAbove(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertFalse(fieldType.ignoreAbove().isSet()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_true_when_ignore_above_is_given_as_logsdb_default_but_index_mod_is_not_logsdb() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + builder.ignoreAbove(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertTrue(fieldType.ignoreAbove().isSet()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_true_when_ignore_above_is_configured_at_index_level() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertTrue(fieldType.ignoreAbove().isSet()); + assertEquals(123, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_false_for_non_primary_constructor() { + // given + KeywordFieldType fieldType1 = new KeywordFieldType("field"); + KeywordFieldType fieldType2 = new KeywordFieldType("field", mock(FieldType.class)); + KeywordFieldType fieldType3 = new KeywordFieldType("field", true, true, Collections.emptyMap()); + KeywordFieldType fieldType4 = new KeywordFieldType("field", mock(NamedAnalyzer.class)); + + // when/then + assertFalse(fieldType1.ignoreAbove().isSet()); + assertFalse(fieldType2.ignoreAbove().isSet()); + assertFalse(fieldType3.ignoreAbove().isSet()); + assertFalse(fieldType4.ignoreAbove().isSet()); + } + private static IndexAnalyzers createIndexAnalyzers() { return IndexAnalyzers.of( Map.of("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer())), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java index fd024c5d23e28..06c3125648309 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java @@ -63,7 +63,6 @@ private KeywordFieldMapper.Builder getKeywordFieldMapperBuilder(boolean isStored "field", IndexAnalyzers.of(Map.of(), Map.of("normalizer", Lucene.STANDARD_ANALYZER), Map.of()), ScriptCompiler.NONE, - Integer.MAX_VALUE, IndexVersion.current() ); if (isStored) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java index 1f8a2a754428b..a16c67f21f84e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java @@ -263,12 +263,12 @@ public void testMergeSameObjectDifferentFields() { ObjectMapper parent0 = (ObjectMapper) mergedAdd0.getMapper("parent"); assertNotNull(parent0.getMapper("child1")); - assertEquals(42, ((KeywordFieldMapper) parent0.getMapper("child1")).fieldType().ignoreAbove()); + assertEquals(42, ((KeywordFieldMapper) parent0.getMapper("child1")).fieldType().ignoreAbove().get()); assertNull(parent0.getMapper("child2")); ObjectMapper parent1 = (ObjectMapper) mergedAdd1.getMapper("parent"); assertNotNull(parent1.getMapper("child1")); - assertEquals(42, ((KeywordFieldMapper) parent1.getMapper("child1")).fieldType().ignoreAbove()); + assertEquals(42, ((KeywordFieldMapper) parent1.getMapper("child1")).fieldType().ignoreAbove().get()); assertNotNull(parent1.getMapper("child2")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java new file mode 100644 index 0000000000000..d27ec182db7e7 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.test.ESTestCase; + +public class ParameterTests extends ESTestCase { + + public void test_ignore_above_param_default() { + // when + FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam((FieldMapper fm) -> 123, 456); + + // then + assertEquals(456, ignoreAbove.getValue().intValue()); + } + + public void test_ignore_above_param_invalid_value() { + // when + FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam((FieldMapper fm) -> -1, 456); + ignoreAbove.setValue(-1); + + // then + assertThrows(IllegalArgumentException.class, ignoreAbove::validate); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 4d246d3c557a6..50074c50d51c4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -8,6 +8,7 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.FieldType; import org.apache.lucene.index.Term; import org.apache.lucene.queries.intervals.Intervals; import org.apache.lucene.queries.intervals.IntervalsSource; @@ -31,10 +32,18 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.AutomatonQueries; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; +import org.elasticsearch.script.ScriptCompiler; import java.io.IOException; import java.util.ArrayList; @@ -44,6 +53,8 @@ import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; public class TextFieldTypeTests extends FieldTypeTestCase { @@ -289,4 +300,131 @@ public void testRangeIntervals() { rangeIntervals ); } + + public void test_block_loader_uses_synthetic_source_delegate_when_ignore_above_is_not_set() { + // given + KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( + "child", + true, + true, + Collections.emptyMap() + ); + + TextFieldType ft = new TextFieldType( + "parent", + true, + false, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + true, + syntheticSourceDelegate, + Collections.singletonMap("potato", "tomato"), + false, + false + ); + + // when + ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + + // then + // verify that we delegate block loading to the synthetic source delegate + assertTrue(blockLoader instanceof BlockLoader.Delegating); + assertThat(((BlockLoader.Delegating) blockLoader).delegatingTo(), equalTo("child")); + } + + public void test_block_loader_does_not_use_synthetic_source_delegate_when_ignore_above_is_set() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + builder.ignoreAbove(123); + + KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + TextFieldType ft = new TextFieldType( + "parent", + true, + false, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + true, + syntheticSourceDelegate, + Collections.singletonMap("potato", "tomato"), + false, + false + ); + + // when + ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + + // then + // verify that we don't delegate anything + assertFalse(blockLoader instanceof BlockLoader.Delegating); + } + + public void test_block_loader_does_not_use_synthetic_source_delegate_when_ignore_above_is_set_at_index_level() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + + KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + TextFieldType ft = new TextFieldType( + "parent", + true, + false, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + true, + syntheticSourceDelegate, + Collections.singletonMap("potato", "tomato"), + false, + false + ); + + // when + ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + + // then + // verify that we don't delegate anything + assertFalse(blockLoader instanceof BlockLoader.Delegating); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index 0628ca47308ea..d9223a157b2ff 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -86,7 +86,10 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("time_series_dimensions", b -> b.field("time_series_dimensions", List.of("one", "two"))); checker.registerUpdateCheck(b -> b.field("eager_global_ordinals", true), m -> assertTrue(m.fieldType().eagerGlobalOrdinals())); - checker.registerUpdateCheck(b -> b.field("ignore_above", 256), m -> assertEquals(256, ((FlattenedFieldMapper) m).ignoreAbove())); + checker.registerUpdateCheck( + b -> b.field("ignore_above", 256), + m -> assertEquals(256, ((FlattenedFieldMapper) m).fieldType().ignoreAbove().get()) + ); checker.registerUpdateCheck( b -> b.field("split_queries_on_whitespace", true), m -> assertEquals("_whitespace", m.fieldType().getTextSearchInfo().searchAnalyzer().name()) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java index 7873458eb46ee..3127b2c60d0f5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java @@ -22,8 +22,10 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType; import java.io.IOException; @@ -33,8 +35,10 @@ public class RootFlattenedFieldTypeTests extends FieldTypeTestCase { + private static final Mapper.IgnoreAbove IGNORE_ABOVE = new Mapper.IgnoreAbove(null, IndexMode.STANDARD); + private static RootFlattenedFieldType createDefaultFieldType(int ignoreAbove) { - return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, ignoreAbove); + return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, new Mapper.IgnoreAbove(ignoreAbove)); } public void testValueForDisplay() { @@ -61,33 +65,17 @@ public void testTermQuery() { Collections.emptyMap(), false, false, - Integer.MAX_VALUE + IGNORE_ABOVE ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } public void testExistsQuery() { - RootFlattenedFieldType ft = new RootFlattenedFieldType( - "field", - true, - false, - Collections.emptyMap(), - false, - false, - Integer.MAX_VALUE - ); + RootFlattenedFieldType ft = new RootFlattenedFieldType("field", true, false, Collections.emptyMap(), false, false, IGNORE_ABOVE); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))), ft.existsQuery(null)); - RootFlattenedFieldType withDv = new RootFlattenedFieldType( - "field", - true, - true, - Collections.emptyMap(), - false, - false, - Integer.MAX_VALUE - ); + RootFlattenedFieldType withDv = new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, IGNORE_ABOVE); assertEquals(new FieldExistsQuery("field"), withDv.existsQuery(null)); } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 3e374cab327d3..60453d7e22596 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -49,6 +49,7 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.AnalyzerScope; @@ -67,6 +68,7 @@ import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; @@ -87,6 +89,7 @@ import java.util.Set; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; +import static org.elasticsearch.index.mapper.Mapper.IgnoreAbove.getIgnoreAboveDefaultValue; /** * A {@link FieldMapper} for indexing fields with ngrams for efficient wildcard matching @@ -200,28 +203,34 @@ private static WildcardFieldMapper toType(FieldMapper in) { public static class Builder extends FieldMapper.Builder { + final int ignoreAboveDefault; final Parameter ignoreAbove; final Parameter nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull(); final Parameter> meta = Parameter.metaParam(); - final IndexVersion indexVersionCreated; - - final int ignoreAboveDefault; + final IndexMode indexMode; + final IndexVersion indexCreatedVersion; public Builder(final String name, IndexVersion indexVersionCreated) { - this(name, Integer.MAX_VALUE, indexVersionCreated); + this(name, getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexVersionCreated), IndexMode.STANDARD, indexVersionCreated); + } + + private Builder(String name, MappingParserContext mappingParserContext) { + this( + name, + IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), + mappingParserContext.getIndexSettings().getMode(), + mappingParserContext.indexVersionCreated() + ); } - private Builder(String name, int ignoreAboveDefault, IndexVersion indexVersionCreated) { + private Builder(String name, int ignoreAboveDefault, IndexMode indexMode, IndexVersion indexCreatedVersion) { super(name); - this.indexVersionCreated = indexVersionCreated; this.ignoreAboveDefault = ignoreAboveDefault; - this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, ignoreAboveDefault).addValidator(v -> { - if (v < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); - } - }); + this.indexMode = indexMode; + this.indexCreatedVersion = indexCreatedVersion; + this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).ignoreAbove.get(), ignoreAboveDefault); } @Override @@ -243,18 +252,15 @@ Builder nullValue(String nullValue) { public WildcardFieldMapper build(MapperBuilderContext context) { return new WildcardFieldMapper( leafName(), - new WildcardFieldType(context.buildFullName(leafName()), indexVersionCreated, meta.get(), this), + new WildcardFieldType(context.buildFullName(leafName()), indexCreatedVersion, meta.get(), this), context.isSourceSynthetic(), builderParams(this, context), - indexVersionCreated, this ); } } - public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, IGNORE_ABOVE_SETTING.get(c.getSettings()), c.indexVersionCreated()) - ); + public static final TypeParser PARSER = createTypeParserWithLegacySupport(Builder::new); public static final char TOKEN_START_OR_END_CHAR = 0; public static final String TOKEN_START_STRING = Character.toString(TOKEN_START_OR_END_CHAR); @@ -266,7 +272,7 @@ public static final class WildcardFieldType extends MappedFieldType { private final String nullValue; private final NamedAnalyzer analyzer; - private final int ignoreAbove; + private final IgnoreAbove ignoreAbove; private WildcardFieldType(String name, IndexVersion version, Map meta, Builder builder) { super(name, true, false, true, Defaults.TEXT_SEARCH_INFO, meta); @@ -276,7 +282,7 @@ private WildcardFieldType(String name, IndexVersion version, Map this.analyzer = WILDCARD_ANALYZER_7_9; } this.nullValue = builder.nullValue.getValue(); - this.ignoreAbove = builder.ignoreAbove.getValue(); + this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion); } @Override @@ -872,7 +878,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { + if (ignoreAbove.isIgnored(keywordValue)) { return null; } return keywordValue; @@ -891,10 +897,10 @@ protected String parseSourceValue(Object value) { assert NGRAM_FIELD_TYPE.indexOptions() == IndexOptions.DOCS; } private final String nullValue; + private final IndexMode indexMode; private final IndexVersion indexVersionCreated; - - private final int ignoreAbove; private final int ignoreAboveDefault; + private final IgnoreAbove ignoreAbove; private final boolean storeIgnored; private WildcardFieldMapper( @@ -902,15 +908,15 @@ private WildcardFieldMapper( WildcardFieldType mappedFieldType, boolean storeIgnored, BuilderParams builderParams, - IndexVersion indexVersionCreated, Builder builder ) { super(simpleName, mappedFieldType, builderParams); this.nullValue = builder.nullValue.getValue(); this.storeIgnored = storeIgnored; - this.indexVersionCreated = indexVersionCreated; - this.ignoreAbove = builder.ignoreAbove.getValue(); + this.indexMode = builder.indexMode; + this.indexVersionCreated = builder.indexCreatedVersion; this.ignoreAboveDefault = builder.ignoreAboveDefault; + this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion); } @Override @@ -921,7 +927,7 @@ public Map indexAnalyzers() { /** Values that have more chars than the return value of this method will * be skipped at parsing time. */ // pkg-private for testing - int ignoreAbove() { + IgnoreAbove ignoreAbove() { return ignoreAbove; } @@ -943,13 +949,13 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio List fields = new ArrayList<>(); if (value != null) { - if (value.length() <= ignoreAbove) { - createFields(value, parseDoc, fields); - } else { + if (ignoreAbove.isIgnored(value)) { context.addIgnoredField(fullPath()); if (storeIgnored) { parseDoc.add(new StoredField(originalName(), new BytesRef(value))); } + } else { + createFields(value, parseDoc, fields); } } parseDoc.addAll(fields); @@ -985,7 +991,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), ignoreAboveDefault, indexVersionCreated).init(this); + return new Builder(leafName(), ignoreAboveDefault, indexMode, indexVersionCreated).init(this); } @Override @@ -993,7 +999,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() { return new SyntheticSourceSupport.Native(() -> { var layers = new ArrayList(); layers.add(new WildcardSyntheticFieldLoader()); - if (ignoreAbove != Integer.MAX_VALUE) { + if (ignoreAbove.isSet()) { layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) { @Override protected void writeValue(Object value, XContentBuilder b) throws IOException { diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index b7a9b8af057e0..43cd2cf070f47 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -724,7 +724,10 @@ protected Object getSampleValueForDocument() { @Override protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("null_value", b -> b.field("null_value", "foo")); - checker.registerUpdateCheck(b -> b.field("ignore_above", 256), m -> assertEquals(256, ((WildcardFieldMapper) m).ignoreAbove())); + checker.registerUpdateCheck( + b -> b.field("ignore_above", 256), + m -> assertEquals(256, ((WildcardFieldMapper) m).ignoreAbove().get()) + ); }