diff --git a/docs/changelog/134253.yaml b/docs/changelog/134253.yaml new file mode 100644 index 0000000000000..fd12f4c6a83b2 --- /dev/null +++ b/docs/changelog/134253.yaml @@ -0,0 +1,5 @@ +pr: 134253 +summary: Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading +area: Mapping +type: bug +issues: [] diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index faea13dac4e31..1d9fd9cb3a13b 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -253,8 +253,7 @@ private IOFunction, IOExcepti String parentField = searchExecutionContext.parentPath(name()); var parent = searchExecutionContext.lookup().fieldType(parentField); - if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent - && keywordParent.ignoreAbove() != Integer.MAX_VALUE) { + if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent && keywordParent.ignoreAbove().isSet()) { if (parent.isStored()) { return storedFieldFetcher(parentField, keywordParent.originalName()); } else if (parent.hasDocValues()) { @@ -278,7 +277,7 @@ private IOFunction, IOExcepti if (kwd != null) { var fieldType = kwd.fieldType(); - if (fieldType.ignoreAbove() != Integer.MAX_VALUE) { + if (fieldType.ignoreAbove().isSet()) { if (fieldType.isStored()) { return storedFieldFetcher(fieldType.name(), fieldType.originalName()); } else if (fieldType.hasDocValues()) { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index 6b5c43ac3daa6..9dd959981701c 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -250,12 +250,10 @@ public static class Builder extends FieldMapper.Builder { false ).acceptsNull(); - final Parameter ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, Integer.MAX_VALUE) - .addValidator(v -> { - if (v < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); - } - }); + final Parameter ignoreAbove = Parameter.ignoreAboveParam( + m -> toType(m).ignoreAbove, + IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE + ); final Parameter nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull(); public Builder(String name) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 636c30027bd0b..b396e1ca206e3 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -805,20 +805,21 @@ public Iterator> settings() { public static final Setting IGNORE_ABOVE_SETTING = Setting.intSetting( "index.mapping.ignore_above", - IndexSettings::getIgnoreAboveDefaultValue, + settings -> String.valueOf(getIgnoreAboveDefaultValue(settings)), 0, Integer.MAX_VALUE, Property.IndexScope, Property.ServerlessPublic ); - private static String getIgnoreAboveDefaultValue(final Settings settings) { - if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB - && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)) { - return "8191"; - } else { - return String.valueOf(Integer.MAX_VALUE); + private static int getIgnoreAboveDefaultValue(final Settings settings) { + if (settings == null) { + return Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE; } + return Mapper.IgnoreAbove.getIgnoreAboveDefaultValue( + IndexSettings.MODE.get(settings), + IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings) + ); } public static final Setting SEQ_NO_INDEX_OPTIONS_SETTING = Setting.enumSetting( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 34362e25bfece..018cd6ab6a287 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1337,6 +1337,14 @@ public static Parameter normsParam(Function initi .setMergeValidator((prev, curr, c) -> prev == curr || (prev && curr == false)); } + public static Parameter ignoreAboveParam(Function initializer, int defaultValue) { + return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> { + if (v < 0) { + throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); + } + }); + } + /** * Defines a script parameter * @param initializer retrieves the equivalent parameter from an existing FieldMapper for use in merges diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index b6d9215db030b..f730e5188f55d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -92,6 +92,7 @@ import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; +import static org.elasticsearch.index.mapper.Mapper.IgnoreAbove.getIgnoreAboveDefaultValue; /** * A field mapper for keywords. This mapper accepts strings and indexes them as-is. @@ -232,7 +233,6 @@ public Builder(final String name, final MappingParserContext mappingParserContex String name, IndexAnalyzers indexAnalyzers, ScriptCompiler scriptCompiler, - int ignoreAboveDefault, IndexVersion indexCreatedVersion, SourceKeepMode sourceKeepMode ) { @@ -240,7 +240,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex name, indexAnalyzers, scriptCompiler, - ignoreAboveDefault, + getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexCreatedVersion), indexCreatedVersion, IndexMode.STANDARD, null, @@ -289,12 +289,7 @@ private Builder( } }).precludesParameters(normalizer); this.ignoreAboveDefault = ignoreAboveDefault; - this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).fieldType().ignoreAbove(), ignoreAboveDefault) - .addValidator(v -> { - if (v < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); - } - }); + this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), ignoreAboveDefault); this.indexSortConfig = indexSortConfig; this.indexMode = indexMode; this.enableDocValuesSkipper = enableDocValuesSkipper; @@ -303,7 +298,7 @@ private Builder( } public Builder(String name, IndexVersion indexCreatedVersion) { - this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE); + this(name, null, ScriptCompiler.NONE, indexCreatedVersion, SourceKeepMode.NONE); } public static Builder buildWithDocValuesSkipper( @@ -316,7 +311,7 @@ public static Builder buildWithDocValuesSkipper( name, null, ScriptCompiler.NONE, - Integer.MAX_VALUE, + getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion), indexCreatedVersion, indexMode, // Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed. @@ -537,14 +532,15 @@ private static boolean indexSortConfigByHostName(final IndexSortConfig indexSort public static final class KeywordFieldType extends StringFieldType { - private final int ignoreAbove; + private static final IgnoreAbove IGNORE_ABOVE_DEFAULT = new IgnoreAbove(null, IndexMode.STANDARD); + + private final IgnoreAbove ignoreAbove; private final String nullValue; private final NamedAnalyzer normalizer; private final boolean eagerGlobalOrdinals; private final FieldValues scriptValues; private final boolean isDimension; private final boolean isSyntheticSource; - private final IndexMode indexMode; private final IndexSortConfig indexSortConfig; private final boolean hasDocValuesSkipper; private final String originalName; @@ -568,36 +564,34 @@ public KeywordFieldType( ); this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue(); this.normalizer = normalizer; - this.ignoreAbove = builder.ignoreAbove.getValue(); + this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion); this.nullValue = builder.nullValue.getValue(); this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); this.isSyntheticSource = isSyntheticSource; - this.indexMode = builder.indexMode; this.indexSortConfig = builder.indexSortConfig; this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; this.originalName = isSyntheticSource ? name + "._original" : null; } + public KeywordFieldType(String name) { + this(name, true, true, Collections.emptyMap()); + } + public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = Integer.MAX_VALUE; + this.ignoreAbove = IGNORE_ABOVE_DEFAULT; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.indexMode = IndexMode.STANDARD; this.indexSortConfig = null; this.hasDocValuesSkipper = false; this.originalName = null; } - public KeywordFieldType(String name) { - this(name, true, true, Collections.emptyMap()); - } - public KeywordFieldType(String name, FieldType fieldType) { super( name, @@ -608,13 +602,12 @@ public KeywordFieldType(String name, FieldType fieldType) { Collections.emptyMap() ); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = Integer.MAX_VALUE; + this.ignoreAbove = IGNORE_ABOVE_DEFAULT; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.indexMode = IndexMode.STANDARD; this.indexSortConfig = null; this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; this.originalName = null; @@ -623,13 +616,12 @@ public KeywordFieldType(String name, FieldType fieldType) { public KeywordFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = Integer.MAX_VALUE; + this.ignoreAbove = IGNORE_ABOVE_DEFAULT; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.indexMode = IndexMode.STANDARD; this.indexSortConfig = null; this.hasDocValuesSkipper = false; this.originalName = null; @@ -938,10 +930,7 @@ protected String parseSourceValue(Object value) { } private String applyIgnoreAboveAndNormalizer(String value) { - if (value.length() > ignoreAbove) { - return null; - } - + if (ignoreAbove.isIgnored(value)) return null; return normalizeValue(normalizer(), name(), value); } @@ -1060,7 +1049,7 @@ public CollapseType collapseType() { /** Values that have more chars than the return value of this method will * be skipped at parsing time. */ - public int ignoreAbove() { + public IgnoreAbove ignoreAbove() { return ignoreAbove; } @@ -1078,10 +1067,6 @@ public boolean hasNormalizer() { return normalizer != Lucene.KEYWORD_ANALYZER; } - public IndexMode getIndexMode() { - return indexMode; - } - public IndexSortConfig getIndexSortConfig() { return indexSortConfig; } @@ -1216,7 +1201,7 @@ private boolean indexValue(DocumentParserContext context, XContentString value) return false; } - if (value.stringLength() > fieldType().ignoreAbove()) { + if (fieldType().ignoreAbove().isIgnored(value)) { context.addIgnoredField(fullPath()); if (isSyntheticSource) { // Save a copy of the field so synthetic source can load it @@ -1385,7 +1370,7 @@ protected BytesRef preserve(BytesRef value) { } } - if (fieldType().ignoreAbove != Integer.MAX_VALUE) { + if (fieldType().ignoreAbove.isSet()) { layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName) { @Override protected void writeValue(Object value, XContentBuilder b) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index cf3261d88bf10..1453d8fcb65a2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.IndexVersions; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentString; import java.io.IOException; import java.util.Arrays; @@ -131,6 +132,78 @@ default boolean supportsVersion(IndexVersion indexCreatedVersion) { } } + /** + * This class models the ignore_above parameter in indices. + */ + public static final class IgnoreAbove { + + public static final int IGNORE_ABOVE_DEFAULT_VALUE = Integer.MAX_VALUE; + public static final int IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES = 8191; + + private final Integer value; + private final Integer defaultValue; + + public IgnoreAbove(Integer value) { + this(Objects.requireNonNull(value), IndexMode.STANDARD, IndexVersion.current()); + } + + public IgnoreAbove(Integer value, IndexMode indexMode) { + this(value, indexMode, IndexVersion.current()); + } + + public IgnoreAbove(Integer value, IndexMode indexMode, IndexVersion indexCreatedVersion) { + if (value != null && value < 0) { + throw new IllegalArgumentException("[ignore_above] must be positive, got [" + value + "]"); + } + + this.value = value; + this.defaultValue = getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion); + } + + public int get() { + return value != null ? value : defaultValue; + } + + /** + * Returns whether ignore_above is set; at field or index level. + */ + public boolean isSet() { + // if ignore_above equals default, its not considered to be set, even if it was explicitly set to the default value + return Integer.valueOf(get()).equals(defaultValue) == false; + } + + /** + * Returns whether the given string will be ignored. + */ + public boolean isIgnored(final String s) { + if (s == null) return false; + return lengthExceedsIgnoreAbove(s.length()); + } + + public boolean isIgnored(final XContentString s) { + if (s == null) return false; + return lengthExceedsIgnoreAbove(s.stringLength()); + } + + private boolean lengthExceedsIgnoreAbove(int strLength) { + return strLength > get(); + } + + public static int getIgnoreAboveDefaultValue(final IndexMode indexMode, final IndexVersion indexCreatedVersion) { + if (diffIgnoreAboveDefaultForLogs(indexMode, indexCreatedVersion)) { + return IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES; + } else { + return IGNORE_ABOVE_DEFAULT_VALUE; + } + } + + private static boolean diffIgnoreAboveDefaultForLogs(final IndexMode indexMode, final IndexVersion indexCreatedVersion) { + return indexMode == IndexMode.LOGSDB + && (indexCreatedVersion != null && indexCreatedVersion.onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)); + } + + } + private final String leafName; @SuppressWarnings("this-escape") diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 8a586d68177e0..1a232ecf09f72 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1021,7 +1021,7 @@ public boolean isAggregatable() { * A delegate by definition must have doc_values or be stored so most of the time it can be used for loading. */ public boolean canUseSyntheticSourceDelegateForLoading() { - return syntheticSourceDelegate != null && syntheticSourceDelegate.ignoreAbove() == Integer.MAX_VALUE; + return syntheticSourceDelegate != null && syntheticSourceDelegate.ignoreAbove().isSet() == false; } /** @@ -1029,7 +1029,7 @@ public boolean canUseSyntheticSourceDelegateForLoading() { */ public boolean canUseSyntheticSourceDelegateForQuerying() { return syntheticSourceDelegate != null - && syntheticSourceDelegate.ignoreAbove() == Integer.MAX_VALUE + && syntheticSourceDelegate.ignoreAbove().isSet() == false && syntheticSourceDelegate.isIndexed(); } @@ -1045,7 +1045,7 @@ public boolean canUseSyntheticSourceDelegateForQueryingEquality(String str) { return false; } // Can't push equality if the field we're checking for is so big we'd ignore it. - return str.length() <= syntheticSourceDelegate.ignoreAbove(); + return syntheticSourceDelegate.ignoreAbove().isIgnored(str) == false; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index f70a47caa080c..9b818570ba543 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -38,6 +38,8 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -55,6 +57,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextParams; @@ -124,9 +127,6 @@ private static Builder builder(Mapper in) { return ((FlattenedFieldMapper) in).builder; } - private final int ignoreAboveDefault; - private final int ignoreAbove; - public static class Builder extends FieldMapper.Builder { final Parameter depthLimit = Parameter.intParam( @@ -180,23 +180,37 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); + private final IndexMode indexMode; + private final IndexVersion indexCreatedVersion; + public static FieldMapper.Parameter> dimensionsParam(Function> initializer) { return FieldMapper.Parameter.stringArrayParam(TIME_SERIES_DIMENSIONS_ARRAY_PARAM, false, initializer); } public Builder(final String name) { - this(name, Integer.MAX_VALUE); + this( + name, + IgnoreAbove.getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current()), + IndexMode.STANDARD, + IndexVersion.current() + ); } - private Builder(String name, int ignoreAboveDefault) { + private Builder(String name, MappingParserContext mappingParserContext) { + this( + name, + IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), + mappingParserContext.getIndexSettings().getMode(), + mappingParserContext.indexVersionCreated() + ); + } + + private Builder(String name, int ignoreAboveDefault, IndexMode indexMode, IndexVersion indexCreatedVersion) { super(name); this.ignoreAboveDefault = ignoreAboveDefault; - this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> builder(m).ignoreAbove.get(), ignoreAboveDefault) - .addValidator(v -> { - if (v < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); - } - }); + this.indexMode = indexMode; + this.indexCreatedVersion = indexCreatedVersion; + this.ignoreAbove = Parameter.ignoreAboveParam(m -> builder(m).ignoreAbove.get(), ignoreAboveDefault); this.dimensions.precludesParameters(ignoreAbove); } @@ -233,13 +247,13 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { splitQueriesOnWhitespace.get(), eagerGlobalOrdinals.get(), dimensions.get(), - ignoreAbove.getValue() + new IgnoreAbove(ignoreAbove.getValue(), indexMode, indexCreatedVersion) ); - return new FlattenedFieldMapper(leafName(), ft, builderParams(this, context), ignoreAboveDefault, this); + return new FlattenedFieldMapper(leafName(), ft, builderParams(this, context), this); } } - public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, IGNORE_ABOVE_SETTING.get(c.getSettings()))); + public static final TypeParser PARSER = createTypeParserWithLegacySupport(FlattenedFieldMapper.Builder::new); /** * A field type that represents the values under a particular JSON key, used @@ -665,7 +679,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme private final boolean eagerGlobalOrdinals; private final List dimensions; private final boolean isDimension; - private final int ignoreAbove; + private final IgnoreAbove ignoreAbove; RootFlattenedFieldType( String name, @@ -674,7 +688,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme Map meta, boolean splitQueriesOnWhitespace, boolean eagerGlobalOrdinals, - int ignoreAbove + IgnoreAbove ignoreAbove ) { this(name, indexed, hasDocValues, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList(), ignoreAbove); } @@ -687,7 +701,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme boolean splitQueriesOnWhitespace, boolean eagerGlobalOrdinals, List dimensions, - int ignoreAbove + IgnoreAbove ignoreAbove ) { super( name, @@ -741,6 +755,10 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); } + public IgnoreAbove ignoreAbove() { + return ignoreAbove; + } + private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { return new SourceValueFetcher(sourcePaths, null) { @Override @@ -750,7 +768,7 @@ protected Object parseSourceValue(Object value) { final Map result = filterIgnoredValues((Map) valueAsMap); return result.isEmpty() ? null : result; } - if (value instanceof String valueAsString && valueAsString.length() <= ignoreAbove) { + if (value instanceof String valueAsString && ignoreAbove.isIgnored(valueAsString) == false) { return valueAsString; } return null; @@ -772,7 +790,7 @@ private Object filterIgnoredValues(final Object entryValue) { final List validValues = new ArrayList<>(); for (Object value : valueAsList) { if (value instanceof String valueAsString) { - if (valueAsString.length() <= ignoreAbove) { + if (ignoreAbove.isIgnored(valueAsString) == false) { validValues.add(valueAsString); } } else { @@ -788,7 +806,7 @@ private Object filterIgnoredValues(final Object entryValue) { } return validValues; } else if (entryValue instanceof String valueAsString) { - if (valueAsString.length() <= ignoreAbove) { + if (ignoreAbove.isIgnored(valueAsString) == false) { return valueAsString; } return null; @@ -828,17 +846,9 @@ public void validateMatchedRoutingPath(final String routingPath) { private final FlattenedFieldParser fieldParser; private final Builder builder; - private FlattenedFieldMapper( - String leafName, - MappedFieldType mappedFieldType, - BuilderParams builderParams, - int ignoreAboveDefault, - Builder builder - ) { + private FlattenedFieldMapper(String leafName, MappedFieldType mappedFieldType, BuilderParams builderParams, Builder builder) { super(leafName, mappedFieldType, builderParams); - this.ignoreAboveDefault = ignoreAboveDefault; this.builder = builder; - this.ignoreAbove = builder.ignoreAbove.get(); this.fieldParser = new FlattenedFieldParser( mappedFieldType.name(), mappedFieldType.name() + KEYED_FIELD_SUFFIX, @@ -864,10 +874,6 @@ int depthLimit() { return builder.depthLimit.get(); } - public int ignoreAbove() { - return ignoreAbove; - } - @Override public RootFlattenedFieldType fieldType() { return (RootFlattenedFieldType) super.fieldType(); @@ -905,7 +911,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), ignoreAboveDefault).init(this); + return new Builder(leafName(), builder.ignoreAboveDefault, builder.indexMode, builder.indexCreatedVersion).init(this); } @Override @@ -915,7 +921,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() { () -> new FlattenedSortedSetDocValuesSyntheticFieldLoader( fullPath(), fullPath() + KEYED_FIELD_SUFFIX, - ignoreAbove() < Integer.MAX_VALUE ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null, + fieldType().ignoreAbove.isSet() ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null, leafName() ) ); diff --git a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java new file mode 100644 index 0000000000000..f43b0eb95683e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index; + +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.Text; + +public class IgnoreAboveTests extends ESTestCase { + + private static final Mapper.IgnoreAbove IGNORE_ABOVE_DEFAULT = new Mapper.IgnoreAbove(null, IndexMode.STANDARD); + private static final Mapper.IgnoreAbove IGNORE_ABOVE_DEFAULT_LOGS = new Mapper.IgnoreAbove(null, IndexMode.LOGSDB); + + public void test_ignore_above_with_value_and_index_mode_and_index_version() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(123, IndexMode.STANDARD); + + // when/then + assertEquals(123, ignoreAbove.get()); + assertTrue(ignoreAbove.isSet()); + } + + public void test_ignore_above_with_value_only() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(123); + + // when/then + assertEquals(123, ignoreAbove.get()); + assertTrue(ignoreAbove.isSet()); + } + + public void test_ignore_above_with_null_value_should_throw() { + assertThrows(NullPointerException.class, () -> new Mapper.IgnoreAbove(null)); + } + + public void test_ignore_above_with_negative_value_should_throw() { + assertThrows(IllegalArgumentException.class, () -> new Mapper.IgnoreAbove(-1)); + assertThrows(IllegalArgumentException.class, () -> new Mapper.IgnoreAbove(-1, IndexMode.STANDARD)); + } + + public void test_ignore_above_with_null_value() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, IndexMode.STANDARD); + + // when/then + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); + } + + public void test_ignore_above_with_null_value_and_logsdb_index_mode() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, IndexMode.LOGSDB); + + // when/then + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); + } + + public void test_ignore_above_with_null_everything() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, null, null); + + // when/then + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); + } + + public void test_ignore_above_default_for_standard_indices() { + // given + Mapper.IgnoreAbove ignoreAbove = IGNORE_ABOVE_DEFAULT; + + // when/then + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); + } + + public void test_ignore_above_default_for_logsdb_indices() { + // given + Mapper.IgnoreAbove ignoreAbove = IGNORE_ABOVE_DEFAULT_LOGS; + + // when/then + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); + } + + public void test_string_isIgnored() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(10); + + // when/then + assertFalse(ignoreAbove.isIgnored("potato")); + assertFalse(ignoreAbove.isIgnored("1234567890")); + assertTrue(ignoreAbove.isIgnored("12345678901")); + assertTrue(ignoreAbove.isIgnored("potato potato tomato tomato")); + } + + public void test_XContentString_isIgnored() { + // given + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(10); + + // when/then + assertFalse(ignoreAbove.isIgnored(new Text("potato"))); + assertFalse(ignoreAbove.isIgnored(new Text("1234567890"))); + assertTrue(ignoreAbove.isIgnored(new Text("12345678901"))); + assertTrue(ignoreAbove.isIgnored(new Text("potato potato tomato tomato"))); + } + + public void test_default_value() { + assertEquals( + Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, + Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current()) + ); + assertEquals( + Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, + Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(IndexMode.LOGSDB, IndexVersion.current()) + ); + assertEquals( + Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, + Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(IndexMode.LOGSDB, IndexVersions.ENABLE_IGNORE_MALFORMED_LOGSDB) + ); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index f6a7148c71091..0449b4ff7dbf4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -179,7 +179,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerUpdateCheck(b -> b.field("eager_global_ordinals", true), m -> assertTrue(m.fieldType().eagerGlobalOrdinals())); checker.registerUpdateCheck( b -> b.field("ignore_above", 256), - m -> assertEquals(256, ((KeywordFieldMapper) m).fieldType().ignoreAbove()) + m -> assertEquals(256, ((KeywordFieldMapper) m).fieldType().ignoreAbove().get()) ); checker.registerUpdateCheck( b -> b.field("split_queries_on_whitespace", true), @@ -243,6 +243,108 @@ public void testIgnoreAbove() throws IOException { assertTrue(doc.rootDoc().getFields("_ignored").stream().anyMatch(field -> "field".equals(field.stringValue()))); } + public void test_ignore_above_index_level_setting() throws IOException { + // given + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(), + mapping(b -> { + b.startObject("field"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + // when + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + + // then + assertEquals(123, mapper.fieldType().ignoreAbove().get()); + assertTrue(mapper.fieldType().ignoreAbove().isSet()); + } + + public void test_ignore_above_index_level_setting_is_overridden_by_field_level_ignore_above_in_standard_indices() throws IOException { + // given + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(), + mapping(b -> { + b.startObject("potato"); + b.field("type", "keyword"); + b.field("ignore_above", 456); + b.endObject(); + + b.startObject("tomato"); + b.field("type", "keyword"); + b.field("ignore_above", 789); + b.endObject(); + + b.startObject("cheese"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + // when + final KeywordFieldMapper fieldMapper1 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("potato"); + final KeywordFieldMapper fieldMapper2 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("tomato"); + final KeywordFieldMapper fieldMapper3 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("cheese"); + + // then + assertEquals(456, fieldMapper1.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper1.fieldType().ignoreAbove().isSet()); + + assertEquals(789, fieldMapper2.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper2.fieldType().ignoreAbove().isSet()); + + assertEquals(123, fieldMapper3.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper3.fieldType().ignoreAbove().isSet()); + } + + public void test_ignore_above_index_level_setting_is_overridden_by_field_level_ignore_above_in_logsdb_indices() throws IOException { + // given + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(), + mapping(b -> { + b.startObject("potato"); + b.field("type", "keyword"); + b.field("ignore_above", 456); + b.endObject(); + + b.startObject("tomato"); + b.field("type", "keyword"); + b.field("ignore_above", 789); + b.endObject(); + + b.startObject("cheese"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + // when + final KeywordFieldMapper fieldMapper1 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("potato"); + final KeywordFieldMapper fieldMapper2 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("tomato"); + final KeywordFieldMapper fieldMapper3 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("cheese"); + + // then + assertEquals(456, fieldMapper1.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper1.fieldType().ignoreAbove().isSet()); + + assertEquals(789, fieldMapper2.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper2.fieldType().ignoreAbove().isSet()); + + assertEquals(123, fieldMapper3.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper3.fieldType().ignoreAbove().isSet()); + } + public void testNullValue() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); ParsedDocument doc = mapper.parse(source(b -> b.nullField("field"))); @@ -331,7 +433,7 @@ public void testDimensionAndIgnoreAbove() throws IOException { b.field("time_series_dimension", true).field("ignore_above", 2048); })); KeywordFieldMapper field = (KeywordFieldMapper) documentMapper.mappers().getMapper("field"); - assertEquals(2048, field.fieldType().ignoreAbove()); + assertEquals(2048, field.fieldType().ignoreAbove().get()); } public void testDimensionAndNormalizer() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 05a6c24a2b743..7cc886d45dc6b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -36,9 +36,13 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.CharFilterFactory; @@ -58,6 +62,9 @@ import java.util.List; import java.util.Map; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + public class KeywordFieldTypeTests extends FieldTypeTestCase { public void testIsFieldWithinQuery() throws IOException { @@ -245,7 +252,6 @@ public void testFetchSourceValue() throws IOException { "field", createIndexAnalyzers(), ScriptCompiler.NONE, - Integer.MAX_VALUE, IndexVersion.current(), randomFrom(Mapper.SourceKeepMode.values()) ).normalizer("lowercase").build(MapperBuilderContext.root(false, false)).fieldType(); @@ -291,6 +297,243 @@ public void testGetTerms() throws IOException { } } + public void test_ignore_above_index_level_setting() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertTrue(fieldType.ignoreAbove().isSet()); + assertEquals(123, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_true_when_ignore_above_is_given() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + builder.ignoreAbove(123); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertTrue(fieldType.ignoreAbove().isSet()); + assertEquals(123, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_false_when_ignore_above_is_not_given() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertFalse(fieldType.ignoreAbove().isSet()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + builder.ignoreAbove(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertFalse(fieldType.ignoreAbove().isSet()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default_for_logsdb_indices() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + builder.ignoreAbove(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertFalse(fieldType.ignoreAbove().isSet()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_true_when_ignore_above_is_given_as_logsdb_default_but_index_mod_is_not_logsdb() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + builder.ignoreAbove(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertTrue(fieldType.ignoreAbove().isSet()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_true_when_ignore_above_is_configured_at_index_level() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertTrue(fieldType.ignoreAbove().isSet()); + assertEquals(123, fieldType.ignoreAbove().get()); + } + + public void test_ignore_above_isSet_returns_false_for_non_primary_constructor() { + // given + KeywordFieldType fieldType1 = new KeywordFieldType("field"); + KeywordFieldType fieldType2 = new KeywordFieldType("field", mock(FieldType.class)); + KeywordFieldType fieldType3 = new KeywordFieldType("field", true, true, Collections.emptyMap()); + KeywordFieldType fieldType4 = new KeywordFieldType("field", mock(NamedAnalyzer.class)); + + // when/then + assertFalse(fieldType1.ignoreAbove().isSet()); + assertFalse(fieldType2.ignoreAbove().isSet()); + assertFalse(fieldType3.ignoreAbove().isSet()); + assertFalse(fieldType4.ignoreAbove().isSet()); + } + private static IndexAnalyzers createIndexAnalyzers() { return IndexAnalyzers.of( Map.of("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer())), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java index 4c5bfeb66b075..e318fe527850a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java @@ -63,7 +63,6 @@ private KeywordFieldMapper.Builder getKeywordFieldMapperBuilder(boolean isStored "field", IndexAnalyzers.of(Map.of(), Map.of("normalizer", Lucene.STANDARD_ANALYZER), Map.of()), ScriptCompiler.NONE, - Integer.MAX_VALUE, IndexVersion.current(), Mapper.SourceKeepMode.NONE ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java index 7402405f5b883..04e8121fdd90e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java @@ -263,12 +263,12 @@ public void testMergeSameObjectDifferentFields() { ObjectMapper parent0 = (ObjectMapper) mergedAdd0.getMapper("parent"); assertNotNull(parent0.getMapper("child1")); - assertEquals(42, ((KeywordFieldMapper) parent0.getMapper("child1")).fieldType().ignoreAbove()); + assertEquals(42, ((KeywordFieldMapper) parent0.getMapper("child1")).fieldType().ignoreAbove().get()); assertNull(parent0.getMapper("child2")); ObjectMapper parent1 = (ObjectMapper) mergedAdd1.getMapper("parent"); assertNotNull(parent1.getMapper("child1")); - assertEquals(42, ((KeywordFieldMapper) parent1.getMapper("child1")).fieldType().ignoreAbove()); + assertEquals(42, ((KeywordFieldMapper) parent1.getMapper("child1")).fieldType().ignoreAbove().get()); assertNotNull(parent1.getMapper("child2")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java new file mode 100644 index 0000000000000..d27ec182db7e7 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.test.ESTestCase; + +public class ParameterTests extends ESTestCase { + + public void test_ignore_above_param_default() { + // when + FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam((FieldMapper fm) -> 123, 456); + + // then + assertEquals(456, ignoreAbove.getValue().intValue()); + } + + public void test_ignore_above_param_invalid_value() { + // when + FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam((FieldMapper fm) -> -1, 456); + ignoreAbove.setValue(-1); + + // then + assertThrows(IllegalArgumentException.class, ignoreAbove::validate); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 4d246d3c557a6..50074c50d51c4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -8,6 +8,7 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.FieldType; import org.apache.lucene.index.Term; import org.apache.lucene.queries.intervals.Intervals; import org.apache.lucene.queries.intervals.IntervalsSource; @@ -31,10 +32,18 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.AutomatonQueries; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; +import org.elasticsearch.script.ScriptCompiler; import java.io.IOException; import java.util.ArrayList; @@ -44,6 +53,8 @@ import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; public class TextFieldTypeTests extends FieldTypeTestCase { @@ -289,4 +300,131 @@ public void testRangeIntervals() { rangeIntervals ); } + + public void test_block_loader_uses_synthetic_source_delegate_when_ignore_above_is_not_set() { + // given + KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( + "child", + true, + true, + Collections.emptyMap() + ); + + TextFieldType ft = new TextFieldType( + "parent", + true, + false, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + true, + syntheticSourceDelegate, + Collections.singletonMap("potato", "tomato"), + false, + false + ); + + // when + ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + + // then + // verify that we delegate block loading to the synthetic source delegate + assertTrue(blockLoader instanceof BlockLoader.Delegating); + assertThat(((BlockLoader.Delegating) blockLoader).delegatingTo(), equalTo("child")); + } + + public void test_block_loader_does_not_use_synthetic_source_delegate_when_ignore_above_is_set() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + builder.ignoreAbove(123); + + KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + TextFieldType ft = new TextFieldType( + "parent", + true, + false, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + true, + syntheticSourceDelegate, + Collections.singletonMap("potato", "tomato"), + false, + false + ); + + // when + ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + + // then + // verify that we don't delegate anything + assertFalse(blockLoader instanceof BlockLoader.Delegating); + } + + public void test_block_loader_does_not_use_synthetic_source_delegate_when_ignore_above_is_set_at_index_level() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + + KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + TextFieldType ft = new TextFieldType( + "parent", + true, + false, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + true, + syntheticSourceDelegate, + Collections.singletonMap("potato", "tomato"), + false, + false + ); + + // when + ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + + // then + // verify that we don't delegate anything + assertFalse(blockLoader instanceof BlockLoader.Delegating); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index 0628ca47308ea..d9223a157b2ff 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -86,7 +86,10 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("time_series_dimensions", b -> b.field("time_series_dimensions", List.of("one", "two"))); checker.registerUpdateCheck(b -> b.field("eager_global_ordinals", true), m -> assertTrue(m.fieldType().eagerGlobalOrdinals())); - checker.registerUpdateCheck(b -> b.field("ignore_above", 256), m -> assertEquals(256, ((FlattenedFieldMapper) m).ignoreAbove())); + checker.registerUpdateCheck( + b -> b.field("ignore_above", 256), + m -> assertEquals(256, ((FlattenedFieldMapper) m).fieldType().ignoreAbove().get()) + ); checker.registerUpdateCheck( b -> b.field("split_queries_on_whitespace", true), m -> assertEquals("_whitespace", m.fieldType().getTextSearchInfo().searchAnalyzer().name()) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java index 7873458eb46ee..3127b2c60d0f5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java @@ -22,8 +22,10 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType; import java.io.IOException; @@ -33,8 +35,10 @@ public class RootFlattenedFieldTypeTests extends FieldTypeTestCase { + private static final Mapper.IgnoreAbove IGNORE_ABOVE = new Mapper.IgnoreAbove(null, IndexMode.STANDARD); + private static RootFlattenedFieldType createDefaultFieldType(int ignoreAbove) { - return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, ignoreAbove); + return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, new Mapper.IgnoreAbove(ignoreAbove)); } public void testValueForDisplay() { @@ -61,33 +65,17 @@ public void testTermQuery() { Collections.emptyMap(), false, false, - Integer.MAX_VALUE + IGNORE_ABOVE ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } public void testExistsQuery() { - RootFlattenedFieldType ft = new RootFlattenedFieldType( - "field", - true, - false, - Collections.emptyMap(), - false, - false, - Integer.MAX_VALUE - ); + RootFlattenedFieldType ft = new RootFlattenedFieldType("field", true, false, Collections.emptyMap(), false, false, IGNORE_ABOVE); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))), ft.existsQuery(null)); - RootFlattenedFieldType withDv = new RootFlattenedFieldType( - "field", - true, - true, - Collections.emptyMap(), - false, - false, - Integer.MAX_VALUE - ); + RootFlattenedFieldType withDv = new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, IGNORE_ABOVE); assertEquals(new FieldExistsQuery("field"), withDv.existsQuery(null)); } diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java new file mode 100644 index 0000000000000..b99c1a69fab0a --- /dev/null +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java @@ -0,0 +1,310 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.upgrades; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.upgrades.StandardToLogsDbIndexModeRollingUpgradeIT.enableLogsdbByDefault; +import static org.elasticsearch.upgrades.StandardToLogsDbIndexModeRollingUpgradeIT.getWriteBackingIndex; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; + +public class TextRollingUpgradeIT extends AbstractRollingUpgradeWithSecurityTestCase { + + private static final String DATA_STREAM = "logs-bwc-test"; + + private static final int IGNORE_ABOVE_MAX = 256; + private static final int NUM_REQUESTS = 4; + private static final int NUM_DOCS_PER_REQUEST = 1024; + + static String BULK_ITEM_TEMPLATE = + """ + { "create": {} } + {"@timestamp": "$now", "host.name": "$host", "method": "$method", "ip": "$ip", "message": "$message", "length": $length, "factor": $factor} + """; + + private static final String TEMPLATE = """ + { + "mappings": { + "properties": { + "@timestamp" : { + "type": "date" + }, + "method": { + "type": "keyword" + }, + "message": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": $IGNORE_ABOVE, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "length": { + "type": "long" + }, + "factor": { + "type": "double" + } + } + } + }"""; + + // when sorted, this message will appear at the top and hence can be used to validate query results + private String smallestMessage; + + public TextRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { + super(upgradedNodes); + } + + public void testIndexing() throws Exception { + + if (isOldCluster()) { + // given - enable logsdb and create a template + startTrial(); + enableLogsdbByDefault(); + String templateId = getClass().getSimpleName().toLowerCase(Locale.ROOT); + createTemplate(DATA_STREAM, templateId, prepareTemplate()); + + // when - index some documents + bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST); + + // then - verify that logsdb and synthetic source are both enabled + String firstBackingIndex = getWriteBackingIndex(client(), DATA_STREAM, 0); + var settings = (Map) getIndexSettingsWithDefaults(firstBackingIndex).get(firstBackingIndex); + assertThat(((Map) settings.get("settings")).get("index.mode"), equalTo("logsdb")); + assertThat(((Map) settings.get("defaults")).get("index.mapping.source.mode"), equalTo("SYNTHETIC")); + + // then continued - verify that the created data stream using the created template + LogsdbIndexingRollingUpgradeIT.assertDataStream(DATA_STREAM, templateId); + + // when/then - run some queries and verify results + ensureGreen(DATA_STREAM); + search(DATA_STREAM); + query(DATA_STREAM); + + } else if (isMixedCluster()) { + // when + bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST); + + // when/then + ensureGreen(DATA_STREAM); + search(DATA_STREAM); + query(DATA_STREAM); + + } else if (isUpgradedCluster()) { + // when/then + ensureGreen(DATA_STREAM); + bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST); + search(DATA_STREAM); + query(DATA_STREAM); + + // when/then continued - force merge all shard segments into one + var forceMergeRequest = new Request("POST", "/" + DATA_STREAM + "/_forcemerge"); + forceMergeRequest.addParameter("max_num_segments", "1"); + assertOK(client().performRequest(forceMergeRequest)); + + // then continued + ensureGreen(DATA_STREAM); + search(DATA_STREAM); + query(DATA_STREAM); + } + } + + private String prepareTemplate() { + boolean shouldSetIgnoreAbove = randomBoolean(); + if (shouldSetIgnoreAbove) { + return TEMPLATE.replace("$IGNORE_ABOVE", String.valueOf(randomInt(IGNORE_ABOVE_MAX))); + } + + // removes the entire line that defines ignore_above + return TEMPLATE.replaceAll("(?m)^\\s*\"ignore_above\":\\s*\\$IGNORE_ABOVE\\s*,?\\s*\\n?", ""); + } + + static void createTemplate(String dataStreamName, String id, String template) throws IOException { + final String INDEX_TEMPLATE = """ + { + "priority": 500, + "index_patterns": ["$DATASTREAM"], + "template": $TEMPLATE, + "data_stream": { + } + }"""; + var putIndexTemplateRequest = new Request("POST", "/_index_template/" + id); + putIndexTemplateRequest.setJsonEntity(INDEX_TEMPLATE.replace("$TEMPLATE", template).replace("$DATASTREAM", dataStreamName)); + assertOK(client().performRequest(putIndexTemplateRequest)); + } + + private void bulkIndex(int numRequest, int numDocs) throws Exception { + String firstIndex = null; + Instant startTime = Instant.now().minusSeconds(60 * 60); + + for (int i = 0; i < numRequest; i++) { + var bulkRequest = new Request("POST", "/" + DATA_STREAM + "/_bulk"); + bulkRequest.setJsonEntity(bulkIndexRequestBody(numDocs, startTime)); + bulkRequest.addParameter("refresh", "true"); + + var response = client().performRequest(bulkRequest); + var responseBody = entityAsMap(response); + + assertOK(response); + assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false)); + if (firstIndex == null) { + firstIndex = (String) ((Map) ((Map) ((List) responseBody.get("items")).get(0)).get("create")).get("_index"); + } + } + } + + private String bulkIndexRequestBody(int numDocs, Instant startTime) { + StringBuilder requestBody = new StringBuilder(); + + for (int j = 0; j < numDocs; j++) { + String hostName = "host" + j % 50; // Not realistic, but makes asserting search / query response easier. + String methodName = "method" + j % 5; + String ip = NetworkAddress.format(randomIp(true)); + String message = randomAlphasDelimitedBySpace(10, 1, 15); + recordSmallestMessage(message); + long length = randomLong(); + double factor = randomDouble(); + + requestBody.append( + BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime)) + .replace("$host", hostName) + .replace("$method", methodName) + .replace("$ip", ip) + .replace("$message", message) + .replace("$length", Long.toString(length)) + .replace("$factor", Double.toString(factor)) + ); + requestBody.append('\n'); + + startTime = startTime.plusMillis(1); + } + + return requestBody.toString(); + } + + /** + * Generates a string containing a random number of random length alphas, all delimited by space. + */ + public static String randomAlphasDelimitedBySpace(int maxAlphas, int minCodeUnits, int maxCodeUnits) { + int numAlphas = randomIntBetween(1, maxAlphas); + List alphas = new ArrayList<>(numAlphas); + for (int i = 0; i < numAlphas; i++) { + alphas.add(randomAlphaOfLengthBetween(minCodeUnits, maxCodeUnits)); + } + return String.join(" ", alphas); + } + + private void recordSmallestMessage(final String message) { + if (smallestMessage == null || message.compareTo(smallestMessage) < 0) { + smallestMessage = message; + } + } + + void search(String dataStreamName) throws Exception { + var searchRequest = new Request("POST", "/" + dataStreamName + "/_search"); + searchRequest.addParameter("pretty", "true"); + searchRequest.setJsonEntity(""" + { + "size": 500 + } + """); + var response = client().performRequest(searchRequest); + assertOK(response); + var responseBody = entityAsMap(response); + logger.info("{}", responseBody); + + Integer totalCount = ObjectPath.evaluate(responseBody, "hits.total.value"); + assertThat(totalCount, greaterThanOrEqualTo(NUM_REQUESTS * NUM_DOCS_PER_REQUEST)); + } + + private void query(String dataStreamName) throws Exception { + var queryRequest = new Request("POST", "/_query"); + queryRequest.addParameter("pretty", "true"); + queryRequest.setJsonEntity(""" + { + "query": "FROM $ds | STATS max(length), max(factor) BY message | SORT message | LIMIT 5" + } + """.replace("$ds", dataStreamName)); + var response = client().performRequest(queryRequest); + assertOK(response); + var responseBody = entityAsMap(response); + logger.info("{}", responseBody); + + String column1 = ObjectPath.evaluate(responseBody, "columns.0.name"); + assertThat(column1, equalTo("max(length)")); + String column2 = ObjectPath.evaluate(responseBody, "columns.1.name"); + assertThat(column2, equalTo("max(factor)")); + String column3 = ObjectPath.evaluate(responseBody, "columns.2.name"); + assertThat(column3, equalTo("message")); + + Long maxRx = ObjectPath.evaluate(responseBody, "values.0.0"); + assertThat(maxRx, notNullValue()); + Double maxTx = ObjectPath.evaluate(responseBody, "values.0.1"); + assertThat(maxTx, notNullValue()); + String key = ObjectPath.evaluate(responseBody, "values.0.2"); + assertThat(key, equalTo(smallestMessage)); + } + + protected static void startTrial() throws IOException { + Request startTrial = new Request("POST", "/_license/start_trial"); + startTrial.addParameter("acknowledge", "true"); + try { + assertOK(client().performRequest(startTrial)); + } catch (ResponseException e) { + var responseBody = entityAsMap(e.getResponse()); + String error = ObjectPath.evaluate(responseBody, "error_message"); + assertThat(error, containsString("Trial was already activated.")); + } + } + + static Map getIndexSettingsWithDefaults(String index) throws IOException { + Request request = new Request("GET", "/" + index + "/_settings"); + request.addParameter("flat_settings", "true"); + request.addParameter("include_defaults", "true"); + Response response = client().performRequest(request); + try (InputStream is = response.getEntity().getContent()) { + return XContentHelper.convertToMap( + XContentType.fromMediaType(response.getEntity().getContentType().getValue()).xContent(), + is, + true + ); + } + } + + static String formatInstant(Instant instant) { + return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); + } + +} diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index ff89074c3f063..874ea05c00e34 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -51,6 +51,7 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.AnalyzerScope; @@ -69,6 +70,7 @@ import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; @@ -92,6 +94,7 @@ import java.util.TreeSet; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; +import static org.elasticsearch.index.mapper.Mapper.IgnoreAbove.getIgnoreAboveDefaultValue; /** * A {@link FieldMapper} for indexing fields with ngrams for efficient wildcard matching @@ -206,28 +209,34 @@ private static WildcardFieldMapper toType(FieldMapper in) { public static class Builder extends FieldMapper.Builder { + final int ignoreAboveDefault; final Parameter ignoreAbove; final Parameter nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull(); final Parameter> meta = Parameter.metaParam(); - final IndexVersion indexVersionCreated; - - final int ignoreAboveDefault; + final IndexMode indexMode; + final IndexVersion indexCreatedVersion; public Builder(final String name, IndexVersion indexVersionCreated) { - this(name, Integer.MAX_VALUE, indexVersionCreated); + this(name, getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexVersionCreated), IndexMode.STANDARD, indexVersionCreated); + } + + private Builder(String name, MappingParserContext mappingParserContext) { + this( + name, + IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), + mappingParserContext.getIndexSettings().getMode(), + mappingParserContext.indexVersionCreated() + ); } - private Builder(String name, int ignoreAboveDefault, IndexVersion indexVersionCreated) { + private Builder(String name, int ignoreAboveDefault, IndexMode indexMode, IndexVersion indexCreatedVersion) { super(name); - this.indexVersionCreated = indexVersionCreated; this.ignoreAboveDefault = ignoreAboveDefault; - this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, ignoreAboveDefault).addValidator(v -> { - if (v < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); - } - }); + this.indexMode = indexMode; + this.indexCreatedVersion = indexCreatedVersion; + this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).ignoreAbove.get(), ignoreAboveDefault); } @Override @@ -249,18 +258,15 @@ Builder nullValue(String nullValue) { public WildcardFieldMapper build(MapperBuilderContext context) { return new WildcardFieldMapper( leafName(), - new WildcardFieldType(context.buildFullName(leafName()), indexVersionCreated, meta.get(), this), + new WildcardFieldType(context.buildFullName(leafName()), indexCreatedVersion, meta.get(), this), context.isSourceSynthetic(), builderParams(this, context), - indexVersionCreated, this ); } } - public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, IGNORE_ABOVE_SETTING.get(c.getSettings()), c.indexVersionCreated()) - ); + public static final TypeParser PARSER = createTypeParserWithLegacySupport(Builder::new); public static final char TOKEN_START_OR_END_CHAR = 0; public static final String TOKEN_START_STRING = Character.toString(TOKEN_START_OR_END_CHAR); @@ -272,7 +278,7 @@ public static final class WildcardFieldType extends MappedFieldType { private final String nullValue; private final NamedAnalyzer analyzer; - private final int ignoreAbove; + private final IgnoreAbove ignoreAbove; private WildcardFieldType(String name, IndexVersion version, Map meta, Builder builder) { super(name, true, false, true, Defaults.TEXT_SEARCH_INFO, meta); @@ -282,7 +288,7 @@ private WildcardFieldType(String name, IndexVersion version, Map this.analyzer = WILDCARD_ANALYZER_7_9; } this.nullValue = builder.nullValue.getValue(); - this.ignoreAbove = builder.ignoreAbove.getValue(); + this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion); } @Override @@ -981,7 +987,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { + if (ignoreAbove.isIgnored(keywordValue)) { return null; } return keywordValue; @@ -1000,10 +1006,10 @@ protected String parseSourceValue(Object value) { assert NGRAM_FIELD_TYPE.indexOptions() == IndexOptions.DOCS; } private final String nullValue; + private final IndexMode indexMode; private final IndexVersion indexVersionCreated; - - private final int ignoreAbove; private final int ignoreAboveDefault; + private final IgnoreAbove ignoreAbove; private final boolean storeIgnored; private final String originalName; @@ -1012,15 +1018,15 @@ private WildcardFieldMapper( WildcardFieldType mappedFieldType, boolean storeIgnored, BuilderParams builderParams, - IndexVersion indexVersionCreated, Builder builder ) { super(simpleName, mappedFieldType, builderParams); this.nullValue = builder.nullValue.getValue(); this.storeIgnored = storeIgnored; - this.indexVersionCreated = indexVersionCreated; - this.ignoreAbove = builder.ignoreAbove.getValue(); + this.indexMode = builder.indexMode; + this.indexVersionCreated = builder.indexCreatedVersion; this.ignoreAboveDefault = builder.ignoreAboveDefault; + this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion); this.originalName = storeIgnored ? fullPath() + "._original" : null; } @@ -1032,7 +1038,7 @@ public Map indexAnalyzers() { /** Values that have more chars than the return value of this method will * be skipped at parsing time. */ // pkg-private for testing - int ignoreAbove() { + IgnoreAbove ignoreAbove() { return ignoreAbove; } @@ -1054,13 +1060,13 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio List fields = new ArrayList<>(); if (value != null) { - if (value.length() <= ignoreAbove) { - createFields(value, parseDoc, fields); - } else { + if (ignoreAbove.isIgnored(value)) { context.addIgnoredField(fullPath()); if (storeIgnored) { parseDoc.add(new StoredField(originalName(), new BytesRef(value))); } + } else { + createFields(value, parseDoc, fields); } } parseDoc.addAll(fields); @@ -1096,7 +1102,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), ignoreAboveDefault, indexVersionCreated).init(this); + return new Builder(leafName(), ignoreAboveDefault, indexMode, indexVersionCreated).init(this); } @Override @@ -1104,7 +1110,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() { return new SyntheticSourceSupport.Native(() -> { var layers = new ArrayList(); layers.add(new WildcardSyntheticFieldLoader()); - if (ignoreAbove != Integer.MAX_VALUE) { + if (ignoreAbove.isSet()) { layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) { @Override protected void writeValue(Object value, XContentBuilder b) throws IOException { diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 15a6efdff47c4..31f228ae6bd7e 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -721,7 +721,10 @@ protected Object getSampleValueForDocument() { @Override protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("null_value", b -> b.field("null_value", "foo")); - checker.registerUpdateCheck(b -> b.field("ignore_above", 256), m -> assertEquals(256, ((WildcardFieldMapper) m).ignoreAbove())); + checker.registerUpdateCheck( + b -> b.field("ignore_above", 256), + m -> assertEquals(256, ((WildcardFieldMapper) m).ignoreAbove().get()) + ); }