diff --git a/docs/changelog/131317.yaml b/docs/changelog/131317.yaml new file mode 100644 index 0000000000000..a18434a68c741 --- /dev/null +++ b/docs/changelog/131317.yaml @@ -0,0 +1,16 @@ +pr: 131317 +summary: Don't enable norms for fields of type text when the index mode is LogsDB or TSDB +area: Mapping +type: breaking +issues: [] +breaking: + title: Don't enable norms for fields of type text when the index mode is LogsDB or TSDB + area: Mapping + details: "This changes the default behavior for norms on `text` fields in logsdb\ + \ and tsdb indices. Prior to this change, norms were enabled by default, with\ + \ the option to disable them via manual configurations. After this change, norms\ + \ will be disabled by default. Note, because we dont support enabling norms from\ + \ a disabled state, users will not be able to enable norms on `text` fields in\ + \ logsdb and tsdb indices." + impact: Text fields will no longer be normalized by default in LogsDB and TSDB indicies. + notable: false diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java index d6ebe24719e43..ba4fb0b644258 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java @@ -138,7 +138,7 @@ public static class Builder extends FieldMapper.Builder { final Parameter similarity = TextParams.similarity(m -> builder(m).similarity.get()); final Parameter indexOptions = TextParams.textIndexOptions(m -> builder(m).indexOptions.get()); - final Parameter norms = TextParams.norms(true, m -> builder(m).norms.get()); + final Parameter norms = Parameter.normsParam(m -> builder(m).norms.get(), true); final Parameter termVectors = TextParams.termVectors(m -> builder(m).termVectors.get()); private final Parameter> meta = Parameter.metaParam(); diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index 6c4a4ee42ed00..6b5c43ac3daa6 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -226,7 +226,7 @@ public static class Builder extends FieldMapper.Builder { final Parameter stored = Parameter.storeParam(m -> toType(m).fieldType.stored(), false); final Parameter indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions); - final Parameter hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false); + final Parameter hasNorms = Parameter.normsParam(m -> toType(m).fieldType.omitNorms() == false, false); final Parameter> meta = Parameter.metaParam(); diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 8ee639ffc8431..8af24dbbd25ab 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -82,7 +82,7 @@ public static class Builder extends FieldMapper.Builder { final Parameter similarity = TextParams.similarity(m -> builder(m).similarity.getValue()); final Parameter indexOptions = TextParams.textIndexOptions(m -> builder(m).indexOptions.getValue()); - final Parameter norms = TextParams.norms(true, m -> builder(m).norms.getValue()); + final Parameter norms = Parameter.normsParam(m -> builder(m).norms.getValue(), true); final Parameter termVectors = TextParams.termVectors(m -> builder(m).termVectors.getValue()); private final Parameter> meta = Parameter.metaParam(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index a43575b8f990c..34362e25bfece 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1325,6 +1325,18 @@ public static Parameter docValuesParam(Function i return Parameter.boolParam("doc_values", false, initializer, defaultValue); } + public static Parameter normsParam(Function initializer, boolean defaultValue) { + // norms can be updated from 'true' to 'false' but not vice-versa + return Parameter.boolParam("norms", true, initializer, defaultValue) + .setMergeValidator((prev, curr, c) -> prev == curr || (prev && curr == false)); + } + + public static Parameter normsParam(Function initializer, Supplier defaultValueSupplier) { + // norms can be updated from 'true' to 'false' but not vice-versa + return Parameter.boolParam("norms", true, initializer, defaultValueSupplier) + .setMergeValidator((prev, curr, c) -> prev == curr || (prev && curr == false)); + } + /** * Defines a script parameter * @param initializer retrieves the equivalent parameter from an existing FieldMapper for use in merges diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index e26c969f7d495..ecf4c47e7cc1d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -183,7 +183,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder { private final IndexSortConfig indexSortConfig; private final IndexMode indexMode; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions); - private final Parameter hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false); + private final Parameter hasNorms = Parameter.normsParam(m -> toType(m).fieldType.omitNorms() == false, false); private final Parameter similarity = TextParams.similarity( m -> toType(m).fieldType().getTextSearchInfo().similarity() ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 3d2b89f5a1d48..edc0bbeed2e5e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -55,6 +55,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.AnalyzerScope; @@ -239,15 +240,17 @@ public static class Builder extends FieldMapper.Builder { private final IndexVersion indexCreatedVersion; private final Parameter store; + private final Parameter norms; private final boolean isSyntheticSourceEnabled; + private final IndexMode indexMode; + private final Parameter index = Parameter.indexParam(m -> ((TextFieldMapper) m).index, true); final Parameter similarity = TextParams.similarity(m -> ((TextFieldMapper) m).similarity); final Parameter indexOptions = TextParams.textIndexOptions(m -> ((TextFieldMapper) m).indexOptions); - final Parameter norms = TextParams.norms(true, m -> ((TextFieldMapper) m).norms); final Parameter termVectors = TextParams.termVectors(m -> ((TextFieldMapper) m).termVectors); final Parameter fieldData = Parameter.boolParam("fielddata", true, m -> ((TextFieldMapper) m).fieldData, false); @@ -290,18 +293,30 @@ public static class Builder extends FieldMapper.Builder { private final boolean withinMultiField; public Builder(String name, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabled) { - this(name, IndexVersion.current(), indexAnalyzers, isSyntheticSourceEnabled, false); + this(name, IndexVersion.current(), null, indexAnalyzers, isSyntheticSourceEnabled, false); } public Builder( String name, IndexVersion indexCreatedVersion, + IndexMode indexMode, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabled, boolean withinMultiField ) { super(name); + this.indexCreatedVersion = indexCreatedVersion; + this.indexMode = indexMode; + this.isSyntheticSourceEnabled = isSyntheticSourceEnabled; + this.withinMultiField = withinMultiField; + + // don't enable norms by default if the index is LOGSDB or TSDB based + this.norms = Parameter.normsParam( + m -> ((TextFieldMapper) m).norms, + () -> indexMode != IndexMode.LOGSDB && indexMode != IndexMode.TIME_SERIES + ); + // If synthetic source is used we need to either store this field // to recreate the source or use keyword multi-fields for that. // So if there are no suitable multi-fields we will default to @@ -309,7 +324,6 @@ public Builder( // // If 'store' parameter was explicitly provided we'll reject the request. // Note that if current builder is a multi field, then we don't need to store, given that responsibility lies with parent field - this.withinMultiField = withinMultiField; this.store = Parameter.storeParam(m -> ((TextFieldMapper) m).store, () -> { if (multiFieldsNotStoredByDefaultIndexVersionCheck(indexCreatedVersion)) { return isSyntheticSourceEnabled @@ -319,14 +333,12 @@ public Builder( return isSyntheticSourceEnabled && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false; } }); - this.indexCreatedVersion = indexCreatedVersion; this.analyzers = new TextParams.Analyzers( indexAnalyzers, m -> ((TextFieldMapper) m).indexAnalyzer, m -> (((TextFieldMapper) m).positionIncrementGap), indexCreatedVersion ); - this.isSyntheticSourceEnabled = isSyntheticSourceEnabled; } public static boolean multiFieldsNotStoredByDefaultIndexVersionCheck(IndexVersion indexCreatedVersion) { @@ -508,6 +520,7 @@ public TextFieldMapper build(MapperBuilderContext context) { (n, c) -> new Builder( n, c.indexVersionCreated(), + c.getIndexSettings().getMode(), c.getIndexAnalyzers(), SourceFieldMapper.isSynthetic(c.getIndexSettings()), c.isWithinMultiField() @@ -1319,6 +1332,7 @@ public Query existsQuery(SearchExecutionContext context) { } private final IndexVersion indexCreatedVersion; + private final IndexMode indexMode; private final boolean index; private final boolean store; private final String indexOptions; @@ -1357,6 +1371,7 @@ private TextFieldMapper( this.prefixFieldInfo = prefixFieldInfo; this.phraseFieldInfo = phraseFieldInfo; this.indexCreatedVersion = builder.indexCreatedVersion; + this.indexMode = builder.indexMode; this.indexAnalyzer = builder.analyzers.getIndexAnalyzer(); this.indexAnalyzers = builder.analyzers.indexAnalyzers; this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue(); @@ -1394,7 +1409,9 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, isSyntheticSourceEnabled, isWithinMultiField).init(this); + return new Builder(leafName(), indexCreatedVersion, indexMode, indexAnalyzers, isSyntheticSourceEnabled, isWithinMultiField).init( + this + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java b/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java index 1d41032067d93..5b8adfd1f29d8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java @@ -123,11 +123,6 @@ private NamedAnalyzer wrapAnalyzer(NamedAnalyzer a) { } } - public static Parameter norms(boolean defaultValue, Function initializer) { - // norms can be updated from 'true' to 'false' but not vv - return Parameter.boolParam("norms", true, initializer, defaultValue).setMergeValidator((o, n, c) -> o == n || (o && n == false)); - } - public static Parameter similarity(Function init) { return new Parameter<>( "similarity", diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 2c8dc48cf0484..7b6f5723e8ef5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -48,6 +48,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -79,6 +80,8 @@ import org.junit.AssumptionViolatedException; import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -1432,4 +1435,143 @@ public void testEmpty() throws Exception { assertFalse(dv.advanceExact(3)); }); } + + public void testNormalizeByDefault() throws IOException { + // given + Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); + indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()); + Settings indexSettings = indexSettingsBuilder.build(); + + XContentBuilder mapping = mapping(b -> { + b.startObject("potato"); + b.field("type", "text"); + b.endObject(); + }); + + var source = source(b -> b.field("potato", "a potato flew around my room")); + + // when + DocumentMapper mapper = createMapperService(indexSettings, mapping).documentMapper(); + ParsedDocument doc = mapper.parse(source); + + List fields = doc.rootDoc().getFields("potato"); + IndexableFieldType fieldType = fields.get(0).fieldType(); + + // then + assertThat(fieldType.omitNorms(), is(false)); + } + + public void testNormalizeWhenIndexModeIsNotGiven() throws IOException { + // given + Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); + Settings indexSettings = indexSettingsBuilder.build(); + + XContentBuilder mapping = mapping(b -> { + b.startObject("potato"); + b.field("type", "text"); + b.endObject(); + }); + + var source = source(b -> b.field("potato", "a potato flew around my room")); + + // when + DocumentMapper mapper = createMapperService(indexSettings, mapping).documentMapper(); + ParsedDocument doc = mapper.parse(source); + + List fields = doc.rootDoc().getFields("potato"); + IndexableFieldType fieldType = fields.get(0).fieldType(); + + // then + assertThat(fieldType.omitNorms(), is(false)); + } + + public void testNormalizeWhenIndexModeIsNull() throws IOException { + // given + Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); + indexSettingsBuilder.put(IndexSettings.MODE.getKey(), (String) null); + Settings indexSettings = indexSettingsBuilder.build(); + + XContentBuilder mapping = mapping(b -> { + b.startObject("potato"); + b.field("type", "text"); + b.endObject(); + }); + + var source = source(b -> b.field("potato", "a potato flew around my room")); + + // when + DocumentMapper mapper = createMapperService(indexSettings, mapping).documentMapper(); + ParsedDocument doc = mapper.parse(source); + + List fields = doc.rootDoc().getFields("potato"); + IndexableFieldType fieldType = fields.get(0).fieldType(); + + // then + assertThat(fieldType.omitNorms(), is(false)); + } + + public void testDontNormalizeWhenIndexModeIsLogsDB() throws IOException { + // given + Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); + indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()); + Settings indexSettings = indexSettingsBuilder.build(); + + XContentBuilder mapping = mapping(b -> { + b.startObject("potato"); + b.field("type", "text"); + b.endObject(); + }); + + var source = source(b -> { + b.field("@timestamp", Instant.now()); + b.field("potato", "a potato flew around my room"); + }); + + // when + DocumentMapper mapper = createMapperService(indexSettings, mapping).documentMapper(); + ParsedDocument doc = mapper.parse(source); + + List fields = doc.rootDoc().getFields("potato"); + IndexableFieldType fieldType = fields.get(0).fieldType(); + + // then + assertThat(fieldType.omitNorms(), is(true)); + } + + public void testDontNormalizeWhenIndexModeIsTSDB() throws IOException { + // given + Instant currentTime = Instant.now(); + Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); + indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), currentTime.minus(1, ChronoUnit.HOURS).toEpochMilli()) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), currentTime.plus(1, ChronoUnit.HOURS).toEpochMilli()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension"); + Settings indexSettings = indexSettingsBuilder.build(); + + XContentBuilder mapping = mapping(b -> { + b.startObject("potato"); + b.field("type", "text"); + b.endObject(); + + b.startObject("@timestamp"); + b.field("type", "date"); + b.endObject(); + }); + + var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> { + b.field("@timestamp", Instant.now()); + b.field("potato", "a potato flew around my room"); + }, null); + + // when + DocumentMapper mapper = createMapperService(indexSettings, mapping).documentMapper(); + ParsedDocument doc = mapper.parse(source); + + List fields = doc.rootDoc().getFields("potato"); + IndexableFieldType fieldType = fields.get(0).fieldType(); + + // then + assertThat(fieldType.omitNorms(), is(true)); + } + }