From d4f41f713f998b3809d8c33fea0ffb184ae0b6d1 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Mon, 25 Aug 2025 12:52:27 -0700 Subject: [PATCH 1/3] Gate disable norms on text fields for logsdb/tsdb indices --- .../elasticsearch/index/IndexVersions.java | 1 + .../index/mapper/TextFieldMapper.java | 15 ++-- .../index/mapper/TextFieldMapperTests.java | 76 +++++++++++++++++-- 3 files changed, 82 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 221bc9264b100..1c710e2e36c14 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -182,6 +182,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion MATCH_ONLY_TEXT_STORED_AS_BYTES = def(9_033_0_00, Version.LUCENE_10_2_2); public static final IndexVersion IGNORED_SOURCE_FIELDS_PER_ENTRY_WITH_FF = def(9_034_0_00, Version.LUCENE_10_2_2); public static final IndexVersion EXCLUDE_SOURCE_VECTORS_DEFAULT = def(9_035_0_00, Version.LUCENE_10_2_2); + public static final IndexVersion DISABLE_NORMS_BY_DEFAULT_FOR_LOGSDB_AND_TSDB = def(9_036_0_00, Version.LUCENE_10_2_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index aa2df200038e7..8a586d68177e0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -311,11 +311,7 @@ public Builder( this.isSyntheticSourceEnabled = isSyntheticSourceEnabled; this.withinMultiField = withinMultiField; - // don't enable norms by default if the index is LOGSDB or TSDB based - this.norms = Parameter.normsParam( - m -> ((TextFieldMapper) m).norms, - () -> indexMode != IndexMode.LOGSDB && indexMode != IndexMode.TIME_SERIES - ); + this.norms = Parameter.normsParam(m -> ((TextFieldMapper) m).norms, this::normsDefault); // If synthetic source is used we need to either store this field // to recreate the source or use keyword multi-fields for that. @@ -341,6 +337,15 @@ public Builder( ); } + private boolean normsDefault() { + if (indexCreatedVersion.onOrAfter(IndexVersions.DISABLE_NORMS_BY_DEFAULT_FOR_LOGSDB_AND_TSDB)) { + // don't enable norms by default if the index is LOGSDB or TSDB based + return indexMode != IndexMode.LOGSDB && indexMode != IndexMode.TIME_SERIES; + } + // bwc - historically, norms were enabled by default on text fields regardless of which index mode was used + return true; + } + public static boolean multiFieldsNotStoredByDefaultIndexVersionCheck(IndexVersion indexCreatedVersion) { return indexCreatedVersion.onOrAfter(IndexVersions.MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED) || indexCreatedVersion.between( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 7b6f5723e8ef5..6b6149ff69a96 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1436,7 +1436,7 @@ public void testEmpty() throws Exception { }); } - public void testNormalizeByDefault() throws IOException { + public void testNormsEnabledByDefault() throws IOException { // given Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()); @@ -1461,7 +1461,7 @@ public void testNormalizeByDefault() throws IOException { assertThat(fieldType.omitNorms(), is(false)); } - public void testNormalizeWhenIndexModeIsNotGiven() throws IOException { + public void testNormsEnabledWhenIndexModeIsNotGiven() throws IOException { // given Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); Settings indexSettings = indexSettingsBuilder.build(); @@ -1485,7 +1485,7 @@ public void testNormalizeWhenIndexModeIsNotGiven() throws IOException { assertThat(fieldType.omitNorms(), is(false)); } - public void testNormalizeWhenIndexModeIsNull() throws IOException { + public void textNormsEnabledWhenIndexModeIsNull() throws IOException { // given Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); indexSettingsBuilder.put(IndexSettings.MODE.getKey(), (String) null); @@ -1510,7 +1510,7 @@ public void testNormalizeWhenIndexModeIsNull() throws IOException { assertThat(fieldType.omitNorms(), is(false)); } - public void testDontNormalizeWhenIndexModeIsLogsDB() throws IOException { + public void testNormsDisabledWhenIndexModeIsLogsDb() throws IOException { // given Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()); @@ -1538,7 +1538,7 @@ public void testDontNormalizeWhenIndexModeIsLogsDB() throws IOException { assertThat(fieldType.omitNorms(), is(true)); } - public void testDontNormalizeWhenIndexModeIsTSDB() throws IOException { + public void testNormsDisabledWhenIndexModeIsTsdb() throws IOException { // given Instant currentTime = Instant.now(); Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); @@ -1574,4 +1574,70 @@ public void testDontNormalizeWhenIndexModeIsTSDB() throws IOException { assertThat(fieldType.omitNorms(), is(true)); } + public void testNormsEnabledWhenIndexModeIsLogsDb_bwcCheck() throws IOException { + // given + Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); + indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()); + Settings indexSettings = indexSettingsBuilder.build(); + + XContentBuilder mapping = mapping(b -> { + b.startObject("potato"); + b.field("type", "text"); + b.endObject(); + }); + + var source = source(b -> { + b.field("@timestamp", Instant.now()); + b.field("potato", "a potato flew around my room"); + }); + + // when + IndexVersion bwcIndexVersion = IndexVersions.EXCLUDE_SOURCE_VECTORS_DEFAULT; + DocumentMapper mapper = createMapperService(bwcIndexVersion, indexSettings, mapping).documentMapper(); + ParsedDocument doc = mapper.parse(source); + + List fields = doc.rootDoc().getFields("potato"); + IndexableFieldType fieldType = fields.get(0).fieldType(); + + // then + assertThat(fieldType.omitNorms(), is(false)); + } + + public void testNormsEnabledWhenIndexModeIsTsdb_bwcCheck() throws IOException { + // given + Instant currentTime = Instant.now(); + Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); + indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), currentTime.minus(1, ChronoUnit.HOURS).toEpochMilli()) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), currentTime.plus(1, ChronoUnit.HOURS).toEpochMilli()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension"); + Settings indexSettings = indexSettingsBuilder.build(); + + XContentBuilder mapping = mapping(b -> { + b.startObject("potato"); + b.field("type", "text"); + b.endObject(); + + b.startObject("@timestamp"); + b.field("type", "date"); + b.endObject(); + }); + + var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> { + b.field("@timestamp", Instant.now()); + b.field("potato", "a potato flew around my room"); + }, null); + + // when + IndexVersion bwcIndexVersion = IndexVersions.EXCLUDE_SOURCE_VECTORS_DEFAULT; + DocumentMapper mapper = createMapperService(bwcIndexVersion, indexSettings, mapping).documentMapper(); + ParsedDocument doc = mapper.parse(source); + + List fields = doc.rootDoc().getFields("potato"); + IndexableFieldType fieldType = fields.get(0).fieldType(); + + // then + assertThat(fieldType.omitNorms(), is(false)); + } + } From 67080d856ec17b14fa8b76eafc9432db98888c23 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Mon, 25 Aug 2025 12:56:55 -0700 Subject: [PATCH 2/3] Update docs/changelog/133493.yaml --- docs/changelog/133493.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/133493.yaml diff --git a/docs/changelog/133493.yaml b/docs/changelog/133493.yaml new file mode 100644 index 0000000000000..41154ad7cac10 --- /dev/null +++ b/docs/changelog/133493.yaml @@ -0,0 +1,5 @@ +pr: 133493 +summary: Gate disable norms on text fields for logsdb/tsdb indices +area: Mapping +type: bug +issues: [] From edc3fec1dfd45f89399bf192b2aaf07e1bc05cd6 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Mon, 25 Aug 2025 13:07:51 -0700 Subject: [PATCH 3/3] Delete docs/changelog/133493.yaml --- docs/changelog/133493.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/133493.yaml diff --git a/docs/changelog/133493.yaml b/docs/changelog/133493.yaml deleted file mode 100644 index 41154ad7cac10..0000000000000 --- a/docs/changelog/133493.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 133493 -summary: Gate disable norms on text fields for logsdb/tsdb indices -area: Mapping -type: bug -issues: []