Gate disable norms on text fields for logsdb/tsdb indices

docs/changelog/133493.yaml

@@ -0,0 +1,5 @@
+pr: 133493
+summary: Gate disable norms on text fields for logsdb/tsdb indices
+area: Mapping
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -182,6 +182,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion MATCH_ONLY_TEXT_STORED_AS_BYTES = def(9_033_0_00, Version.LUCENE_10_2_2);
     public static final IndexVersion IGNORED_SOURCE_FIELDS_PER_ENTRY_WITH_FF = def(9_034_0_00, Version.LUCENE_10_2_2);
     public static final IndexVersion EXCLUDE_SOURCE_VECTORS_DEFAULT = def(9_035_0_00, Version.LUCENE_10_2_2);
+    public static final IndexVersion DISABLE_NORMS_BY_DEFAULT_FOR_LOGSDB_AND_TSDB = def(9_036_0_00, Version.LUCENE_10_2_2);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -311,11 +311,7 @@ public Builder(
             this.isSyntheticSourceEnabled = isSyntheticSourceEnabled;
             this.withinMultiField = withinMultiField;
 
-            // don't enable norms by default if the index is LOGSDB or TSDB based
-            this.norms = Parameter.normsParam(
-                m -> ((TextFieldMapper) m).norms,
-                () -> indexMode != IndexMode.LOGSDB && indexMode != IndexMode.TIME_SERIES
-            );
+            this.norms = Parameter.normsParam(m -> ((TextFieldMapper) m).norms, this::normsDefault);
 
             // If synthetic source is used we need to either store this field
             // to recreate the source or use keyword multi-fields for that.
@@ -341,6 +337,15 @@ public Builder(
             );
         }
 
+        private boolean normsDefault() {
+            if (indexCreatedVersion.onOrAfter(IndexVersions.DISABLE_NORMS_BY_DEFAULT_FOR_LOGSDB_AND_TSDB)) {
+                // don't enable norms by default if the index is LOGSDB or TSDB based
+                return indexMode != IndexMode.LOGSDB && indexMode != IndexMode.TIME_SERIES;
+            }
+            // bwc - historically, norms were enabled by default on text fields regardless of which index mode was used
+            return true;
+        }
+
         public static boolean multiFieldsNotStoredByDefaultIndexVersionCheck(IndexVersion indexCreatedVersion) {
             return indexCreatedVersion.onOrAfter(IndexVersions.MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED)
                 || indexCreatedVersion.between(

server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java

@@ -1436,7 +1436,7 @@ public void testEmpty() throws Exception {
         });
     }
 
-    public void testNormalizeByDefault() throws IOException {
+    public void testNormsEnabledByDefault() throws IOException {
         // given
         Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
         indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName());
@@ -1461,7 +1461,7 @@ public void testNormalizeByDefault() throws IOException {
         assertThat(fieldType.omitNorms(), is(false));
     }
 
-    public void testNormalizeWhenIndexModeIsNotGiven() throws IOException {
+    public void testNormsEnabledWhenIndexModeIsNotGiven() throws IOException {
         // given
         Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
         Settings indexSettings = indexSettingsBuilder.build();
@@ -1485,7 +1485,7 @@ public void testNormalizeWhenIndexModeIsNotGiven() throws IOException {
         assertThat(fieldType.omitNorms(), is(false));
     }
 
-    public void testNormalizeWhenIndexModeIsNull() throws IOException {
+    public void textNormsEnabledWhenIndexModeIsNull() throws IOException {
         // given
         Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
         indexSettingsBuilder.put(IndexSettings.MODE.getKey(), (String) null);
@@ -1510,7 +1510,7 @@ public void testNormalizeWhenIndexModeIsNull() throws IOException {
         assertThat(fieldType.omitNorms(), is(false));
     }
 
-    public void testDontNormalizeWhenIndexModeIsLogsDB() throws IOException {
+    public void testNormsDisabledWhenIndexModeIsLogsDb() throws IOException {
         // given
         Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
         indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName());
@@ -1538,7 +1538,7 @@ public void testDontNormalizeWhenIndexModeIsLogsDB() throws IOException {
         assertThat(fieldType.omitNorms(), is(true));
     }
 
-    public void testDontNormalizeWhenIndexModeIsTSDB() throws IOException {
+    public void testNormsDisabledWhenIndexModeIsTsdb() throws IOException {
         // given
         Instant currentTime = Instant.now();
         Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
@@ -1574,4 +1574,70 @@ public void testDontNormalizeWhenIndexModeIsTSDB() throws IOException {
         assertThat(fieldType.omitNorms(), is(true));
     }
 
+    public void testNormsEnabledWhenIndexModeIsLogsDb_bwcCheck() throws IOException {
+        // given
+        Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
+        indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName());
+        Settings indexSettings = indexSettingsBuilder.build();
+
+        XContentBuilder mapping = mapping(b -> {
+            b.startObject("potato");
+            b.field("type", "text");
+            b.endObject();
+        });
+
+        var source = source(b -> {
+            b.field("@timestamp", Instant.now());
+            b.field("potato", "a potato flew around my room");
+        });
+
+        // when
+        IndexVersion bwcIndexVersion = IndexVersions.EXCLUDE_SOURCE_VECTORS_DEFAULT;
+        DocumentMapper mapper = createMapperService(bwcIndexVersion, indexSettings, mapping).documentMapper();
+        ParsedDocument doc = mapper.parse(source);
+
+        List<IndexableField> fields = doc.rootDoc().getFields("potato");
+        IndexableFieldType fieldType = fields.get(0).fieldType();
+
+        // then
+        assertThat(fieldType.omitNorms(), is(false));
+    }
+
+    public void testNormsEnabledWhenIndexModeIsTsdb_bwcCheck() throws IOException {
+        // given
+        Instant currentTime = Instant.now();
+        Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
+        indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName())
+            .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), currentTime.minus(1, ChronoUnit.HOURS).toEpochMilli())
+            .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), currentTime.plus(1, ChronoUnit.HOURS).toEpochMilli())
+            .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension");
+        Settings indexSettings = indexSettingsBuilder.build();
+
+        XContentBuilder mapping = mapping(b -> {
+            b.startObject("potato");
+            b.field("type", "text");
+            b.endObject();
+
+            b.startObject("@timestamp");
+            b.field("type", "date");
+            b.endObject();
+        });
+
+        var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> {
+            b.field("@timestamp", Instant.now());
+            b.field("potato", "a potato flew around my room");
+        }, null);
+
+        // when
+        IndexVersion bwcIndexVersion = IndexVersions.EXCLUDE_SOURCE_VECTORS_DEFAULT;
+        DocumentMapper mapper = createMapperService(bwcIndexVersion, indexSettings, mapping).documentMapper();
+        ParsedDocument doc = mapper.parse(source);
+
+        List<IndexableField> fields = doc.rootDoc().getFields("potato");
+        IndexableFieldType fieldType = fields.get(0).fieldType();
+
+        // then
+        assertThat(fieldType.omitNorms(), is(false));
+    }
+
 }