Remove index.dimensions feature flag

docs/changelog/135402.yaml

@@ -0,0 +1,5 @@
+pr: 135402
+summary: Improve TSDB ingestion by hashing dimensions only once, using a new auto-populeted `index.dimensions` private index setting
+area: TSDB
+type: enhancement
+issues: []

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

@@ -65,7 +65,6 @@
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 
-import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG;
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -331,10 +330,7 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception {
                     new Template(
                         Settings.builder()
                             .put("index.mode", "time_series")
-                            .put(
-                                "index.routing_path",
-                                randomBoolean() && INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? null : "metricset"
-                            )
+                            .put("index.routing_path", randomBoolean() ? null : "metricset")
                             .build(),
                         new CompressedXContent(mappingTemplate),
                         null
@@ -664,13 +660,8 @@ public void testAddDimensionToMapping() throws Exception {
             "my-ds"
         );
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest));
-        if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), equalTo(List.of("metricset")));
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty());
-        } else {
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty());
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset")));
-        }
+        assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), equalTo(List.of("metricset")));
+        assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty());
 
         // put mapping with k8s.pod.uid as another time series dimension
         var putMappingRequest = new PutMappingRequest(dataStreamName).source("""
@@ -684,13 +675,8 @@ public void testAddDimensionToMapping() throws Exception {
             }
             """, XContentType.JSON);
         assertAcked(client().execute(TransportPutMappingAction.TYPE, putMappingRequest).actionGet());
-        if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name"));
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty());
-        } else {
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty());
-            assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset")));
-        }
+        assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name"));
+        assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty());
 
         // put dynamic template defining time series dimensions
         // we don't support index.dimensions in that case

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java

@@ -44,7 +44,6 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG;
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -186,13 +185,8 @@ public void testIndexingGettingAndSearching() throws Exception {
 
         // validate index:
         var getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(index)).actionGet();
-        if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
-            assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), equalTo("[attributes.*]"));
-            assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), nullValue());
-        } else {
-            assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), nullValue());
-            assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), equalTo("[attributes.*]"));
-        }
+        assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), equalTo("[attributes.*]"));
+        assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), nullValue());
         // validate mapping
         var mapping = getIndexResponse.mappings().get(index).getSourceAsMap();
         assertMap(

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java

@@ -16,7 +16,6 @@
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
@@ -50,8 +49,6 @@
  */
 public class DataStreamIndexSettingsProvider implements IndexSettingProvider {
 
-    public static final boolean INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG = new FeatureFlag("index_dimensions_tsid_optimization")
-        .isEnabled();
     static final DateFormatter FORMATTER = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER;
 
     private final CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory;
@@ -133,9 +130,7 @@ public void provideAdditionalSettings(
                             dimensions
                         );
                         if (dimensions.isEmpty() == false) {
-                            if (matchesAllDimensions
-                                && INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG
-                                && indexVersion.onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING)) {
+                            if (matchesAllDimensions && indexVersion.onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING)) {
                                 // Only set index.dimensions if the paths in the dimensions list match all potential dimension fields.
                                 // This is not the case e.g. if a dynamic template matches by match_mapping_type instead of path_match
                                 additionalSettings.putList(INDEX_DIMENSIONS.getKey(), dimensions);

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java

@@ -39,7 +39,6 @@
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance;
 import static org.elasticsearch.common.settings.Settings.builder;
 import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.FORMATTER;
-import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -61,8 +60,7 @@ public void setup() {
         indexVersion = randomBoolean()
             ? IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.TSID_CREATED_DURING_ROUTING)
             : IndexVersionUtils.randomVersionBetween(random(), IndexVersions.TSID_CREATED_DURING_ROUTING, IndexVersion.current());
-        indexDimensionsTsidOptimizationEnabled = INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG
-            && indexVersion.onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING);
+        indexDimensionsTsidOptimizationEnabled = indexVersion.onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING);
     }
 
     public void testGetAdditionalIndexSettings() throws Exception {

x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java

@@ -64,7 +64,6 @@
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD;
-import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
@@ -200,19 +199,11 @@ public void testDataStreamDownsample() throws ExecutionException, InterruptedExc
                     assertThat(setting.hasValue(IndexMetadata.INDEX_DIMENSIONS.getKey()), equalTo(false));
                     assertThat(setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()), containsInAnyOrder("routing_field"));
                 } else {
-                    if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
-                        assertThat(
-                            setting.getAsList(IndexMetadata.INDEX_DIMENSIONS.getKey()),
-                            containsInAnyOrder("routing_field", "dimension")
-                        );
-                        assertThat(setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()), empty());
-                    } else {
-                        assertThat(setting.getAsList(IndexMetadata.INDEX_DIMENSIONS.getKey()), empty());
-                        assertThat(
-                            setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()),
-                            containsInAnyOrder("routing_field", "dimension")
-                        );
-                    }
+                    assertThat(
+                        setting.getAsList(IndexMetadata.INDEX_DIMENSIONS.getKey()),
+                        containsInAnyOrder("routing_field", "dimension")
+                    );
+                    assertThat(setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()), empty());
                 }
             });
         });