diff --git a/docs/changelog/135402.yaml b/docs/changelog/135402.yaml new file mode 100644 index 0000000000000..429862a3854fd --- /dev/null +++ b/docs/changelog/135402.yaml @@ -0,0 +1,5 @@ +pr: 135402 +summary: Improve TSDB ingestion by hashing dimensions only once, using a new auto-populeted `index.dimensions` private index setting +area: TSDB +type: enhancement +issues: [] diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index 200e7c8785262..e45956f79295f 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -65,7 +65,6 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; -import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -331,10 +330,7 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception { new Template( Settings.builder() .put("index.mode", "time_series") - .put( - "index.routing_path", - randomBoolean() && INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? null : "metricset" - ) + .put("index.routing_path", randomBoolean() ? null : "metricset") .build(), new CompressedXContent(mappingTemplate), null @@ -664,13 +660,8 @@ public void testAddDimensionToMapping() throws Exception { "my-ds" ); assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); - if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { - assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), equalTo(List.of("metricset"))); - assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty()); - } else { - assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty()); - assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); - } + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), equalTo(List.of("metricset"))); + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty()); // put mapping with k8s.pod.uid as another time series dimension var putMappingRequest = new PutMappingRequest(dataStreamName).source(""" @@ -684,13 +675,8 @@ public void testAddDimensionToMapping() throws Exception { } """, XContentType.JSON); assertAcked(client().execute(TransportPutMappingAction.TYPE, putMappingRequest).actionGet()); - if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { - assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name")); - assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty()); - } else { - assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty()); - assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); - } + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name")); + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty()); // put dynamic template defining time series dimensions // we don't support index.dimensions in that case diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index fa0c2f76d8517..27cfc62822441 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -44,7 +44,6 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -186,13 +185,8 @@ public void testIndexingGettingAndSearching() throws Exception { // validate index: var getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(index)).actionGet(); - if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { - assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), equalTo("[attributes.*]")); - assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), nullValue()); - } else { - assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), nullValue()); - assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), equalTo("[attributes.*]")); - } + assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), equalTo("[attributes.*]")); + assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), nullValue()); // validate mapping var mapping = getIndexResponse.mappings().get(index).getSourceAsMap(); assertMap( diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index 268ef3455cd41..2741809a67b24 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; @@ -50,8 +49,6 @@ */ public class DataStreamIndexSettingsProvider implements IndexSettingProvider { - public static final boolean INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG = new FeatureFlag("index_dimensions_tsid_optimization") - .isEnabled(); static final DateFormatter FORMATTER = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; private final CheckedFunction mapperServiceFactory; @@ -133,9 +130,7 @@ public void provideAdditionalSettings( dimensions ); if (dimensions.isEmpty() == false) { - if (matchesAllDimensions - && INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG - && indexVersion.onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING)) { + if (matchesAllDimensions && indexVersion.onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING)) { // Only set index.dimensions if the paths in the dimensions list match all potential dimension fields. // This is not the case e.g. if a dynamic template matches by match_mapping_type instead of path_match additionalSettings.putList(INDEX_DIMENSIONS.getKey(), dimensions); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index d896f6c7afa64..48e562d2329ef 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -39,7 +39,6 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.FORMATTER; -import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -61,8 +60,7 @@ public void setup() { indexVersion = randomBoolean() ? IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.TSID_CREATED_DURING_ROUTING) : IndexVersionUtils.randomVersionBetween(random(), IndexVersions.TSID_CREATED_DURING_ROUTING, IndexVersion.current()); - indexDimensionsTsidOptimizationEnabled = INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG - && indexVersion.onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING); + indexDimensionsTsidOptimizationEnabled = indexVersion.onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING); } public void testGetAdditionalIndexSettings() throws Exception { diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java index 900689ff3de7d..9046afa168ce0 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java @@ -64,7 +64,6 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; -import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; @@ -200,19 +199,11 @@ public void testDataStreamDownsample() throws ExecutionException, InterruptedExc assertThat(setting.hasValue(IndexMetadata.INDEX_DIMENSIONS.getKey()), equalTo(false)); assertThat(setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()), containsInAnyOrder("routing_field")); } else { - if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { - assertThat( - setting.getAsList(IndexMetadata.INDEX_DIMENSIONS.getKey()), - containsInAnyOrder("routing_field", "dimension") - ); - assertThat(setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()), empty()); - } else { - assertThat(setting.getAsList(IndexMetadata.INDEX_DIMENSIONS.getKey()), empty()); - assertThat( - setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()), - containsInAnyOrder("routing_field", "dimension") - ); - } + assertThat( + setting.getAsList(IndexMetadata.INDEX_DIMENSIONS.getKey()), + containsInAnyOrder("routing_field", "dimension") + ); + assertThat(setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()), empty()); } }); });