diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 28a7d6981a4cb..5749cfe46272e 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -271,6 +271,11 @@ + + + + + diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index 51bb04185cfde..32b3e44866a6d 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -52,6 +52,10 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("data_stream/30_auto_create_data_stream/Don't initialize failure store during data stream auto-creation on successful index", "Configuring the failure store via data stream templates is not supported anymore.") task.skipTest("data_stream/150_tsdb/TSDB failures go to failure store", "Configuring the failure store via data stream templates is not supported anymore.") + // TODO remove these after removing exact _tsid assertions in 8.x + task.skipTest("data_stream/150_tsdb/dynamic templates", "The _tsid has changed in a new index version. This tests verifies the exact _tsid value with is too brittle for compatibility testing.") + task.skipTest("data_stream/150_tsdb/dynamic templates - conflicting aliases", "The _tsid has changed in a new index version. This tests verifies the exact _tsid value with is too brittle for compatibility testing.") + task.skipTest("data_stream/150_tsdb/dynamic templates with nesting", "The _tsid has changed in a new index version. This tests verifies the exact _tsid value with is too brittle for compatibility testing.") task.skipTest("data_stream/170_modify_data_stream/Modify a data stream's failure store", "Configuring the failure store via data stream templates is not supported anymore.") diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index eaef99d86a86e..50e612903248c 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -14,6 +14,8 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; @@ -24,6 +26,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; @@ -34,6 +37,7 @@ import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; @@ -58,12 +62,14 @@ import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -320,16 +326,12 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception { ComposableIndexTemplate.builder() .indexPatterns(List.of("k8s*")) .template( - new Template( - Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset").build(), - new CompressedXContent(mappingTemplate), - null - ) + new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(mappingTemplate), null) ) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build() ); - client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + assertResponse(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request), ElasticsearchAssertions::assertAcked); } public void testInvalidTsdbTemplatesMissingSettings() throws Exception { @@ -621,6 +623,192 @@ public void testReindexing() throws Exception { ); } + public void testAddDimensionToMapping() throws Exception { + String dataStreamName = "my-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .template( + new Template( + Settings.builder().put("index.mode", "time_series").build(), + new CompressedXContent(MAPPING_TEMPLATE), + null + ) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + // create data stream + CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "my-ds" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), equalTo(List.of("metricset"))); + + // put mapping with k8s.pod.uid as another time series dimension + var putMappingRequest = new PutMappingRequest(dataStreamName).source(""" + { + "properties": { + "k8s.pod.name": { + "type": "keyword", + "time_series_dimension": true + } + } + } + """, XContentType.JSON); + assertAcked(client().execute(TransportPutMappingAction.TYPE, putMappingRequest).actionGet()); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name")); + + indexWithPodNames(dataStreamName, Instant.now(), Map.of(), "dog", "cat"); + } + + public void testDynamicStringDimensions() throws Exception { + String dataStreamName = "my-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .template(new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(""" + + { + "_doc": { + "dynamic_templates": [ + { + "labels": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword", + "time_series_dimension": true + } + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""), null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "my-ds" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); + + // doesn't populate index.dimensions because the "labels" dynamic template doesn't have a path_math + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + + // index doc + BulkResponse bulkResponse = client().prepareBulk() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .add( + client().prepareIndex(dataStreamName) + .setOpType(DocWriteRequest.OpType.CREATE) + .setSource(DOC.replace("$time", formatInstant(Instant.now())), XContentType.JSON) + ) + .get(); + assertThat(bulkResponse.hasFailures(), is(false)); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + } + + public void testDynamicDimensions() throws Exception { + String dataStreamName = "my-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .template(new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(""" + + { + "_doc": { + "dynamic_templates": [ + { + "label": { + "mapping": { + "type": "keyword", + "time_series_dimension": true + } + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""), null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "my-ds" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); + + // doesn't populate index.dimensions because the "label" dynamic template doesn't have a path_math + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + + // index doc + indexWithPodNames(dataStreamName, Instant.now(), Map.of("k8s.pod.name", "label"), "dog", "cat"); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + } + + private void indexWithPodNames(String dataStreamName, Instant timestamp, Map dynamicTemplates, String... podNames) { + // index doc + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (String podName : podNames) { + bulkRequestBuilder.add( + client().prepareIndex(dataStreamName) + .setOpType(DocWriteRequest.OpType.CREATE) + .setSource(DOC.replace("$time", formatInstant(timestamp)).replace("dog", podName), XContentType.JSON) + .request() + .setDynamicTemplates(dynamicTemplates) + ); + } + + BulkResponse bulkResponse = bulkRequestBuilder.get(); + assertThat(bulkResponse.hasFailures(), is(false)); + } + + private T getSetting(String dataStreamName, Setting setting) { + GetIndexResponse getIndexResponse = safeGet( + indicesAdmin().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(dataStreamName)) + ); + assertThat(getIndexResponse.getIndices().length, equalTo(1)); + Settings settings = getIndexResponse.getSettings().get(getIndexResponse.getIndices()[0]); + return setting.get(settings); + } + static String formatInstant(Instant instant) { return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index a76dac5db4540..c32eb05ddda65 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -184,7 +184,7 @@ public void testIndexingGettingAndSearching() throws Exception { // validate index: var getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(index)).actionGet(); - assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), equalTo("[attributes.*]")); + assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), equalTo("[attributes.*]")); // validate mapping var mapping = getIndexResponse.mappings().get(index).getSourceAsMap(); assertMap( diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java index 9be0c18d18213..a7fc4b102866f 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java @@ -433,9 +433,10 @@ public void testSimulateTsdbDataStreamTemplate() throws Exception { assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.time_series.start_time"), notNullValue()); assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.time_series.end_time"), notNullValue()); assertThat( - ObjectPath.evaluate(responseBody, "template.settings.index.routing_path"), + ObjectPath.evaluate(responseBody, "template.settings.index.dimensions"), containsInAnyOrder("metricset", "k8s.pod.uid", "pod.labels.*") ); + assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.routing_path"), nullValue()); assertThat(ObjectPath.evaluate(responseBody, "overlapping"), empty()); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index ac3987dfa2771..8c10266907c57 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.core.CheckedFunction; @@ -23,21 +24,20 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.PassThroughObjectMapper; import java.io.IOException; import java.io.UncheckedIOException; import java.time.Instant; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.Locale; +import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_DIMENSIONS; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; /** @@ -119,13 +119,17 @@ public Settings getAdditionalIndexSettings( if (indexTemplateAndCreateRequestSettings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey()) == false && combinedTemplateMappings.isEmpty() == false) { - List routingPaths = findRoutingPaths( + List dimensions = new ArrayList<>(); + boolean matchesAllDimensions = findDimensionFields( indexName, indexTemplateAndCreateRequestSettings, - combinedTemplateMappings + combinedTemplateMappings, + dimensions ); - if (routingPaths.isEmpty() == false) { - builder.putList(INDEX_ROUTING_PATH.getKey(), routingPaths); + if (dimensions.isEmpty() == false) { + // TODO handle the case when adding a dimension field to the mappings of an existing index + // at the moment, the index.dimensions setting is only set when an index is created + builder.putList(matchesAllDimensions ? INDEX_DIMENSIONS.getKey() : INDEX_ROUTING_PATH.getKey(), dimensions); } } return builder.build(); @@ -137,15 +141,46 @@ public Settings getAdditionalIndexSettings( } /** - * Find fields in mapping that are of type keyword and time_series_dimension enabled. + * This is called when mappings are updated, so that the {@link IndexMetadata#INDEX_DIMENSIONS} + * and {@link IndexMetadata#INDEX_ROUTING_PATH} settings are updated to match the new mappings. + * Updates {@link IndexMetadata#INDEX_DIMENSIONS} if a new dimension field is added to the mappings, + * or sets {@link IndexMetadata#INDEX_ROUTING_PATH} if a new dimension field is added that doesn't allow for matching all + * dimension fields via a wildcard pattern. + */ + @Override + public Settings onUpdateMappings(IndexMetadata indexMetadata, DocumentMapper documentMapper) { + List indexDimensions = INDEX_DIMENSIONS.get(indexMetadata.getSettings()); + if (indexDimensions.isEmpty()) { + return Settings.EMPTY; + } + assert indexMetadata.getIndexMode() == IndexMode.TIME_SERIES; + List newIndexDimensions = new ArrayList<>(indexDimensions.size()); + boolean matchesAllDimensions = findDimensionFields(newIndexDimensions, documentMapper); + if (indexDimensions.equals(newIndexDimensions)) { + return Settings.EMPTY; + } + if (matchesAllDimensions) { + return Settings.builder().putList(INDEX_DIMENSIONS.getKey(), newIndexDimensions).build(); + } else { + return Settings.builder().putList(INDEX_ROUTING_PATH.getKey(), newIndexDimensions).build(); + } + } + + /** + * Find fields in mapping that are time_series_dimension enabled. * Using MapperService here has an overhead, but allows the mappings from template to * be merged correctly and fetching the fields without manually parsing the mappings. - * + *

* Alternatively this method can instead parse mappings into map of maps and merge that and * iterate over all values to find the field that can serve as routing value. But this requires * mapping specific logic to exist here. */ - private List findRoutingPaths(String indexName, Settings allSettings, List combinedTemplateMappings) { + private boolean findDimensionFields( + String indexName, + Settings allSettings, + List combinedTemplateMappings, + List dimensions + ) { var tmpIndexMetadata = IndexMetadata.builder(indexName); int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(allSettings); @@ -169,57 +204,52 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li // Create MapperService just to extract keyword dimension fields: try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata.build())) { mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); - List routingPaths = new ArrayList<>(); - for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) { - extractPath(routingPaths, fieldMapper); - } - for (var objectMapper : mapperService.documentMapper().mappers().objectMappers().values()) { - if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { - if (passThroughObjectMapper.containsDimensions()) { - routingPaths.add(passThroughObjectMapper.fullPath() + ".*"); - } - } - } - for (var template : mapperService.getAllDynamicTemplates()) { - if (template.pathMatch().isEmpty()) { - continue; - } - - var templateName = "__dynamic__" + template.name(); - var mappingSnippet = template.mappingForName(templateName, KeywordFieldMapper.CONTENT_TYPE); - String mappingSnippetType = (String) mappingSnippet.get("type"); - if (mappingSnippetType == null) { - continue; - } + DocumentMapper documentMapper = mapperService.documentMapper(); + return findDimensionFields(dimensions, documentMapper); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } - MappingParserContext parserContext = mapperService.parserContext(); - for (Iterator iterator = template.pathMatch().iterator(); iterator.hasNext();) { - var mapper = parserContext.typeParser(mappingSnippetType) - .parse(iterator.next(), mappingSnippet, parserContext) - .build(MapperBuilderContext.root(false, false)); - extractPath(routingPaths, mapper); - if (iterator.hasNext()) { - // Since FieldMapper.parse modifies the Map passed in (removing entries for "type"), that means - // that only the first pathMatch passed in gets recognized as a time_series_dimension. - // To avoid this, each parsing call uses a new mapping snippet. - // Note that a shallow copy of the mappingSnippet map is not enough if there are multi-fields. - mappingSnippet = template.mappingForName(templateName, KeywordFieldMapper.CONTENT_TYPE); - } + private static boolean findDimensionFields(List dimensions, DocumentMapper documentMapper) { + for (var objectMapper : documentMapper.mappers().objectMappers().values()) { + if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { + if (passThroughObjectMapper.containsDimensions()) { + dimensions.add(passThroughObjectMapper.fullPath() + ".*"); } } - return routingPaths; - } catch (IOException e) { - throw new UncheckedIOException(e); } + boolean matchesAllDimensions = true; + for (var template : documentMapper.mapping().getRoot().dynamicTemplates()) { + if (template.isTimeSeriesDimension() == false) { + continue; + } + if (template.isSimplePathMath() == false) { + matchesAllDimensions = false; + } + if (template.pathMatch().isEmpty() == false) { + dimensions.addAll(template.pathMatch()); + } + } + + for (var fieldMapper : documentMapper.mappers().fieldMappers()) { + extractPath(dimensions, fieldMapper); + } + return matchesAllDimensions; } /** - * Helper method that adds the name of the mapper to the provided list if it is a keyword dimension field. + * Helper method that adds the name of the mapper to the provided list. */ - private static void extractPath(List routingPaths, Mapper mapper) { - if (mapper instanceof KeywordFieldMapper keywordFieldMapper) { - if (keywordFieldMapper.fieldType().isDimension()) { - routingPaths.add(mapper.fullPath()); + private static void extractPath(List dimensions, Mapper mapper) { + if (mapper instanceof FieldMapper fieldMapper) { + if (fieldMapper.fieldType().isDimension()) { + String path = mapper.fullPath(); + // don't add if the path already matches via a wildcard pattern in the list + // e.g. if "path.*" is already added, "path.foo" should not be added + if (Regex.simpleMatch(dimensions, path) == false) { + dimensions.add(path); + } } } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index b85b3f6e7ae39..bdd94de1f92c6 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -33,7 +33,6 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.FORMATTER; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -70,6 +69,18 @@ public void testGetAdditionalIndexSettings() throws Exception { "field3": { "type": "keyword", "time_series_dimension": true + }, + "field4": { + "type": "long", + "time_series_dimension": true + }, + "field5": { + "type": "ip", + "time_series_dimension": true + }, + "field6": { + "type": "boolean", + "time_series_dimension": true } } } @@ -91,7 +102,7 @@ public void testGetAdditionalIndexSettings() throws Exception { assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("field3")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field3", "field4", "field5", "field6")); } public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throws Exception { @@ -105,7 +116,7 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw "_doc": { "properties": { "field1": { - "type": "keyword" + "type": "keyword", "time_series_dimension": true }, "field2": { @@ -206,7 +217,7 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception { assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field1", "field3")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field3")); } public void testGetAdditionalIndexSettingsNoMappings() { @@ -461,7 +472,7 @@ public void testGenerateRoutingPathFromDynamicTemplate() throws Exception { assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); } public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntries() throws Exception { @@ -502,10 +513,10 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); assertThat( - IndexMetadata.INDEX_ROUTING_PATH.get(result), + IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*") ); - List routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result); + List routingPathList = IndexMetadata.INDEX_DIMENSIONS.get(result); assertEquals(3, routingPathList.size()); } @@ -552,10 +563,10 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); assertThat( - IndexMetadata.INDEX_ROUTING_PATH.get(result), + IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*") ); - List routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result); + List routingPathList = IndexMetadata.INDEX_DIMENSIONS.get(result); assertEquals(3, routingPathList.size()); } @@ -605,7 +616,7 @@ public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch() assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); } public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() throws Exception { @@ -652,8 +663,8 @@ public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() thro Settings result = generateTsdbSettings(mapping, now); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); - assertEquals(2, IndexMetadata.INDEX_ROUTING_PATH.get(result).size()); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + assertEquals(2, IndexMetadata.INDEX_DIMENSIONS.get(result).size()); } public void testGenerateRoutingPathFromPassThroughObject() throws Exception { @@ -665,7 +676,12 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception { "labels": { "type": "passthrough", "time_series_dimension": true, - "priority": 2 + "priority": 2, + "properties": { + "label1": { + "type": "keyword" + } + } }, "metrics": { "type": "passthrough", @@ -683,7 +699,7 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception { assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("labels.*")); } private Settings generateTsdbSettings(String mapping, Instant now) throws IOException { diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 598bc90217574..f393fafb5067e 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -148,13 +148,12 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" ]} --- "aggregate the tsid": - requires: cluster_features: ["gte_v8.13.0"] - reason: _tsid hahing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: @@ -169,9 +168,8 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} + - length: {aggregations.tsids.buckets: 2} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -368,7 +366,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -387,7 +384,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -406,7 +402,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -425,7 +420,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: search: @@ -443,7 +437,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -520,7 +513,6 @@ dynamic templates - conflicting aliases: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KGejYryCnrIkXYZdIF_Q8F8X2dfFIGKYisFh7t1RGGWOWgWU7C0RiFE" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -539,7 +531,6 @@ dynamic templates - conflicting aliases: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KGejYryCnrIkXYZdIF_Q8F8X2dfFIGKYisFh7t1RGGWOWgWU7C0RiFE" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -699,7 +690,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -718,7 +708,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -737,7 +726,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -756,7 +744,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -775,7 +762,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -794,7 +780,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- diff --git a/server/build.gradle b/server/build.gradle index 59615e5f42ac5..2c2c9771adcb5 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -59,6 +59,7 @@ dependencies { // utilities api project(":libs:cli") implementation 'com.carrotsearch:hppc:0.8.1' + api 'com.dynatrace.hash4j:hash4j:0.25.0' // precentil ranks aggregation api 'org.hdrhistogram:HdrHistogram:2.1.9' diff --git a/server/licenses/hash4j-LICENSE.txt b/server/licenses/hash4j-LICENSE.txt new file mode 100644 index 0000000000000..261eeb9e9f8b2 --- /dev/null +++ b/server/licenses/hash4j-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/server/licenses/hash4j-NOTICE.txt b/server/licenses/hash4j-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 90cd3c669a52c..393d180be05f9 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -54,6 +54,7 @@ requires org.apache.lucene.queryparser; requires org.apache.lucene.sandbox; requires org.apache.lucene.suggest; + requires hash4j; exports org.elasticsearch; exports org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index f1f892f11ecf1..d863922f81d73 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -360,6 +360,7 @@ static TransportVersion def(int id) { public static final TransportVersion INDEX_TEMPLATE_TRACKING_INFO = def(9_136_0_00); public static final TransportVersion EXTENDED_SNAPSHOT_STATS_IN_NODE_INFO = def(9_137_0_00); public static final TransportVersion SIMULATE_INGEST_MAPPING_MERGE_TYPE = def(9_138_0_00); + public static final TransportVersion INGEST_REQUEST_INCLUDE_TSID = def(9_139_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index e8bd38b01414f..2bc51b252386c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -52,6 +52,8 @@ public class CreateIndexClusterStateUpdateRequest { private ComposableIndexTemplate matchingTemplate; + private boolean settingsSystemProvided = false; + /** * @deprecated project id ought always be specified */ @@ -223,6 +225,19 @@ public CreateIndexClusterStateUpdateRequest setMatchingTemplate(ComposableIndexT return this; } + /** + * Indicates whether the {@link #settings} of this request are system provided. + * If this is true, private settings will be allowed to be set in the request. + */ + public CreateIndexClusterStateUpdateRequest settingsSystemProvided(boolean settingsSystemProvided) { + this.settingsSystemProvided = settingsSystemProvided; + return this; + } + + public boolean settingsSystemProvided() { + return settingsSystemProvided; + } + @Override public String toString() { return "CreateIndexClusterStateUpdateRequest{" diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 01968596db932..357504fe87d1b 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -415,7 +415,8 @@ static boolean executeBulkItemRequest( request.routing(), request.getDynamicTemplates(), request.getIncludeSourceOnError(), - meteringParserDecorator + meteringParserDecorator, + request.tsid() ); result = primary.applyIndexOperationOnPrimary( version, @@ -743,7 +744,11 @@ private static Engine.Result performOpOnReplica( indexRequest.id(), indexRequest.source(), indexRequest.getContentType(), - indexRequest.routing() + indexRequest.routing(), + Map.of(), + true, + XContentMeteringParserDecorator.NOOP, + indexRequest.tsid() ); result = replica.applyIndexOperationOnReplica( primaryResponse.getSeqNo(), diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 7da12e05925af..47c435ada645a 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -204,7 +204,8 @@ private Tuple, Exception> validateMappings( request.routing(), request.getDynamicTemplates(), request.getIncludeSourceOnError(), - XContentMeteringParserDecorator.NOOP + XContentMeteringParserDecorator.NOOP, + request.tsid() ); ProjectMetadata project = projectResolver.getProjectMetadata(clusterService.state()); diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 707ea0919a91f..a3210bcbb4983 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.index; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchGenerationException; @@ -152,6 +153,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement * rawTimestamp field is used on the coordinate node, it doesn't need to be serialised. */ private Object rawTimestamp; + private BytesRef tsid; public IndexRequest(StreamInput in) throws IOException { this(null, in); @@ -216,6 +218,13 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio if (in.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR)) { includeSourceOnError = in.readBoolean(); } // else default value is true + + if (in.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_TSID)) { + tsid = in.readBytesRef(); + if (tsid.length == 0) { + tsid = null; // no tsid set + } + } } public IndexRequest() { @@ -353,6 +362,22 @@ public String routing() { return this.routing; } + /** + * When {@link org.elasticsearch.cluster.metadata.IndexMetadata#INDEX_DIMENSIONS} is populated, + * the coordinating node will calculate _tsid during routing and set it on the request. + * For time series indices where the setting is not populated, the _tsid will be created in the data node during document parsing. + *

+ * The _tsid can not be directly set by a user, it is set by the coordinating node. + */ + public IndexRequest tsid(BytesRef tsid) { + this.tsid = tsid; + return this; + } + + public BytesRef tsid() { + return this.tsid; + } + /** * Sets the ingest pipeline to be executed before indexing the document */ @@ -815,6 +840,9 @@ private void writeBody(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR)) { out.writeBoolean(includeSourceOnError); } + if (out.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_TSID)) { + out.writeBytesRef(tsid); + } } @Override @@ -917,7 +945,7 @@ public Index getConcreteWriteIndex(IndexAbstraction ia, ProjectMetadata project) @Override public int route(IndexRouting indexRouting) { - return indexRouting.indexShard(id, routing, contentType, source); + return indexRouting.indexShard(id, routing, tsid, contentType, source); } public IndexRequest setRequireAlias(boolean requireAlias) { diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index 3041e6bf5e274..620cd90cbebfc 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.index; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; @@ -50,6 +51,7 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder> settings() { Property.ServerlessPublic ); + /** + * Populated when an index that belongs to a time_series data stream is created or its mappings are updated. + * This setting is used so that the coordinating node knows which fields are time series dimensions + * as it doesn't have access to mappings. + * When this setting is populated, an optimization kicks in that allows the coordinating node to create the tsid and the routing hash + * in one go. + * Otherwise, the coordinating node only creates the routing hash based on {@link #INDEX_ROUTING_PATH} and the tsid is created + * during document parsing, effectively requiring two passes over the document. + *

+ * The condition for this optimization to kick in is that all possible dimension fields can be identified + * via a list of wildcard patterns. + * If that's not the case (for example when certain types of dynamic templates are used), + * the {@link #INDEX_ROUTING_PATH} is populated instead. + */ + public static final Setting> INDEX_DIMENSIONS = Setting.stringListSetting( + "index.dimensions", + Setting.Property.IndexScope, + Property.Dynamic, + Property.PrivateIndex + ); + /** * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use. * TODO: Remove in 9.0 @@ -576,6 +597,7 @@ public Iterator> settings() { private final int routingFactor; private final int routingPartitionSize; private final List routingPaths; + private final List dimensions; private final int numberOfShards; private final int numberOfReplicas; @@ -689,6 +711,7 @@ private IndexMetadata( final int routingNumShards, final int routingPartitionSize, final List routingPaths, + final List dimensions, final ActiveShardCount waitForActiveShards, final ImmutableOpenMap rolloverInfos, final boolean isSystem, @@ -744,6 +767,7 @@ private IndexMetadata( this.routingFactor = routingNumShards / numberOfShards; this.routingPartitionSize = routingPartitionSize; this.routingPaths = routingPaths; + this.dimensions = dimensions; this.waitForActiveShards = waitForActiveShards; this.rolloverInfos = rolloverInfos; this.isSystem = isSystem; @@ -803,6 +827,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) { this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -865,6 +890,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set inSyncSet) this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -935,6 +961,7 @@ public IndexMetadata withSetPrimaryTerm(int shardId, long primaryTerm) { this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -996,6 +1023,7 @@ public IndexMetadata withTimestampRanges(IndexLongFieldRange timestampRange, Ind this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -1052,6 +1080,7 @@ public IndexMetadata withIncrementedVersion() { this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -1166,6 +1195,10 @@ public List getRoutingPaths() { return routingPaths; } + public List getDimensions() { + return dimensions; + } + public int getTotalNumberOfShards() { return totalNumberOfShards; } @@ -2378,6 +2411,7 @@ IndexMetadata build(boolean repair) { } final List routingPaths = INDEX_ROUTING_PATH.get(settings); + final List dimensions = INDEX_DIMENSIONS.get(settings); final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE); @@ -2457,6 +2491,7 @@ IndexMetadata build(boolean repair) { getRoutingNumShards(), routingPartitionSize, routingPaths, + dimensions, waitForActiveShards, rolloverInfos.build(), isSystem, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index ca612eb20747a..4f4263dcaff04 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1561,12 +1561,12 @@ private static void validateActiveShardCount(ActiveShardCount waitForActiveShard private void validate(CreateIndexClusterStateUpdateRequest request, ProjectMetadata projectMetadata, RoutingTable routingTable) { validateIndexName(request.index(), projectMetadata, routingTable); - validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings); + validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings && request.settingsSystemProvided() == false); } public void validateIndexSettings(String indexName, final Settings settings, final boolean forbidPrivateIndexSettings) throws IndexCreationException { - List validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings); + List validationErrors = getIndexSettingsValidationErrors(settings, null, forbidPrivateIndexSettings); if (validationErrors.isEmpty() == false) { ValidationException validationException = new ValidationException(); @@ -1575,21 +1575,30 @@ public void validateIndexSettings(String indexName, final Settings settings, fin } } - List getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings) { + List getIndexSettingsValidationErrors( + final Settings settings, + @Nullable Settings systemProvided, + final boolean forbidPrivateIndexSettings + ) { List validationErrors = validateIndexCustomPath(settings, env.sharedDataDir()); if (forbidPrivateIndexSettings) { - validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings)); + validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, systemProvided, indexScopedSettings)); } return validationErrors; } - private static List validatePrivateSettingsNotExplicitlySet(Settings settings, IndexScopedSettings indexScopedSettings) { + private static List validatePrivateSettingsNotExplicitlySet( + Settings settings, + @Nullable Settings systemProvided, + IndexScopedSettings indexScopedSettings + ) { List validationErrors = new ArrayList<>(); for (final String key : settings.keySet()) { final Setting setting = indexScopedSettings.get(key); if (setting == null) { assert indexScopedSettings.isPrivateSetting(key) : "expected [" + key + "] to be private but it was not"; - } else if (setting.isPrivateIndex()) { + } else if (setting.isPrivateIndex() && (systemProvided == null || settings.get(key).equals(systemProvided.get(key)) == false)) { + // if the setting is system provided, they're not set by the user validationErrors.add("private index setting [" + key + "] can not be set explicitly"); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index d490ba527ad65..bafda76b8cf12 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -774,6 +774,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo final var combinedSettings = resolveSettings(indexTemplate, projectMetadata.componentTemplates()); // First apply settings sourced from index setting providers: var finalSettings = Settings.builder(); + var additionalSettingsBuilder = Settings.builder(); for (var provider : indexSettingProviders) { var newAdditionalSettings = provider.getAdditionalIndexSettings( VALIDATE_INDEX_NAME, @@ -784,9 +785,11 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo combinedSettings, combinedMappings ); - MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, finalSettings); - finalSettings.put(newAdditionalSettings); + MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, additionalSettingsBuilder); + additionalSettingsBuilder.put(newAdditionalSettings); } + Settings additionalSettings = additionalSettingsBuilder.build(); + finalSettings.put(additionalSettings); // Then apply setting from component templates: finalSettings.put(combinedSettings); // Then finally apply settings resolved from index template: @@ -796,7 +799,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo var templateToValidate = indexTemplate.toBuilder().template(Template.builder(finalTemplate).settings(finalSettings)).build(); - validate(name, templateToValidate); + validate(name, templateToValidate, additionalSettings); validateDataStreamsStillReferenced(projectMetadata, name, templateToValidate); validateLifecycle(projectMetadata, name, templateToValidate, globalRetentionSettings.get(false)); validateDataStreamOptions(projectMetadata, name, templateToValidate, globalRetentionSettings.get(true)); @@ -2025,18 +2028,19 @@ static void validateTemplate(Settings validateSettings, CompressedXContent mappi } private void validate(String name, ComponentTemplate template) { - validate(name, template.template(), Collections.emptyList()); + validate(name, template.template(), Collections.emptyList(), null); } - private void validate(String name, ComposableIndexTemplate template) { - validate(name, template.template(), template.indexPatterns()); + private void validate(String name, ComposableIndexTemplate template, Settings additionalSettings) { + validate(name, template.template(), template.indexPatterns(), additionalSettings); } - private void validate(String name, Template template, List indexPatterns) { + private void validate(String name, Template template, List indexPatterns, @Nullable Settings systemProvided) { Optional