Skip to content

Commit 8eb0f7a

Browse files
felixbarnyfelixbarny
committed
Hash once to create routing hash and _tsid
1 parent 6dae011 commit 8eb0f7a

File tree

51 files changed

+1508
-263
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+1508
-263
lines changed

‎gradle/verification-metadata.xml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,11 @@
272272
<sha256 value="a3609eeb7173837a589a4ad865e6feaf71ee6139d061eae2e698401485f7589c" origin="Generated by Gradle"/>
273273
</artifact>
274274
</component>
275+
<component group="com.dynatrace.hash4j" name="hash4j" version="0.25.0">
276+
<artifact name="hash4j-0.25.0.jar">
277+
<sha256 value="a9f1ebf6dab3fecfb36eed4e084ca587e5cdf9dcdd723b8adb7d17b6fc874862" origin="Generated by Gradle"/>
278+
</artifact>
279+
</component>
275280
<component group="com.ethlo.time" name="itu" version="1.7.0">
276281
<artifact name="itu-1.7.0.jar">
277282
<sha256 value="55ceb418c9e8138c4fcf62e213c4c814d89e8a84c827d395407cbecba5d791e7" origin="Generated by Gradle"/>

‎modules/data-streams/build.gradle

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
5252
task.skipTest("data_stream/30_auto_create_data_stream/Don't initialize failure store during data stream auto-creation on successful index", "Configuring the failure store via data stream templates is not supported anymore.")
5353

5454
task.skipTest("data_stream/150_tsdb/TSDB failures go to failure store", "Configuring the failure store via data stream templates is not supported anymore.")
55+
// TODO remove these after removing exact _tsid assertions in 8.x
56+
task.skipTest("data_stream/150_tsdb/dynamic templates", "The _tsid has changed in a new index version. This tests verifies the exact _tsid value with is too brittle for compatibility testing.")
57+
task.skipTest("data_stream/150_tsdb/dynamic templates - conflicting aliases", "The _tsid has changed in a new index version. This tests verifies the exact _tsid value with is too brittle for compatibility testing.")
58+
task.skipTest("data_stream/150_tsdb/dynamic templates with nesting", "The _tsid has changed in a new index version. This tests verifies the exact _tsid value with is too brittle for compatibility testing.")
5559

5660
task.skipTest("data_stream/170_modify_data_stream/Modify a data stream's failure store", "Configuring the failure store via data stream templates is not supported anymore.")
5761

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

Lines changed: 193 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
1515
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
1616
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
17+
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
18+
import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction;
1719
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
1820
import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
1921
import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest;
@@ -24,6 +26,7 @@
2426
import org.elasticsearch.action.bulk.BulkRequestBuilder;
2527
import org.elasticsearch.action.bulk.BulkResponse;
2628
import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus;
29+
import org.elasticsearch.action.datastreams.CreateDataStreamAction;
2730
import org.elasticsearch.action.get.GetRequest;
2831
import org.elasticsearch.action.index.IndexRequest;
2932
import org.elasticsearch.action.search.SearchRequest;
@@ -34,6 +37,7 @@
3437
import org.elasticsearch.cluster.metadata.Template;
3538
import org.elasticsearch.common.Strings;
3639
import org.elasticsearch.common.compress.CompressedXContent;
40+
import org.elasticsearch.common.settings.Setting;
3741
import org.elasticsearch.common.settings.Settings;
3842
import org.elasticsearch.common.time.DateFormatter;
3943
import org.elasticsearch.common.time.FormatNames;
@@ -58,12 +62,14 @@
5862
import java.time.temporal.ChronoUnit;
5963
import java.util.Collection;
6064
import java.util.List;
65+
import java.util.Map;
6166
import java.util.concurrent.CountDownLatch;
6267

6368
import static org.elasticsearch.test.MapMatcher.assertMap;
6469
import static org.elasticsearch.test.MapMatcher.matchesMap;
6570
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
6671
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
72+
import static org.hamcrest.Matchers.containsInAnyOrder;
6773
import static org.hamcrest.Matchers.containsString;
6874
import static org.hamcrest.Matchers.equalTo;
6975
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -320,16 +326,12 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception {
320326
ComposableIndexTemplate.builder()
321327
.indexPatterns(List.of("k8s*"))
322328
.template(
323-
new Template(
324-
Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset").build(),
325-
new CompressedXContent(mappingTemplate),
326-
null
327-
)
329+
new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(mappingTemplate), null)
328330
)
329331
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
330332
.build()
331333
);
332-
client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet();
334+
assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request));
333335
}
334336

335337
public void testInvalidTsdbTemplatesMissingSettings() throws Exception {
@@ -621,6 +623,191 @@ public void testReindexing() throws Exception {
621623
);
622624
}
623625

626+
public void testAddDimensionToMapping() throws Exception {
627+
String dataStreamName = "my-ds";
628+
var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id");
629+
putTemplateRequest.indexTemplate(
630+
ComposableIndexTemplate.builder()
631+
.indexPatterns(List.of(dataStreamName))
632+
.template(
633+
new Template(
634+
Settings.builder().put("index.mode", "time_series").build(),
635+
new CompressedXContent(MAPPING_TEMPLATE),
636+
null
637+
)
638+
)
639+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
640+
.build()
641+
);
642+
assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest));
643+
644+
// create data stream
645+
CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request(
646+
TEST_REQUEST_TIMEOUT,
647+
TEST_REQUEST_TIMEOUT,
648+
"my-ds"
649+
);
650+
assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest));
651+
652+
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), equalTo(List.of("metricset")));
653+
654+
// put mapping with k8s.pod.uid as another time series dimension
655+
var putMappingRequest = new PutMappingRequest(dataStreamName).source("""
656+
{
657+
"properties": {
658+
"k8s.pod.name": {
659+
"type": "keyword",
660+
"time_series_dimension": true
661+
}
662+
}
663+
}
664+
""", XContentType.JSON);
665+
assertAcked(client().execute(TransportPutMappingAction.TYPE, putMappingRequest).actionGet());
666+
667+
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name"));
668+
669+
indexWithPodNames(dataStreamName, Instant.now(), Map.of(), "dog", "cat");
670+
}
671+
672+
public void testDynamicStringDimensions() throws Exception {
673+
String dataStreamName = "my-ds";
674+
var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id");
675+
putTemplateRequest.indexTemplate(
676+
ComposableIndexTemplate.builder()
677+
.indexPatterns(List.of(dataStreamName))
678+
.template(new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent("""
679+
{
680+
"_doc": {
681+
"dynamic_templates": [
682+
{
683+
"labels": {
684+
"match_mapping_type": "string",
685+
"mapping": {
686+
"type": "keyword",
687+
"time_series_dimension": true
688+
}
689+
}
690+
}
691+
],
692+
"properties": {
693+
"@timestamp": {
694+
"type": "date"
695+
},
696+
"metricset": {
697+
"type": "keyword",
698+
"time_series_dimension": true
699+
}
700+
}
701+
}
702+
}"""), null))
703+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
704+
.build()
705+
);
706+
assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest));
707+
708+
CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request(
709+
TEST_REQUEST_TIMEOUT,
710+
TEST_REQUEST_TIMEOUT,
711+
"my-ds"
712+
);
713+
assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest));
714+
715+
// doesn't populate index.dimensions because the "labels" dynamic template doesn't have a path_math
716+
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset")));
717+
718+
// index doc
719+
BulkResponse bulkResponse = client().prepareBulk()
720+
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
721+
.add(
722+
client().prepareIndex(dataStreamName)
723+
.setOpType(DocWriteRequest.OpType.CREATE)
724+
.setSource(DOC.replace("$time", formatInstant(Instant.now())), XContentType.JSON)
725+
)
726+
.get();
727+
assertThat(bulkResponse.hasFailures(), is(false));
728+
729+
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset")));
730+
}
731+
732+
public void testDynamicDimensions() throws Exception {
733+
String dataStreamName = "my-ds";
734+
var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id");
735+
putTemplateRequest.indexTemplate(
736+
ComposableIndexTemplate.builder()
737+
.indexPatterns(List.of(dataStreamName))
738+
.template(new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent("""
739+
740+
{
741+
"_doc": {
742+
"dynamic_templates": [
743+
{
744+
"label": {
745+
"mapping": {
746+
"type": "keyword",
747+
"time_series_dimension": true
748+
}
749+
}
750+
}
751+
],
752+
"properties": {
753+
"@timestamp": {
754+
"type": "date"
755+
},
756+
"metricset": {
757+
"type": "keyword",
758+
"time_series_dimension": true
759+
}
760+
}
761+
}
762+
}"""), null))
763+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
764+
.build()
765+
);
766+
assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest));
767+
768+
CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request(
769+
TEST_REQUEST_TIMEOUT,
770+
TEST_REQUEST_TIMEOUT,
771+
"my-ds"
772+
);
773+
assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest));
774+
775+
// doesn't populate index.dimensions because the "label" dynamic template doesn't have a path_math
776+
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset")));
777+
778+
// index doc
779+
indexWithPodNames(dataStreamName, Instant.now(), Map.of("k8s.pod.name", "label"), "dog", "cat");
780+
781+
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset")));
782+
}
783+
784+
private void indexWithPodNames(String dataStreamName, Instant timestamp, Map<String, String> dynamicTemplates, String... podNames) {
785+
// index doc
786+
BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
787+
bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
788+
for (String podName : podNames) {
789+
bulkRequestBuilder.add(
790+
client().prepareIndex(dataStreamName)
791+
.setOpType(DocWriteRequest.OpType.CREATE)
792+
.setSource(DOC.replace("$time", formatInstant(timestamp)).replace("dog", podName), XContentType.JSON)
793+
.request()
794+
.setDynamicTemplates(dynamicTemplates)
795+
);
796+
}
797+
798+
BulkResponse bulkResponse = bulkRequestBuilder.get();
799+
assertThat(bulkResponse.hasFailures(), is(false));
800+
}
801+
802+
private <T> T getSetting(String dataStreamName, Setting<T> setting) {
803+
GetIndexResponse getIndexResponse = safeGet(
804+
indicesAdmin().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(dataStreamName))
805+
);
806+
assertThat(getIndexResponse.getIndices().length, equalTo(1));
807+
Settings settings = getIndexResponse.getSettings().get(getIndexResponse.getIndices()[0]);
808+
return setting.get(settings);
809+
}
810+
624811
static String formatInstant(Instant instant) {
625812
return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant);
626813
}

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ public void testIndexingGettingAndSearching() throws Exception {
184184

185185
// validate index:
186186
var getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(index)).actionGet();
187-
assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), equalTo("[attributes.*]"));
187+
assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), equalTo("[attributes.*]"));
188188
// validate mapping
189189
var mapping = getIndexResponse.mappings().get(index).getSourceAsMap();
190190
assertMap(

‎modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -433,9 +433,10 @@ public void testSimulateTsdbDataStreamTemplate() throws Exception {
433433
assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.time_series.start_time"), notNullValue());
434434
assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.time_series.end_time"), notNullValue());
435435
assertThat(
436-
ObjectPath.evaluate(responseBody, "template.settings.index.routing_path"),
436+
ObjectPath.evaluate(responseBody, "template.settings.index.dimensions"),
437437
containsInAnyOrder("metricset", "k8s.pod.uid", "pod.labels.*")
438438
);
439+
assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.routing_path"), nullValue());
439440
assertThat(ObjectPath.evaluate(responseBody, "overlapping"), empty());
440441
}
441442

0 commit comments

Comments
 (0)