Skip to content

Commit ec975a9

Browse files
felixbarnyfelixbarny
committed
Move back to index.dimensions index setting
1 parent 5ab5361 commit ec975a9

File tree

15 files changed

+100
-207
lines changed

15 files changed

+100
-207
lines changed

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.elasticsearch.index.reindex.BulkByScrollResponse;
4949
import org.elasticsearch.index.reindex.ReindexAction;
5050
import org.elasticsearch.index.reindex.ReindexRequest;
51+
import org.elasticsearch.indices.InvalidIndexTemplateException;
5152
import org.elasticsearch.plugins.Plugin;
5253
import org.elasticsearch.reindex.ReindexPlugin;
5354
import org.elasticsearch.rest.RestStatus;
@@ -300,10 +301,10 @@ public void testInvalidTsdbTemplatesNoTimeSeriesDimensionAttribute() throws Exce
300301
.build()
301302
);
302303
var e = expectThrows(
303-
IllegalArgumentException.class,
304+
InvalidIndexTemplateException.class,
304305
() -> client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet()
305306
);
306-
assertThat(e.getCause().getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]"));
307+
assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]"));
307308
}
308309
}
309310

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
import org.elasticsearch.cluster.metadata.DataStream;
1212
import org.elasticsearch.cluster.metadata.IndexMetadata;
1313
import org.elasticsearch.cluster.metadata.ProjectMetadata;
14-
import org.elasticsearch.cluster.routing.TimeSeriesDimensionsMetadataAccessor;
1514
import org.elasticsearch.common.UUIDs;
1615
import org.elasticsearch.common.compress.CompressedXContent;
1716
import org.elasticsearch.common.regex.Regex;
@@ -40,6 +39,7 @@
4039
import java.util.Map;
4140
import java.util.function.BiConsumer;
4241

42+
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_DIMENSIONS;
4343
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
4444

4545
/**
@@ -131,7 +131,7 @@ public void provideAdditionalMetadata(
131131
);
132132
if (dimensions.isEmpty() == false) {
133133
if (matchesAllDimensions) {
134-
TimeSeriesDimensionsMetadataAccessor.addToCustomMetadata(additionalCustomMetadata, dimensions);
134+
additionalSettings.putList(INDEX_DIMENSIONS.getKey(), dimensions);
135135
} else {
136136
// Fall back to setting index.routing_path if the paths in the dimensions list don't match all potential
137137
// dimension fields (e.g. if a dynamic template matches by type instead of path).
@@ -169,10 +169,10 @@ public void onUpdateMappings(
169169
return;
170170
}
171171
if (matchesAllDimensions) {
172-
TimeSeriesDimensionsMetadataAccessor.addToCustomMetadata(additionalCustomMetadata, newIndexDimensions);
172+
additionalSettings.putList(INDEX_DIMENSIONS.getKey(), newIndexDimensions);
173173
} else {
174174
// If the new dimensions don't match all potential dimension fields, we need to set index.routing_path
175-
additionalSettings.putList(INDEX_ROUTING_PATH.getKey(), newIndexDimensions).build();
175+
additionalSettings.putList(INDEX_ROUTING_PATH.getKey(), newIndexDimensions);
176176
}
177177
}
178178

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java‎

Lines changed: 21 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
1313
import org.elasticsearch.cluster.metadata.IndexMetadata;
1414
import org.elasticsearch.cluster.metadata.ProjectMetadata;
15-
import org.elasticsearch.cluster.routing.TimeSeriesDimensionsMetadataAccessor;
1615
import org.elasticsearch.common.collect.ImmutableOpenMap;
1716
import org.elasticsearch.common.compress.CompressedXContent;
1817
import org.elasticsearch.common.settings.Settings;
@@ -105,14 +104,11 @@ public void testGetAdditionalIndexSettings() throws Exception {
105104
// The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
106105
// (in production the index.mode setting is usually provided in an index or component template)
107106
result = builder().put(result).put("index.mode", "time_series").build();
108-
assertThat(result.size(), equalTo(3));
107+
assertThat(result.size(), equalTo(4));
109108
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
110109
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
111110
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
112-
assertThat(
113-
TimeSeriesDimensionsMetadataAccessor.fromCustomMetadata(customMetadataBuilder.build()),
114-
containsInAnyOrder("field3", "field4", "field5", "field6")
115-
);
111+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field3", "field4", "field5", "field6"));
116112
}
117113

118114
public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throws Exception {
@@ -234,14 +230,11 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception {
234230
// The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
235231
// (in production the index.mode setting is usually provided in an index or component template)
236232
result = builder().put(result).put("index.mode", "time_series").build();
237-
assertThat(result.size(), equalTo(3));
233+
assertThat(result.size(), equalTo(4));
238234
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
239235
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
240236
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
241-
assertThat(
242-
TimeSeriesDimensionsMetadataAccessor.fromCustomMetadata(customMetadataBuilder.build()),
243-
containsInAnyOrder("field1", "field3")
244-
);
237+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field3"));
245238
}
246239

247240
public void testGetAdditionalIndexSettingsNoMappings() {
@@ -536,14 +529,11 @@ public void testGenerateRoutingPathFromDynamicTemplate() throws Exception {
536529
""";
537530
ImmutableOpenMap.Builder<String, Map<String, String>> customMetadataBuilder = ImmutableOpenMap.builder();
538531
Settings result = generateTsdbSettings(mapping, now, customMetadataBuilder);
539-
assertThat(result.size(), equalTo(3));
532+
assertThat(result.size(), equalTo(4));
540533
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
541534
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
542535
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
543-
assertThat(
544-
TimeSeriesDimensionsMetadataAccessor.fromCustomMetadata(customMetadataBuilder.build()),
545-
containsInAnyOrder("host.id", "prometheus.labels.*")
546-
);
536+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
547537
}
548538

549539
public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntries() throws Exception {
@@ -580,13 +570,14 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri
580570
""";
581571
ImmutableOpenMap.Builder<String, Map<String, String>> customMetadataBuilder = ImmutableOpenMap.builder();
582572
Settings result = generateTsdbSettings(mapping, now, customMetadataBuilder);
583-
assertThat(result.size(), equalTo(3));
573+
assertThat(result.size(), equalTo(4));
584574
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
585575
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
586576
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
587-
List<String> routingPathList = TimeSeriesDimensionsMetadataAccessor.fromCustomMetadata(customMetadataBuilder.build());
588-
assertThat(routingPathList, containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*"));
589-
assertEquals(3, routingPathList.size());
577+
assertThat(
578+
IndexMetadata.INDEX_DIMENSIONS.get(result),
579+
containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*")
580+
);
590581
}
591582

592583
public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntriesMultiFields() throws Exception {
@@ -628,13 +619,14 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri
628619
""";
629620
ImmutableOpenMap.Builder<String, Map<String, String>> customMetadataBuilder = ImmutableOpenMap.builder();
630621
Settings result = generateTsdbSettings(mapping, now, customMetadataBuilder);
631-
assertThat(result.size(), equalTo(3));
622+
assertThat(result.size(), equalTo(4));
632623
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
633624
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
634625
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
635-
List<String> dimensions = TimeSeriesDimensionsMetadataAccessor.fromCustomMetadata(customMetadataBuilder.build());
636-
assertThat(dimensions, containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*"));
637-
assertEquals(3, dimensions.size());
626+
assertThat(
627+
IndexMetadata.INDEX_DIMENSIONS.get(result),
628+
containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*")
629+
);
638630
}
639631

640632
public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch() throws Exception {
@@ -680,14 +672,11 @@ public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch()
680672
""";
681673
ImmutableOpenMap.Builder<String, Map<String, String>> customMetadataBuilder = ImmutableOpenMap.builder();
682674
Settings result = generateTsdbSettings(mapping, now, customMetadataBuilder);
683-
assertThat(result.size(), equalTo(3));
675+
assertThat(result.size(), equalTo(4));
684676
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
685677
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
686678
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
687-
assertThat(
688-
TimeSeriesDimensionsMetadataAccessor.fromCustomMetadata(customMetadataBuilder.build()),
689-
containsInAnyOrder("host.id", "prometheus.labels.*")
690-
);
679+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
691680
}
692681

693682
public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() throws Exception {
@@ -735,9 +724,7 @@ public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() thro
735724
Settings result = generateTsdbSettings(mapping, now, customMetadataBuilder);
736725
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
737726
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
738-
List<String> dimensions = TimeSeriesDimensionsMetadataAccessor.fromCustomMetadata(customMetadataBuilder.build());
739-
assertThat(dimensions, containsInAnyOrder("host.id", "prometheus.labels.*"));
740-
assertEquals(2, dimensions.size());
727+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
741728
}
742729

743730
public void testGenerateRoutingPathFromPassThroughObject() throws Exception {
@@ -769,11 +756,11 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception {
769756
""";
770757
ImmutableOpenMap.Builder<String, Map<String, String>> customMetadataBuilder = ImmutableOpenMap.builder();
771758
Settings result = generateTsdbSettings(mapping, now, customMetadataBuilder);
772-
assertThat(result.size(), equalTo(3));
759+
assertThat(result.size(), equalTo(4));
773760
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
774761
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
775762
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
776-
assertThat(TimeSeriesDimensionsMetadataAccessor.fromCustomMetadata(customMetadataBuilder.build()), containsInAnyOrder("labels.*"));
763+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("labels.*"));
777764
}
778765

779766
private Settings generateTsdbSettings(String mapping, Instant now, ImmutableOpenMap.Builder<String, Map<String, String>> builder)

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.elasticsearch.index.IndexSettingProviders;
3030
import org.elasticsearch.indices.EmptySystemIndices;
3131
import org.elasticsearch.indices.IndicesService;
32+
import org.elasticsearch.indices.InvalidIndexTemplateException;
3233
import org.elasticsearch.indices.ShardLimitValidator;
3334
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
3435
import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -68,8 +69,8 @@ public void testRequireRoutingPath() throws Exception {
6869
.priority(100L)
6970
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
7071
.build();
71-
var e = expectThrows(IllegalArgumentException.class, () -> service.addIndexTemplateV2(project, false, "1", indexTemplate));
72-
assertThat(e.getCause().getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]"));
72+
var e = expectThrows(InvalidIndexTemplateException.class, () -> service.addIndexTemplateV2(project, false, "1", indexTemplate));
73+
assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]"));
7374
}
7475
{
7576
// Routing path fetched from mapping of component template

‎server/src/main/java/org/elasticsearch/action/index/IndexRequest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@ public String routing() {
364364
}
365365

366366
/**
367-
* When {@link IndexMetadata#getTimeSeriesDimensions()} is populated,
367+
* When {@link IndexMetadata#INDEX_DIMENSIONS} is populated,
368368
* the coordinating node will calculate _tsid during routing and set it on the request.
369369
* For time series indices where the setting is not populated, the _tsid will be created in the data node during document parsing.
370370
* <p>

0 commit comments

Comments
 (0)