Skip to content

Commit fc68a81

Browse files
felixbarnyfelixbarny
authored
Set either index.dimensions or index.routing_path (#135351)
This is to reduce size of the cluster state by avoiding unnecessary/redundant index settings.
1 parent 5b45a28 commit fc68a81

File tree

5 files changed

+89
-40
lines changed

5 files changed

+89
-40
lines changed

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -666,10 +666,11 @@ public void testAddDimensionToMapping() throws Exception {
666666
assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest));
667667
if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
668668
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), equalTo(List.of("metricset")));
669+
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty());
669670
} else {
670671
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty());
672+
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset")));
671673
}
672-
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset")));
673674

674675
// put mapping with k8s.pod.uid as another time series dimension
675676
var putMappingRequest = new PutMappingRequest(dataStreamName).source("""
@@ -685,10 +686,35 @@ public void testAddDimensionToMapping() throws Exception {
685686
assertAcked(client().execute(TransportPutMappingAction.TYPE, putMappingRequest).actionGet());
686687
if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
687688
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name"));
689+
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty());
688690
} else {
689691
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty());
692+
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset")));
690693
}
691-
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset")));
694+
695+
// put dynamic template defining time series dimensions
696+
// we don't support index.dimensions in that case
697+
putMappingRequest = new PutMappingRequest(dataStreamName).source("""
698+
{
699+
"dynamic_templates": [
700+
{
701+
"labels": {
702+
"path_match": "labels.*",
703+
"mapping": {
704+
"type": "keyword",
705+
"time_series_dimension": true
706+
}
707+
}
708+
}
709+
]
710+
}
711+
""", XContentType.JSON);
712+
assertAcked(client().execute(TransportPutMappingAction.TYPE, putMappingRequest).actionGet());
713+
assertThat(
714+
getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH),
715+
containsInAnyOrder("metricset", "labels.*", "k8s.pod.name")
716+
);
717+
assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty());
692718

693719
indexWithPodNames(dataStreamName, Instant.now(), Map.of(), "dog", "cat");
694720
}

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,11 +186,12 @@ public void testIndexingGettingAndSearching() throws Exception {
186186

187187
// validate index:
188188
var getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(index)).actionGet();
189-
assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), equalTo("[attributes.*]"));
190189
if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
191190
assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), equalTo("[attributes.*]"));
191+
assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), nullValue());
192192
} else {
193193
assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), nullValue());
194+
assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), equalTo("[attributes.*]"));
194195
}
195196
// validate mapping
196197
var mapping = getIndexResponse.mappings().get(index).getSourceAsMap();

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.elasticsearch.index.IndexSettingProvider;
2525
import org.elasticsearch.index.IndexSettings;
2626
import org.elasticsearch.index.IndexVersion;
27+
import org.elasticsearch.index.IndexVersions;
2728
import org.elasticsearch.index.mapper.DateFieldMapper;
2829
import org.elasticsearch.index.mapper.DocumentMapper;
2930
import org.elasticsearch.index.mapper.FieldMapper;
@@ -132,14 +133,19 @@ public void provideAdditionalSettings(
132133
dimensions
133134
);
134135
if (dimensions.isEmpty() == false) {
135-
if (matchesAllDimensions && INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
136+
if (matchesAllDimensions
137+
&& INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG
138+
&& indexVersion.onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING)) {
136139
// Only set index.dimensions if the paths in the dimensions list match all potential dimension fields.
137140
// This is not the case e.g. if a dynamic template matches by match_mapping_type instead of path_match
138141
additionalSettings.putList(INDEX_DIMENSIONS.getKey(), dimensions);
142+
} else {
143+
// For older index versions, or when not all dimension fields can be matched via the dimensions list,
144+
// we fall back to use index.routing_path.
145+
// This is less efficient, because the dimensions need to be hashed twice:
146+
// once to determine the shard during routing, and once to create the tsid during document parsing.
147+
additionalSettings.putList(INDEX_ROUTING_PATH.getKey(), dimensions);
139148
}
140-
// always populate index.routing_path, so that routing works for older index versions
141-
// this applies to indices created during a rolling upgrade
142-
additionalSettings.putList(INDEX_ROUTING_PATH.getKey(), dimensions);
143149
}
144150
}
145151
}
@@ -167,9 +173,10 @@ public void onUpdateMappings(IndexMetadata indexMetadata, DocumentMapper documen
167173
&& new HashSet<>(indexDimensions).equals(new HashSet<>(newIndexDimensions)) == false;
168174
if (matchesAllDimensions == false) {
169175
// If the new dimensions don't match all potential dimension fields, we need to unset index.dimensions
170-
// so that index.routing_path is used instead.
171-
// This can happen if a new dynamic template is added to an existing index that matches by mapping type instead of path_match.
176+
// and set index.routing_path instead.
177+
// This can happen if a new dynamic template with time_series_dimension: true is added to an existing index.
172178
additionalSettings.putList(INDEX_DIMENSIONS.getKey(), List.of());
179+
additionalSettings.putList(INDEX_ROUTING_PATH.getKey(), newIndexDimensions);
173180
} else if (hasChanges) {
174181
additionalSettings.putList(INDEX_DIMENSIONS.getKey(), newIndexDimensions);
175182
}

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java‎

Lines changed: 41 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,12 @@
2121
import org.elasticsearch.index.IndexMode;
2222
import org.elasticsearch.index.IndexSettings;
2323
import org.elasticsearch.index.IndexVersion;
24+
import org.elasticsearch.index.IndexVersions;
2425
import org.elasticsearch.index.MapperTestUtils;
2526
import org.elasticsearch.index.mapper.DocumentMapper;
2627
import org.elasticsearch.index.mapper.MapperService;
2728
import org.elasticsearch.test.ESTestCase;
29+
import org.elasticsearch.test.index.IndexVersionUtils;
2830
import org.junit.Before;
2931

3032
import java.io.IOException;
@@ -48,12 +50,19 @@ public class DataStreamIndexSettingsProviderTests extends ESTestCase {
4850
private static final TimeValue DEFAULT_LOOK_AHEAD_TIME = TimeValue.timeValueMinutes(30); // default
4951

5052
DataStreamIndexSettingsProvider provider;
53+
private boolean indexDimensionsTsidOptimizationEnabled;
54+
private IndexVersion indexVersion;
5155

5256
@Before
5357
public void setup() {
5458
provider = new DataStreamIndexSettingsProvider(
5559
im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName())
5660
);
61+
indexVersion = randomBoolean()
62+
? IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.TSID_CREATED_DURING_ROUTING)
63+
: IndexVersionUtils.randomVersionBetween(random(), IndexVersions.TSID_CREATED_DURING_ROUTING, IndexVersion.current());
64+
indexDimensionsTsidOptimizationEnabled = INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG
65+
&& indexVersion.onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING);
5766
}
5867

5968
public void testGetAdditionalIndexSettings() throws Exception {
@@ -101,21 +110,22 @@ public void testGetAdditionalIndexSettings() throws Exception {
101110
now,
102111
settings,
103112
List.of(new CompressedXContent(mapping)),
104-
IndexVersion.current(),
113+
indexVersion,
105114
additionalSettings
106115
);
107116
Settings result = additionalSettings.build();
108117
// The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
109118
// (in production the index.mode setting is usually provided in an index or component template)
110119
result = builder().put(result).put("index.mode", "time_series").build();
111-
assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4));
120+
assertThat(result.size(), equalTo(4));
112121
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
113122
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
114123
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
115-
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field3", "field4", "field5", "field6"));
116-
if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
124+
if (indexDimensionsTsidOptimizationEnabled) {
117125
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field3", "field4", "field5", "field6"));
126+
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty());
118127
} else {
128+
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field3", "field4", "field5", "field6"));
119129
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty());
120130
}
121131
}
@@ -155,7 +165,7 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw
155165
now,
156166
settings,
157167
List.of(new CompressedXContent(mapping)),
158-
IndexVersion.current(),
168+
indexVersion,
159169
additionalSettings
160170
);
161171
Settings result = additionalSettings.build();
@@ -229,21 +239,22 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception {
229239
now,
230240
settings,
231241
List.of(new CompressedXContent(mapping1), new CompressedXContent(mapping2), new CompressedXContent(mapping3)),
232-
IndexVersion.current(),
242+
indexVersion,
233243
additionalSettings
234244
);
235245
Settings result = additionalSettings.build();
236246
// The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
237247
// (in production the index.mode setting is usually provided in an index or component template)
238248
result = builder().put(result).put("index.mode", "time_series").build();
239-
assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4));
249+
assertThat(result.size(), equalTo(4));
240250
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
241251
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
242252
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
243-
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field1", "field3"));
244-
if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
253+
if (indexDimensionsTsidOptimizationEnabled) {
245254
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field3"));
255+
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty());
246256
} else {
257+
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field1", "field3"));
247258
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty());
248259
}
249260
}
@@ -263,7 +274,7 @@ public void testGetAdditionalIndexSettingsNoMappings() {
263274
now,
264275
settings,
265276
List.of(),
266-
IndexVersion.current(),
277+
indexVersion,
267278
additionalSettings
268279
);
269280
Settings result = additionalSettings.build();
@@ -292,7 +303,7 @@ public void testGetAdditionalIndexSettingsLookAheadTime() throws Exception {
292303
now,
293304
settings,
294305
List.of(new CompressedXContent("{}")),
295-
IndexVersion.current(),
306+
indexVersion,
296307
additionalSettings
297308
);
298309
Settings result = additionalSettings.build();
@@ -321,7 +332,7 @@ public void testGetAdditionalIndexSettingsLookBackTime() throws Exception {
321332
now,
322333
settings,
323334
List.of(new CompressedXContent("{}")),
324-
IndexVersion.current(),
335+
indexVersion,
325336
additionalSettings
326337
);
327338
Settings result = additionalSettings.build();
@@ -357,7 +368,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() throws Exce
357368
now,
358369
settings,
359370
List.of(new CompressedXContent("{}")),
360-
IndexVersion.current(),
371+
indexVersion,
361372
additionalSettings
362373
);
363374
var result = additionalSettings.build();
@@ -397,7 +408,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi
397408
now,
398409
settings,
399410
null,
400-
IndexVersion.current(),
411+
indexVersion,
401412
builder()
402413
)
403414
);
@@ -426,7 +437,7 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() {
426437
Instant.ofEpochMilli(1L),
427438
settings,
428439
null,
429-
IndexVersion.current(),
440+
indexVersion,
430441
additionalSettings
431442
);
432443
Settings result = additionalSettings.build();
@@ -452,7 +463,7 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() {
452463
now,
453464
settings,
454465
List.of(),
455-
IndexVersion.current(),
466+
indexVersion,
456467
additionalSettings
457468
);
458469
Settings result = additionalSettings.build();
@@ -485,7 +496,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromTsdb() {
485496
Instant.ofEpochMilli(1L),
486497
Settings.EMPTY,
487498
List.of(),
488-
IndexVersion.current(),
499+
indexVersion,
489500
additionalSettings
490501
);
491502
Settings result = additionalSettings.build();
@@ -706,14 +717,15 @@ public void testGenerateNonDimensionDynamicTemplate() throws Exception {
706717
}
707718
""";
708719
Settings result = generateTsdbSettings(mapping, now);
709-
assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4));
720+
assertThat(result.size(), equalTo(4));
710721
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
711722
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
712723
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
713-
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id"));
714-
if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
724+
if (indexDimensionsTsidOptimizationEnabled) {
715725
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id"));
726+
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty());
716727
} else {
728+
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id"));
717729
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty());
718730
}
719731
}
@@ -793,14 +805,15 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception {
793805
}
794806
""";
795807
Settings result = generateTsdbSettings(mapping, now);
796-
assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4));
808+
assertThat(result.size(), equalTo(4));
797809
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
798810
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
799811
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
800-
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*"));
801-
if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) {
812+
if (indexDimensionsTsidOptimizationEnabled) {
802813
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("labels.*"));
814+
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty());
803815
} else {
816+
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*"));
804817
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty());
805818
}
806819
}
@@ -943,9 +956,10 @@ public void testAddDynamicTemplate() throws Exception {
943956
}
944957
""";
945958
// we don't support index.dimensions with dynamic templates so we'll unset index.dimensions
946-
Settings result = onUpdateMappings("labels.*", "labels.*", mapping);
947-
assertThat(result.size(), equalTo(1));
959+
Settings result = onUpdateMappings(null, "labels.*", mapping);
960+
assertThat(result.size(), equalTo(2));
948961
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty());
962+
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*"));
949963
}
950964

951965
private Settings generateTsdbSettings(String mapping, Instant now) throws IOException {
@@ -962,7 +976,7 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce
962976
now,
963977
settings,
964978
List.of(new CompressedXContent(mapping)),
965-
IndexVersion.current(),
979+
indexVersion,
966980
additionalSettings
967981
);
968982
var result = additionalSettings.build();
@@ -976,7 +990,7 @@ private Settings onUpdateMappings(String routingPath, String dimensions, String
976990
Settings.Builder currentSettings = Settings.builder()
977991
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath)
978992
.put(IndexMetadata.INDEX_DIMENSIONS.getKey(), dimensions)
979-
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
993+
.put(IndexMetadata.SETTING_VERSION_CREATED, indexVersion)
980994
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
981995
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
982996
.put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())

0 commit comments

Comments
 (0)