Skip to content

Commit 6b8212a

Browse files
felixbarnyfelixbarny
committed
Hash once to create routing hash and _tsid
1 parent 231d051 commit 6b8212a

35 files changed

+943
-169
lines changed

‎gradle/verification-metadata.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,11 @@
271271
<sha256 value="a3609eeb7173837a589a4ad865e6feaf71ee6139d061eae2e698401485f7589c" origin="Generated by Gradle"/>
272272
</artifact>
273273
</component>
274+
<component group="com.dynatrace.hash4j" name="hash4j" version="0.25.0">
275+
<artifact name="hash4j-0.25.0.jar">
276+
<sha256 value="a9f1ebf6dab3fecfb36eed4e084ca587e5cdf9dcdd723b8adb7d17b6fc874862" origin="Generated by Gradle"/>
277+
</artifact>
278+
</component>
274279
<component group="com.ethlo.time" name="itu" version="1.7.0">
275280
<artifact name="itu-1.7.0.jar">
276281
<sha256 value="55ceb418c9e8138c4fcf62e213c4c814d89e8a84c827d395407cbecba5d791e7" origin="Generated by Gradle"/>

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java‎

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.cluster.metadata.ProjectMetadata;
1414
import org.elasticsearch.common.UUIDs;
1515
import org.elasticsearch.common.compress.CompressedXContent;
16+
import org.elasticsearch.common.regex.Regex;
1617
import org.elasticsearch.common.settings.Settings;
1718
import org.elasticsearch.common.time.DateFormatter;
1819
import org.elasticsearch.core.CheckedFunction;
@@ -23,6 +24,7 @@
2324
import org.elasticsearch.index.IndexSettings;
2425
import org.elasticsearch.index.IndexVersion;
2526
import org.elasticsearch.index.mapper.DateFieldMapper;
27+
import org.elasticsearch.index.mapper.FieldMapper;
2628
import org.elasticsearch.index.mapper.KeywordFieldMapper;
2729
import org.elasticsearch.index.mapper.Mapper;
2830
import org.elasticsearch.index.mapper.MapperBuilderContext;
@@ -38,6 +40,7 @@
3840
import java.util.List;
3941
import java.util.Locale;
4042

43+
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_DIMENSIONS;
4144
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
4245

4346
/**
@@ -117,15 +120,16 @@ public Settings getAdditionalIndexSettings(
117120
builder.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), FORMATTER.format(start));
118121
builder.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), FORMATTER.format(end));
119122

120-
if (indexTemplateAndCreateRequestSettings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey()) == false
121-
&& combinedTemplateMappings.isEmpty() == false) {
122-
List<String> routingPaths = findRoutingPaths(
123+
if (combinedTemplateMappings.isEmpty() == false) {
124+
List<String> dimensions = findDimensionFields(
123125
indexName,
124126
indexTemplateAndCreateRequestSettings,
125127
combinedTemplateMappings
126128
);
127-
if (routingPaths.isEmpty() == false) {
128-
builder.putList(INDEX_ROUTING_PATH.getKey(), routingPaths);
129+
if (dimensions.isEmpty() == false) {
130+
// TODO handle the case when adding a dimension field to the mappings of an existing index
131+
// at the moment, the index.dimensions setting is only set when an index is created
132+
builder.putList(INDEX_DIMENSIONS.getKey(), dimensions);
129133
}
130134
}
131135
return builder.build();
@@ -137,15 +141,15 @@ public Settings getAdditionalIndexSettings(
137141
}
138142

139143
/**
140-
* Find fields in mapping that are of type keyword and time_series_dimension enabled.
144+
* Find fields in mapping that are time_series_dimension enabled.
141145
* Using MapperService here has an overhead, but allows the mappings from template to
142146
* be merged correctly and fetching the fields without manually parsing the mappings.
143-
*
147+
* <p>
144148
* Alternatively this method can instead parse mappings into map of maps and merge that and
145149
* iterate over all values to find the field that can serve as routing value. But this requires
146150
* mapping specific logic to exist here.
147151
*/
148-
private List<String> findRoutingPaths(String indexName, Settings allSettings, List<CompressedXContent> combinedTemplateMappings) {
152+
private List<String> findDimensionFields(String indexName, Settings allSettings, List<CompressedXContent> combinedTemplateMappings) {
149153
var tmpIndexMetadata = IndexMetadata.builder(indexName);
150154

151155
int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(allSettings);
@@ -170,16 +174,16 @@ private List<String> findRoutingPaths(String indexName, Settings allSettings, Li
170174
try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata.build())) {
171175
mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE);
172176
List<String> routingPaths = new ArrayList<>();
173-
for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) {
174-
extractPath(routingPaths, fieldMapper);
175-
}
176177
for (var objectMapper : mapperService.documentMapper().mappers().objectMappers().values()) {
177178
if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) {
178179
if (passThroughObjectMapper.containsDimensions()) {
179180
routingPaths.add(passThroughObjectMapper.fullPath() + ".*");
180181
}
181182
}
182183
}
184+
for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) {
185+
extractPath(routingPaths, fieldMapper);
186+
}
183187
for (var template : mapperService.getAllDynamicTemplates()) {
184188
if (template.pathMatch().isEmpty()) {
185189
continue;
@@ -214,12 +218,17 @@ private List<String> findRoutingPaths(String indexName, Settings allSettings, Li
214218
}
215219

216220
/**
217-
* Helper method that adds the name of the mapper to the provided list if it is a keyword dimension field.
221+
* Helper method that adds the name of the mapper to the provided list.
218222
*/
219223
private static void extractPath(List<String> routingPaths, Mapper mapper) {
220-
if (mapper instanceof KeywordFieldMapper keywordFieldMapper) {
221-
if (keywordFieldMapper.fieldType().isDimension()) {
222-
routingPaths.add(mapper.fullPath());
224+
if (mapper instanceof FieldMapper fieldMapper) {
225+
if (fieldMapper.fieldType().isDimension()) {
226+
String path = mapper.fullPath();
227+
// don't add if the path already matches via a wildcard pattern in the list
228+
// e.g. if "path.*" is already added, "path.foo" should not be added
229+
if (Regex.simpleMatch(routingPaths, path) == false) {
230+
routingPaths.add(path);
231+
}
223232
}
224233
}
225234
}

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java‎

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance;
3434
import static org.elasticsearch.common.settings.Settings.builder;
3535
import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.FORMATTER;
36-
import static org.hamcrest.Matchers.contains;
3736
import static org.hamcrest.Matchers.containsInAnyOrder;
3837
import static org.hamcrest.Matchers.equalTo;
3938

@@ -70,6 +69,18 @@ public void testGetAdditionalIndexSettings() throws Exception {
7069
"field3": {
7170
"type": "keyword",
7271
"time_series_dimension": true
72+
},
73+
"field4": {
74+
"type": "long",
75+
"time_series_dimension": true
76+
},
77+
"field5": {
78+
"type": "ip",
79+
"time_series_dimension": true
80+
},
81+
"field6": {
82+
"type": "boolean",
83+
"time_series_dimension": true
7384
}
7485
}
7586
}
@@ -91,7 +102,7 @@ public void testGetAdditionalIndexSettings() throws Exception {
91102
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
92103
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
93104
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
94-
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("field3"));
105+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field3", "field4", "field5", "field6"));
95106
}
96107

97108
public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throws Exception {
@@ -105,7 +116,7 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw
105116
"_doc": {
106117
"properties": {
107118
"field1": {
108-
"type": "keyword"
119+
"type": "keyword",
109120
"time_series_dimension": true
110121
},
111122
"field2": {
@@ -132,10 +143,11 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw
132143
// The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
133144
// (in production the index.mode setting is usually provided in an index or component template)
134145
result = builder().put(result).put("index.mode", "time_series").build();
135-
assertThat(result.size(), equalTo(3));
146+
assertThat(result.size(), equalTo(4));
136147
assertThat(result.get(IndexSettings.MODE.getKey()), equalTo("time_series"));
137148
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
138149
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
150+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field2", "field3"));
139151
}
140152

141153
public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception {
@@ -206,7 +218,7 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception {
206218
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
207219
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
208220
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
209-
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field1", "field3"));
221+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field3"));
210222
}
211223

212224
public void testGetAdditionalIndexSettingsNoMappings() {
@@ -461,7 +473,7 @@ public void testGenerateRoutingPathFromDynamicTemplate() throws Exception {
461473
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
462474
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
463475
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
464-
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
476+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
465477
}
466478

467479
public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntries() throws Exception {
@@ -502,10 +514,10 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri
502514
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
503515
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
504516
assertThat(
505-
IndexMetadata.INDEX_ROUTING_PATH.get(result),
517+
IndexMetadata.INDEX_DIMENSIONS.get(result),
506518
containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*")
507519
);
508-
List<String> routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result);
520+
List<String> routingPathList = IndexMetadata.INDEX_DIMENSIONS.get(result);
509521
assertEquals(3, routingPathList.size());
510522
}
511523

@@ -552,10 +564,10 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri
552564
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
553565
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
554566
assertThat(
555-
IndexMetadata.INDEX_ROUTING_PATH.get(result),
567+
IndexMetadata.INDEX_DIMENSIONS.get(result),
556568
containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*")
557569
);
558-
List<String> routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result);
570+
List<String> routingPathList = IndexMetadata.INDEX_DIMENSIONS.get(result);
559571
assertEquals(3, routingPathList.size());
560572
}
561573

@@ -605,7 +617,7 @@ public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch()
605617
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
606618
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
607619
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
608-
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
620+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
609621
}
610622

611623
public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() throws Exception {
@@ -652,8 +664,8 @@ public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() thro
652664
Settings result = generateTsdbSettings(mapping, now);
653665
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
654666
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
655-
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
656-
assertEquals(2, IndexMetadata.INDEX_ROUTING_PATH.get(result).size());
667+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
668+
assertEquals(2, IndexMetadata.INDEX_DIMENSIONS.get(result).size());
657669
}
658670

659671
public void testGenerateRoutingPathFromPassThroughObject() throws Exception {
@@ -665,7 +677,12 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception {
665677
"labels": {
666678
"type": "passthrough",
667679
"time_series_dimension": true,
668-
"priority": 2
680+
"priority": 2,
681+
"properties": {
682+
"label1": {
683+
"type": "keyword"
684+
}
685+
}
669686
},
670687
"metrics": {
671688
"type": "passthrough",
@@ -683,7 +700,7 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception {
683700
assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
684701
assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
685702
assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
686-
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*"));
703+
assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("labels.*"));
687704
}
688705

689706
private Settings generateTsdbSettings(String mapping, Instant now) throws IOException {

0 commit comments

Comments
 (0)