Skip to content

Commit ad77f8b

Browse files
masseykejoegallo
masseyke
authored and
joegallo
committed
Using DataStream::getEffectiveSettings (#127282)
1 parent a3fe59f commit ad77f8b

File tree

8 files changed

+273
-29
lines changed

8 files changed

+273
-29
lines changed

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -238,15 +238,15 @@ static GetDataStreamAction.Response innerOperation(
238238
} else {
239239
indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
240240
if (indexTemplate != null) {
241-
Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
241+
Settings settings = dataStream.getEffectiveSettings(state.metadata());
242242
ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
243243
if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
244244
indexMode = resolveMode(
245245
state,
246246
indexSettingProviders,
247247
dataStream,
248248
settings,
249-
state.metadata().templatesV2().get(indexTemplate)
249+
dataStream.getEffectiveIndexTemplate(state.metadata())
250250
);
251251
}
252252
indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java‎

Lines changed: 119 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,16 @@
1111
import org.elasticsearch.action.datastreams.GetDataStreamAction;
1212
import org.elasticsearch.cluster.ClusterName;
1313
import org.elasticsearch.cluster.ClusterState;
14+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
1415
import org.elasticsearch.cluster.metadata.DataStream;
1516
import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
1617
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
1718
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
1819
import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
20+
import org.elasticsearch.cluster.metadata.IndexMetadata;
1921
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
2022
import org.elasticsearch.cluster.metadata.Metadata;
23+
import org.elasticsearch.cluster.metadata.Template;
2124
import org.elasticsearch.common.settings.ClusterSettings;
2225
import org.elasticsearch.common.settings.Settings;
2326
import org.elasticsearch.core.TimeValue;
@@ -26,15 +29,20 @@
2629
import org.elasticsearch.index.IndexMode;
2730
import org.elasticsearch.index.IndexNotFoundException;
2831
import org.elasticsearch.index.IndexSettingProviders;
32+
import org.elasticsearch.index.IndexSettings;
2933
import org.elasticsearch.indices.SystemIndices;
3034
import org.elasticsearch.indices.TestIndexNameExpressionResolver;
3135
import org.elasticsearch.test.ESTestCase;
3236

3337
import java.time.Instant;
3438
import java.time.temporal.ChronoUnit;
39+
import java.util.ArrayList;
3540
import java.util.List;
3641
import java.util.Set;
42+
import java.util.stream.Collectors;
3743

44+
import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName;
45+
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createIndexMetadata;
3846
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.getClusterStateWithDataStreams;
3947
import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch;
4048
import static org.elasticsearch.test.LambdaMatchers.transformedMatch;
@@ -312,9 +320,9 @@ public void testGetTimeSeriesMixedDataStream() {
312320
null
313321
);
314322

315-
var name1 = DataStream.getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli());
316-
var name2 = DataStream.getDefaultBackingIndexName("ds-1", 2, instant.toEpochMilli());
317-
var name3 = DataStream.getDefaultBackingIndexName("ds-1", 3, twoHoursAgo.toEpochMilli());
323+
var name1 = getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli());
324+
var name2 = getDefaultBackingIndexName("ds-1", 2, instant.toEpochMilli());
325+
var name3 = getDefaultBackingIndexName("ds-1", 3, twoHoursAgo.toEpochMilli());
318326
assertThat(
319327
response.getDataStreams(),
320328
contains(
@@ -534,4 +542,112 @@ public void testProvidersAffectMode() {
534542
equalTo("standard")
535543
);
536544
}
545+
546+
public void testGetEffectiveSettingsTemplateOnlySettings() {
547+
// Set a lifecycle only in the template, and make sure that is in the response:
548+
GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
549+
final String templatePolicy = "templatePolicy";
550+
final String templateIndexMode = IndexMode.LOOKUP.getName();
551+
final String dataStreamPolicy = "dataStreamPolicy";
552+
final String dataStreamIndexMode = IndexMode.LOGSDB.getName();
553+
554+
ClusterState state = getClusterStateWithDataStreamWithSettings(
555+
Settings.builder()
556+
.put(IndexMetadata.LIFECYCLE_NAME, templatePolicy)
557+
.put(IndexSettings.MODE.getKey(), templateIndexMode)
558+
.build(),
559+
Settings.EMPTY
560+
);
561+
562+
GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation(
563+
state,
564+
req,
565+
resolver,
566+
systemIndices,
567+
ClusterSettings.createBuiltInClusterSettings(),
568+
dataStreamGlobalRetentionSettings,
569+
emptyDataStreamFailureStoreSettings,
570+
new IndexSettingProviders(Set.of()),
571+
null
572+
);
573+
assertNotNull(response.getDataStreams());
574+
assertThat(response.getDataStreams().size(), equalTo(1));
575+
assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(templatePolicy));
576+
assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(templateIndexMode));
577+
}
578+
579+
public void testGetEffectiveSettings() {
580+
GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
581+
final String templatePolicy = "templatePolicy";
582+
final String templateIndexMode = IndexMode.LOOKUP.getName();
583+
final String dataStreamPolicy = "dataStreamPolicy";
584+
final String dataStreamIndexMode = IndexMode.LOGSDB.getName();
585+
// Now set a lifecycle in both the template and the data stream, and make sure the response has the data stream one:
586+
ClusterState state = getClusterStateWithDataStreamWithSettings(
587+
Settings.builder()
588+
.put(IndexMetadata.LIFECYCLE_NAME, templatePolicy)
589+
.put(IndexSettings.MODE.getKey(), templateIndexMode)
590+
.build(),
591+
Settings.builder()
592+
.put(IndexMetadata.LIFECYCLE_NAME, dataStreamPolicy)
593+
.put(IndexSettings.MODE.getKey(), dataStreamIndexMode)
594+
.build()
595+
);
596+
GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation(
597+
state,
598+
req,
599+
resolver,
600+
systemIndices,
601+
ClusterSettings.createBuiltInClusterSettings(),
602+
dataStreamGlobalRetentionSettings,
603+
emptyDataStreamFailureStoreSettings,
604+
new IndexSettingProviders(Set.of()),
605+
null
606+
);
607+
assertNotNull(response.getDataStreams());
608+
assertThat(response.getDataStreams().size(), equalTo(1));
609+
assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(dataStreamPolicy));
610+
assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(dataStreamIndexMode));
611+
}
612+
613+
private static ClusterState getClusterStateWithDataStreamWithSettings(Settings templateSettings, Settings dataStreamSettings) {
614+
String dataStreamName = "data-stream-1";
615+
int numberOfBackingIndices = randomIntBetween(1, 5);
616+
long currentTime = System.currentTimeMillis();
617+
int replicas = 0;
618+
boolean replicated = false;
619+
Metadata.Builder builder = Metadata.builder();
620+
builder.put(
621+
"template_1",
622+
ComposableIndexTemplate.builder()
623+
.indexPatterns(List.of("*"))
624+
.template(Template.builder().settings(templateSettings))
625+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
626+
.build()
627+
);
628+
629+
List<IndexMetadata> backingIndices = new ArrayList<>();
630+
for (int backingIndexNumber = 1; backingIndexNumber <= numberOfBackingIndices; backingIndexNumber++) {
631+
backingIndices.add(
632+
createIndexMetadata(
633+
getDefaultBackingIndexName(dataStreamName, backingIndexNumber, currentTime),
634+
true,
635+
templateSettings,
636+
replicas
637+
)
638+
);
639+
}
640+
List<IndexMetadata> allIndices = new ArrayList<>(backingIndices);
641+
642+
DataStream ds = DataStream.builder(
643+
dataStreamName,
644+
backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList())
645+
).setGeneration(numberOfBackingIndices).setSettings(dataStreamSettings).setReplicated(replicated).build();
646+
builder.put(ds);
647+
648+
for (IndexMetadata index : allIndices) {
649+
builder.put(index, false);
650+
}
651+
return ClusterState.builder(new ClusterName("_name")).metadata(builder.build()).build();
652+
}
537653
}

‎server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.cluster.metadata;
11+
12+
import org.elasticsearch.action.ActionListener;
13+
import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
14+
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
15+
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
16+
import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse;
17+
import org.elasticsearch.common.settings.Settings;
18+
import org.elasticsearch.core.TimeValue;
19+
import org.elasticsearch.test.ESIntegTestCase;
20+
21+
import java.util.Locale;
22+
import java.util.concurrent.CountDownLatch;
23+
import java.util.concurrent.TimeUnit;
24+
25+
import static org.hamcrest.Matchers.equalTo;
26+
27+
public class MetadataCreateIndexServiceIT extends ESIntegTestCase {
28+
29+
public void testRequestTemplateIsRespected() throws InterruptedException {
30+
/*
31+
* This test passes a template in the CreateIndexClusterStateUpdateRequest, and makes sure that the settings from that template
32+
* are used when creating the index.
33+
*/
34+
MetadataCreateIndexService metadataCreateIndexService = internalCluster().getCurrentMasterNodeInstance(
35+
MetadataCreateIndexService.class
36+
);
37+
final String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
38+
final int numberOfReplicas = randomIntBetween(1, 7);
39+
CreateIndexClusterStateUpdateRequest request = new CreateIndexClusterStateUpdateRequest(
40+
"testRequestTemplateIsRespected",
41+
indexName,
42+
randomAlphaOfLength(20)
43+
);
44+
request.setMatchingTemplate(
45+
ComposableIndexTemplate.builder()
46+
.template(Template.builder().settings(Settings.builder().put("index.number_of_replicas", numberOfReplicas)))
47+
.build()
48+
);
49+
final CountDownLatch listenerCalledLatch = new CountDownLatch(1);
50+
ActionListener<ShardsAcknowledgedResponse> listener = new ActionListener<>() {
51+
@Override
52+
public void onResponse(ShardsAcknowledgedResponse shardsAcknowledgedResponse) {
53+
listenerCalledLatch.countDown();
54+
}
55+
56+
@Override
57+
public void onFailure(Exception e) {
58+
logger.error(e);
59+
listenerCalledLatch.countDown();
60+
}
61+
};
62+
63+
metadataCreateIndexService.createIndex(
64+
TimeValue.THIRTY_SECONDS,
65+
TimeValue.THIRTY_SECONDS,
66+
TimeValue.THIRTY_SECONDS,
67+
request,
68+
listener
69+
);
70+
listenerCalledLatch.await(10, TimeUnit.SECONDS);
71+
GetIndexResponse response = admin().indices().getIndex(new GetIndexRequest().indices(indexName)).actionGet();
72+
Settings settings = response.getSettings().get(indexName);
73+
assertThat(settings.get("index.number_of_replicas"), equalTo(Integer.toString(numberOfReplicas)));
74+
}
75+
}

‎server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@
6363

6464
import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.ALIAS;
6565
import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.DATA_STREAM;
66-
import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream;
6766
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates;
6867
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template;
6968
import static org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener.rerouteCompletionIsNotRequired;
@@ -312,7 +311,7 @@ private RolloverResult rolloverDataStream(
312311
final SystemDataStreamDescriptor systemDataStreamDescriptor;
313312
if (dataStream.isSystem() == false) {
314313
systemDataStreamDescriptor = null;
315-
templateV2 = lookupTemplateForDataStream(dataStreamName, metadata);
314+
templateV2 = dataStream.getEffectiveIndexTemplate(currentState.metadata());
316315
} else {
317316
systemDataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStreamName);
318317
if (systemDataStreamDescriptor == null) {

‎server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -261,8 +261,8 @@ public static Template resolveTemplate(
261261
: indexName;
262262
List<CompressedXContent> mappings = MetadataCreateIndexService.collectV2Mappings(
263263
null, // empty request mapping as the user can't specify any explicit mappings via the simulate api
264-
simulatedState,
265-
matchingTemplate,
264+
simulatedState.metadata(),
265+
template,
266266
xContentRegistry,
267267
simulatedIndexName
268268
);

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -376,7 +376,12 @@ public ComposableIndexTemplate getEffectiveIndexTemplate(Metadata metadata) {
376376

377377
public Settings getEffectiveSettings(Metadata metadata) {
378378
ComposableIndexTemplate template = getMatchingIndexTemplate(metadata);
379-
Settings templateSettings = template.template() == null ? Settings.EMPTY : template.template().settings();
379+
final Settings templateSettings;
380+
if (template.template() == null || template.template().settings() == null) {
381+
templateSettings = Settings.EMPTY;
382+
} else {
383+
templateSettings = template.template().settings();
384+
}
380385
return templateSettings.merge(settings);
381386
}
382387

0 commit comments

Comments
 (0)