Skip to content

Commit ce626b3

Browse files
masseykemasseyke
authored
Taking additional settings providers into account for data stream effective settings (#137407)
1 parent 5ae8eac commit ce626b3

File tree

11 files changed

+654
-156
lines changed

11 files changed

+654
-156
lines changed

‎docs/changelog/137407.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
pr: 137407
2+
summary: Taking additional settings providers into account for data stream effective
3+
settings
4+
area: Data streams
5+
type: bug
6+
issues:
7+
- 137381

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.elasticsearch.cluster.block.ClusterBlockLevel;
2121
import org.elasticsearch.cluster.metadata.DataStream;
2222
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
23+
import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
2324
import org.elasticsearch.cluster.project.ProjectResolver;
2425
import org.elasticsearch.cluster.service.ClusterService;
2526
import org.elasticsearch.common.settings.Settings;
@@ -38,6 +39,7 @@ public class TransportGetDataStreamSettingsAction extends TransportLocalProjectM
3839
GetDataStreamSettingsAction.Response> {
3940
private final IndexNameExpressionResolver indexNameExpressionResolver;
4041
private final SettingsFilter settingsFilter;
42+
private final MetadataDataStreamsService metadataDataStreamsService;
4143

4244
@Inject
4345
public TransportGetDataStreamSettingsAction(
@@ -47,7 +49,8 @@ public TransportGetDataStreamSettingsAction(
4749
SettingsFilter settingsFilter,
4850
ActionFilters actionFilters,
4951
ProjectResolver projectResolver,
50-
IndexNameExpressionResolver indexNameExpressionResolver
52+
IndexNameExpressionResolver indexNameExpressionResolver,
53+
MetadataDataStreamsService metadataDataStreamsService
5154
) {
5255
super(
5356
GetSettingsAction.NAME,
@@ -59,6 +62,7 @@ public TransportGetDataStreamSettingsAction(
5962
);
6063
this.indexNameExpressionResolver = indexNameExpressionResolver;
6164
this.settingsFilter = settingsFilter;
65+
this.metadataDataStreamsService = metadataDataStreamsService;
6266
}
6367

6468
@Override
@@ -83,7 +87,9 @@ protected void localClusterStateOperation(
8387
for (String dataStreamName : dataStreamNames) {
8488
DataStream dataStream = dataStreamMap.get(dataStreamName);
8589
Settings settings = settingsFilter.filter(dataStream.getSettings());
86-
Settings effectiveSettings = settingsFilter.filter(dataStream.getEffectiveSettings(project.metadata()));
90+
Settings effectiveSettings = settingsFilter.filter(
91+
metadataDataStreamsService.getEffectiveSettings(project.metadata(), dataStream)
92+
);
8793
responseList.add(new GetDataStreamSettingsAction.DataStreamSettingsResponse(dataStreamName, settings, effectiveSettings));
8894
}
8995
listener.onResponse(new GetDataStreamSettingsAction.Response(responseList));

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
3333
import org.elasticsearch.cluster.metadata.IndexMetadata;
3434
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
35+
import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
3536
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
3637
import org.elasticsearch.cluster.metadata.ProjectMetadata;
3738
import org.elasticsearch.cluster.project.ProjectResolver;
@@ -80,6 +81,7 @@ public class TransportGetDataStreamsAction extends TransportLocalProjectMetadata
8081
private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
8182
private final IndexSettingProviders indexSettingProviders;
8283
private final Client client;
84+
private final MetadataDataStreamsService metadataDataStreamsService;
8385

8486
/**
8587
* NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until
@@ -99,7 +101,8 @@ public TransportGetDataStreamsAction(
99101
DataStreamGlobalRetentionSettings globalRetentionSettings,
100102
DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
101103
IndexSettingProviders indexSettingProviders,
102-
Client client
104+
Client client,
105+
MetadataDataStreamsService metadataDataStreamsService
103106
) {
104107
super(
105108
GetDataStreamAction.NAME,
@@ -116,6 +119,7 @@ public TransportGetDataStreamsAction(
116119
this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
117120
this.indexSettingProviders = indexSettingProviders;
118121
this.client = new OriginSettingClient(client, "stack");
122+
this.metadataDataStreamsService = metadataDataStreamsService;
119123

120124
transportService.registerRequestHandler(
121125
actionName,
@@ -159,7 +163,8 @@ public void onResponse(DataStreamsStatsAction.Response response) {
159163
globalRetentionSettings,
160164
dataStreamFailureStoreSettings,
161165
indexSettingProviders,
162-
maxTimestamps
166+
maxTimestamps,
167+
metadataDataStreamsService
163168
)
164169
);
165170
}
@@ -180,7 +185,8 @@ public void onFailure(Exception e) {
180185
globalRetentionSettings,
181186
dataStreamFailureStoreSettings,
182187
indexSettingProviders,
183-
null
188+
null,
189+
metadataDataStreamsService
184190
)
185191
);
186192
}
@@ -230,7 +236,8 @@ static GetDataStreamAction.Response innerOperation(
230236
DataStreamGlobalRetentionSettings globalRetentionSettings,
231237
DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
232238
IndexSettingProviders indexSettingProviders,
233-
@Nullable Map<String, Long> maxTimestamps
239+
@Nullable Map<String, Long> maxTimestamps,
240+
MetadataDataStreamsService metadataDataStreamsService
234241
) {
235242
List<DataStream> dataStreams = getDataStreams(state.metadata(), indexNameExpressionResolver, request);
236243
List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = new ArrayList<>(dataStreams.size());
@@ -265,7 +272,12 @@ static GetDataStreamAction.Response innerOperation(
265272
} else {
266273
indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
267274
if (indexTemplate != null) {
268-
Settings settings = dataStream.getEffectiveSettings(state.metadata());
275+
final Settings settings;
276+
try {
277+
settings = metadataDataStreamsService.getEffectiveSettings(state.metadata(), dataStream);
278+
} catch (IOException e) {
279+
throw new RuntimeException("Failed to get effective settings for data stream: " + dataStream.getName(), e);
280+
}
269281
ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
270282
if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
271283
try {
@@ -277,7 +289,7 @@ static GetDataStreamAction.Response innerOperation(
277289
dataStream.getEffectiveIndexTemplate(state.metadata())
278290
);
279291
} catch (IOException e) {
280-
throw new RuntimeException(e);
292+
throw new RuntimeException("Failed to determine indexMode for data stream: " + dataStream.getName(), e);
281293
}
282294
}
283295
indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.elasticsearch.threadpool.ThreadPool;
4141
import org.elasticsearch.transport.TransportService;
4242

43+
import java.io.IOException;
4344
import java.util.ArrayList;
4445
import java.util.HashMap;
4546
import java.util.HashSet;
@@ -231,7 +232,13 @@ private void updateSettingsOnIndices(
231232
List<String> appliedToDataStreamOnly = new ArrayList<>();
232233
List<String> appliedToDataStreamAndWriteIndexOnly = new ArrayList<>();
233234
List<String> appliedToDataStreamAndBackingIndices = new ArrayList<>();
234-
Settings effectiveSettings = dataStream.getEffectiveSettings(projectResolver.getProjectMetadata(clusterService.state()));
235+
ProjectMetadata projectMetadata = projectResolver.getProjectMetadata(clusterService.state());
236+
final Settings effectiveSettings;
237+
try {
238+
effectiveSettings = metadataDataStreamsService.getEffectiveSettings(projectMetadata, dataStream);
239+
} catch (IOException e) {
240+
throw new RuntimeException(e);
241+
}
235242
for (String settingName : requestSettings.keySet()) {
236243
if (APPLY_TO_WRITE_INDEX.contains(settingName)) {
237244
settingsToApplyToWriteIndex.put(settingName, effectiveSettings.get(settingName));

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java‎

Lines changed: 59 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,10 @@
2020
import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
2121
import org.elasticsearch.cluster.metadata.IndexMetadata;
2222
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
23+
import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
2324
import org.elasticsearch.cluster.metadata.ProjectMetadata;
2425
import org.elasticsearch.cluster.metadata.Template;
26+
import org.elasticsearch.cluster.service.ClusterService;
2527
import org.elasticsearch.common.settings.ClusterSettings;
2628
import org.elasticsearch.common.settings.Settings;
2729
import org.elasticsearch.core.TimeValue;
@@ -31,10 +33,17 @@
3133
import org.elasticsearch.index.IndexNotFoundException;
3234
import org.elasticsearch.index.IndexSettingProviders;
3335
import org.elasticsearch.index.IndexSettings;
36+
import org.elasticsearch.indices.IndicesService;
3437
import org.elasticsearch.indices.SystemIndices;
3538
import org.elasticsearch.indices.TestIndexNameExpressionResolver;
39+
import org.elasticsearch.test.ClusterServiceUtils;
3640
import org.elasticsearch.test.ESTestCase;
41+
import org.elasticsearch.threadpool.TestThreadPool;
42+
import org.elasticsearch.threadpool.ThreadPool;
43+
import org.junit.After;
44+
import org.junit.Before;
3745

46+
import java.io.IOException;
3847
import java.time.Instant;
3948
import java.time.temporal.ChronoUnit;
4049
import java.util.ArrayList;
@@ -55,6 +64,7 @@
5564
import static org.hamcrest.Matchers.hasSize;
5665
import static org.hamcrest.Matchers.is;
5766
import static org.hamcrest.Matchers.nullValue;
67+
import static org.mockito.Mockito.mock;
5868

5969
public class TransportGetDataStreamsActionTests extends ESTestCase {
6070

@@ -66,6 +76,29 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
6676
private final DataStreamFailureStoreSettings emptyDataStreamFailureStoreSettings = DataStreamFailureStoreSettings.create(
6777
ClusterSettings.createBuiltInClusterSettings()
6878
);
79+
private ThreadPool testThreadPool;
80+
private MetadataDataStreamsService metadataDataStreamsService;
81+
82+
@Before
83+
public void setup() throws IOException {
84+
testThreadPool = new TestThreadPool(getTestName());
85+
ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool);
86+
IndicesService indicesService = mock(IndicesService.class);
87+
metadataDataStreamsService = new MetadataDataStreamsService(
88+
clusterService,
89+
indicesService,
90+
DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()),
91+
IndexSettingProviders.EMPTY
92+
);
93+
}
94+
95+
@After
96+
public void tearDown() throws Exception {
97+
super.tearDown();
98+
if (testThreadPool != null) {
99+
testThreadPool.shutdown();
100+
}
101+
}
69102

70103
public void testGetDataStream() {
71104
final String dataStreamName = "my-data-stream";
@@ -186,7 +219,8 @@ public void testGetTimeSeriesDataStream() {
186219
dataStreamGlobalRetentionSettings,
187220
emptyDataStreamFailureStoreSettings,
188221
new IndexSettingProviders(Set.of()),
189-
null
222+
null,
223+
metadataDataStreamsService
190224
);
191225
assertThat(
192226
response.getDataStreams(),
@@ -219,7 +253,8 @@ public void testGetTimeSeriesDataStream() {
219253
dataStreamGlobalRetentionSettings,
220254
emptyDataStreamFailureStoreSettings,
221255
new IndexSettingProviders(Set.of()),
222-
null
256+
null,
257+
metadataDataStreamsService
223258
);
224259
assertThat(
225260
response.getDataStreams(),
@@ -273,7 +308,8 @@ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() {
273308
dataStreamGlobalRetentionSettings,
274309
emptyDataStreamFailureStoreSettings,
275310
new IndexSettingProviders(Set.of()),
276-
null
311+
null,
312+
metadataDataStreamsService
277313
);
278314
assertThat(
279315
response.getDataStreams(),
@@ -313,7 +349,8 @@ public void testGetTimeSeriesMixedDataStream() {
313349
dataStreamGlobalRetentionSettings,
314350
emptyDataStreamFailureStoreSettings,
315351
new IndexSettingProviders(Set.of()),
316-
null
352+
null,
353+
metadataDataStreamsService
317354
);
318355

319356
var name1 = getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli());
@@ -349,7 +386,8 @@ public void testPassingGlobalRetention() {
349386
dataStreamGlobalRetentionSettings,
350387
emptyDataStreamFailureStoreSettings,
351388
new IndexSettingProviders(Set.of()),
352-
null
389+
null,
390+
metadataDataStreamsService
353391
);
354392
assertThat(response.getDataGlobalRetention(), nullValue());
355393
DataStreamGlobalRetention dataGlobalRetention = new DataStreamGlobalRetention(
@@ -376,7 +414,8 @@ public void testPassingGlobalRetention() {
376414
withGlobalRetentionSettings,
377415
emptyDataStreamFailureStoreSettings,
378416
new IndexSettingProviders(Set.of()),
379-
null
417+
null,
418+
metadataDataStreamsService
380419
);
381420
assertThat(response.getDataGlobalRetention(), equalTo(dataGlobalRetention));
382421
// We used the default failures retention here which is greater than the max
@@ -404,7 +443,8 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_disabled() {
404443
dataStreamGlobalRetentionSettings,
405444
emptyDataStreamFailureStoreSettings,
406445
new IndexSettingProviders(Set.of()),
407-
null
446+
null,
447+
metadataDataStreamsService
408448
);
409449
assertThat(response.getDataStreams(), hasSize(1));
410450
assertThat(response.getDataStreams().getFirst().isFailureStoreEffectivelyEnabled(), is(false));
@@ -431,7 +471,8 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledExplicitly() {
431471
dataStreamGlobalRetentionSettings,
432472
emptyDataStreamFailureStoreSettings,
433473
new IndexSettingProviders(Set.of()),
434-
null
474+
null,
475+
metadataDataStreamsService
435476
);
436477
assertThat(response.getDataStreams(), hasSize(1));
437478
assertThat(response.getDataStreams().getFirst().isFailureStoreEffectivelyEnabled(), is(true));
@@ -464,7 +505,8 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledByClusterSetti
464505
)
465506
),
466507
new IndexSettingProviders(Set.of()),
467-
null
508+
null,
509+
metadataDataStreamsService
468510
);
469511
assertThat(response.getDataStreams(), hasSize(1));
470512
assertThat(response.getDataStreams().getFirst().isFailureStoreEffectivelyEnabled(), is(true));
@@ -491,7 +533,8 @@ public void testProvidersAffectMode() {
491533
dataStreamGlobalRetentionSettings,
492534
emptyDataStreamFailureStoreSettings,
493535
IndexSettingProviders.of((additionalSettings) -> additionalSettings.put("index.mode", IndexMode.LOOKUP)),
494-
null
536+
null,
537+
metadataDataStreamsService
495538
);
496539
assertThat(response.getDataStreams().getFirst().getIndexModeName(), equalTo("lookup"));
497540
assertThat(
@@ -531,7 +574,8 @@ public void testGetEffectiveSettingsTemplateOnlySettings() {
531574
dataStreamGlobalRetentionSettings,
532575
emptyDataStreamFailureStoreSettings,
533576
new IndexSettingProviders(Set.of()),
534-
null
577+
null,
578+
metadataDataStreamsService
535579
);
536580
assertNotNull(response.getDataStreams());
537581
assertThat(response.getDataStreams().size(), equalTo(1));
@@ -563,7 +607,8 @@ public void testGetEffectiveSettingsComponentTemplateOnlySettings() {
563607
dataStreamGlobalRetentionSettings,
564608
emptyDataStreamFailureStoreSettings,
565609
new IndexSettingProviders(Set.of()),
566-
null
610+
null,
611+
metadataDataStreamsService
567612
);
568613
assertNotNull(response.getDataStreams());
569614
assertThat(response.getDataStreams().size(), equalTo(1));
@@ -601,7 +646,8 @@ public void testGetEffectiveSettings() {
601646
dataStreamGlobalRetentionSettings,
602647
emptyDataStreamFailureStoreSettings,
603648
new IndexSettingProviders(Set.of()),
604-
null
649+
null,
650+
metadataDataStreamsService
605651
);
606652
assertNotNull(response.getDataStreams());
607653
assertThat(response.getDataStreams().size(), equalTo(1));

0 commit comments

Comments
 (0)