Skip to content

Commit ed79e17

Browse files
dakronedakrone
committed
Add index mode to get data stream API
This commit adds the `index_mode` for both the data stream and each backing index to the output of `GET /_data_stream`. An example looks like: ``` { "data_streams" : [ { "name" : "foo-things", "indices" : [ { "index_name" : ".ds-foo-things-2025.02.13-000001", ... "index_mode" : "standard" } ], ... "index_mode" : "standard" }, { "name" : "logs-foo-bar", "indices" : [ { "index_name" : ".ds-logs-foo-bar-2025.02.13-000001", ... "index_mode" : "logsdb" }, { "index_name" : ".ds-logs-foo-bar-2025.02.13-000002", ... "index_mode" : "logsdb" } ], ... "index_mode" : "logsdb", } ] } ```
1 parent bfaffe9 commit ed79e17

File tree

8 files changed

+147
-25
lines changed

8 files changed

+147
-25
lines changed

‎docs/reference/indices/get-data-stream.asciidoc‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -304,21 +304,24 @@ The API returns the following response:
304304
"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
305305
"prefer_ilm": true,
306306
"ilm_policy": "my-lifecycle-policy",
307-
"managed_by": "Index Lifecycle Management"
307+
"managed_by": "Index Lifecycle Management",
308+
"index_mode": "standard"
308309
},
309310
{
310311
"index_name": ".ds-my-data-stream-2099.03.08-000002",
311312
"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
312313
"prefer_ilm": true,
313314
"ilm_policy": "my-lifecycle-policy",
314-
"managed_by": "Index Lifecycle Management"
315+
"managed_by": "Index Lifecycle Management",
316+
"index_mode": "standard"
315317
}
316318
],
317319
"generation": 2,
318320
"_meta": {
319321
"my-meta-field": "foo"
320322
},
321323
"status": "GREEN",
324+
"index_mode": "standard",
322325
"next_generation_managed_by": "Index Lifecycle Management",
323326
"prefer_ilm": true,
324327
"template": "my-index-template",
@@ -340,14 +343,16 @@ The API returns the following response:
340343
"index_uuid": "3liBu2SYS5axasRt6fUIpA",
341344
"prefer_ilm": true,
342345
"ilm_policy": "my-lifecycle-policy",
343-
"managed_by": "Index Lifecycle Management"
346+
"managed_by": "Index Lifecycle Management",
347+
"index_mode": "standard"
344348
}
345349
],
346350
"generation": 1,
347351
"_meta": {
348352
"my-meta-field": "foo"
349353
},
350354
"status": "YELLOW",
355+
"index_mode": "standard",
351356
"next_generation_managed_by": "Index Lifecycle Management",
352357
"prefer_ilm": true,
353358
"template": "my-index-template",

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.elasticsearch.cluster.block.ClusterBlockException;
2525
import org.elasticsearch.cluster.block.ClusterBlockLevel;
2626
import org.elasticsearch.cluster.health.ClusterStateHealth;
27+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
2728
import org.elasticsearch.cluster.metadata.DataStream;
2829
import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
2930
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
@@ -39,6 +40,9 @@
3940
import org.elasticsearch.core.Tuple;
4041
import org.elasticsearch.index.Index;
4142
import org.elasticsearch.index.IndexMode;
43+
import org.elasticsearch.index.IndexSettingProvider;
44+
import org.elasticsearch.index.IndexSettingProviders;
45+
import org.elasticsearch.index.IndexSettings;
4246
import org.elasticsearch.indices.SystemDataStreamDescriptor;
4347
import org.elasticsearch.indices.SystemIndices;
4448
import org.elasticsearch.injection.guice.Inject;
@@ -52,6 +56,7 @@
5256
import java.util.Comparator;
5357
import java.util.HashMap;
5458
import java.util.List;
59+
import java.util.Locale;
5560
import java.util.Map;
5661
import java.util.stream.Collectors;
5762

@@ -67,6 +72,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadAction
6772
private final ClusterSettings clusterSettings;
6873
private final DataStreamGlobalRetentionSettings globalRetentionSettings;
6974
private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
75+
private final IndexSettingProviders indexSettingProviders;
7076
private final Client client;
7177

7278
@Inject
@@ -79,6 +85,7 @@ public TransportGetDataStreamsAction(
7985
SystemIndices systemIndices,
8086
DataStreamGlobalRetentionSettings globalRetentionSettings,
8187
DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
88+
IndexSettingProviders indexSettingProviders,
8289
Client client
8390
) {
8491
super(
@@ -96,6 +103,7 @@ public TransportGetDataStreamsAction(
96103
this.globalRetentionSettings = globalRetentionSettings;
97104
clusterSettings = clusterService.getClusterSettings();
98105
this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
106+
this.indexSettingProviders = indexSettingProviders;
99107
this.client = new OriginSettingClient(client, "stack");
100108
}
101109

@@ -128,6 +136,7 @@ public void onResponse(DataStreamsStatsAction.Response response) {
128136
clusterSettings,
129137
globalRetentionSettings,
130138
dataStreamFailureStoreSettings,
139+
indexSettingProviders,
131140
maxTimestamps
132141
)
133142
);
@@ -148,12 +157,43 @@ public void onFailure(Exception e) {
148157
clusterSettings,
149158
globalRetentionSettings,
150159
dataStreamFailureStoreSettings,
160+
indexSettingProviders,
151161
null
152162
)
153163
);
154164
}
155165
}
156166

167+
/**
168+
* Resolves the index mode ("index.mode" setting) for the given data stream, from the template or additional setting providers
169+
*/
170+
@Nullable
171+
static IndexMode resolveMode(
172+
ClusterState state,
173+
IndexSettingProviders indexSettingProviders,
174+
DataStream dataStream,
175+
Settings settings,
176+
ComposableIndexTemplate indexTemplate
177+
) {
178+
IndexMode indexMode = state.metadata().retrieveIndexModeFromTemplate(indexTemplate);
179+
for (IndexSettingProvider provider : indexSettingProviders.getIndexSettingProviders()) {
180+
Settings addlSettinsg = provider.getAdditionalIndexSettings(
181+
MetadataIndexTemplateService.VALIDATE_INDEX_NAME,
182+
dataStream.getName(),
183+
indexMode,
184+
state.metadata(),
185+
Instant.now(),
186+
settings,
187+
List.of()
188+
);
189+
var rawMode = addlSettinsg.get(IndexSettings.MODE.getKey());
190+
if (rawMode != null) {
191+
indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT));
192+
}
193+
}
194+
return indexMode;
195+
}
196+
157197
static GetDataStreamAction.Response innerOperation(
158198
ClusterState state,
159199
GetDataStreamAction.Request request,
@@ -162,6 +202,7 @@ static GetDataStreamAction.Response innerOperation(
162202
ClusterSettings clusterSettings,
163203
DataStreamGlobalRetentionSettings globalRetentionSettings,
164204
DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
205+
IndexSettingProviders indexSettingProviders,
165206
@Nullable Map<String, Long> maxTimestamps
166207
) {
167208
List<DataStream> dataStreams = getDataStreams(state, indexNameExpressionResolver, request);
@@ -174,6 +215,7 @@ static GetDataStreamAction.Response innerOperation(
174215
final String indexTemplate;
175216
boolean indexTemplatePreferIlmValue = true;
176217
String ilmPolicyName = null;
218+
IndexMode indexMode = dataStream.getIndexMode();
177219
if (dataStream.isSystem()) {
178220
SystemDataStreamDescriptor dataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStream.getName());
179221
indexTemplate = dataStreamDescriptor != null ? dataStreamDescriptor.getDataStreamName() : null;
@@ -183,13 +225,31 @@ static GetDataStreamAction.Response innerOperation(
183225
dataStreamDescriptor.getComponentTemplates()
184226
);
185227
ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
228+
if (indexMode == null) {
229+
indexMode = resolveMode(
230+
state,
231+
indexSettingProviders,
232+
dataStream,
233+
settings,
234+
dataStreamDescriptor.getComposableIndexTemplate()
235+
);
236+
}
186237
indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);
187238
}
188239
} else {
189240
indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
190241
if (indexTemplate != null) {
191242
Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
192243
ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
244+
if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
245+
indexMode = resolveMode(
246+
state,
247+
indexSettingProviders,
248+
dataStream,
249+
settings,
250+
state.metadata().templatesV2().get(indexTemplate)
251+
);
252+
}
193253
indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);
194254
} else {
195255
LOGGER.warn(
@@ -281,7 +341,9 @@ public int compareTo(IndexInfo o) {
281341
timeSeries,
282342
backingIndicesSettingsValues,
283343
indexTemplatePreferIlmValue,
284-
maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName())
344+
maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName()),
345+
// Default to standard mode if not specified; should we set this to "unset" or "unspecified" instead?
346+
indexMode == null ? IndexMode.STANDARD.getName() : indexMode.getName()
285347
)
286348
);
287349
}
@@ -310,7 +372,11 @@ private static void collectIndexSettingsValues(
310372
} else {
311373
managedBy = ManagedBy.UNMANAGED;
312374
}
313-
backingIndicesSettingsValues.put(index, new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy));
375+
String indexMode = IndexSettings.MODE.get(indexMetadata.getSettings()).getName();
376+
backingIndicesSettingsValues.put(
377+
index,
378+
new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy, indexMode)
379+
);
314380
}
315381
}
316382

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -91,13 +91,13 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
9191
String ilmPolicyName = "rollover-30days";
9292
Map<Index, Response.IndexProperties> indexSettingsValues = Map.of(
9393
firstGenerationIndex,
94-
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM),
94+
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null),
9595
secondGenerationIndex,
96-
new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE),
96+
new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE, null),
9797
writeIndex,
98-
new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE),
98+
new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE, null),
9999
failureStoreIndex,
100-
new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE)
100+
new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE, null)
101101
);
102102

103103
Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo(
@@ -109,6 +109,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
109109
null,
110110
indexSettingsValues,
111111
false,
112+
null,
112113
null
113114
);
114115
Response response = new Response(List.of(dataStreamInfo));
@@ -195,13 +196,13 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
195196
String ilmPolicyName = "rollover-30days";
196197
Map<Index, Response.IndexProperties> indexSettingsValues = Map.of(
197198
firstGenerationIndex,
198-
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM),
199+
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null),
199200
secondGenerationIndex,
200-
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM),
201+
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null),
201202
writeIndex,
202-
new Response.IndexProperties(false, null, ManagedBy.UNMANAGED),
203+
new Response.IndexProperties(false, null, ManagedBy.UNMANAGED, null),
203204
failureStoreIndex,
204-
new Response.IndexProperties(false, null, ManagedBy.UNMANAGED)
205+
new Response.IndexProperties(false, null, ManagedBy.UNMANAGED, null)
205206
);
206207

207208
Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo(
@@ -213,6 +214,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
213214
null,
214215
indexSettingsValues,
215216
false,
217+
null,
216218
null
217219
);
218220
Response response = new Response(List.of(dataStreamInfo));
@@ -309,7 +311,8 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance)
309311
new Response.IndexProperties(
310312
randomBoolean(),
311313
randomAlphaOfLengthBetween(50, 100),
312-
randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE
314+
randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE,
315+
null
313316
)
314317
)
315318
);
@@ -328,7 +331,8 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance)
328331
timeSeries,
329332
indexSettings,
330333
templatePreferIlm,
331-
maximumTimestamp
334+
maximumTimestamp,
335+
null
332336
);
333337
}
334338

@@ -349,7 +353,8 @@ private Map<Index, Response.IndexProperties> generateRandomIndexSettingsValues()
349353
new Response.IndexProperties(
350354
randomBoolean(),
351355
randomAlphaOfLengthBetween(50, 100),
352-
randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE
356+
randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE,
357+
randomBoolean() ? randomFrom(IndexMode.values()).getName() : null
353358
)
354359
);
355360
}
@@ -367,7 +372,8 @@ private Response.DataStreamInfo generateRandomDataStreamInfo() {
367372
timeSeries != null ? new Response.TimeSeries(timeSeries) : null,
368373
generateRandomIndexSettingsValues(),
369374
randomBoolean(),
370-
usually() ? randomNonNegativeLong() : null
375+
usually() ? randomNonNegativeLong() : null,
376+
usually() ? randomFrom(IndexMode.values()).getName() : null
371377
);
372378
}
373379
}

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,15 @@
2424
import org.elasticsearch.core.Tuple;
2525
import org.elasticsearch.index.Index;
2626
import org.elasticsearch.index.IndexNotFoundException;
27+
import org.elasticsearch.index.IndexSettingProviders;
2728
import org.elasticsearch.indices.SystemIndices;
2829
import org.elasticsearch.indices.TestIndexNameExpressionResolver;
2930
import org.elasticsearch.test.ESTestCase;
3031

3132
import java.time.Instant;
3233
import java.time.temporal.ChronoUnit;
3334
import java.util.List;
35+
import java.util.Set;
3436

3537
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.getClusterStateWithDataStreams;
3638
import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch;
@@ -173,6 +175,7 @@ public void testGetTimeSeriesDataStream() {
173175
ClusterSettings.createBuiltInClusterSettings(),
174176
dataStreamGlobalRetentionSettings,
175177
emptyDataStreamFailureStoreSettings,
178+
new IndexSettingProviders(Set.of()),
176179
null
177180
);
178181
assertThat(
@@ -205,6 +208,7 @@ public void testGetTimeSeriesDataStream() {
205208
ClusterSettings.createBuiltInClusterSettings(),
206209
dataStreamGlobalRetentionSettings,
207210
emptyDataStreamFailureStoreSettings,
211+
new IndexSettingProviders(Set.of()),
208212
null
209213
);
210214
assertThat(
@@ -257,6 +261,7 @@ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() {
257261
ClusterSettings.createBuiltInClusterSettings(),
258262
dataStreamGlobalRetentionSettings,
259263
emptyDataStreamFailureStoreSettings,
264+
new IndexSettingProviders(Set.of()),
260265
null
261266
);
262267
assertThat(
@@ -302,6 +307,7 @@ public void testGetTimeSeriesMixedDataStream() {
302307
ClusterSettings.createBuiltInClusterSettings(),
303308
dataStreamGlobalRetentionSettings,
304309
emptyDataStreamFailureStoreSettings,
310+
new IndexSettingProviders(Set.of()),
305311
null
306312
);
307313

@@ -349,6 +355,7 @@ public void testPassingGlobalRetention() {
349355
ClusterSettings.createBuiltInClusterSettings(),
350356
dataStreamGlobalRetentionSettings,
351357
emptyDataStreamFailureStoreSettings,
358+
new IndexSettingProviders(Set.of()),
352359
null
353360
);
354361
assertThat(response.getGlobalRetention(), nullValue());
@@ -375,6 +382,7 @@ public void testPassingGlobalRetention() {
375382
ClusterSettings.createBuiltInClusterSettings(),
376383
withGlobalRetentionSettings,
377384
emptyDataStreamFailureStoreSettings,
385+
new IndexSettingProviders(Set.of()),
378386
null
379387
);
380388
assertThat(response.getGlobalRetention(), equalTo(globalRetention));
@@ -403,6 +411,7 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_disabled() {
403411
ClusterSettings.createBuiltInClusterSettings(),
404412
dataStreamGlobalRetentionSettings,
405413
emptyDataStreamFailureStoreSettings,
414+
new IndexSettingProviders(Set.of()),
406415
null
407416
);
408417
assertThat(response.getDataStreams(), hasSize(1));
@@ -432,6 +441,7 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledExplicitly() {
432441
ClusterSettings.createBuiltInClusterSettings(),
433442
dataStreamGlobalRetentionSettings,
434443
emptyDataStreamFailureStoreSettings,
444+
new IndexSettingProviders(Set.of()),
435445
null
436446
);
437447
assertThat(response.getDataStreams(), hasSize(1));
@@ -467,6 +477,7 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledByClusterSetti
467477
.build()
468478
)
469479
),
480+
new IndexSettingProviders(Set.of()),
470481
null
471482
);
472483
assertThat(response.getDataStreams(), hasSize(1));

0 commit comments

Comments
 (0)