Skip to content

Commit 42b9a41

Browse files
dakronedakrone
authored
[8.19] Add index mode to get data stream API (elastic#122486) (elastic#128154)
* Add index mode to get data stream API (elastic#122486) This commit adds the `index_mode` for both the data stream and each backing index to the output of `GET /_data_stream`. An example looks like: ``` { "data_streams" : [ { "name" : "foo-things", "indices" : [ { "index_name" : ".ds-foo-things-2025.02.13-000001", ... "index_mode" : "standard" } ], ... "index_mode" : "standard" }, { "name" : "logs-foo-bar", "indices" : [ { "index_name" : ".ds-logs-foo-bar-2025.02.13-000001", ... "index_mode" : "logsdb" }, { "index_name" : ".ds-logs-foo-bar-2025.02.13-000002", ... "index_mode" : "logsdb" } ], ... "index_mode" : "logsdb", } ] } ``` (cherry picked from commit 47706b5) # Conflicts: # docs/reference/indices/get-data-stream.asciidoc # server/src/main/java/org/elasticsearch/TransportVersions.java # server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java # server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java # x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java # x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java * Fix compilation for backport * Fix docs
1 parent 42ae399 commit 42b9a41

File tree

14 files changed

+242
-39
lines changed

14 files changed

+242
-39
lines changed

‎docs/changelog/122486.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 122486
2+
summary: Add index mode to get data stream API
3+
area: Data streams
4+
type: enhancement
5+
issues: []

‎docs/reference/data-streams/change-mappings-and-settings.asciidoc‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -575,13 +575,15 @@ stream's oldest backing index.
575575
"index_name": ".ds-my-data-stream-2099.03.07-000001", <1>
576576
"index_uuid": "Gpdiyq8sRuK9WuthvAdFbw",
577577
"prefer_ilm": true,
578-
"managed_by": "Unmanaged"
578+
"managed_by": "Unmanaged",
579+
"index_mode" : "standard"
579580
},
580581
{
581582
"index_name": ".ds-my-data-stream-2099.03.08-000002",
582583
"index_uuid": "_eEfRrFHS9OyhqWntkgHAQ",
583584
"prefer_ilm": true,
584-
"managed_by": "Unmanaged"
585+
"managed_by": "Unmanaged",
586+
"index_mode" : "standard"
585587
}
586588
],
587589
"generation": 2,
@@ -593,6 +595,7 @@ stream's oldest backing index.
593595
"system": false,
594596
"allow_custom_routing": false,
595597
"replicated": false,
598+
"index_mode": "standard",
596599
"rollover_on_write": false
597600
}
598601
]

‎docs/reference/data-streams/downsampling-manual.asciidoc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,7 @@ This returns:
360360
"index_name": ".ds-my-data-stream-2023.07.26-000001", <1>
361361
"index_uuid": "ltOJGmqgTVm4T-Buoe7Acg",
362362
"prefer_ilm": true,
363+
"index_mode": "time_series",
363364
"managed_by": "Unmanaged"
364365
}
365366
],
@@ -373,6 +374,7 @@ This returns:
373374
"allow_custom_routing": false,
374375
"replicated": false,
375376
"rollover_on_write": false,
377+
"index_mode": "time_series",
376378
"time_series": {
377379
"temporal_ranges": [
378380
{

‎docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -118,14 +118,16 @@ and that the next generation index will also be managed by {ilm-init}:
118118
"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
119119
"prefer_ilm": true, <2>
120120
"ilm_policy": "pre-dsl-ilm-policy", <3>
121-
"managed_by": "Index Lifecycle Management" <4>
121+
"managed_by": "Index Lifecycle Management", <4>
122+
"index_mode": "standard"
122123
},
123124
{
124125
"index_name": ".ds-dsl-data-stream-2023.10.19-000002",
125126
"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
126127
"prefer_ilm": true,
127128
"ilm_policy": "pre-dsl-ilm-policy",
128-
"managed_by": "Index Lifecycle Management"
129+
"managed_by": "Index Lifecycle Management",
130+
"index_mode": "standard"
129131
}
130132
],
131133
"generation": 2,
@@ -138,6 +140,7 @@ and that the next generation index will also be managed by {ilm-init}:
138140
"system": false,
139141
"allow_custom_routing": false,
140142
"replicated": false,
143+
"index_mode": "standard",
141144
"rollover_on_write": false
142145
}
143146
]
@@ -251,14 +254,16 @@ GET _data_stream/dsl-data-stream
251254
"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
252255
"prefer_ilm": true,
253256
"ilm_policy": "pre-dsl-ilm-policy",
254-
"managed_by": "Index Lifecycle Management" <1>
257+
"managed_by": "Index Lifecycle Management", <1>
258+
"index_mode": "standard"
255259
},
256260
{
257261
"index_name": ".ds-dsl-data-stream-2023.10.19-000002",
258262
"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
259263
"prefer_ilm": true,
260264
"ilm_policy": "pre-dsl-ilm-policy",
261-
"managed_by": "Index Lifecycle Management" <2>
265+
"managed_by": "Index Lifecycle Management", <2>
266+
"index_mode": "standard"
262267
}
263268
],
264269
"generation": 2,
@@ -277,6 +282,7 @@ GET _data_stream/dsl-data-stream
277282
"system": false,
278283
"allow_custom_routing": false,
279284
"replicated": false,
285+
"index_mode": "standard",
280286
"rollover_on_write": false
281287
}
282288
]
@@ -324,20 +330,23 @@ GET _data_stream/dsl-data-stream
324330
"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
325331
"prefer_ilm": true,
326332
"ilm_policy": "pre-dsl-ilm-policy",
333+
"index_mode": "standard",
327334
"managed_by": "Index Lifecycle Management" <1>
328335
},
329336
{
330337
"index_name": ".ds-dsl-data-stream-2023.10.19-000002",
331338
"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
332339
"prefer_ilm": true,
333340
"ilm_policy": "pre-dsl-ilm-policy",
341+
"index_mode": "standard",
334342
"managed_by": "Index Lifecycle Management" <2>
335343
},
336344
{
337345
"index_name": ".ds-dsl-data-stream-2023.10.19-000003",
338346
"index_uuid": "PA_JquKGSiKcAKBA8abcd1",
339347
"prefer_ilm": false, <3>
340348
"ilm_policy": "pre-dsl-ilm-policy",
349+
"index_mode": "standard",
341350
"managed_by": "Data stream lifecycle" <4>
342351
}
343352
],
@@ -357,6 +366,7 @@ GET _data_stream/dsl-data-stream
357366
"system": false,
358367
"allow_custom_routing": false,
359368
"replicated": false,
369+
"index_mode": "standard",
360370
"rollover_on_write": false
361371
}
362372
]
@@ -424,20 +434,23 @@ GET _data_stream/dsl-data-stream
424434
"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
425435
"prefer_ilm": true,
426436
"ilm_policy": "pre-dsl-ilm-policy",
437+
"index_mode": "standard",
427438
"managed_by": "Index Lifecycle Management"
428439
},
429440
{
430441
"index_name": ".ds-dsl-data-stream-2023.10.19-000002",
431442
"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
432443
"prefer_ilm": true,
433444
"ilm_policy": "pre-dsl-ilm-policy",
445+
"index_mode": "standard",
434446
"managed_by": "Index Lifecycle Management"
435447
},
436448
{
437449
"index_name": ".ds-dsl-data-stream-2023.10.19-000003",
438450
"index_uuid": "PA_JquKGSiKcAKBA8abcd1",
439451
"prefer_ilm": false,
440452
"ilm_policy": "pre-dsl-ilm-policy",
453+
"index_mode": "standard",
441454
"managed_by": "Index Lifecycle Management" <1>
442455
}
443456
],
@@ -455,6 +468,7 @@ GET _data_stream/dsl-data-stream
455468
"system": false,
456469
"allow_custom_routing": false,
457470
"replicated": false,
471+
"index_mode": "standard",
458472
"rollover_on_write": false
459473
}
460474
]

‎docs/reference/indices/get-data-stream.asciidoc‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -304,21 +304,24 @@ The API returns the following response:
304304
"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
305305
"prefer_ilm": true,
306306
"ilm_policy": "my-lifecycle-policy",
307-
"managed_by": "Index Lifecycle Management"
307+
"managed_by": "Index Lifecycle Management",
308+
"index_mode": "standard"
308309
},
309310
{
310311
"index_name": ".ds-my-data-stream-2099.03.08-000002",
311312
"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
312313
"prefer_ilm": true,
313314
"ilm_policy": "my-lifecycle-policy",
314-
"managed_by": "Index Lifecycle Management"
315+
"managed_by": "Index Lifecycle Management",
316+
"index_mode": "standard"
315317
}
316318
],
317319
"generation": 2,
318320
"_meta": {
319321
"my-meta-field": "foo"
320322
},
321323
"status": "GREEN",
324+
"index_mode": "standard",
322325
"next_generation_managed_by": "Index Lifecycle Management",
323326
"prefer_ilm": true,
324327
"template": "my-index-template",
@@ -340,14 +343,16 @@ The API returns the following response:
340343
"index_uuid": "3liBu2SYS5axasRt6fUIpA",
341344
"prefer_ilm": true,
342345
"ilm_policy": "my-lifecycle-policy",
343-
"managed_by": "Index Lifecycle Management"
346+
"managed_by": "Index Lifecycle Management",
347+
"index_mode": "standard"
344348
}
345349
],
346350
"generation": 1,
347351
"_meta": {
348352
"my-meta-field": "foo"
349353
},
350354
"status": "YELLOW",
355+
"index_mode": "standard",
351356
"next_generation_managed_by": "Index Lifecycle Management",
352357
"prefer_ilm": true,
353358
"template": "my-index-template",

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.elasticsearch.cluster.block.ClusterBlockException;
2525
import org.elasticsearch.cluster.block.ClusterBlockLevel;
2626
import org.elasticsearch.cluster.health.ClusterStateHealth;
27+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
2728
import org.elasticsearch.cluster.metadata.DataStream;
2829
import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
2930
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
@@ -39,6 +40,9 @@
3940
import org.elasticsearch.core.Tuple;
4041
import org.elasticsearch.index.Index;
4142
import org.elasticsearch.index.IndexMode;
43+
import org.elasticsearch.index.IndexSettingProvider;
44+
import org.elasticsearch.index.IndexSettingProviders;
45+
import org.elasticsearch.index.IndexSettings;
4246
import org.elasticsearch.indices.SystemDataStreamDescriptor;
4347
import org.elasticsearch.indices.SystemIndices;
4448
import org.elasticsearch.injection.guice.Inject;
@@ -52,6 +56,7 @@
5256
import java.util.Comparator;
5357
import java.util.HashMap;
5458
import java.util.List;
59+
import java.util.Locale;
5560
import java.util.Map;
5661
import java.util.stream.Collectors;
5762

@@ -67,6 +72,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadAction
6772
private final ClusterSettings clusterSettings;
6873
private final DataStreamGlobalRetentionSettings globalRetentionSettings;
6974
private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
75+
private final IndexSettingProviders indexSettingProviders;
7076
private final Client client;
7177

7278
@Inject
@@ -79,6 +85,7 @@ public TransportGetDataStreamsAction(
7985
SystemIndices systemIndices,
8086
DataStreamGlobalRetentionSettings globalRetentionSettings,
8187
DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
88+
IndexSettingProviders indexSettingProviders,
8289
Client client
8390
) {
8491
super(
@@ -96,6 +103,7 @@ public TransportGetDataStreamsAction(
96103
this.globalRetentionSettings = globalRetentionSettings;
97104
clusterSettings = clusterService.getClusterSettings();
98105
this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
106+
this.indexSettingProviders = indexSettingProviders;
99107
this.client = new OriginSettingClient(client, "stack");
100108
}
101109

@@ -128,6 +136,7 @@ public void onResponse(DataStreamsStatsAction.Response response) {
128136
clusterSettings,
129137
globalRetentionSettings,
130138
dataStreamFailureStoreSettings,
139+
indexSettingProviders,
131140
maxTimestamps
132141
)
133142
);
@@ -148,12 +157,43 @@ public void onFailure(Exception e) {
148157
clusterSettings,
149158
globalRetentionSettings,
150159
dataStreamFailureStoreSettings,
160+
indexSettingProviders,
151161
null
152162
)
153163
);
154164
}
155165
}
156166

167+
/**
168+
* Resolves the index mode ("index.mode" setting) for the given data stream, from the template or additional setting providers
169+
*/
170+
@Nullable
171+
static IndexMode resolveMode(
172+
ClusterState state,
173+
IndexSettingProviders indexSettingProviders,
174+
DataStream dataStream,
175+
Settings settings,
176+
ComposableIndexTemplate indexTemplate
177+
) {
178+
IndexMode indexMode = state.metadata().retrieveIndexModeFromTemplate(indexTemplate);
179+
for (IndexSettingProvider provider : indexSettingProviders.getIndexSettingProviders()) {
180+
Settings addlSettinsg = provider.getAdditionalIndexSettings(
181+
MetadataIndexTemplateService.VALIDATE_INDEX_NAME,
182+
dataStream.getName(),
183+
indexMode,
184+
state.metadata(),
185+
Instant.now(),
186+
settings,
187+
List.of()
188+
);
189+
var rawMode = addlSettinsg.get(IndexSettings.MODE.getKey());
190+
if (rawMode != null) {
191+
indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT));
192+
}
193+
}
194+
return indexMode;
195+
}
196+
157197
static GetDataStreamAction.Response innerOperation(
158198
ClusterState state,
159199
GetDataStreamAction.Request request,
@@ -162,6 +202,7 @@ static GetDataStreamAction.Response innerOperation(
162202
ClusterSettings clusterSettings,
163203
DataStreamGlobalRetentionSettings globalRetentionSettings,
164204
DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
205+
IndexSettingProviders indexSettingProviders,
165206
@Nullable Map<String, Long> maxTimestamps
166207
) {
167208
List<DataStream> dataStreams = getDataStreams(state, indexNameExpressionResolver, request);
@@ -173,6 +214,7 @@ static GetDataStreamAction.Response innerOperation(
173214
final String indexTemplate;
174215
boolean indexTemplatePreferIlmValue = true;
175216
String ilmPolicyName = null;
217+
IndexMode indexMode = dataStream.getIndexMode();
176218
if (dataStream.isSystem()) {
177219
SystemDataStreamDescriptor dataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStream.getName());
178220
indexTemplate = dataStreamDescriptor != null ? dataStreamDescriptor.getDataStreamName() : null;
@@ -182,13 +224,31 @@ static GetDataStreamAction.Response innerOperation(
182224
dataStreamDescriptor.getComponentTemplates()
183225
);
184226
ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
227+
if (indexMode == null) {
228+
indexMode = resolveMode(
229+
state,
230+
indexSettingProviders,
231+
dataStream,
232+
settings,
233+
dataStreamDescriptor.getComposableIndexTemplate()
234+
);
235+
}
185236
indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);
186237
}
187238
} else {
188239
indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
189240
if (indexTemplate != null) {
190241
Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
191242
ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
243+
if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
244+
indexMode = resolveMode(
245+
state,
246+
indexSettingProviders,
247+
dataStream,
248+
settings,
249+
state.metadata().templatesV2().get(indexTemplate)
250+
);
251+
}
192252
indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);
193253
} else {
194254
LOGGER.warn(
@@ -280,7 +340,9 @@ public int compareTo(IndexInfo o) {
280340
timeSeries,
281341
backingIndicesSettingsValues,
282342
indexTemplatePreferIlmValue,
283-
maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName())
343+
maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName()),
344+
// Default to standard mode if not specified; should we set this to "unset" or "unspecified" instead?
345+
indexMode == null ? IndexMode.STANDARD.getName() : indexMode.getName()
284346
)
285347
);
286348
}
@@ -310,7 +372,11 @@ private static void collectIndexSettingsValues(
310372
} else {
311373
managedBy = ManagedBy.UNMANAGED;
312374
}
313-
backingIndicesSettingsValues.put(index, new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy));
375+
String indexMode = IndexSettings.MODE.get(indexMetadata.getSettings()).getName();
376+
backingIndicesSettingsValues.put(
377+
index,
378+
new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy, indexMode)
379+
);
314380
}
315381
}
316382

0 commit comments

Comments
 (0)