Skip to content

Commit dc5f3cd

Browse files
gmarouligmarouli
committed
Expose failure store lifecycle information via the GET data stream API
1 parent 6842203 commit dc5f3cd

File tree

4 files changed

+228
-9
lines changed

4 files changed

+228
-9
lines changed

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
import org.elasticsearch.action.datastreams.GetDataStreamAction;
1212
import org.elasticsearch.action.support.IndicesOptions;
1313
import org.elasticsearch.client.internal.node.NodeClient;
14+
import org.elasticsearch.cluster.metadata.DataStream;
15+
import org.elasticsearch.cluster.metadata.DataStreamFailureStore;
1416
import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
1517
import org.elasticsearch.common.Strings;
1618
import org.elasticsearch.common.util.set.Sets;
@@ -23,7 +25,10 @@
2325
import org.elasticsearch.rest.action.RestToXContentListener;
2426

2527
import java.util.List;
28+
import java.util.Objects;
2629
import java.util.Set;
30+
import java.util.stream.Collectors;
31+
import java.util.stream.Stream;
2732

2833
import static org.elasticsearch.rest.RestRequest.Method.GET;
2934

@@ -45,6 +50,10 @@ public class RestGetDataStreamsAction extends BaseRestHandler {
4550
)
4651
)
4752
);
53+
private static final Set<String> CAPABILITIES = Stream.of(
54+
DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY,
55+
DataStream.isFailureStoreFeatureFlagEnabled() ? DataStreamFailureStore.FAILURES_LIFECYCLE_API_CAPABILITY : null
56+
).filter(Objects::nonNull).collect(Collectors.toSet());
4857

4958
@Override
5059
public String getName() {
@@ -79,7 +88,7 @@ public boolean allowSystemIndexAccessByDefault() {
7988

8089
@Override
8190
public Set<String> supportedCapabilities() {
82-
return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY);
91+
return CAPABILITIES;
8392
}
8493

8594
@Override

‎modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
setup:
2+
- requires:
3+
test_runner_features: [ capabilities, allowed_warnings ]
4+
reason: "Exposing failures lifecycle config in templates was added in 9.1+"
5+
capabilities:
6+
- method: GET
7+
path: /_data_stream/{target}
8+
capabilities: [ 'failures_lifecycle' ]
9+
- do:
10+
allowed_warnings:
11+
- "index template [my-template1] has index patterns [fs-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
12+
indices.put_index_template:
13+
name: my-template1
14+
body:
15+
index_patterns: [fs-data-stream]
16+
template:
17+
settings:
18+
index.number_of_replicas: 1
19+
mappings:
20+
properties:
21+
'@timestamp':
22+
type: date
23+
count:
24+
type: long
25+
lifecycle: {}
26+
data_stream_options:
27+
failure_store:
28+
enabled: true
29+
data_stream: {}
30+
31+
- do:
32+
allowed_warnings:
33+
- "index template [my-template2] has index patterns [fs-default-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
34+
indices.put_index_template:
35+
name: my-template2
36+
body:
37+
index_patterns: [ fs-default-* ]
38+
template:
39+
settings:
40+
index.number_of_replicas: 1
41+
mappings:
42+
properties:
43+
'@timestamp':
44+
type: date
45+
count:
46+
type: long
47+
data_stream: { }
48+
49+
- do:
50+
allowed_warnings:
51+
- "index template [my-template3] has index patterns [no-fs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template3] will take precedence during new index creation"
52+
indices.put_index_template:
53+
name: my-template3
54+
body:
55+
index_patterns: [ no-fs-* ]
56+
template:
57+
settings:
58+
index.number_of_replicas: 1
59+
mappings:
60+
properties:
61+
'@timestamp':
62+
type: date
63+
count:
64+
type: long
65+
data_stream: { }
66+
67+
- do:
68+
cluster.put_settings:
69+
body:
70+
persistent:
71+
data_streams.failure_store.enabled: 'fs-default*'
72+
73+
---
74+
teardown:
75+
- do:
76+
indices.delete_data_stream:
77+
name: fs-data-stream
78+
ignore: 404
79+
80+
- do:
81+
indices.delete_index_template:
82+
name: fs-default-data-stream
83+
ignore: 404
84+
85+
- do:
86+
indices.delete_index_template:
87+
name: no-fs-data-stream
88+
ignore: 404
89+
90+
---
91+
"Get failure store info from explicitly enabled failure store":
92+
- do:
93+
indices.create_data_stream:
94+
name: fs-data-stream
95+
- is_true: acknowledged
96+
97+
- do:
98+
indices.get_data_stream:
99+
name: "fs-data-stream"
100+
- match: { data_streams.0.name: fs-data-stream }
101+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
102+
- match: { data_streams.0.generation: 1 }
103+
- length: { data_streams.0.indices: 1 }
104+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
105+
- match: { data_streams.0.template: 'my-template1' }
106+
- match: { data_streams.0.failure_store.enabled: true }
107+
- match: { data_streams.0.failure_store.lifecycle.enabled: true}
108+
- match: { data_streams.0.failure_store.indices: [] }
109+
110+
# Initialize failure store
111+
- do:
112+
index:
113+
index: fs-data-stream
114+
refresh: true
115+
body:
116+
'@timestamp': '2020-12-12'
117+
count: 'invalid value'
118+
119+
- do:
120+
indices.get_data_stream:
121+
name: "fs-data-stream"
122+
- match: { data_streams.0.name: fs-data-stream }
123+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
124+
- match: { data_streams.0.generation: 2 }
125+
- length: { data_streams.0.indices: 1 }
126+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
127+
- match: { data_streams.0.template: 'my-template1' }
128+
- match: { data_streams.0.failure_store.enabled: true }
129+
- match: { data_streams.0.failure_store.lifecycle.enabled: true }
130+
- length: { data_streams.0.failure_store.indices: 1 }
131+
- match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
132+
- is_false: data_streams.0.failure_store.indices.0.prefer_ilm
133+
- match: { data_streams.0.failure_store.indices.0.managed_by: 'Data stream lifecycle' }
134+
135+
---
136+
"Get failure store info from disabled failure store":
137+
- do:
138+
indices.create_data_stream:
139+
name: no-fs-data-stream
140+
- is_true: acknowledged
141+
142+
- do:
143+
indices.get_data_stream:
144+
name: "no-fs-data-stream"
145+
- match: { data_streams.0.name: no-fs-data-stream }
146+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
147+
- match: { data_streams.0.generation: 1 }
148+
- length: { data_streams.0.indices: 1 }
149+
- match: { data_streams.0.indices.0.index_name: '/\.ds-no-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
150+
- match: { data_streams.0.template: 'my-template3' }
151+
- match: { data_streams.0.failure_store.enabled: false }
152+
- is_false: data_streams.0.failure_store.lifecycle
153+
- match: { data_streams.0.failure_store.indices: [] }
154+
155+
---
156+
"Get failure store info from explicitly enabled failure store and disabled lifecycle":
157+
- do:
158+
indices.create_data_stream:
159+
name: fs-data-stream
160+
- is_true: acknowledged
161+
162+
- do:
163+
indices.put_data_lifecycle:
164+
name: "fs-data-stream"
165+
body:
166+
enabled: false
167+
168+
- is_true: acknowledged
169+
170+
# Initialize failure store
171+
- do:
172+
index:
173+
index: fs-data-stream
174+
refresh: true
175+
body:
176+
'@timestamp': '2020-12-12'
177+
count: 'invalid value'
178+
179+
- do:
180+
indices.get_data_stream:
181+
name: "fs-data-stream"
182+
- match: { data_streams.0.name: fs-data-stream }
183+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
184+
- match: { data_streams.0.generation: 2 }
185+
- length: { data_streams.0.indices: 1 }
186+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
187+
- match: { data_streams.0.template: 'my-template1' }
188+
- match: { data_streams.0.failure_store.enabled: true }
189+
- match: { data_streams.0.failure_store.lifecycle.enabled: false }
190+
- length: { data_streams.0.failure_store.indices: 1 }
191+
- match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
192+
- is_false: data_streams.0.failure_store.indices.0.prefer_ilm
193+
- match: { data_streams.0.failure_store.indices.0.managed_by: 'Unmanaged' }

‎server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -371,7 +371,7 @@ public XContentBuilder toXContent(
371371
.field(DataStream.NAME_FIELD.getPreferredName(), DataStream.TIMESTAMP_FIELD_NAME)
372372
.endObject();
373373

374-
indicesToXContent(builder, dataStream.getIndices());
374+
indicesToXContent(builder, dataStream.getIndices(), false);
375375
builder.field(DataStream.GENERATION_FIELD.getPreferredName(), dataStream.getGeneration());
376376
if (dataStream.getMetadata() != null) {
377377
builder.field(DataStream.METADATA_FIELD.getPreferredName(), dataStream.getMetadata());
@@ -423,28 +423,44 @@ public XContentBuilder toXContent(
423423
DataStream.ROLLOVER_ON_WRITE_FIELD.getPreferredName(),
424424
dataStream.getFailureComponent().isRolloverOnWrite()
425425
);
426-
indicesToXContent(builder, dataStream.getFailureIndices());
426+
indicesToXContent(builder, dataStream.getFailureIndices(), true);
427427
addAutoShardingEvent(builder, params, dataStream.getFailureComponent().getAutoShardingEvent());
428+
DataStreamLifecycle failuresLifecycle = dataStream.getFailuresLifecycle();
429+
if (failuresLifecycle != null) {
430+
builder.field(LIFECYCLE_FIELD.getPreferredName());
431+
failuresLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention, dataStream.isInternal());
432+
}
428433
builder.endObject();
429434
}
430435
builder.endObject();
431436
return builder;
432437
}
433438

434-
private XContentBuilder indicesToXContent(XContentBuilder builder, List<Index> indices) throws IOException {
439+
private XContentBuilder indicesToXContent(XContentBuilder builder, List<Index> indices, boolean failureIndices)
440+
throws IOException {
435441
builder.field(DataStream.INDICES_FIELD.getPreferredName());
436442
builder.startArray();
437443
for (Index index : indices) {
438444
builder.startObject();
439445
index.toXContentFragment(builder);
440446
IndexProperties indexProperties = indexSettingsValues.get(index);
441447
if (indexProperties != null) {
442-
builder.field(PREFER_ILM.getPreferredName(), indexProperties.preferIlm());
443-
if (indexProperties.ilmPolicyName() != null) {
444-
builder.field(ILM_POLICY_FIELD.getPreferredName(), indexProperties.ilmPolicyName());
445-
}
446448
builder.field(MANAGED_BY.getPreferredName(), indexProperties.managedBy.displayValue);
447-
builder.field(INDEX_MODE.getPreferredName(), indexProperties.indexMode);
449+
// Failure indices have more limitation than backing indices,
450+
// so we hide some index properties that are less relevant
451+
if (failureIndices) {
452+
// We only display ILM info, if this index has an ILM policy
453+
if (indexProperties.ilmPolicyName() != null) {
454+
builder.field(PREFER_ILM.getPreferredName(), indexProperties.preferIlm());
455+
builder.field(ILM_POLICY_FIELD.getPreferredName(), indexProperties.ilmPolicyName());
456+
}
457+
} else {
458+
builder.field(PREFER_ILM.getPreferredName(), indexProperties.preferIlm());
459+
if (indexProperties.ilmPolicyName() != null) {
460+
builder.field(ILM_POLICY_FIELD.getPreferredName(), indexProperties.ilmPolicyName());
461+
}
462+
builder.field(INDEX_MODE.getPreferredName(), indexProperties.indexMode);
463+
}
448464
}
449465
builder.endObject();
450466
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
* supports the following configurations only explicitly enabling or disabling the failure store
3030
*/
3131
public record DataStreamFailureStore(Boolean enabled) implements SimpleDiffable<DataStreamFailureStore>, ToXContentObject {
32+
public static final String FAILURES_LIFECYCLE_API_CAPABILITY = "failures_lifecycle";
3233
public static final String FAILURE_STORE = "failure_store";
3334
public static final String ENABLED = "enabled";
3435

0 commit comments

Comments
 (0)