Skip to content

Commit 2056170

Browse files
gmarouligmarouli
committed
[Failure Store] Expose failure store lifecycle information via the GET data stream API (elastic#126668)
To retrieve the effective configuration you need to use the `GET` data streams API, for example, if a data stream has empty data stream options, it might still have failure store enabled from a cluster setting. The failure store is managed by default with a lifecycle with infinite (for now) retention, so the response will look like this: ``` GET _data_stream/* { "data_streams": [ { "name": "my-data-stream", "timestamp_field": { "name": "@timestamp" }, ..... "failure_store": { "enabled": true, "lifecycle": { "enabled": true }, "rollover_on_write": false, "indices": [ { "index_name": ".fs-my-data-stream-2099.03.08-000003", "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw", "managed_by": "Data stream lifecycle" } ] } },... ] ``` In case there is a failure indexed managed by ILM the failure index info will be displayed as follows. ``` { "index_name": ".fs-my-data-stream-2099.03.08-000002", "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw", "prefer_ilm": true, "ilm_policy": "my-lifecycle-policy", "managed_by": "Index Lifecycle Management" } ``` (cherry picked from commit db2992f) # Conflicts: # modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java # server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java
1 parent ef154a6 commit 2056170

File tree

4 files changed

+353
-47
lines changed

4 files changed

+353
-47
lines changed

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ public class RestGetDataStreamsAction extends BaseRestHandler {
4444
)
4545
)
4646
);
47+
public static final String FAILURES_LIFECYCLE_API_CAPABILITY = "failure_store.lifecycle";
48+
private static final Set<String> CAPABILITIES = Set.of(
49+
DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY,
50+
FAILURES_LIFECYCLE_API_CAPABILITY
51+
);
4752

4853
@Override
4954
public String getName() {
@@ -74,7 +79,7 @@ public boolean allowSystemIndexAccessByDefault() {
7479

7580
@Override
7681
public Set<String> supportedCapabilities() {
77-
return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY);
82+
return CAPABILITIES;
7883
}
7984

8085
@Override

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java‎

Lines changed: 80 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName;
3939
import static org.elasticsearch.cluster.metadata.DataStream.getDefaultFailureStoreName;
4040
import static org.hamcrest.Matchers.is;
41+
import static org.hamcrest.Matchers.notNullValue;
4142
import static org.hamcrest.Matchers.nullValue;
4243

4344
public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase<Response> {
@@ -163,21 +164,16 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
163164
is(ManagedBy.LIFECYCLE.displayValue)
164165
);
165166

166-
if (DataStream.isFailureStoreFeatureFlagEnabled()) {
167-
var failureStore = (Map<String, Object>) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName());
168-
List<Object> failureStoresRepresentation = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
169-
Map<String, Object> failureStoreRepresentation = (Map<String, Object>) failureStoresRepresentation.get(0);
170-
assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName()));
171-
assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), is(false));
172-
assertThat(
173-
failureStoreRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()),
174-
is(nullValue())
175-
);
176-
assertThat(
177-
failureStoreRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
178-
is(ManagedBy.LIFECYCLE.displayValue)
179-
);
180-
}
167+
var failureStore = (Map<String, Object>) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName());
168+
List<Object> failureIndices = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
169+
Map<String, Object> failureIndexRepresentation = (Map<String, Object>) failureIndices.get(0);
170+
assertThat(failureIndexRepresentation.get("index_name"), is(failureStoreIndex.getName()));
171+
assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), nullValue());
172+
assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue()));
173+
assertThat(
174+
failureIndexRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
175+
is(ManagedBy.LIFECYCLE.displayValue)
176+
);
181177
}
182178
}
183179

@@ -256,21 +252,75 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
256252
is(ManagedBy.UNMANAGED.displayValue)
257253
);
258254

259-
if (DataStream.isFailureStoreFeatureFlagEnabled()) {
260-
var failureStore = (Map<String, Object>) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName());
261-
List<Object> failureStoresRepresentation = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
262-
Map<String, Object> failureStoreRepresentation = (Map<String, Object>) failureStoresRepresentation.get(0);
263-
assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName()));
264-
assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), is(false));
265-
assertThat(
266-
failureStoreRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()),
267-
is(nullValue())
268-
);
269-
assertThat(
270-
failureStoreRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
271-
is(ManagedBy.UNMANAGED.displayValue)
272-
);
273-
}
255+
var failureStore = (Map<String, Object>) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName());
256+
List<Object> failureStoresRepresentation = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
257+
Map<String, Object> failureStoreRepresentation = (Map<String, Object>) failureStoresRepresentation.get(0);
258+
assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName()));
259+
assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), nullValue());
260+
assertThat(failureStoreRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(nullValue()));
261+
assertThat(
262+
failureStoreRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
263+
is(ManagedBy.UNMANAGED.displayValue)
264+
);
265+
}
266+
}
267+
268+
{
269+
// one failure index that have ILM policy
270+
DataStream logs = DataStream.builder("logs", indices)
271+
.setGeneration(3)
272+
.setAllowCustomRouting(true)
273+
.setIndexMode(IndexMode.STANDARD)
274+
.setLifecycle(DataStreamLifecycle.DEFAULT_DATA_LIFECYCLE)
275+
.setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED)
276+
.setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build())
277+
.build();
278+
279+
String ilmPolicyName = "rollover-30days";
280+
Map<Index, Response.IndexProperties> indexSettingsValues = Map.of(
281+
firstGenerationIndex,
282+
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM),
283+
secondGenerationIndex,
284+
new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE),
285+
writeIndex,
286+
new Response.IndexProperties(true, null, ManagedBy.LIFECYCLE),
287+
failureStoreIndex,
288+
new Response.IndexProperties(randomBoolean(), ilmPolicyName, ManagedBy.LIFECYCLE)
289+
);
290+
291+
Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo(
292+
logs,
293+
true,
294+
ClusterHealthStatus.GREEN,
295+
"index-template",
296+
null,
297+
null,
298+
indexSettingsValues,
299+
false,
300+
null
301+
);
302+
Response response = new Response(List.of(dataStreamInfo));
303+
XContentBuilder contentBuilder = XContentFactory.jsonBuilder();
304+
response.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
305+
306+
BytesReference bytes = BytesReference.bytes(contentBuilder);
307+
try (XContentParser parser = createParser(JsonXContent.jsonXContent, bytes)) {
308+
Map<String, Object> map = parser.map();
309+
List<Object> dataStreams = (List<Object>) map.get(Response.DATA_STREAMS_FIELD.getPreferredName());
310+
assertThat(dataStreams.size(), is(1));
311+
Map<String, Object> dataStreamMap = (Map<String, Object>) dataStreams.get(0);
312+
assertThat(dataStreamMap.get(DataStream.NAME_FIELD.getPreferredName()), is(dataStreamName));
313+
314+
var failureStore = (Map<String, Object>) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName());
315+
List<Object> failureIndices = (List<Object>) failureStore.get(DataStream.INDICES_FIELD.getPreferredName());
316+
Map<String, Object> failureIndexRepresentation = (Map<String, Object>) failureIndices.get(0);
317+
assertThat(failureIndexRepresentation.get("index_name"), is(failureStoreIndex.getName()));
318+
assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.PREFER_ILM.getPreferredName()), notNullValue());
319+
assertThat(failureIndexRepresentation.get(Response.DataStreamInfo.ILM_POLICY_FIELD.getPreferredName()), is(ilmPolicyName));
320+
assertThat(
321+
failureIndexRepresentation.get(Response.DataStreamInfo.MANAGED_BY.getPreferredName()),
322+
is(ManagedBy.LIFECYCLE.displayValue)
323+
);
274324
}
275325
}
276326
}

‎modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
setup:
2+
- requires:
3+
test_runner_features: [ capabilities, allowed_warnings ]
4+
reason: "Exposing failures lifecycle config in templates was added in 9.1+"
5+
capabilities:
6+
- method: GET
7+
path: /_data_stream/{target}
8+
capabilities: [ 'failure_store.lifecycle' ]
9+
- do:
10+
allowed_warnings:
11+
- "index template [my-template1] has index patterns [fs-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
12+
indices.put_index_template:
13+
name: my-template1
14+
body:
15+
index_patterns: [fs-data-stream]
16+
template:
17+
settings:
18+
index.number_of_replicas: 1
19+
mappings:
20+
properties:
21+
'@timestamp':
22+
type: date
23+
count:
24+
type: long
25+
lifecycle: {}
26+
data_stream_options:
27+
failure_store:
28+
enabled: true
29+
data_stream: {}
30+
31+
- do:
32+
allowed_warnings:
33+
- "index template [my-template2] has index patterns [fs-default-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
34+
indices.put_index_template:
35+
name: my-template2
36+
body:
37+
index_patterns: [ fs-default-* ]
38+
template:
39+
settings:
40+
index.number_of_replicas: 1
41+
mappings:
42+
properties:
43+
'@timestamp':
44+
type: date
45+
count:
46+
type: long
47+
lifecycle: {}
48+
data_stream: { }
49+
50+
- do:
51+
allowed_warnings:
52+
- "index template [my-template3] has index patterns [no-fs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template3] will take precedence during new index creation"
53+
indices.put_index_template:
54+
name: my-template3
55+
body:
56+
index_patterns: [ no-fs-* ]
57+
template:
58+
settings:
59+
index.number_of_replicas: 1
60+
mappings:
61+
properties:
62+
'@timestamp':
63+
type: date
64+
count:
65+
type: long
66+
data_stream: { }
67+
68+
- do:
69+
cluster.put_settings:
70+
body:
71+
persistent:
72+
data_streams.failure_store.enabled: 'fs-default*'
73+
74+
---
75+
teardown:
76+
- do:
77+
indices.delete_data_stream:
78+
name: fs-data-stream
79+
ignore: 404
80+
81+
- do:
82+
indices.delete_index_template:
83+
name: fs-default-data-stream
84+
ignore: 404
85+
86+
- do:
87+
indices.delete_index_template:
88+
name: no-fs-data-stream
89+
ignore: 404
90+
91+
---
92+
"Get failure store info from explicitly enabled failure store":
93+
- do:
94+
indices.create_data_stream:
95+
name: fs-data-stream
96+
- is_true: acknowledged
97+
98+
- do:
99+
indices.get_data_stream:
100+
name: "fs-data-stream"
101+
- match: { data_streams.0.name: fs-data-stream }
102+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
103+
- match: { data_streams.0.generation: 1 }
104+
- length: { data_streams.0.indices: 1 }
105+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
106+
- match: { data_streams.0.template: 'my-template1' }
107+
- match: { data_streams.0.failure_store.enabled: true }
108+
- match: { data_streams.0.failure_store.lifecycle.enabled: true}
109+
- match: { data_streams.0.failure_store.indices: [] }
110+
111+
# Initialize failure store
112+
- do:
113+
index:
114+
index: fs-data-stream
115+
refresh: true
116+
body:
117+
'@timestamp': '2020-12-12'
118+
count: 'invalid value'
119+
120+
- do:
121+
indices.get_data_stream:
122+
name: "fs-data-stream"
123+
- match: { data_streams.0.name: fs-data-stream }
124+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
125+
- match: { data_streams.0.generation: 2 }
126+
- length: { data_streams.0.indices: 1 }
127+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
128+
- match: { data_streams.0.template: 'my-template1' }
129+
- match: { data_streams.0.failure_store.enabled: true }
130+
- match: { data_streams.0.failure_store.lifecycle.enabled: true }
131+
- length: { data_streams.0.failure_store.indices: 1 }
132+
- match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
133+
- is_false: data_streams.0.failure_store.indices.0.prefer_ilm
134+
- match: { data_streams.0.failure_store.indices.0.managed_by: 'Data stream lifecycle' }
135+
136+
---
137+
"Get failure store info from disabled failure store":
138+
- do:
139+
indices.create_data_stream:
140+
name: no-fs-data-stream
141+
- is_true: acknowledged
142+
143+
- do:
144+
indices.get_data_stream:
145+
name: "no-fs-data-stream"
146+
- match: { data_streams.0.name: no-fs-data-stream }
147+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
148+
- match: { data_streams.0.generation: 1 }
149+
- length: { data_streams.0.indices: 1 }
150+
- match: { data_streams.0.indices.0.index_name: '/\.ds-no-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
151+
- match: { data_streams.0.template: 'my-template3' }
152+
- match: { data_streams.0.failure_store.enabled: false }
153+
- is_false: data_streams.0.failure_store.lifecycle
154+
- match: { data_streams.0.failure_store.indices: [] }
155+
156+
---
157+
"Get failure store info from explicitly enabled failure store and disabled lifecycle":
158+
- do:
159+
indices.create_data_stream:
160+
name: fs-data-stream
161+
- is_true: acknowledged
162+
163+
- do:
164+
indices.put_data_lifecycle:
165+
name: "fs-data-stream"
166+
body:
167+
enabled: false
168+
169+
- is_true: acknowledged
170+
171+
# Initialize failure store
172+
- do:
173+
index:
174+
index: fs-data-stream
175+
refresh: true
176+
body:
177+
'@timestamp': '2020-12-12'
178+
count: 'invalid value'
179+
180+
- do:
181+
indices.get_data_stream:
182+
name: "fs-data-stream"
183+
- match: { data_streams.0.name: fs-data-stream }
184+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
185+
- match: { data_streams.0.generation: 2 }
186+
- length: { data_streams.0.indices: 1 }
187+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
188+
- match: { data_streams.0.template: 'my-template1' }
189+
- match: { data_streams.0.failure_store.enabled: true }
190+
- match: { data_streams.0.failure_store.lifecycle.enabled: false }
191+
- length: { data_streams.0.failure_store.indices: 1 }
192+
- match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
193+
- is_false: data_streams.0.failure_store.indices.0.prefer_ilm
194+
- match: { data_streams.0.failure_store.indices.0.managed_by: 'Unmanaged' }
195+
196+
---
197+
"Get failure store info from cluster setting enabled failure store":
198+
- do:
199+
indices.create_data_stream:
200+
name: fs-default-data-stream
201+
- is_true: acknowledged
202+
203+
- do:
204+
indices.get_data_stream:
205+
name: "fs-default-data-stream"
206+
- match: { data_streams.0.name: fs-default-data-stream }
207+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
208+
- match: { data_streams.0.generation: 1 }
209+
- length: { data_streams.0.indices: 1 }
210+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
211+
- match: { data_streams.0.template: 'my-template2' }
212+
- match: { data_streams.0.failure_store.enabled: true }
213+
- match: { data_streams.0.failure_store.lifecycle.enabled: true }
214+
- match: { data_streams.0.failure_store.indices: [] }
215+
216+
# Initialize failure store
217+
- do:
218+
index:
219+
index: fs-default-data-stream
220+
refresh: true
221+
body:
222+
'@timestamp': '2020-12-12'
223+
count: 'invalid value'
224+
225+
- do:
226+
indices.get_data_stream:
227+
name: "fs-default-data-stream"
228+
- match: { data_streams.0.name: fs-default-data-stream }
229+
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
230+
- match: { data_streams.0.generation: 2 }
231+
- length: { data_streams.0.indices: 1 }
232+
- match: { data_streams.0.indices.0.index_name: '/\.ds-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
233+
- match: { data_streams.0.template: 'my-template2' }
234+
- match: { data_streams.0.failure_store.enabled: true }
235+
- match: { data_streams.0.failure_store.lifecycle.enabled: true }
236+
- length: { data_streams.0.failure_store.indices: 1 }
237+
- match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
238+
- is_false: data_streams.0.failure_store.indices.0.prefer_ilm
239+
- match: { data_streams.0.failure_store.indices.0.managed_by: 'Data stream lifecycle' }

0 commit comments

Comments
 (0)