Skip to content

Commit 9c626f0

Browse files
nielsbaumannielsbauman
committed
Fix data stream stats testst
Occasional shard allocation issues were causing the YAML tests to fail because the shard that had the document in it would be unavailable TODO
1 parent 1edf77c commit 9c626f0

File tree

3 files changed

+61
-84
lines changed

3 files changed

+61
-84
lines changed

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java‎ renamed to ‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsStatsIT.java‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
import org.elasticsearch.common.compress.CompressedXContent;
3232
import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
3333
import org.elasticsearch.plugins.Plugin;
34-
import org.elasticsearch.test.ESSingleNodeTestCase;
34+
import org.elasticsearch.test.ESIntegTestCase;
3535
import org.elasticsearch.xcontent.json.JsonXContent;
3636
import org.junit.After;
3737

@@ -49,10 +49,11 @@
4949
import static org.hamcrest.Matchers.greaterThan;
5050
import static org.hamcrest.Matchers.is;
5151

52-
public class DataStreamsStatsTests extends ESSingleNodeTestCase {
52+
@ESIntegTestCase.ClusterScope(minNumDataNodes = 2)
53+
public class DataStreamsStatsIT extends ESIntegTestCase {
5354

5455
@Override
55-
protected Collection<Class<? extends Plugin>> getPlugins() {
56+
protected Collection<Class<? extends Plugin>> nodePlugins() {
5657
return List.of(DataStreamsPlugin.class, MapperExtrasPlugin.class);
5758
}
5859

@@ -92,7 +93,8 @@ public void testStatsEmptyDataStream() throws Exception {
9293
String dataStreamName = createDataStream();
9394

9495
DataStreamsStatsAction.Response stats = getDataStreamsStats();
95-
assertEquals(1, stats.getSuccessfulShards());
96+
assertEquals(2, stats.getTotalShards());
97+
assertEquals(2, stats.getSuccessfulShards());
9698
assertEquals(0, stats.getFailedShards());
9799
assertEquals(1, stats.getDataStreamCount());
98100
assertEquals(1, stats.getBackingIndices());
@@ -111,7 +113,8 @@ public void testStatsExistingDataStream() throws Exception {
111113
long timestamp = createDocument(dataStreamName);
112114

113115
DataStreamsStatsAction.Response stats = getDataStreamsStats();
114-
assertEquals(1, stats.getSuccessfulShards());
116+
assertEquals(2, stats.getTotalShards());
117+
assertEquals(2, stats.getSuccessfulShards());
115118
assertEquals(0, stats.getFailedShards());
116119
assertEquals(1, stats.getDataStreamCount());
117120
assertEquals(1, stats.getBackingIndices());
@@ -131,7 +134,8 @@ public void testStatsExistingDataStreamWithFailureStores() throws Exception {
131134

132135
DataStreamsStatsAction.Response stats = getDataStreamsStats();
133136

134-
assertEquals(2, stats.getSuccessfulShards());
137+
assertEquals(4, stats.getTotalShards());
138+
assertEquals(4, stats.getSuccessfulShards());
135139
assertEquals(0, stats.getFailedShards());
136140
assertEquals(1, stats.getDataStreamCount());
137141
assertEquals(2, stats.getBackingIndices());
@@ -153,7 +157,8 @@ public void testStatsExistingHiddenDataStream() throws Exception {
153157
long timestamp = createDocument(dataStreamName);
154158

155159
DataStreamsStatsAction.Response stats = getDataStreamsStats(true);
156-
assertEquals(1, stats.getSuccessfulShards());
160+
assertEquals(2, stats.getTotalShards());
161+
assertEquals(2, stats.getSuccessfulShards());
157162
assertEquals(0, stats.getFailedShards());
158163
assertEquals(1, stats.getDataStreamCount());
159164
assertEquals(1, stats.getBackingIndices());
@@ -181,7 +186,8 @@ public void testStatsClosedBackingIndexDataStream() throws Exception {
181186
);
182187

183188
DataStreamsStatsAction.Response stats = getDataStreamsStats();
184-
assertEquals(2, stats.getSuccessfulShards());
189+
assertEquals(4, stats.getTotalShards());
190+
assertEquals(4, stats.getSuccessfulShards());
185191
assertEquals(0, stats.getFailedShards());
186192
assertEquals(1, stats.getDataStreamCount());
187193
assertEquals(2, stats.getBackingIndices());
@@ -197,7 +203,8 @@ public void testStatsClosedBackingIndexDataStream() throws Exception {
197203
long timestamp = createDocument(dataStreamName);
198204

199205
stats = getDataStreamsStats();
200-
assertEquals(2, stats.getSuccessfulShards());
206+
assertEquals(4, stats.getTotalShards());
207+
assertEquals(4, stats.getSuccessfulShards());
201208
assertEquals(0, stats.getFailedShards());
202209
assertEquals(1, stats.getDataStreamCount());
203210
assertEquals(2, stats.getBackingIndices());
@@ -218,7 +225,8 @@ public void testStatsRolledDataStream() throws Exception {
218225
timestamp = max(timestamp, createDocument(dataStreamName));
219226

220227
DataStreamsStatsAction.Response stats = getDataStreamsStats();
221-
assertEquals(2, stats.getSuccessfulShards());
228+
assertEquals(4, stats.getTotalShards());
229+
assertEquals(4, stats.getSuccessfulShards());
222230
assertEquals(0, stats.getFailedShards());
223231
assertEquals(1, stats.getDataStreamCount());
224232
assertEquals(2, stats.getBackingIndices());
@@ -249,7 +257,8 @@ public void testStatsMultipleDataStreams() throws Exception {
249257

250258
DataStreamsStatsAction.Response stats = getDataStreamsStats();
251259
logger.error(stats.toString());
252-
assertEquals(createdDataStreams.size(), stats.getSuccessfulShards());
260+
assertEquals(createdDataStreams.size() * 2, stats.getTotalShards());
261+
assertEquals(createdDataStreams.size() * 2, stats.getSuccessfulShards());
253262
assertEquals(0, stats.getFailedShards());
254263
assertEquals(createdDataStreams.size(), stats.getDataStreamCount());
255264
assertEquals(createdDataStreams.size(), stats.getBackingIndices());
@@ -273,7 +282,7 @@ private String createDataStream(boolean hidden, boolean failureStore) throws Exc
273282
DataStreamOptions.Template failureStoreOptions = failureStore == false
274283
? null
275284
: new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate());
276-
Template idxTemplate = new Template(null, new CompressedXContent("""
285+
Template idxTemplate = new Template(indexSettings(2, 0).build(), new CompressedXContent("""
277286
{"properties":{"@timestamp":{"type":"date"},"data":{"type":"keyword"}}}
278287
"""), null, null, failureStoreOptions);
279288
ComposableIndexTemplate template = ComposableIndexTemplate.builder()
@@ -293,6 +302,7 @@ private String createDataStream(boolean hidden, boolean failureStore) throws Exc
293302
new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName)
294303
)
295304
);
305+
ensureGreen(dataStreamName);
296306
createdDataStreams.add(dataStreamName);
297307
return dataStreamName;
298308
}

‎modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/120_data_streams_stats.yml‎

Lines changed: 39 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
setup:
2+
- requires:
3+
cluster_features: ["gte_v7.9.0"]
4+
reason: "data streams only supported in 7.9+"
5+
26
- skip:
37
features: allowed_warnings
48
- do:
@@ -12,24 +16,26 @@ setup:
1216
settings:
1317
index.number_of_replicas: 0
1418
data_stream: {}
19+
1520
- do:
16-
allowed_warnings:
17-
- "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
18-
indices.put_index_template:
19-
name: my-template2
20-
body:
21-
index_patterns: [simple-data-stream2]
22-
template:
23-
settings:
24-
index.number_of_replicas: 0
25-
data_stream: {}
21+
indices.create_data_stream:
22+
name: simple-data-stream1
23+
24+
- do:
25+
cluster.health:
26+
wait_for_status: green
2627

2728
---
2829
"No data streams":
2930
- requires:
3031
cluster_features: ["gte_v7.9.0"]
3132
reason: "data streams only supported in 7.9+"
3233

34+
- do:
35+
indices.delete_data_stream:
36+
name: simple-data-stream1
37+
- is_true: acknowledged
38+
3339
- do:
3440
indices.data_streams_stats: {}
3541
- match: { data_stream_count: 0 }
@@ -38,15 +44,6 @@ setup:
3844

3945
---
4046
"Empty data stream":
41-
- requires:
42-
cluster_features: ["gte_v7.9.0"]
43-
reason: "data streams only supported in 7.9+"
44-
45-
- do:
46-
indices.create_data_stream:
47-
name: simple-data-stream1
48-
- is_true: acknowledged
49-
5047
- do:
5148
indices.rollover:
5249
alias: "simple-data-stream1"
@@ -60,22 +57,8 @@ setup:
6057
- match: { data_streams.0.backing_indices: 2 }
6158
- match: { data_streams.0.maximum_timestamp: 0 }
6259

63-
- do:
64-
indices.delete_data_stream:
65-
name: simple-data-stream1
66-
- is_true: acknowledged
67-
6860
---
6961
"Single data stream":
70-
- requires:
71-
cluster_features: ["gte_v7.9.0"]
72-
reason: "data streams only supported in 7.9+"
73-
74-
- do:
75-
indices.create_data_stream:
76-
name: simple-data-stream1
77-
- is_true: acknowledged
78-
7962
- do:
8063
index:
8164
index: simple-data-stream1
@@ -85,29 +68,18 @@ setup:
8568

8669
- do:
8770
indices.data_streams_stats: {}
71+
- match: { _shards.total: 1}
72+
- match: { _shards.successful: 1}
73+
- match: { _shards.failed: 0}
8874
- match: { data_stream_count: 1 }
8975
- match: { backing_indices: 1 }
9076
- length: { data_streams: 1 }
9177
- match: { data_streams.0.data_stream: 'simple-data-stream1' }
9278
- match: { data_streams.0.backing_indices: 1 }
9379
- match: { data_streams.0.maximum_timestamp: 1593639273740 }
9480

95-
- do:
96-
indices.delete_data_stream:
97-
name: simple-data-stream1
98-
- is_true: acknowledged
99-
10081
---
10182
"Rolled over data stream":
102-
- requires:
103-
cluster_features: ["gte_v7.9.0"]
104-
reason: "data streams only supported in 7.9+"
105-
106-
- do:
107-
indices.create_data_stream:
108-
name: simple-data-stream1
109-
- is_true: acknowledged
110-
11183
- do:
11284
index:
11385
index: simple-data-stream1
@@ -130,34 +102,39 @@ setup:
130102

131103
- do:
132104
indices.data_streams_stats: {}
105+
- match: { _shards.total: 2}
106+
- match: { _shards.successful: 2}
107+
- match: { _shards.failed: 0}
133108
- match: { data_stream_count: 1 }
134109
- match: { backing_indices: 2 }
135110
- length: { data_streams: 1 }
136111
- match: { data_streams.0.data_stream: 'simple-data-stream1' }
137112
- match: { data_streams.0.backing_indices: 2 }
138113
- match: { data_streams.0.maximum_timestamp: 1593639345064 }
139114

140-
- do:
141-
indices.delete_data_stream:
142-
name: simple-data-stream1
143-
- is_true: acknowledged
144-
145115
---
146116
"Multiple data stream":
147-
- requires:
148-
cluster_features: ["gte_v7.9.0"]
149-
reason: "data streams only supported in 7.9+"
150-
151117
- do:
152-
indices.create_data_stream:
153-
name: simple-data-stream1
154-
- is_true: acknowledged
118+
allowed_warnings:
119+
- "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
120+
indices.put_index_template:
121+
name: my-template2
122+
body:
123+
index_patterns: [simple-data-stream2]
124+
template:
125+
settings:
126+
index.number_of_replicas: 0
127+
data_stream: {}
155128

156129
- do:
157130
indices.create_data_stream:
158131
name: simple-data-stream2
159132
- is_true: acknowledged
160133

134+
- do:
135+
cluster.health:
136+
wait_for_status: green
137+
161138
- do:
162139
index:
163140
index: simple-data-stream1
@@ -187,6 +164,9 @@ setup:
187164

188165
- do:
189166
indices.data_streams_stats: {}
167+
- match: { _shards.total: 3}
168+
- match: { _shards.successful: 3}
169+
- match: { _shards.failed: 0}
190170
- match: { data_stream_count: 2 }
191171
- match: { backing_indices: 3 }
192172
- length: { data_streams: 2 }
@@ -196,13 +176,3 @@ setup:
196176
- match: { data_streams.1.data_stream: 'simple-data-stream2' }
197177
- match: { data_streams.1.backing_indices: 1 }
198178
- match: { data_streams.1.maximum_timestamp: 1593639450943 }
199-
200-
- do:
201-
indices.delete_data_stream:
202-
name: simple-data-stream1
203-
- is_true: acknowledged
204-
205-
- do:
206-
indices.delete_data_stream:
207-
name: simple-data-stream2
208-
- is_true: acknowledged

‎muted-tests.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,9 +102,6 @@ tests:
102102
- class: org.elasticsearch.packaging.test.ArchiveTests
103103
method: test51AutoConfigurationWithPasswordProtectedKeystore
104104
issue: https://github.com/elastic/elasticsearch/issues/118212
105-
- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
106-
method: test {p0=data_stream/120_data_streams_stats/Multiple data stream}
107-
issue: https://github.com/elastic/elasticsearch/issues/118217
108105
- class: org.elasticsearch.xpack.ccr.rest.ShardChangesRestIT
109106
method: testShardChangesNoOperation
110107
issue: https://github.com/elastic/elasticsearch/issues/118800

0 commit comments

Comments
 (0)