Skip to content

Commit 8904282

Browse files
gmarouligmarouli
committed
Refactor DataStreamLifecycleDriver to DownsamplingIntegTestCase
We collect helper methods to an base test case so all tests can benefit.
1 parent 711d37d commit 8904282

File tree

4 files changed

+176
-238
lines changed

4 files changed

+176
-238
lines changed

‎x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,33 +17,21 @@
1717
import org.elasticsearch.cluster.service.ClusterService;
1818
import org.elasticsearch.common.settings.Settings;
1919
import org.elasticsearch.core.TimeValue;
20-
import org.elasticsearch.datastreams.DataStreamsPlugin;
2120
import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService;
22-
import org.elasticsearch.plugins.Plugin;
2321
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
2422
import org.elasticsearch.test.ClusterServiceUtils;
2523
import org.elasticsearch.test.ESIntegTestCase;
2624
import org.elasticsearch.test.InternalTestCluster;
27-
import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin;
28-
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
2925

30-
import java.util.Collection;
3126
import java.util.List;
3227

3328
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_DOWNSAMPLE_STATUS;
34-
import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.getBackingIndices;
35-
import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.putTSDBIndexTemplate;
3629

3730
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 4)
38-
public class DataStreamLifecycleDownsampleDisruptionIT extends ESIntegTestCase {
31+
public class DataStreamLifecycleDownsampleDisruptionIT extends DownsamplingIntegTestCase {
3932
private static final Logger logger = LogManager.getLogger(DataStreamLifecycleDownsampleDisruptionIT.class);
4033
public static final int DOC_COUNT = 25_000;
4134

42-
@Override
43-
protected Collection<Class<? extends Plugin>> nodePlugins() {
44-
return List.of(DataStreamsPlugin.class, LocalStateCompositeXPackPlugin.class, Downsample.class, AggregateMetricMapperPlugin.class);
45-
}
46-
4735
@Override
4836
protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
4937
Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings));
@@ -69,8 +57,7 @@ public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception {
6957
)
7058
)
7159
.buildTemplate();
72-
DataStreamLifecycleDriver.setupTSDBDataStreamAndIngestDocs(
73-
client(),
60+
setupTSDBDataStreamAndIngestDocs(
7461
dataStreamName,
7562
"1986-01-08T23:40:53.384Z",
7663
"2022-01-08T23:40:53.384Z",
@@ -81,9 +68,9 @@ public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception {
8168

8269
// before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with
8370
// testing so DSL doesn't have to wait for the end_time to lapse)
84-
putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle);
85-
client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet();
86-
String sourceIndex = getBackingIndices(client(), dataStreamName).get(0);
71+
putTSDBIndexTemplate(dataStreamName, null, null, lifecycle);
72+
safeGet(client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)));
73+
String sourceIndex = getDataStreamBackingIndexNames(dataStreamName).get(0);
8774
final String targetIndex = "downsample-5m-" + sourceIndex;
8875

8976
/**

‎x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java‎

Lines changed: 26 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -9,40 +9,26 @@
99

1010
import org.elasticsearch.action.admin.indices.rollover.RolloverAction;
1111
import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
12-
import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction;
1312
import org.elasticsearch.action.downsample.DownsampleConfig;
1413
import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
1514
import org.elasticsearch.common.settings.Settings;
1615
import org.elasticsearch.core.TimeValue;
17-
import org.elasticsearch.datastreams.DataStreamsPlugin;
1816
import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService;
19-
import org.elasticsearch.plugins.Plugin;
2017
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
21-
import org.elasticsearch.test.ESIntegTestCase;
2218
import org.elasticsearch.test.junit.annotations.TestLogging;
23-
import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin;
24-
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
2519

26-
import java.util.Collection;
2720
import java.util.HashSet;
2821
import java.util.List;
2922
import java.util.Set;
3023
import java.util.concurrent.TimeUnit;
3124

3225
import static org.elasticsearch.cluster.metadata.ClusterChangedEventUtils.indicesCreated;
3326
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo;
34-
import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.getBackingIndices;
35-
import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.putTSDBIndexTemplate;
3627
import static org.hamcrest.Matchers.is;
3728

38-
public class DataStreamLifecycleDownsampleIT extends ESIntegTestCase {
29+
public class DataStreamLifecycleDownsampleIT extends DownsamplingIntegTestCase {
3930
public static final int DOC_COUNT = 50_000;
4031

41-
@Override
42-
protected Collection<Class<? extends Plugin>> nodePlugins() {
43-
return List.of(DataStreamsPlugin.class, LocalStateCompositeXPackPlugin.class, Downsample.class, AggregateMetricMapperPlugin.class);
44-
}
45-
4632
@Override
4733
protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
4834
Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings));
@@ -69,8 +55,7 @@ public void testDownsampling() throws Exception {
6955
)
7056
.buildTemplate();
7157

72-
DataStreamLifecycleDriver.setupTSDBDataStreamAndIngestDocs(
73-
client(),
58+
setupTSDBDataStreamAndIngestDocs(
7459
dataStreamName,
7560
"1986-01-08T23:40:53.384Z",
7661
"2022-01-08T23:40:53.384Z",
@@ -79,7 +64,7 @@ public void testDownsampling() throws Exception {
7964
"1990-09-09T18:00:00"
8065
);
8166

82-
List<String> backingIndices = getBackingIndices(client(), dataStreamName);
67+
List<String> backingIndices = getDataStreamBackingIndexNames(dataStreamName);
8368
String firstGenerationBackingIndex = backingIndices.get(0);
8469
String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex;
8570
String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex;
@@ -96,7 +81,7 @@ public void testDownsampling() throws Exception {
9681
});
9782
// before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with
9883
// testing so DSL doesn't have to wait for the end_time to lapse)
99-
putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle);
84+
putTSDBIndexTemplate(dataStreamName, null, null, lifecycle);
10085

10186
client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet();
10287

@@ -112,7 +97,7 @@ public void testDownsampling() throws Exception {
11297
}, 30, TimeUnit.SECONDS);
11398

11499
assertBusy(() -> {
115-
List<String> dsBackingIndices = getBackingIndices(client(), dataStreamName);
100+
List<String> dsBackingIndices = getDataStreamBackingIndexNames(dataStreamName);
116101

117102
assertThat(dsBackingIndices.size(), is(2));
118103
String writeIndex = dsBackingIndices.get(1);
@@ -143,8 +128,7 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception
143128
)
144129
)
145130
.buildTemplate();
146-
DataStreamLifecycleDriver.setupTSDBDataStreamAndIngestDocs(
147-
client(),
131+
setupTSDBDataStreamAndIngestDocs(
148132
dataStreamName,
149133
"1986-01-08T23:40:53.384Z",
150134
"2022-01-08T23:40:53.384Z",
@@ -153,7 +137,7 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception
153137
"1990-09-09T18:00:00"
154138
);
155139

156-
List<String> backingIndices = getBackingIndices(client(), dataStreamName);
140+
List<String> backingIndices = getDataStreamBackingIndexNames(dataStreamName);
157141
String firstGenerationBackingIndex = backingIndices.get(0);
158142
String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex;
159143
String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex;
@@ -170,7 +154,7 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception
170154
});
171155
// before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with
172156
// testing so DSL doesn't have to wait for the end_time to lapse)
173-
putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle);
157+
putTSDBIndexTemplate(dataStreamName, null, null, lifecycle);
174158
client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet();
175159

176160
assertBusy(() -> {
@@ -180,7 +164,7 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception
180164
}, 30, TimeUnit.SECONDS);
181165

182166
assertBusy(() -> {
183-
List<String> dsBackingIndices = getBackingIndices(client(), dataStreamName);
167+
List<String> dsBackingIndices = getDataStreamBackingIndexNames(dataStreamName);
184168

185169
assertThat(dsBackingIndices.size(), is(2));
186170
String writeIndex = dsBackingIndices.get(1);
@@ -212,8 +196,7 @@ public void testUpdateDownsampleRound() throws Exception {
212196
)
213197
.buildTemplate();
214198

215-
DataStreamLifecycleDriver.setupTSDBDataStreamAndIngestDocs(
216-
client(),
199+
setupTSDBDataStreamAndIngestDocs(
217200
dataStreamName,
218201
"1986-01-08T23:40:53.384Z",
219202
"2022-01-08T23:40:53.384Z",
@@ -222,7 +205,7 @@ public void testUpdateDownsampleRound() throws Exception {
222205
"1990-09-09T18:00:00"
223206
);
224207

225-
List<String> backingIndices = getBackingIndices(client(), dataStreamName);
208+
List<String> backingIndices = getDataStreamBackingIndexNames(dataStreamName);
226209
String firstGenerationBackingIndex = backingIndices.get(0);
227210
String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex;
228211
String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex;
@@ -239,8 +222,8 @@ public void testUpdateDownsampleRound() throws Exception {
239222
});
240223
// before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with
241224
// testing so DSL doesn't have to wait for the end_time to lapse)
242-
putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle);
243-
client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet();
225+
putTSDBIndexTemplate(dataStreamName, null, null, lifecycle);
226+
safeGet(client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)));
244227

245228
assertBusy(() -> {
246229
assertThat(witnessedDownsamplingIndices.size(), is(1));
@@ -249,7 +232,7 @@ public void testUpdateDownsampleRound() throws Exception {
249232
}, 30, TimeUnit.SECONDS);
250233

251234
assertBusy(() -> {
252-
List<String> dsBackingIndices = getBackingIndices(client(), dataStreamName);
235+
List<String> dsBackingIndices = getDataStreamBackingIndexNames(dataStreamName);
253236
assertThat(dsBackingIndices.size(), is(2));
254237
String writeIndex = dsBackingIndices.get(1);
255238
assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2));
@@ -258,34 +241,27 @@ public void testUpdateDownsampleRound() throws Exception {
258241

259242
// update the lifecycle so that it only has one round, for the same `after` parameter as before, but a different interval
260243
// the different interval should yield a different downsample index name so we expect the data stream lifecycle to get the previous
261-
// `10s` interval downsample index, downsample it to `30s` and replace it in the data stream instead of the `10s` one.
262-
DataStreamLifecycle updatedLifecycle = DataStreamLifecycle.dataLifecycleBuilder()
263-
.downsampling(
264-
List.of(
265-
new DataStreamLifecycle.DownsamplingRound(
266-
TimeValue.timeValueMillis(10),
267-
new DownsampleConfig(new DateHistogramInterval("20m"))
244+
// `10s` interval downsample index, downsample it to `20m` and replace it in the data stream instead of the `10s` one.
245+
updateDataLifecycle(
246+
dataStreamName,
247+
DataStreamLifecycle.dataLifecycleBuilder()
248+
.downsampling(
249+
List.of(
250+
new DataStreamLifecycle.DownsamplingRound(
251+
TimeValue.timeValueMillis(10),
252+
new DownsampleConfig(new DateHistogramInterval("20m"))
253+
)
268254
)
269255
)
270-
)
271-
.build();
272-
273-
client().execute(
274-
PutDataStreamLifecycleAction.INSTANCE,
275-
new PutDataStreamLifecycleAction.Request(
276-
TEST_REQUEST_TIMEOUT,
277-
TEST_REQUEST_TIMEOUT,
278-
new String[] { dataStreamName },
279-
updatedLifecycle
280-
)
256+
.build()
281257
);
282258

283259
String thirtySecondsDownsampleIndex = "downsample-20m-" + firstGenerationBackingIndex;
284260

285261
assertBusy(() -> {
286262
assertThat(indexExists(tenSecondsDownsampleIndex), is(false));
287263

288-
List<String> dsBackingIndices = getBackingIndices(client(), dataStreamName);
264+
List<String> dsBackingIndices = getDataStreamBackingIndexNames(dataStreamName);
289265
assertThat(dsBackingIndices.size(), is(2));
290266
String writeIndex = dsBackingIndices.get(1);
291267
assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2));

0 commit comments

Comments
 (0)