Skip to content

Commit 41489ab

Browse files
kfarazkfaraz
committed
Remove ITAppendBatchIndexTest
1 parent 1bcf023 commit 41489ab

File tree

9 files changed

+54
-608
lines changed

9 files changed

+54
-608
lines changed

‎.github/workflows/cron-job-its.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ jobs:
6060
strategy:
6161
fail-fast: false
6262
matrix:
63-
testing_group: [kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, realtime-index, append-ingestion]
63+
testing_group: [kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, realtime-index]
6464
uses: ./.github/workflows/reusable-standard-its.yml
6565
needs: build
6666
with:
@@ -74,7 +74,7 @@ jobs:
7474
strategy:
7575
fail-fast: false
7676
matrix:
77-
testing_group: [ kafka-index, kafka-transactional-index, kafka-index-slow, kafka-transactional-index-slow, kafka-data-format, append-ingestion ]
77+
testing_group: [ kafka-index, kafka-transactional-index, kafka-index-slow, kafka-transactional-index-slow, kafka-data-format ]
7878
uses: ./.github/workflows/reusable-standard-its.yml
7979
needs: build
8080
with:

‎.github/workflows/standard-its.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ jobs:
4747
strategy:
4848
fail-fast: false
4949
matrix:
50-
testing_group: [kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, realtime-index, append-ingestion, cds-task-schema-publish-disabled, cds-coordinator-metadata-query-disabled]
50+
testing_group: [kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, realtime-index, cds-task-schema-publish-disabled, cds-coordinator-metadata-query-disabled]
5151
uses: ./.github/workflows/reusable-standard-its.yml
5252
if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
5353
with:
@@ -63,7 +63,7 @@ jobs:
6363
strategy:
6464
fail-fast: false
6565
matrix:
66-
testing_group: [kafka-index, append-ingestion]
66+
testing_group: [kafka-index]
6767
uses: ./.github/workflows/reusable-standard-its.yml
6868
if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
6969
with:

‎embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java‎

Lines changed: 50 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -129,30 +129,7 @@ public void test_runIndexTask_andReindexIntoAnotherDatasource(PartitionsSpec par
129129
{
130130
final boolean isRollup = partitionsSpec.isForceGuaranteedRollupCompatible();
131131

132-
final TaskBuilder.IndexParallel indexTask =
133-
TaskBuilder.ofTypeIndexParallel()
134-
.dataSource(dataSource)
135-
.timestampColumn("timestamp")
136-
.jsonInputFormat()
137-
.localInputSourceWithFiles(
138-
Resources.DataFile.tinyWiki1Json(),
139-
Resources.DataFile.tinyWiki2Json(),
140-
Resources.DataFile.tinyWiki3Json()
141-
)
142-
.segmentGranularity("DAY")
143-
.dimensions("namespace", "page", "language")
144-
.metricAggregates(
145-
new DoubleSumAggregatorFactory("added", "added"),
146-
new DoubleSumAggregatorFactory("deleted", "deleted"),
147-
new DoubleSumAggregatorFactory("delta", "delta"),
148-
new CountAggregatorFactory("count")
149-
)
150-
.tuningConfig(
151-
t -> t.withPartitionsSpec(partitionsSpec)
152-
.withForceGuaranteedRollup(isRollup)
153-
.withMaxNumConcurrentSubTasks(10)
154-
.withSplitHintSpec(new MaxSizeSplitHintSpec(1, null))
155-
);
132+
final TaskBuilder.IndexParallel indexTask = buildIndexParallelTask(partitionsSpec, false);
156133

157134
runTask(indexTask, dataSource);
158135
cluster.callApi().waitForAllSegmentsToBeAvailable(dataSource, coordinator, broker);
@@ -211,6 +188,55 @@ public void test_runIndexTask_andReindexIntoAnotherDatasource(PartitionsSpec par
211188
runQueries(dataSource3);
212189
}
213190

191+
@MethodSource("getTestParamPartitionsSpec")
192+
@ParameterizedTest(name = "partitionsSpec={0}")
193+
public void test_runIndexTask_andAppendData(PartitionsSpec partitionsSpec)
194+
{
195+
final TaskBuilder.IndexParallel initialTask = buildIndexParallelTask(partitionsSpec, false);
196+
runTask(initialTask, dataSource);
197+
cluster.callApi().waitForAllSegmentsToBeAvailable(dataSource, coordinator, broker);
198+
cluster.callApi().verifySqlQuery("SELECT COUNT(*) FROM %s", dataSource, "10");
199+
200+
final TaskBuilder.IndexParallel appendTask
201+
= buildIndexParallelTask(new DynamicPartitionsSpec(null, null), true);
202+
runTask(appendTask, dataSource);
203+
cluster.callApi().waitForAllSegmentsToBeAvailable(dataSource, coordinator, broker);
204+
cluster.callApi().verifySqlQuery("SELECT COUNT(*) FROM %s", dataSource, "20");
205+
}
206+
207+
private TaskBuilder.IndexParallel buildIndexParallelTask(
208+
PartitionsSpec partitionsSpec,
209+
boolean appendToExisting
210+
)
211+
{
212+
final boolean isRollup = partitionsSpec.isForceGuaranteedRollupCompatible();
213+
214+
return TaskBuilder.ofTypeIndexParallel()
215+
.dataSource(dataSource)
216+
.timestampColumn("timestamp")
217+
.jsonInputFormat()
218+
.localInputSourceWithFiles(
219+
Resources.DataFile.tinyWiki1Json(),
220+
Resources.DataFile.tinyWiki2Json(),
221+
Resources.DataFile.tinyWiki3Json()
222+
)
223+
.segmentGranularity("DAY")
224+
.dimensions("namespace", "page", "language")
225+
.metricAggregates(
226+
new DoubleSumAggregatorFactory("added", "added"),
227+
new DoubleSumAggregatorFactory("deleted", "deleted"),
228+
new DoubleSumAggregatorFactory("delta", "delta"),
229+
new CountAggregatorFactory("count")
230+
)
231+
.appendToExisting(appendToExisting)
232+
.tuningConfig(
233+
t -> t.withPartitionsSpec(partitionsSpec)
234+
.withForceGuaranteedRollup(isRollup)
235+
.withMaxNumConcurrentSubTasks(10)
236+
.withSplitHintSpec(new MaxSizeSplitHintSpec(1, null))
237+
);
238+
}
239+
214240
private String runTask(TaskBuilder.IndexParallel taskBuilder, String dataSource)
215241
{
216242
final String taskId = EmbeddedClusterApis.newTaskId(dataSource);

‎integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@ public class TestNGGroup
3333

3434
public static final String TRANSACTIONAL_KAFKA_INDEX_SLOW = "kafka-transactional-index-slow";
3535

36-
public static final String APPEND_INGESTION = "append-ingestion";
37-
3836
/**
3937
* This group can only be run individually using -Dgroups=query since it requires specific test data setup.
4038
*/

‎integration-tests/src/test/java/org/apache/druid/tests/indexer/ITAppendBatchIndexTest.java‎

Lines changed: 0 additions & 189 deletions
This file was deleted.

‎integration-tests/src/test/resources/indexer/wikipedia_compaction_task.json‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

0 commit comments

Comments
 (0)