Skip to content

Commit e604110

Browse files
adoroszlaiadoroszlai
authored
RATIS-2217. Automatically re-try flaky tests in CI (#1229)
1 parent 6238fe3 commit e604110

File tree

19 files changed

+148
-0
lines changed

19 files changed

+148
-0
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ jobs:
8383
- grpc
8484
- server
8585
- misc
86+
- flaky
8687
fail-fast: false
8788
uses: ./.github/workflows/check.yaml
8889
with:

‎dev-support/checks/unit.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ mkdir -p "$REPORT_DIR"
3636
export MAVEN_OPTS="-Xmx4096m"
3737
MAVEN_OPTIONS='-V -B'
3838

39+
if [[ "$@" =~ "-Pflaky-tests" ]]; then
40+
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.timeout=1200"
41+
fi
42+
3943
if [[ "${FAIL_FAST}" == "true" ]]; then
4044
MAVEN_OPTIONS="${MAVEN_OPTIONS} --fail-fast -Dsurefire.skipAfterFailureCount=1"
4145
else

‎pom.xml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,7 @@
222222
<slf4j.version>2.0.7</slf4j.version>
223223
<junit-bom.version>5.11.2</junit-bom.version>
224224
<jacoco.version>0.8.12</jacoco.version>
225+
<flaky-test-groups>flaky | org.apache.ratis.test.tag.FlakyTest</flaky-test-groups>
225226
</properties>
226227

227228
<dependencyManagement>
@@ -1118,6 +1119,7 @@
11181119
<includes>
11191120
<include>org.apache.ratis.grpc.**</include>
11201121
</includes>
1122+
<excludedGroups>${flaky-test-groups}</excludedGroups>
11211123
</configuration>
11221124
</plugin>
11231125
</plugins>
@@ -1135,6 +1137,7 @@
11351137
<include>org.apache.ratis.datastream.**</include>
11361138
<include>org.apache.ratis.server.**</include>
11371139
</includes>
1140+
<excludedGroups>${flaky-test-groups}</excludedGroups>
11381141
</configuration>
11391142
</plugin>
11401143
</plugins>
@@ -1153,6 +1156,21 @@
11531156
<exclude>org.apache.ratis.grpc.**</exclude>
11541157
<exclude>org.apache.ratis.server.**</exclude>
11551158
</excludes>
1159+
<excludedGroups>${flaky-test-groups}</excludedGroups>
1160+
</configuration>
1161+
</plugin>
1162+
</plugins>
1163+
</build>
1164+
</profile>
1165+
<profile>
1166+
<id>flaky-tests</id>
1167+
<build>
1168+
<plugins>
1169+
<plugin>
1170+
<groupId>org.apache.maven.plugins</groupId>
1171+
<artifactId>maven-surefire-plugin</artifactId>
1172+
<configuration>
1173+
<groups>${flaky-test-groups}</groups>
11561174
</configuration>
11571175
</plugin>
11581176
</plugins>

‎ratis-assembly/pom.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
<test.cache.data>${project.build.directory}/test-classes</test.cache.data>
3434
<test.build.classes>${project.build.directory}/test-classes</test.build.classes>
3535
<license.bundles.dependencies>true</license.bundles.dependencies>
36+
<!-- no testable code in this module -->
37+
<skipTests>true</skipTests>
3638
</properties>
3739

3840
<build>

‎ratis-client/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@
2323
<artifactId>ratis-client</artifactId>
2424
<name>Apache Ratis Client</name>
2525

26+
<properties>
27+
<!-- no tests in this module so far -->
28+
<skipTests>true</skipTests>
29+
</properties>
30+
2631
<dependencies>
2732
<dependency>
2833
<groupId>org.apache.ratis</groupId>

‎ratis-common/src/test/java/org/apache/ratis/test/tag/Flaky.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
* <p>
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
* <p>
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.ratis.test.tag;
19+
20+
import java.lang.annotation.ElementType;
21+
import java.lang.annotation.Retention;
22+
import java.lang.annotation.RetentionPolicy;
23+
import java.lang.annotation.Target;
24+
25+
import org.junit.jupiter.api.Tag;
26+
27+
/**
28+
* Annotation to mark JUnit5 test classes or methods that exhibit intermittent
29+
* issues. These are run separately from the normal tests in CI. In case of
30+
* failure they may be repeated a few times.
31+
* Usage: <code>@Flaky("RATIS-123")</code>
32+
*/
33+
@Target({ ElementType.TYPE, ElementType.METHOD })
34+
@Retention(RetentionPolicy.RUNTIME)
35+
@Tag("flaky")
36+
public @interface Flaky {
37+
/**
38+
* The issue(s) tracking the flaky test.
39+
*/
40+
String[] value();
41+
}

‎ratis-common/src/test/java/org/apache/ratis/test/tag/FlakyTest.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
* <p>
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
* <p>
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.ratis.test.tag;
19+
20+
/**
21+
* Interface to mark JUnit4 test classes or methods that exhibit intermittent
22+
* issues. These are run separately from the normal tests in CI. In case of
23+
* failure they may be repeated a few times.
24+
* Usage: <code>@Category(FlakyTest.class) @Flaky("RATIS-123")</code>
25+
*/
26+
public interface FlakyTest {
27+
// category marker
28+
}

‎ratis-docs/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
2626
<artifactId>ratis-docs</artifactId>
2727
<name>Apache Ratis Documentation</name>
2828
<packaging>jar</packaging>
29+
30+
<properties>
31+
<!-- no testable code in this module -->
32+
<skipTests>true</skipTests>
33+
</properties>
34+
2935
</project>

‎ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.ratis.protocol.RaftGroup;
2525
import org.apache.ratis.server.impl.GroupManagementBaseTest;
2626
import org.apache.ratis.server.impl.MiniRaftCluster;
27+
import org.apache.ratis.test.tag.Flaky;
2728
import org.apache.ratis.util.function.CheckedBiConsumer;
2829
import org.junit.jupiter.api.Timeout;
2930
import org.junit.jupiter.params.ParameterizedTest;
@@ -33,6 +34,7 @@
3334
import java.util.Collection;
3435
import java.util.concurrent.atomic.AtomicInteger;
3536

37+
@Flaky("RATIS-2218")
3638
@Timeout(value = 300)
3739
public class TestMultiRaftGroup extends BaseTest {
3840
public static Collection<Object[]> data() {

‎ratis-experiments/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@
2323
<artifactId>ratis-experiments</artifactId>
2424
<name>Apache Ratis Experiments</name>
2525

26+
<properties>
27+
<!-- no tests in this module so far -->
28+
<skipTests>true</skipTests>
29+
</properties>
30+
2631
<dependencies>
2732
<dependency>
2833
<groupId>org.apache.ratis</groupId>

0 commit comments

Comments
 (0)