[POC] Flaky test retries

Author: smola

.circleci/config.continue.yml.j2

@@ -1122,7 +1122,6 @@ build_test_jobs: &build_test_jobs
       name: z_test_<< matrix.testJvm >>_base
       triggeredBy: *core_modules
       gradleTarget: ":baseTest"
-      gradleParameters: "-PskipFlakyTests"
       stage: core
       cacheType: base
       parallelism: 4
@@ -1136,7 +1135,7 @@ build_test_jobs: &build_test_jobs
       name: z_test_8_base
       triggeredBy: *core_modules
       gradleTarget: :baseTest
-      gradleParameters: "-PskipFlakyTests -PcheckCoverage"
+      gradleParameters: "-PcheckCoverage"
       stage: core
       cacheType: base
       parallelism: 4
@@ -1148,7 +1147,6 @@ build_test_jobs: &build_test_jobs
         - ok_to_test
       name: z_test_<< matrix.testJvm >>_inst
       gradleTarget: ":instrumentationTest"
-      gradleParameters: "-PskipFlakyTests"
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: inst
@@ -1162,7 +1160,6 @@ build_test_jobs: &build_test_jobs
         - ok_to_test
       name: z_test_8_inst
       gradleTarget: ":instrumentationTest"
-      gradleParameters: "-PskipFlakyTests"
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: inst
@@ -1176,7 +1173,6 @@ build_test_jobs: &build_test_jobs
         - build_latestdep
       name: test_8_inst_latest
       gradleTarget: ":instrumentationLatestDepTest"
-      gradleParameters: "-PskipFlakyTests"
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: latestdep
@@ -1190,7 +1186,6 @@ build_test_jobs: &build_test_jobs
         - build_latestdep
       name: test_17_inst_latest
       gradleTarget: ":instrumentationLatestDepTest"
-      gradleParameters: "-PskipFlakyTests"
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: latestdep
@@ -1204,77 +1199,18 @@ build_test_jobs: &build_test_jobs
         - build_latestdep
       name: test_21_inst_latest
       gradleTarget: ":instrumentationLatestDepTest"
-      gradleParameters: "-PskipFlakyTests"
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: latestdep
       parallelism: 12
       maxWorkers: 3
       testJvm: "21"
 
-{% if flaky %}
-  - tests:
-      requires:
-        - ok_to_test
-      name: z_test_8_flaky_base
-      gradleTarget: ":baseTest"
-      gradleParameters: "-PrunFlakyTests"
-      continueOnFailure: true
-      triggeredBy: *core_modules
-      stage: core
-      cacheType: base
-      parallelism: 4
-      maxWorkers: 4
-      testJvm: "8"
-
-  - xlarge_tests:
-      requires:
-        - ok_to_test
-      name: z_test_8_flaky_inst
-      gradleTarget: ":instrumentationTest"
-      gradleParameters: "-PrunFlakyTests"
-      continueOnFailure: true
-      triggeredBy: *instrumentation_modules
-      stage: instrumentation
-      cacheType: inst
-      parallelism: 12
-      maxWorkers: 4
-      testJvm: "8"
-
-  - tests:
-      requires:
-        - ok_to_test
-      name: z_test_8_flaky_smoke
-      gradleTarget: ":smokeTest"
-      gradleParameters: "-PrunFlakyTests"
-      continueOnFailure: true
-      stage: smoke
-      cacheType: smoke
-      parallelism: 4
-      maxWorkers: 4
-      testJvm: "8"
-
-  - tests:
-      requires:
-        - ok_to_test
-      name: z_test_8_flaky_debugger
-      gradleTarget: ":debuggerTest"
-      gradleParameters: "-PrunFlakyTests"
-      continueOnFailure: true
-      triggeredBy: *debugger_modules
-      stage: debugger
-      cacheType: base
-      parallelism: 4
-      maxWorkers: 4
-      testJvm: "8"
-{% endif %}
-
   - tests:
       requires:
         - ok_to_test
       maxWorkers: 4
       gradleTarget: ":profilingTest"
-      gradleParameters: "-PskipFlakyTests"
       triggeredBy: *profiling_modules
       stage: profiling
       cacheType: profiling
@@ -1288,7 +1224,6 @@ build_test_jobs: &build_test_jobs
       name: test_<< matrix.testJvm >>_debugger
       maxWorkers: 4
       gradleTarget: ":debuggerTest"
-      gradleParameters: "-PskipFlakyTests"
       triggeredBy: *debugger_modules
       stage: debugger
       cacheType: base
@@ -1300,7 +1235,6 @@ build_test_jobs: &build_test_jobs
         - ok_to_test
       name: z_test_<< matrix.testJvm >>_smoke
       gradleTarget: "stageMainDist :smokeTest"
-      gradleParameters: "-PskipFlakyTests"
       stage: smoke
       cacheType: smoke
       parallelism: 4
@@ -1314,7 +1248,6 @@ build_test_jobs: &build_test_jobs
       name: z_test_<< matrix.testJvm >>_ssi_smoke
       environment: "DD_INJECT_FORCE=true DD_INJECTION_ENABLED=tracer"
       gradleTarget: "stageMainDist :smokeTest"
-      gradleParameters: "-PskipFlakyTests"
       stage: smoke
       cacheType: smoke
       parallelism: 4
@@ -1328,7 +1261,6 @@ build_test_jobs: &build_test_jobs
       name: test_semeru8_debugger_smoke
       maxWorkers: 4
       gradleTarget: "stageMainDist dd-smoke-tests:debugger-integration-tests:test"
-      gradleParameters: "-PskipFlakyTests"
       triggeredBy: *debugger_modules
       stage: debugger
       cacheType: smoke
@@ -1357,7 +1289,6 @@ build_test_jobs: &build_test_jobs
         - ok_to_test
       name: z_test_8_smoke
       gradleTarget: "stageMainDist :smokeTest"
-      gradleParameters: "-PskipFlakyTests"
       stage: smoke
       cacheType: smoke
       parallelism: 4
@@ -1370,7 +1301,6 @@ build_test_jobs: &build_test_jobs
       name: z_test_8_ssi_smoke
       environment: "DD_INJECT_FORCE=true DD_INJECTION_ENABLED=tracer"
       gradleTarget: "stageMainDist :smokeTest"
-      gradleParameters: "-PskipFlakyTests"
       stage: smoke
       cacheType: smoke
       parallelism: 4

gradle/configure_tests.gradle

@@ -84,8 +84,21 @@ tasks.withType(Test).configureEach {
 
 Task allTestsTask = tasks.maybeCreate('allTests')
 Task allLatestDepTestsTask = tasks.maybeCreate('allLatestDepTests')
-project.afterEvaluate {
-  tasks.withType(Test).each {
+project.afterEvaluate { proj ->
+  if (System.getenv("CI") != null) {
+    proj.tasks.withType(Test).configureEach { task ->
+      task.extensions.configure('develocity') { it ->
+        it.testRetry.maxRetries = 3
+        it.testRetry.maxFailures = 20
+        it.testRetry.failOnPassedAfterRetry = false
+        it.testRetry.failOnSkippedAfterRetry = true
+        it.testRetry.filter { f ->
+          f.includeAnnotationClasses.add('datadog.trace.test.util.Flaky')
+        }
+      }
+    }
+  }
+  proj.tasks.withType(Test).each {
     if (it.name.containsIgnoreCase('latest')) {
       allLatestDepTestsTask.dependsOn it
     } else if (it.name != 'traceAgentTest') {

settings.gradle

@@ -1,5 +1,5 @@
 plugins {
-  id 'com.gradle.develocity' version '3.18'
+  id 'com.gradle.develocity' version '3.19'
 }
 
 def isCI = System.getenv("CI") != null