Unified timeout of 15 minutes. Improved logic of taking thread and heap dumps. (#9552)

Author: AlexeyKuznetsov-DD

dd-java-agent/agent-ci-visibility/build.gradle

@@ -1,6 +1,3 @@
-import java.time.Duration
-import java.time.temporal.ChronoUnit
-
 buildscript {
   repositories {
     mavenLocal()
@@ -67,7 +64,3 @@ shadowJar {
 jar {
   archiveClassifier = 'unbundled'
 }
-
-tasks.withType(Test).configureEach {
-  timeout = Duration.of(12, ChronoUnit.MINUTES)
-}

dd-java-agent/instrumentation/aws-java/aws-java-sqs-1.0/build.gradle

@@ -1,6 +1,3 @@
-import java.time.Duration
-import java.time.temporal.ChronoUnit
-
 muzzle {
   pass {
     group = "com.amazonaws"
@@ -18,15 +15,6 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test')
 
 apply from: "$rootDir/gradle/configure_tests.gradle"
 
-// Increased time needed in CI due to migration from JUnit 4 SpockRunner to JUnit 5 SpockExtension
-forkedTest {
-  timeout = Duration.of(15, ChronoUnit.MINUTES)
-}
-
-latestDepForkedTest {
-  timeout = Duration.of(15, ChronoUnit.MINUTES)
-}
-
 dependencies {
   compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-sqs', version: '1.11.0'
   compileOnly group: 'com.amazonaws', name: 'amazon-sqs-java-messaging-lib', version: '1.0.0'

dd-java-agent/instrumentation/aws-java/aws-java-sqs-2.0/build.gradle

@@ -1,6 +1,3 @@
-import java.time.Duration
-import java.time.temporal.ChronoUnit
-
 muzzle {
   pass {
     group = "software.amazon.awssdk"
@@ -19,16 +16,6 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test')
 
 apply from: "$rootDir/gradle/configure_tests.gradle"
 
-// Increased time needed in CI due to migration from JUnit 4 SpockRunner to JUnit 5 SpockExtension
-forkedTest {
-  timeout = Duration.of(15, ChronoUnit.MINUTES)
-}
-
-latestDepForkedTest {
-  timeout = Duration.of(15, ChronoUnit.MINUTES)
-}
-
-
 dependencies {
   compileOnly group: 'software.amazon.awssdk', name: 'sqs', version: '2.2.0'
   compileOnly group: 'com.amazonaws', name: 'amazon-sqs-java-messaging-lib', version: '2.0.0'

dd-java-agent/instrumentation/kafka/kafka-clients-0.11/build.gradle

@@ -1,6 +1,3 @@
-import java.time.Duration
-import java.time.temporal.ChronoUnit
-
 muzzle {
   pass {
     name = "since-0.11"
@@ -25,11 +22,6 @@ addTestSuite('iastLatestDepTest3')
 
 apply from: "$rootDir/gradle/configure_tests.gradle"
 
-// Increased time needed in CI due to migration from JUnit 4 SpockRunner to JUnit 5 SpockExtension
-forkedTest {
-  timeout = Duration.of(15, ChronoUnit.MINUTES)
-}
-
 dependencies {
   compileOnly group: 'org.apache.kafka', name: 'kafka-clients', version: '0.11.0.0'
   implementation project(':dd-java-agent:instrumentation:kafka:kafka-common')

dd-java-agent/instrumentation/vertx/vertx-rx-3.5/build.gradle

@@ -1,6 +1,3 @@
-import java.time.Duration
-import java.time.temporal.ChronoUnit
-
 apply from: "$rootDir/gradle/java.gradle"
 
 addTestSuiteForDir('latestDepTest', 'test')
@@ -12,15 +9,6 @@ latestDepTest {
   finalizedBy 'latestDepForkedTest'
 }
 
-// Increased time needed in CI due to migration from JUnit 4 SpockRunner to JUnit 5 SpockExtension
-forkedTest {
-  timeout = Duration.of(15, ChronoUnit.MINUTES)
-}
-
-latestDepForkedTest {
-  timeout = Duration.of(15, ChronoUnit.MINUTES)
-}
-
 dependencies {
   testImplementation project(':dd-java-agent:instrumentation:netty:netty-4.1')
   testImplementation project(':dd-java-agent:instrumentation:trace-annotation')

dd-smoke-tests/gradle/build.gradle

@@ -25,8 +25,7 @@ test {
     environment "MAVEN_REPOSITORY_PROXY", project.property("mavenRepositoryProxy")
   }
 
-  // overriding the default timeout of 9 minutes set in configure_tests.gradle,
-  // as Gradle smoke tests might run for a longer duration
+  // Overriding the default timeout as Gradle smoke tests might run for a longer duration.
   timeout = Duration.of(20, ChronoUnit.MINUTES)
 }
 

dd-smoke-tests/maven/build.gradle

@@ -26,8 +26,7 @@ tasks.withType(Test).configureEach {
   dependsOn "shadowJar"
   jvmArgs "-Ddatadog.smoketest.maven.jar.path=${tasks.shadowJar.archiveFile.get()}"
 
-  // overriding the default timeout set in configure_tests.gradle, as Maven smoke
-  // tests might run for a longer duration
+  // Overriding the default timeout as Maven smoke tests might run for a longer duration.
   timeout = Duration.of(25, ChronoUnit.MINUTES)
 
   if (project.hasProperty("mavenRepositoryProxy")) {

gradle/configure_tests.gradle

@@ -27,9 +27,6 @@ def forkedTestLimit = gradle.sharedServices.registerIfAbsent("forkedTestLimit",
   maxParallelUsages = 3
 }
 
-// Force timeout after 9 minutes (The timeout is configurable per job, default job timeout is 1h)
-def testTimeoutDuration = Duration.of(9, ChronoUnit.MINUTES)
-
 testing {
   suites.configureEach {
     // Use JUnit 5 to run tests
@@ -39,10 +36,6 @@ testing {
 
 // Go through the Test tasks and configure them
 tasks.withType(Test).configureEach {
-  // All tests must complete within 15 minutes.
-  // This value is quite big because with lower values (3 mins) we were experiencing large number of false positives
-  timeout = Duration.ofMinutes(15)
-
   // Disable all tests if skipTests property was specified
   onlyIf { !project.rootProject.hasProperty("skipTests") }
 
@@ -82,7 +75,8 @@ tasks.withType(Test).configureEach {
     exclude("**/*ForkedTest*")
   }
 
-  timeout = testTimeoutDuration
+  // Set test timeout for 15 minutes. Default job timeout is 1h (configured on CI level).
+  timeout = Duration.of(15, ChronoUnit.MINUTES)
 
   check.dependsOn(it)
 }

gradle/dump_hanging_test.gradle

@@ -10,13 +10,15 @@ tasks.withType(Test).configureEach { testTask ->
       t
     })
 
-    // Calculate delay for taking dumps as test timeout minus 2 minutes, but no less than 1 minute.
-    def delayMinutes = Math.max(1L, timeout.get().minusMinutes(2).toMinutes())
+    // Calculate delay for taking dumps as test timeout minus 1 minutes, but no less than 1 minute.
+    def delayMinutes = Math.max(1L, timeout.get().minusMinutes(1).toMinutes())
 
     def future = scheduler.schedule({
+      logger.warn("Taking dumps for: ${testTask.getPath()} after ${delayMinutes} minutes.")
+
       try {
         // Use Gradle's build dir and adjust for CI artifacts collection if needed.
-        def dumpDir = layout.buildDirectory.dir('dumps').map {
+        def dumpsDir = layout.buildDirectory.dir('dumps').map {
           if (providers.environmentVariable("CI").isPresent()) {
             // Move reports into the folder collected by the collect_reports.sh script.
             new File(it.getAsFile().absolutePath.replace('dd-trace-java/dd-java-agent', 'dd-trace-java/workspace/dd-java-agent'))
@@ -25,7 +27,15 @@ tasks.withType(Test).configureEach { testTask ->
           }
         }.get()
 
-        dumpDir.mkdirs()
+        dumpsDir.mkdirs()
+
+        // For simplicity, use `0` as the PID, which collects all thread dumps across JVMs.
+        // Single file can be useful for quick search.
+        def threadDumpsFile = new File(dumpsDir, "all-thread-dumps-${System.currentTimeMillis()}.log")
+        new ProcessBuilder("jcmd", "0", "Thread.print", "-l")
+          .redirectErrorStream(true)
+          .redirectOutput(threadDumpsFile)
+          .start().waitFor()
 
         // Collect PIDs of all Java processes.
         def jvmProcesses = 'jcmd -l'.execute().text.readLines()
@@ -36,20 +46,18 @@ tasks.withType(Test).configureEach { testTask ->
           .collect({ it.substring(0, it.indexOf(' ')) })
 
         pids.each { pid ->
-          logger.warn("Taking dumps for: ${testTask.getPath()}")
+          // Collect heap dump by pid.
+          def heapDumpFile = new File(dumpsDir, "${pid}-heap-dump-${System.currentTimeMillis()}.hprof").absolutePath
+          def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}"
+          cmd.execute().waitFor()
 
-          // Collect thread dump.
-          def threadDumpFile = new File(dumpDir, "${pid}-thread-dump-${System.currentTimeMillis()}.log")
+          // Collect thread dump by pid.
+          def threadDumpFile = new File(dumpsDir, "${pid}-thread-dump-${System.currentTimeMillis()}.log")
           new ProcessBuilder('jcmd', pid, 'Thread.print', '-l')
             .redirectErrorStream(true)
             .redirectOutput(threadDumpFile)
             .start()
             .waitFor()
-
-          // Collect heap dump.
-          def heapDumpFile = new File(dumpDir, "${pid}-heap-dump-${System.currentTimeMillis()}.hprof").absolutePath
-          def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}"
-          cmd.execute().waitFor()
         }
       } catch (Throwable e) {
         logger.warn("Dumping failed: ${e.message}")