diff --git a/gradle/configure_tests.gradle b/gradle/configure_tests.gradle index d2e9ccf71e6..b9883887481 100644 --- a/gradle/configure_tests.gradle +++ b/gradle/configure_tests.gradle @@ -1,6 +1,8 @@ import java.time.Duration import java.time.temporal.ChronoUnit +apply from: "$rootDir/gradle/dump_hanging_test.gradle" + def isTestingInstrumentation(Project project) { return [ "junit-4.10", @@ -128,7 +130,7 @@ if (!project.property("activePartition")) { } } -tasks.withType(Test) { +tasks.withType(Test).configureEach { // https://docs.gradle.com/develocity/flaky-test-detection/ // https://docs.gradle.com/develocity/gradle-plugin/current/#test_retry develocity.testRetry { diff --git a/gradle/dump_hanging_test.gradle b/gradle/dump_hanging_test.gradle new file mode 100644 index 00000000000..56bbe0a638f --- /dev/null +++ b/gradle/dump_hanging_test.gradle @@ -0,0 +1,75 @@ +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +// Schedule thread and heap dumps collection near test timeout. +tasks.withType(Test).configureEach { testTask -> + doFirst { + def scheduler = Executors.newSingleThreadScheduledExecutor({ r -> + Thread t = new Thread(r, 'dump-scheduler') + t.daemon = true + t + }) + + // Calculate delay for taking dumps as test timeout minus 2 minutes, but no less than 1 minute. + def delayMinutes = Math.max(1L, timeout.get().minusMinutes(2).toMinutes()) + + def future = scheduler.schedule({ + try { + // Use Gradle's build dir and adjust for CI artifacts collection if needed. + def dumpDir = layout.buildDirectory.dir('dumps').map { + if (providers.environmentVariable("CI").isPresent()) { + // Move reports into the folder collected by the collect_reports.sh script. + new File(it.getAsFile().absolutePath.replace('dd-trace-java/dd-java-agent', 'dd-trace-java/workspace/dd-java-agent')) + } else { + it.asFile + } + }.get() + + dumpDir.mkdirs() + + // Collect PIDs of all Java processes. + def jvmProcesses = 'jcmd -l'.execute().text.readLines() + + // Collect pids for 'Gradle test executors'. + def pids = jvmProcesses + .findAll({ it.contains('Gradle Test Executor') }) + .collect({ it.substring(0, it.indexOf(' ')) }) + + pids.each { pid -> + logger.warn("Taking dumps for: ${testTask.getPath()}") + + // Collect thread dump. + def threadDumpFile = new File(dumpDir, "${pid}-thread-dump-${System.currentTimeMillis()}.log") + new ProcessBuilder('jcmd', pid, 'Thread.print', '-l') + .redirectErrorStream(true) + .redirectOutput(threadDumpFile) + .start() + .waitFor() + + // Collect heap dump. + def heapDumpFile = new File(dumpDir, "${pid}-heap-dump-${System.currentTimeMillis()}.hprof").absolutePath + def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}" + cmd.execute().waitFor() + } + } catch (Throwable e) { + logger.warn("Dumping failed: ${e.message}") + } + finally { + scheduler.shutdown() + } + }, delayMinutes, TimeUnit.MINUTES) + + // Store handles for cancellation in doLast. + ext.dumpFuture = future + ext.dumpScheduler = scheduler + } + + doLast { + // Cancel if the task finished before the scheduled dump. + try { + ext.dumpFuture?.cancel(false) + } finally { + ext.dumpScheduler?.shutdownNow() + } + } +}