Skip to content

Debug freezing test on CI. #9350

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 11 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitlab/collect_reports.sh
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ function process_reports () {
cp -r workspace/$project_to_save/build/reports/* $report_path/ 2>/dev/null || true
cp workspace/$project_to_save/build/hs_err_pid*.log $report_path/ 2>/dev/null || true
cp workspace/$project_to_save/build/javacore*.txt $report_path/ 2>/dev/null || true
cp workspace/$project_to_save/build/*.* $report_path/ 2>/dev/null || true
fi
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,46 @@
import com.sun.management.HotSpotDiagnosticMXBean
import datadog.trace.agent.test.AgentTestRunner
import datadog.trace.api.Trace

import javax.management.MBeanServer
import java.lang.management.ManagementFactory
import java.util.concurrent.Callable
import java.util.concurrent.Executors
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.ScheduledFuture
import java.util.concurrent.StructuredTaskScope
import java.util.concurrent.TimeUnit

import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace
import static datadog.trace.agent.test.utils.TraceUtils.runnableUnderTrace
import static java.time.Instant.now

class StructuredConcurrencyTest extends AgentTestRunner {
ThreadDumpLogger threadDumpLogger

def setup() {
File reportDir = new File("build")
String fullPath = reportDir.absolutePath.replace("dd-trace-java/dd-java-agent",
"dd-trace-java/workspace/dd-java-agent")

reportDir = new File(fullPath)
if (!reportDir.exists()) {
println("Folder not found: " + fullPath)
reportDir.mkdirs()
} else println("Folder found: " + fullPath)

// Use the current feature name as the test name
String testName = "${specificationContext?.currentSpec?.name ?: "unknown-spec"} : ${specificationContext?.currentFeature?.name ?: "unknown-test"}"

threadDumpLogger = new ThreadDumpLogger(testName, reportDir)
threadDumpLogger.start()
}

def cleanup() {
threadDumpLogger.stop()
}


/**
* Tests the structured task scope with a single task.
*/
Expand All @@ -20,12 +52,12 @@ class StructuredConcurrencyTest extends AgentTestRunner {
when:
runUnderTrace("parent") {
def task = taskScope.fork(new Callable<Boolean>() {
@Trace(operationName = "child")
@Override
Boolean call() throws Exception {
return true
}
})
@Trace(operationName = "child")
@Override
Boolean call() throws Exception {
return true
}
})
taskScope.joinUntil(now() + 10) // Wait for 10 seconds at maximum
result = task.get()
}
Expand Down Expand Up @@ -164,4 +196,48 @@ class StructuredConcurrencyTest extends AgentTestRunner {
}
}
}
// 🔒 Private helper class for thread dump logging
private static class ThreadDumpLogger {
private final String testName
private final File outputDir
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor()
private ScheduledFuture<?> task

ThreadDumpLogger(String testName, File outputDir) {
this.testName = testName
this.outputDir = outputDir
}

void start() {
task = scheduler.scheduleAtFixedRate({
heapDump("test_1")

def reportFile = new File(outputDir, "${System.currentTimeMillis()}-thread-dump.log")
try (def writer = new FileWriter(reportFile)) {
writer.write("=== Test: ${testName} ===\n")
writer.write("=== Thread Dump Triggered at ${new Date()} ===\n")
Thread.getAllStackTraces().each { thread, stack ->
writer.write("Thread: ${thread.name}, daemon: ${thread.daemon}\n")
stack.each { writer.write("\tat ${it}\n") }
}
writer.write("==============================================\n")
}

heapDump("test_2")
}, 10005, 60005, TimeUnit.MILLISECONDS)
}

void heapDump(String kind) {
def heapDumpFile = new File(outputDir, "${System.currentTimeMillis()}-heap-dump-${kind}.hprof").absolutePath
MBeanServer server = ManagementFactory.getPlatformMBeanServer()
HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy(
server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class)
mxBean.dumpHeap(heapDumpFile, true)
}

void stop() {
task?.cancel(false)
scheduler.shutdownNow()
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,19 @@ import com.lambdaworks.redis.RedisClient
import com.lambdaworks.redis.api.StatefulConnection
import com.lambdaworks.redis.api.async.RedisAsyncCommands
import com.lambdaworks.redis.api.sync.RedisCommands
import com.sun.management.HotSpotDiagnosticMXBean
import datadog.trace.agent.test.naming.VersionedNamingTestBase
import datadog.trace.agent.test.utils.PortUtils
import redis.embedded.RedisServer
import spock.lang.Shared

import javax.management.MBeanServer
import java.lang.management.ManagementFactory
import java.util.concurrent.Executors
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.ScheduledFuture
import java.util.concurrent.TimeUnit

import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace

abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase {
Expand All @@ -32,6 +40,8 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase {
@Shared
RedisServer redisServer

ThreadDumpLogger threadDumpLogger

@Shared
Map<String, String> testHashMap = [
firstname: "John",
Expand All @@ -53,14 +63,30 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase {
embeddedDbUri = "redis://" + dbAddr

redisServer = RedisServer.newRedisServer()
// bind to localhost to avoid firewall popup
.setting("bind " + HOST)
// set max memory to avoid problems in CI
.setting("maxmemory 128M")
.port(port).build()
// bind to localhost to avoid firewall popup
.setting("bind " + HOST)
// set max memory to avoid problems in CI
.setting("maxmemory 128M")
.port(port).build()
}

def setup() {
File reportDir = new File("build")
String fullPath = reportDir.absolutePath.replace("dd-trace-java/dd-java-agent",
"dd-trace-java/workspace/dd-java-agent")

reportDir = new File(fullPath)
if (!reportDir.exists()) {
println("Folder not found: " + fullPath)
reportDir.mkdirs()
} else println("Folder found: " + fullPath)

// Use the current feature name as the test name
String testName = "${specificationContext?.currentSpec?.name ?: "unknown-spec"} : ${specificationContext?.currentFeature?.name ?: "unknown-test"}"

threadDumpLogger = new ThreadDumpLogger(testName, reportDir)
threadDumpLogger.start()

redisServer.start()

redisClient = RedisClient.create(embeddedDbUri)
Expand All @@ -79,8 +105,55 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase {
}

def cleanup() {
threadDumpLogger.stop()

connection.close()
redisClient.shutdown()
redisServer.stop()
}

// 🔒 Private helper class for thread dump logging
private static class ThreadDumpLogger {
private final String testName
private final File outputDir
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor()
private ScheduledFuture<?> task

ThreadDumpLogger(String testName, File outputDir) {
this.testName = testName
this.outputDir = outputDir
}

void start() {
// new File(outputDir, "${System.currentTimeMillis()}-start-mark.txt") << testName

task = scheduler.scheduleAtFixedRate({
heapDump("test")

def reportFile = new File(outputDir, "${System.currentTimeMillis()}-thread-dump.log")
try (def writer = new FileWriter(reportFile)) {
writer.write("=== Test: ${testName} ===\n")
writer.write("=== Thread Dump Triggered at ${new Date()} ===\n")
Thread.getAllStackTraces().each { thread, stack ->
writer.write("Thread: ${thread.name}, daemon: ${thread.daemon}\n")
stack.each { writer.write("\tat ${it}\n") }
}
writer.write("==============================================\n")
}
}, 10003, 60000, TimeUnit.MILLISECONDS)
}

void heapDump(String kind) {
def heapDumpFile = new File(outputDir, "${System.currentTimeMillis()}-heap-dump-${kind}.hprof").absolutePath
MBeanServer server = ManagementFactory.getPlatformMBeanServer()
HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy(
server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class)
mxBean.dumpHeap(heapDumpFile, true)
}

void stop() {
task?.cancel(false)
scheduler.shutdownNow()
}
}
}
Loading