diff --git a/dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy b/dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy index e71a8a5d8c9..930f53da4ac 100644 --- a/dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy +++ b/dd-java-agent/agent-ci-visibility/civisibility-instrumentation-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy @@ -353,7 +353,7 @@ abstract class CiVisibilityInstrumentationTest extends InstrumentationSpecificat settings.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled } - def assertSpansData(String testcaseName, Map replacements = [:], List ignoredTags = []) { + def assertSpansData(String testcaseName, Map replacements = [:], List ignoredTags = [], List dynamicTags = []) { Predicate sessionSpan = span -> span.spanType == "test_session_end" spanFilter.waitForSpan(sessionSpan, TimeUnit.SECONDS.toMillis(20)) @@ -369,13 +369,13 @@ abstract class CiVisibilityInstrumentationTest extends InstrumentationSpecificat def additionalIgnoredTags = CiVisibilityTestUtils.IGNORED_TAGS + ignoredTags if (System.getenv().get("GENERATE_TEST_FIXTURES") != null) { - return generateTestFixtures(testcaseName, events, coverages, additionalReplacements, additionalIgnoredTags) + return generateTestFixtures(testcaseName, events, coverages, additionalReplacements, additionalIgnoredTags, dynamicTags) } - return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements, additionalIgnoredTags) + return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements, additionalIgnoredTags, [], dynamicTags) } - def generateTestFixtures(String testcaseName, List events, List coverages, Map additionalReplacements, List additionalIgnoredTags) { + def generateTestFixtures(String testcaseName, List events, List coverages, Map additionalReplacements, List additionalIgnoredTags, List dynamicTags = []) { def clazz = this.getClass() def resourceName = "/" + clazz.name.replace('.', '/') + ".class" def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart @@ -387,7 +387,7 @@ abstract class CiVisibilityInstrumentationTest extends InstrumentationSpecificat submoduleName = "test" } def baseTemplatesPath = modulePath + "/src/" + submoduleName + "/resources/" + testcaseName - CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements.keySet(), additionalIgnoredTags) + CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements.keySet(), additionalIgnoredTags, dynamicTags) return [:] } diff --git a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTestUtils.java b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTestUtils.java index fb9ff951a88..b71de7356e0 100644 --- a/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTestUtils.java +++ b/dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/java/datadog/trace/civisibility/CiVisibilityTestUtils.java @@ -120,6 +120,22 @@ public static void generateTemplates( List> coverages, Collection additionalDynamicPaths, List ignoredTags) { + generateTemplates( + baseTemplatesPath, + events, + coverages, + additionalDynamicPaths, + ignoredTags, + Collections.emptyList()); + } + + public static void generateTemplates( + String baseTemplatesPath, + List> events, + List> coverages, + Collection additionalDynamicPaths, + List ignoredTags, + Collection additionalNonUniqueDynamicPaths) { List> mutableEvents = new ArrayList<>(events); if (!ignoredTags.isEmpty()) { mutableEvents = removeTags(mutableEvents, ignoredTags); @@ -128,6 +144,7 @@ public static void generateTemplates( TemplateGenerator templateGenerator = new TemplateGenerator(new LabelGenerator()); List compiledAdditionalReplacements = compile(additionalDynamicPaths); + compiledAdditionalReplacements.addAll(compileNonUnique(additionalNonUniqueDynamicPaths)); try { Files.createDirectories(Paths.get(baseTemplatesPath)); @@ -219,6 +236,24 @@ public static Map assertData( Map additionalReplacements, List ignoredTags, List additionalDynamicPaths) { + return assertData( + baseTemplatesPath, + events, + coverages, + additionalReplacements, + ignoredTags, + additionalDynamicPaths, + Collections.emptyList()); + } + + public static Map assertData( + String baseTemplatesPath, + List> events, + List> coverages, + Map additionalReplacements, + List ignoredTags, + List additionalDynamicPaths, + List additionalNonUniqueDynamicPaths) { List> mutableEvents = new ArrayList<>(events); mutableEvents.sort(EVENT_RESOURCE_COMPARATOR); @@ -227,6 +262,7 @@ public static Map assertData( List eventPaths = new ArrayList<>(EVENT_DYNAMIC_PATHS); eventPaths.addAll(compile(additionalDynamicPaths)); + eventPaths.addAll(compileNonUnique(additionalNonUniqueDynamicPaths)); templateGenerator.generateReplacementMap(mutableEvents, eventPaths); Map replacementMap = templateGenerator.generateReplacementMap(coverages, COVERAGE_DYNAMIC_PATHS); @@ -512,6 +548,14 @@ private static List compile(Iterable rawPaths) { return compiledPaths; } + private static List compileNonUnique(Iterable rawPaths) { + List compiledPaths = new ArrayList<>(); + for (String rawPath : rawPaths) { + compiledPaths.add(path(rawPath, false)); + } + return compiledPaths; + } + private static DynamicPath path(String rawPath) { return path(rawPath, true); } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java index 91f3c5e8101..7236e8c823f 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java @@ -4,6 +4,7 @@ public interface TestDecorator { String TEST_TYPE = "test"; + String TEST_TYPE_BENCHMARK = "benchmark"; AgentSpan afterStart(final AgentSpan span); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecoratorImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecoratorImpl.java index de2ab526728..d6b97384059 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecoratorImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecoratorImpl.java @@ -6,12 +6,17 @@ import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.util.Strings; +import java.util.Collections; import java.util.Map; +import java.util.Set; public class TestDecoratorImpl implements TestDecorator { private static final UTF8BytesString CIAPP_TEST_ORIGIN = UTF8BytesString.create("ciapp-test"); + // components whose tests represent benchmarks (test.type = "benchmark") + private static final Set BENCHMARK_COMPONENTS = Collections.singleton("jmh"); + private final String component; private final String sessionName; private final Map ciTags; @@ -32,7 +37,7 @@ public TestDecoratorImpl( } protected String testType() { - return TEST_TYPE; + return BENCHMARK_COMPONENTS.contains(component) ? TEST_TYPE_BENCHMARK : TEST_TYPE; } protected UTF8BytesString origin() { diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle b/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle new file mode 100644 index 00000000000..a5256396007 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle @@ -0,0 +1,23 @@ +plugins { + id 'dd-trace-java.instrumentation.testing-framework-tests' +} + +apply from: "$rootDir/gradle/java.gradle" + +muzzle { + pass { + group = 'org.openjdk.jmh' + module = 'jmh-core' + versions = '[1.0,)' + } +} + +dependencies { + compileOnly group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.0' + + testImplementation group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.37' + testImplementation project(':dd-java-agent:agent-ci-visibility:civisibility-instrumentation-test-fixtures') + + // JMH annotation processor generates META-INF/BenchmarkList from @Benchmark annotations + testAnnotationProcessor group: 'org.openjdk.jmh', name: 'jmh-generator-annprocess', version: '1.37' +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/gradle.lockfile b/dd-java-agent/instrumentation/jmh/jmh-1.0/gradle.lockfile new file mode 100644 index 00000000000..010265b6f6f --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/gradle.lockfile @@ -0,0 +1,144 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +cafe.cryptography:curve25519-elisabeth:0.1.0=testRuntimeClasspath +cafe.cryptography:ed25519-elisabeth:0.1.0=testRuntimeClasspath +ch.qos.logback:logback-classic:1.2.13=testCompileClasspath,testRuntimeClasspath +ch.qos.logback:logback-core:1.2.13=testCompileClasspath,testRuntimeClasspath +com.blogspot.mydailyjava:weak-lock-free:0.17=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq.okhttp3:okhttp:3.12.15=testCompileClasspath,testRuntimeClasspath +com.datadoghq.okio:okio:1.17.6=testCompileClasspath,testRuntimeClasspath +com.datadoghq:dd-instrument-java:0.0.3=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq:dd-javac-plugin-client:0.2.2=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq:java-dogstatsd-client:4.4.5=testRuntimeClasspath +com.datadoghq:sketches-java:0.8.3=testRuntimeClasspath +com.fasterxml.jackson.core:jackson-annotations:2.20=testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.20.0=testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.20.0=testCompileClasspath,testRuntimeClasspath +com.github.javaparser:javaparser-core:3.25.6=codenarc +com.github.jnr:jffi:1.3.14=testRuntimeClasspath +com.github.jnr:jnr-a64asm:1.0.0=testRuntimeClasspath +com.github.jnr:jnr-constants:0.10.4=testRuntimeClasspath +com.github.jnr:jnr-enxio:0.32.19=testRuntimeClasspath +com.github.jnr:jnr-ffi:2.2.18=testRuntimeClasspath +com.github.jnr:jnr-posix:3.1.21=testRuntimeClasspath +com.github.jnr:jnr-unixsocket:0.38.24=testRuntimeClasspath +com.github.jnr:jnr-x86asm:1.0.2=testRuntimeClasspath +com.github.spotbugs:spotbugs-annotations:4.9.8=compileClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +com.github.spotbugs:spotbugs:4.9.8=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.1.1=annotationProcessor,compileClasspath,testAnnotationProcessor,testCompileClasspath +com.google.auto.service:auto-service:1.1.1=annotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.1=annotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,compileClasspath,spotbugs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotations:2.18.0=annotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.guava:failureaccess:1.0.1=annotationProcessor,testAnnotationProcessor +com.google.guava:guava:20.0=testCompileClasspath,testRuntimeClasspath +com.google.guava:guava:32.0.1-jre=annotationProcessor,testAnnotationProcessor +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:2.8=annotationProcessor,testAnnotationProcessor +com.google.re2j:re2j:1.7=testRuntimeClasspath +com.jayway.jsonpath:json-path:2.8.0=testCompileClasspath,testRuntimeClasspath +com.squareup.moshi:moshi:1.11.0=testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:logging-interceptor:3.12.12=testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:okhttp:3.12.12=testCompileClasspath,testRuntimeClasspath +com.squareup.okio:okio:1.17.5=testCompileClasspath,testRuntimeClasspath +com.thoughtworks.qdox:qdox:1.12.1=codenarc +com.vaadin.external.google:android-json:0.0.20131108.vaadin1=testCompileClasspath,testRuntimeClasspath +commons-fileupload:commons-fileupload:1.5=testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.11.0=testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.20.0=spotbugs +de.thetaphi:forbiddenapis:3.10=compileClasspath,testCompileClasspath,testRuntimeClasspath +io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath +io.sqreen:libsqreen:17.3.0=testRuntimeClasspath +javax.servlet:javax.servlet-api:3.1.0=testCompileClasspath,testRuntimeClasspath +jaxen:jaxen:2.0.0=spotbugs +junit:junit:4.13.2=testRuntimeClasspath +net.bytebuddy:byte-buddy-agent:1.18.8=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +net.bytebuddy:byte-buddy:1.18.8=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna-platform:5.8.0=testRuntimeClasspath +net.java.dev.jna:jna:5.8.0=testRuntimeClasspath +net.minidev:accessors-smart:2.4.9=testRuntimeClasspath +net.minidev:json-smart:2.4.10=testRuntimeClasspath +net.sf.jopt-simple:jopt-simple:4.6=compileClasspath +net.sf.jopt-simple:jopt-simple:5.0.4=testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=spotbugs +org.apache.ant:ant-antlr:1.10.14=codenarc +org.apache.ant:ant-junit:1.10.14=codenarc +org.apache.bcel:bcel:6.11.0=spotbugs +org.apache.commons:commons-lang3:3.19.0=spotbugs +org.apache.commons:commons-math3:3.2=compileClasspath +org.apache.commons:commons-math3:3.6.1=testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-text:1.14.0=spotbugs +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.checkerframework:checker-qual:3.33.0=annotationProcessor,testAnnotationProcessor +org.codehaus.groovy:groovy-ant:3.0.23=codenarc +org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc +org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.25=testCompileClasspath,testRuntimeClasspath +org.codehaus.groovy:groovy-templates:3.0.23=codenarc +org.codehaus.groovy:groovy-xml:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.25=testCompileClasspath,testRuntimeClasspath +org.codenarc:CodeNarc:3.7.0=codenarc +org.dom4j:dom4j:2.2.0=spotbugs +org.freemarker:freemarker:2.3.31=testCompileClasspath,testRuntimeClasspath +org.gmetrics:GMetrics:2.1.0=codenarc +org.hamcrest:hamcrest-core:1.3=testRuntimeClasspath +org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath +org.jacoco:org.jacoco.core:0.8.14=testRuntimeClasspath +org.jacoco:org.jacoco.report:0.8.14=testRuntimeClasspath +org.jctools:jctools-core-jdk11:4.0.6=testRuntimeClasspath +org.jctools:jctools-core:4.0.6=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:5.14.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:1.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:1.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-launcher:1.14.1=testRuntimeClasspath +org.junit.platform:junit-platform-runner:1.14.1=testRuntimeClasspath +org.junit.platform:junit-platform-suite-api:1.14.1=testRuntimeClasspath +org.junit.platform:junit-platform-suite-commons:1.14.1=testRuntimeClasspath +org.junit:junit-bom:5.14.0=spotbugs +org.junit:junit-bom:5.14.1=testCompileClasspath,testRuntimeClasspath +org.mockito:mockito-core:4.4.0=testRuntimeClasspath +org.msgpack:jackson-dataformat-msgpack:0.9.6=testCompileClasspath,testRuntimeClasspath +org.msgpack:msgpack-core:0.9.6=testCompileClasspath,testRuntimeClasspath +org.objenesis:objenesis:3.3=testCompileClasspath,testRuntimeClasspath +org.openjdk.jmh:jmh-core:1.0=compileClasspath +org.openjdk.jmh:jmh-core:1.37=testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +org.openjdk.jmh:jmh-generator-annprocess:1.37=testAnnotationProcessor +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.7.1=testRuntimeClasspath +org.ow2.asm:asm-analysis:9.9=spotbugs +org.ow2.asm:asm-commons:9.9=spotbugs +org.ow2.asm:asm-commons:9.9.1=testRuntimeClasspath +org.ow2.asm:asm-tree:9.9=spotbugs +org.ow2.asm:asm-tree:9.9.1=testRuntimeClasspath +org.ow2.asm:asm-util:9.7.1=testRuntimeClasspath +org.ow2.asm:asm-util:9.9=spotbugs +org.ow2.asm:asm:9.9=spotbugs +org.ow2.asm:asm:9.9.1=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.skyscreamer:jsonassert:1.5.1=testCompileClasspath,testRuntimeClasspath +org.slf4j:jcl-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:jul-to-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:log4j-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:1.7.30=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath +org.slf4j:slf4j-api:1.7.32=testCompileClasspath +org.slf4j:slf4j-api:1.7.36=testRuntimeClasspath +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j +org.snakeyaml:snakeyaml-engine:2.9=buildTimeInstrumentationPlugin,muzzleTooling,runtimeClasspath,testRuntimeClasspath +org.spockframework:spock-bom:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath +org.spockframework:spock-core:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-junit:1.2.1=testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-parser:1.2.0=testCompileClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=spotbugs +org.xmlunit:xmlunit-core:2.10.3=testCompileClasspath,testRuntimeClasspath +empty=spotbugsPlugins diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java new file mode 100644 index 00000000000..097f4de2c41 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java @@ -0,0 +1,100 @@ +package datadog.trace.instrumentation.jmh; + +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.IterationParams; +import org.openjdk.jmh.results.BenchmarkResult; +import org.openjdk.jmh.results.IterationResult; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.format.OutputFormat; + +/** + * Wraps a JMH {@link OutputFormat} to emit CI Visibility spans for each benchmark method. + * + *

Hooks only fire once per benchmark method (after all forks and iterations complete), so there + * is zero overhead on the benchmark hot path. + */ +public class DDOutputFormat implements OutputFormat { + + private final OutputFormat delegate; + + public DDOutputFormat(OutputFormat delegate) { + this.delegate = delegate; + } + + @Override + public void startBenchmark(BenchmarkParams benchParams) { + delegate.startBenchmark(benchParams); + DatadogJmhReporter.onBenchmarkStart(benchParams); + } + + @Override + public void endBenchmark(BenchmarkResult result) { + try { + DatadogJmhReporter.onBenchmarkEnd(result); + } finally { + delegate.endBenchmark(result); + } + } + + @Override + public void endRun(java.util.Collection result) { + try { + DatadogJmhReporter.onRunEnd(); + } finally { + delegate.endRun(result); + } + } + + // ---- Delegation-only methods ---- + + @Override + public void iteration(BenchmarkParams benchParams, IterationParams params, int iteration) { + delegate.iteration(benchParams, params, iteration); + } + + @Override + public void iterationResult( + BenchmarkParams benchParams, IterationParams params, int iteration, IterationResult data) { + delegate.iterationResult(benchParams, params, iteration, data); + } + + @Override + public void startRun() { + delegate.startRun(); + } + + @Override + public void print(String s) { + delegate.print(s); + } + + @Override + public void println(String s) { + delegate.println(s); + } + + @Override + public void verbosePrintln(String s) { + delegate.verbosePrintln(s); + } + + @Override + public void write(int b) { + delegate.write(b); + } + + @Override + public void write(byte[] b) throws java.io.IOException { + delegate.write(b); + } + + @Override + public void flush() { + delegate.flush(); + } + + @Override + public void close() { + delegate.close(); + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DatadogJmhReporter.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DatadogJmhReporter.java new file mode 100644 index 00000000000..f1df49a2015 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DatadogJmhReporter.java @@ -0,0 +1,215 @@ +package datadog.trace.instrumentation.jmh; + +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_ERROR; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_FORKS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_ITERATIONS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MAX; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MIN; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MODE; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P50; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P90; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P95; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P99; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_SAMPLE_COUNT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_THREADS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_TIME_UNIT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_UNIT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_VALUE; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_WARMUP_ITERATIONS; + +import datadog.trace.api.civisibility.InstrumentationBridge; +import datadog.trace.api.civisibility.config.TestSourceData; +import datadog.trace.api.civisibility.events.TestDescriptor; +import datadog.trace.api.civisibility.events.TestEventsHandler; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; +import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.results.BenchmarkResult; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.util.Statistics; + +public final class DatadogJmhReporter { + + private static final String TEST_FRAMEWORK = "jmh"; + private static final String TEST_FRAMEWORK_VERSION = JmhUtils.frameworkVersion(); + + private static volatile TestEventsHandler + TEST_EVENTS_HANDLER; + + // Suites opened so far in this run, keyed by suite (class) name. JMH has no class-level + // start/finish callbacks, so we open a suite the first time we see a benchmark of that class, + // keep it open while every benchmark of the class runs under it, and finish all suites together + // in onRunEnd. + private static final Map OPEN_SUITES = new ConcurrentHashMap<>(); + + // The single benchmark (test) currently in flight. startBenchmark/endBenchmark only ever fire in + // the launcher JVM, strictly sequentially (forked JVMs never call them), so a single reference is + // enough. If it is still set when the next benchmark starts (or the run ends) JMH skipped + // endBenchmark for a failed benchmark, and we finish it as a failure rather than leaking it. + private static volatile TestDescriptor currentTest; + + private DatadogJmhReporter() {} + + public static synchronized void start() { + if (TEST_EVENTS_HANDLER == null) { + TEST_EVENTS_HANDLER = + InstrumentationBridge.createTestEventsHandler( + TEST_FRAMEWORK, null, null, JmhUtils.CAPABILITIES); + } + } + + public static synchronized void stop() { + if (TEST_EVENTS_HANDLER != null) { + TEST_EVENTS_HANDLER.close(); + TEST_EVENTS_HANDLER = null; + } + } + + public static void onBenchmarkStart(BenchmarkParams benchParams) { + // lazy load handler + start(); + + // A previous benchmark whose endBenchmark was skipped (JMH swallows benchmark errors when + // fail-on-error is off) is still open. Finish it as a failure before starting the next one. + finishOpenTest(true); + + String fullName = benchParams.getBenchmark(); + String[] parts = JmhUtils.splitBenchmarkName(fullName); + String suiteName = parts[0]; + String testName = parts[1]; + String testParameters = JmhUtils.testParameters(benchParams); + + // JMH gives us only fully-qualified names, not Class objects, so testClass is null. The + // TestDescriptor includes testParameters in its identity, so each @Param variant is distinct. + TestSuiteDescriptor suite = new TestSuiteDescriptor(suiteName, null); + TestDescriptor test = new TestDescriptor(suiteName, null, testName, testParameters, null); + currentTest = test; + + // Open the suite the first time we see this class; later benchmarks of the class reuse it. + if (OPEN_SUITES.putIfAbsent(suiteName, suite) == null) { + // A run can have benchmarks from several classes, so multiple suites stay open at once and + // are finished together in onRunEnd (in arbitrary map order). Mark them parallelized so the + // suite spans are not pushed onto the active-span stack — otherwise finishing an outer suite + // while an inner one is still active throws IllegalStateException (TestSuiteImpl.end). Test + // spans still self-activate, so tagBenchmarkMetrics' activeSpan() is unaffected. + boolean parallelized = true; + TEST_EVENTS_HANDLER.onTestSuiteStart( + suite, + suiteName, + TEST_FRAMEWORK, + TEST_FRAMEWORK_VERSION, + null, + Collections.emptyList(), + parallelized, + TestFrameworkInstrumentation.JMH, + null); + } + + TEST_EVENTS_HANDLER.onTestStart( + suite, + test, + testName, + TEST_FRAMEWORK, + TEST_FRAMEWORK_VERSION, + testParameters, + Collections.emptyList(), + TestSourceData.UNKNOWN, + null, + null); + } + + public static void onBenchmarkEnd(BenchmarkResult result) { + if (currentTest != null) { + tagBenchmarkMetrics(result); + finishOpenTest(false); + } + } + + /** + * Finishes the in-flight benchmark's test span, if any, and clears it so a spurious second call + * is a no-op. When {@code failed} is true the test is marked failed first (the failure propagates + * to its suite when both are finished), so a benchmark whose {@code endBenchmark} was skipped + * surfaces as {@code test.status = fail} instead of being silently leaked. Suites are left open + * and finished together in {@link #onRunEnd()}. + */ + private static void finishOpenTest(boolean failed) { + TestDescriptor test = currentTest; + if (test == null) { + return; + } + currentTest = null; + + TestEventsHandler handler = TEST_EVENTS_HANDLER; + if (handler == null) { + return; + } + if (failed) { + handler.onTestFailure(test, null); + } + handler.onTestFinish(test, null, null); + } + + private static void tagBenchmarkMetrics(BenchmarkResult result) { + AgentSpan span = AgentTracer.activeSpan(); + if (span == null) { + return; + } + + BenchmarkParams params = result.getParams(); + span.setTag(BENCHMARK_MODE, params.getMode().shortLabel()); + span.setTag(BENCHMARK_ITERATIONS, params.getMeasurement().getCount()); + span.setTag(BENCHMARK_WARMUP_ITERATIONS, params.getWarmup().getCount()); + span.setTag(BENCHMARK_FORKS, params.getForks()); + span.setTag(BENCHMARK_THREADS, params.getThreads()); + span.setTag(BENCHMARK_TIME_UNIT, params.getTimeUnit().name()); + + Result primary = result.getPrimaryResult(); + span.setMetric(BENCHMARK_VALUE, primary.getScore()); + span.setTag(BENCHMARK_UNIT, primary.getScoreUnit()); + + double error = primary.getScoreError(); + if (!Double.isNaN(error)) { + span.setMetric(BENCHMARK_ERROR, error); + } + + // Single-shot mode has no per-invocation distribution: any spread is across forks, not + // samples, so the percentiles would be misleading. Only emit them when there is a real + // sample distribution. + Statistics stats = primary.getStatistics(); + if (params.getMode() != Mode.SingleShotTime && stats.getN() > 1) { + span.setMetric(BENCHMARK_P50, stats.getPercentile(50)); + span.setMetric(BENCHMARK_P90, stats.getPercentile(90)); + span.setMetric(BENCHMARK_P95, stats.getPercentile(95)); + span.setMetric(BENCHMARK_P99, stats.getPercentile(99)); + span.setMetric(BENCHMARK_MIN, stats.getMin()); + span.setMetric(BENCHMARK_MAX, stats.getMax()); + span.setMetric(BENCHMARK_SAMPLE_COUNT, stats.getN()); + } + } + + public static void onRunEnd() { + // Flush a test left open by a swallowed failure, finish every suite opened during the run (JMH + // gives us no per-class finish callback), then close the session/module. + // + // Note: onRunEnd is only invoked on the normal path (Runner.runBenchmarks -> out.endRun). With + // fail-on-error enabled (non-default) a benchmark exception aborts the run before endRun, so + // the open suites/tests for that run are not flushed — same data loss as a JVM crash. The + // common + // path (fail-on-error off, the JMH default) always reaches endRun and closes cleanly. + finishOpenTest(true); + TestEventsHandler handler = TEST_EVENTS_HANDLER; + if (handler != null) { + for (TestSuiteDescriptor suite : OPEN_SUITES.values()) { + handler.onTestSuiteFinish(suite, null); + } + } + OPEN_SUITES.clear(); + stop(); + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java new file mode 100644 index 00000000000..98ac021132e --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java @@ -0,0 +1,57 @@ +package datadog.trace.instrumentation.jmh; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isConstructor; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import net.bytebuddy.asm.Advice; +import org.openjdk.jmh.runner.format.OutputFormat; + +@AutoService(InstrumenterModule.class) +public class JmhInstrumentation extends InstrumenterModule.CiVisibility + implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + + public JmhInstrumentation() { + super("ci-visibility", "jmh"); + } + + @Override + public String instrumentedType() { + // Instrument BaseRunner (where the 'out' field is declared) so that the final-field write + // in RunnerConstructorAdvice is legal: JDK 17+ rejects writing a final field declared in a + // superclass from advice injected into the subclass (Runner). + return "org.openjdk.jmh.runner.BaseRunner"; + } + + @Override + public String[] helperClassNames() { + return new String[] { + packageName + ".JmhUtils", + packageName + ".DDOutputFormat", + packageName + ".DatadogJmhReporter", + }; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isConstructor() + .and(takesArgument(0, named("org.openjdk.jmh.runner.options.Options"))) + .and(takesArgument(1, named("org.openjdk.jmh.runner.format.OutputFormat"))), + JmhInstrumentation.class.getName() + "$RunnerConstructorAdvice"); + } + + public static class RunnerConstructorAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void onExit( + @Advice.FieldValue(value = "out", readOnly = false) OutputFormat out) { + if (out instanceof DDOutputFormat) { + return; + } + out = new DDOutputFormat(out); + } + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java new file mode 100644 index 00000000000..e668f525e38 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java @@ -0,0 +1,98 @@ +package datadog.trace.instrumentation.jmh; + +import datadog.trace.api.civisibility.config.LibraryCapability; +import java.io.InputStream; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import javax.annotation.Nullable; +import org.openjdk.jmh.infra.BenchmarkParams; + +public final class JmhUtils { + + public static final List CAPABILITIES = Collections.emptyList(); + + /** + * Resolves the running JMH version. JMH does not publish an {@code Implementation-Version} in its + * jar manifest, so {@code Package.getImplementationVersion()} returns {@code null}; the version + * actually lives in the {@code jmh.properties} classpath resource (the same source JMH's own + * {@code org.openjdk.jmh.util.Version} reads, which we cannot reference because it is absent in + * older versions within the supported range). Falls back to the manifest, then {@code null}. + */ + @Nullable + public static String frameworkVersion() { + try (InputStream is = org.openjdk.jmh.Main.class.getResourceAsStream("/jmh.properties")) { + if (is != null) { + Properties props = new Properties(); + props.load(is); + String version = props.getProperty("jmh.version"); + if (version != null && !version.isEmpty()) { + return version; + } + } + } catch (Throwable ignored) { + // fall through to the manifest lookup + } + try { + return org.openjdk.jmh.Main.class.getPackage().getImplementationVersion(); + } catch (Throwable ignored) { + return null; + } + } + + /** + * Splits a JMH benchmark name into suite (class) and method parts. + * + *

JMH benchmark names have the form {@code "com.example.MyBenchmark.myMethod"}. {@code @Param} + * values are not part of the name — they are exposed separately via {@link + * BenchmarkParams#getParamsKeys()} (see {@link #testParameters(BenchmarkParams)}). + */ + public static String[] splitBenchmarkName(String fullName) { + int lastDot = fullName.lastIndexOf('.'); + if (lastDot < 0) { + return new String[] {"", fullName}; + } + return new String[] {fullName.substring(0, lastDot), fullName.substring(lastDot + 1)}; + } + + /** + * Returns the {@code test.parameters} JSON string for a parameterized benchmark (one declaring + * {@code @Param} fields), or {@code null} for an unparameterized one. + */ + @Nullable + public static String testParameters(BenchmarkParams params) { + // getParamsKeys() is a raw Collection in older JMH versions, so iterate as Object and cast to + // avoid an unchecked-conversion warning when compiling against the minimum supported version. + Map values = new LinkedHashMap<>(); + for (Object key : params.getParamsKeys()) { + String name = (String) key; + values.put(name, params.getParam(name)); + } + if (values.isEmpty()) { + return null; + } + return paramsToJson(values); + } + + static String paramsToJson(Map params) { + StringBuilder displayName = new StringBuilder(); + boolean first = true; + for (Map.Entry entry : params.entrySet()) { + if (!first) { + displayName.append(", "); + } + first = false; + displayName.append(entry.getKey()).append('=').append(entry.getValue()); + } + return "{\"metadata\":{\"test_name\":\"" + escapeJson(displayName.toString()) + "\"}}"; + } + + /** Minimal JSON string escaping for benchmark names (no unicode escaping needed). */ + private static String escapeJson(String s) { + return s.replace("\\", "\\\\").replace("\"", "\\\""); + } + + private JmhUtils() {} +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy new file mode 100644 index 00000000000..72ca436715a --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy @@ -0,0 +1,53 @@ +import datadog.trace.api.DisableTestTrace +import datadog.trace.civisibility.CiVisibilityInstrumentationTest +import datadog.trace.instrumentation.jmh.JmhUtils +import datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark +import datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark +import datadog.trace.instrumentation.jmh.benchmarks.DistributionBenchmark +import org.openjdk.jmh.runner.Runner +import org.openjdk.jmh.runner.options.OptionsBuilder + +@DisableTestTrace(reason = "avoid self-tracing") +class JmhInstrumentationTest extends CiVisibilityInstrumentationTest { + + static final List BENCHMARK_METRIC_TAGS = [ + "content.metrics.['benchmark.value']", + "content.metrics.['benchmark.error']", + "content.metrics.['benchmark.p50']", + "content.metrics.['benchmark.p90']", + "content.metrics.['benchmark.p95']", + "content.metrics.['benchmark.p99']", + "content.metrics.['benchmark.min']", + "content.metrics.['benchmark.max']", + "content.metrics.['benchmark.sample_count']", + ] + + def "test #testcaseName"() { + runBenchmarks(*benchmarkClasses) + assertSpansData(testcaseName, [:], [], BENCHMARK_METRIC_TAGS) + + where: + testcaseName | benchmarkClasses + "test-benchmark-simple" | [SimpleBenchmark] + "test-benchmark-parameterized" | [ParameterizedBenchmark] + "test-benchmark-multi-class" | [SimpleBenchmark, ParameterizedBenchmark] + "test-benchmark-distribution" | [DistributionBenchmark] + } + + private void runBenchmarks(Class... benchmarkClasses) { + def builder = new OptionsBuilder() + .jvmArgsAppend("-Djmh.ignoreLock=true") + benchmarkClasses.each { builder.include(it.getName()) } + new Runner(builder.build()).run() + } + + @Override + String instrumentedLibraryName() { + "jmh" + } + + @Override + String instrumentedLibraryVersion() { + JmhUtils.frameworkVersion() + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java new file mode 100644 index 00000000000..7efbd4b19ea --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java @@ -0,0 +1,54 @@ +package datadog.trace.instrumentation.jmh; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.LinkedHashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class JmhUtilsTest { + + @Test + void splitBenchmarkName_simple() { + assertArrayEquals( + new String[] {"com.example.MyBenchmark", "myMethod"}, + JmhUtils.splitBenchmarkName("com.example.MyBenchmark.myMethod")); + } + + @Test + void splitBenchmarkName_noPackage() { + assertArrayEquals( + new String[] {"MyBenchmark", "myMethod"}, + JmhUtils.splitBenchmarkName("MyBenchmark.myMethod")); + } + + @Test + void splitBenchmarkName_noDot() { + assertArrayEquals(new String[] {"", "noDot"}, JmhUtils.splitBenchmarkName("noDot")); + } + + @Test + void paramsToJson_singleParam() { + Map params = new LinkedHashMap<>(); + params.put("size", "1000"); + assertEquals("{\"metadata\":{\"test_name\":\"size=1000\"}}", JmhUtils.paramsToJson(params)); + } + + @Test + void paramsToJson_multipleParams() { + Map params = new LinkedHashMap<>(); + params.put("size", "1000"); + params.put("threads", "4"); + assertEquals( + "{\"metadata\":{\"test_name\":\"size=1000, threads=4\"}}", JmhUtils.paramsToJson(params)); + } + + @Test + void paramsToJson_escapesQuotes() { + Map params = new LinkedHashMap<>(); + params.put("key", "\"value\""); + assertEquals( + "{\"metadata\":{\"test_name\":\"key=\\\"value\\\"\"}}", JmhUtils.paramsToJson(params)); + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/DistributionBenchmark.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/DistributionBenchmark.java new file mode 100644 index 00000000000..5783f582e11 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/DistributionBenchmark.java @@ -0,0 +1,28 @@ +package datadog.trace.instrumentation.jmh.benchmarks; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Runs two measurement iterations so the primary result's statistics hold more than one data point + * ({@code getN() > 1}) — the gate for emitting the distribution metrics (p50/p90/p95/p99/min/max/ + * sample_count). The single-iteration benchmarks cover the complementary path where they are + * omitted. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 2, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Fork(0) +public class DistributionBenchmark { + @Benchmark + public int measure() { + return 42; + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/ParameterizedBenchmark.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/ParameterizedBenchmark.java new file mode 100644 index 00000000000..2a752e58cab --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/ParameterizedBenchmark.java @@ -0,0 +1,30 @@ +package datadog.trace.instrumentation.jmh.benchmarks; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Fork(0) +@State(Scope.Benchmark) +public class ParameterizedBenchmark { + + @Param({"1", "2"}) + int size; + + @Benchmark + public int measure() { + return size * 2; + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/SimpleBenchmark.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/SimpleBenchmark.java new file mode 100644 index 00000000000..2b2b5b6ef8a --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/SimpleBenchmark.java @@ -0,0 +1,22 @@ +package datadog.trace.instrumentation.jmh.benchmarks; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Fork(0) +public class SimpleBenchmark { + @Benchmark + public int measure() { + return 42; + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/coverages.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/events.ftl new file mode 100644 index 00000000000..040fb1e2b6a --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-distribution/events.ftl @@ -0,0 +1,157 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.DistributionBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.DistributionBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.DistributionBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.max" : ${content_metrics_benchmark_max}, + "benchmark.min" : ${content_metrics_benchmark_min}, + "benchmark.p50" : ${content_metrics_benchmark_p50}, + "benchmark.p90" : ${content_metrics_benchmark_p90}, + "benchmark.p95" : ${content_metrics_benchmark_p95}, + "benchmark.p99" : ${content_metrics_benchmark_p99}, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 2, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "benchmark.sample_count" : ${content_metrics_benchmark_sample_count}, + "benchmark.value" : ${content_metrics_benchmark_value}, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.DistributionBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "jmh-1.0", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test_session", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "jmh.test_module", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/coverages.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/events.ftl new file mode 100644 index 00000000000..055ee09b26d --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-multi-class/events.ftl @@ -0,0 +1,286 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.parameters" : "{\"metadata\":{\"test_name\":\"size=1\"}}", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value}, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.parameters" : "{\"metadata\":{\"test_name\":\"size=2\"}}", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value_2}, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id_2} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value_3}, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id_2}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "jmh-1.0", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test_session", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_7}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_4}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_7} + }, + "name" : "jmh.test_module", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_7}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/coverages.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl new file mode 100644 index 00000000000..1446a701e79 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl @@ -0,0 +1,203 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.parameters" : "{\"metadata\":{\"test_name\":\"size=1\"}}", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value}, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.parameters" : "{\"metadata\":{\"test_name\":\"size=2\"}}", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value_2}, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "jmh-1.0", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test_session", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5} + }, + "name" : "jmh.test_module", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/coverages.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl new file mode 100644 index 00000000000..2864f3d89be --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl @@ -0,0 +1,150 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value}, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "jmh-1.0", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test_session", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "jmh", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "jmh.test_module", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-smoke-tests/jmh/build.gradle b/dd-smoke-tests/jmh/build.gradle new file mode 100644 index 00000000000..e66d7588f54 --- /dev/null +++ b/dd-smoke-tests/jmh/build.gradle @@ -0,0 +1,40 @@ +apply from: "$rootDir/gradle/java.gradle" +description = 'JMH CI Visibility Smoke Tests.' + +dependencies { + // jmh-core is on the test classpath so org.openjdk.jmh.Main (and its runtime deps) are available + // to the spawned benchmark process; its jar path is also passed to the test-time javac below. + implementation group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.37' + // Not used to process this module's sources (it has none) — resolved only so its jar path can be + // passed to the test-time javac, which compiles the example benchmark project under + // src/test/resources and runs the JMH annotation processor to generate META-INF/BenchmarkList. + annotationProcessor group: 'org.openjdk.jmh', name: 'jmh-generator-annprocess', version: '1.37' + + testImplementation project(':dd-smoke-tests:backend-mock') +} + +tasks.withType(Test).configureEach { + jvmArgumentProviders.add(new CommandLineArgumentProvider() { + @Override + Iterable asArguments() { + def jmhJar = configurations.named("runtimeClasspath") + .get() + .find { it.name.contains("jmh-core") } + return ["-Ddatadog.smoketest.jmh.core.jar.path=${jmhJar}"] + } + }) + jvmArgumentProviders.add(new CommandLineArgumentProvider() { + @Override + Iterable asArguments() { + def annprocJar = configurations.named("annotationProcessor") + .get() + .find { it.name.contains("jmh-generator-annprocess") } + return ["-Ddatadog.smoketest.jmh.annproc.jar.path=${annprocJar}"] + } + }) + + if (project.hasProperty("mavenRepositoryProxy")) { + // propagate proxy URL to tests, to then propagate it to nested Gradle builds + environment "MAVEN_REPOSITORY_PROXY", project.property("mavenRepositoryProxy") + } +} diff --git a/dd-smoke-tests/jmh/gradle.lockfile b/dd-smoke-tests/jmh/gradle.lockfile new file mode 100644 index 00000000000..cff54e73834 --- /dev/null +++ b/dd-smoke-tests/jmh/gradle.lockfile @@ -0,0 +1,131 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +cafe.cryptography:curve25519-elisabeth:0.1.0=testRuntimeClasspath +cafe.cryptography:ed25519-elisabeth:0.1.0=testRuntimeClasspath +ch.qos.logback:logback-classic:1.2.13=testCompileClasspath,testRuntimeClasspath +ch.qos.logback:logback-core:1.2.13=testCompileClasspath,testRuntimeClasspath +com.blogspot.mydailyjava:weak-lock-free:0.17=testCompileClasspath,testRuntimeClasspath +com.datadoghq.okhttp3:okhttp:3.12.15=testCompileClasspath,testRuntimeClasspath +com.datadoghq.okio:okio:1.17.6=testCompileClasspath,testRuntimeClasspath +com.datadoghq:dd-instrument-java:0.0.3=testCompileClasspath,testRuntimeClasspath +com.datadoghq:dd-javac-plugin-client:0.2.2=testCompileClasspath,testRuntimeClasspath +com.datadoghq:java-dogstatsd-client:4.4.5=testRuntimeClasspath +com.datadoghq:sketches-java:0.8.3=testRuntimeClasspath +com.fasterxml.jackson.core:jackson-annotations:2.20=testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.20.0=testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.20.0=testCompileClasspath,testRuntimeClasspath +com.github.javaparser:javaparser-core:3.25.6=codenarc +com.github.jnr:jffi:1.3.14=testRuntimeClasspath +com.github.jnr:jnr-a64asm:1.0.0=testRuntimeClasspath +com.github.jnr:jnr-constants:0.10.4=testRuntimeClasspath +com.github.jnr:jnr-enxio:0.32.19=testRuntimeClasspath +com.github.jnr:jnr-ffi:2.2.18=testRuntimeClasspath +com.github.jnr:jnr-posix:3.1.21=testRuntimeClasspath +com.github.jnr:jnr-unixsocket:0.38.24=testRuntimeClasspath +com.github.jnr:jnr-x86asm:1.0.2=testRuntimeClasspath +com.github.spotbugs:spotbugs-annotations:4.9.8=compileClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +com.github.spotbugs:spotbugs:4.9.8=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.code.findbugs:jsr305:3.0.2=compileClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.google.guava:guava:20.0=testCompileClasspath,testRuntimeClasspath +com.google.re2j:re2j:1.7=testRuntimeClasspath +com.jayway.jsonpath:json-path:2.8.0=testCompileClasspath,testRuntimeClasspath +com.squareup.moshi:moshi:1.11.0=testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:logging-interceptor:3.12.12=testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:okhttp:3.12.12=testCompileClasspath,testRuntimeClasspath +com.squareup.okio:okio:1.17.5=testCompileClasspath,testRuntimeClasspath +com.thoughtworks.qdox:qdox:1.12.1=codenarc +com.vaadin.external.google:android-json:0.0.20131108.vaadin1=testCompileClasspath,testRuntimeClasspath +commons-fileupload:commons-fileupload:1.5=testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.11.0=testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.20.0=spotbugs +de.thetaphi:forbiddenapis:3.10=compileClasspath,testCompileClasspath,testRuntimeClasspath +io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath +io.sqreen:libsqreen:17.3.0=testRuntimeClasspath +javax.servlet:javax.servlet-api:3.1.0=testCompileClasspath,testRuntimeClasspath +jaxen:jaxen:2.0.0=spotbugs +junit:junit:4.13.2=testRuntimeClasspath +net.bytebuddy:byte-buddy-agent:1.18.8=testCompileClasspath,testRuntimeClasspath +net.bytebuddy:byte-buddy:1.18.8=testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna-platform:5.8.0=testRuntimeClasspath +net.java.dev.jna:jna:5.8.0=testRuntimeClasspath +net.minidev:accessors-smart:2.4.9=testRuntimeClasspath +net.minidev:json-smart:2.4.10=testRuntimeClasspath +net.sf.jopt-simple:jopt-simple:5.0.4=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=spotbugs +org.apache.ant:ant-antlr:1.10.14=codenarc +org.apache.ant:ant-junit:1.10.14=codenarc +org.apache.bcel:bcel:6.11.0=spotbugs +org.apache.commons:commons-lang3:3.19.0=spotbugs +org.apache.commons:commons-math3:3.6.1=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-text:1.14.0=spotbugs +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.codehaus.groovy:groovy-ant:3.0.23=codenarc +org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc +org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.25=testCompileClasspath,testRuntimeClasspath +org.codehaus.groovy:groovy-templates:3.0.23=codenarc +org.codehaus.groovy:groovy-xml:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.25=testCompileClasspath,testRuntimeClasspath +org.codenarc:CodeNarc:3.7.0=codenarc +org.dom4j:dom4j:2.2.0=spotbugs +org.freemarker:freemarker:2.3.31=testCompileClasspath,testRuntimeClasspath +org.gmetrics:GMetrics:2.1.0=codenarc +org.hamcrest:hamcrest-core:1.3=testRuntimeClasspath +org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath +org.jacoco:org.jacoco.core:0.8.14=testRuntimeClasspath +org.jacoco:org.jacoco.report:0.8.14=testRuntimeClasspath +org.jctools:jctools-core-jdk11:4.0.6=testRuntimeClasspath +org.jctools:jctools-core:4.0.6=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:5.14.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:1.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:1.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-launcher:1.14.1=testRuntimeClasspath +org.junit.platform:junit-platform-runner:1.14.1=testRuntimeClasspath +org.junit.platform:junit-platform-suite-api:1.14.1=testRuntimeClasspath +org.junit.platform:junit-platform-suite-commons:1.14.1=testRuntimeClasspath +org.junit:junit-bom:5.14.0=spotbugs +org.junit:junit-bom:5.14.1=testCompileClasspath,testRuntimeClasspath +org.mockito:mockito-core:4.4.0=testRuntimeClasspath +org.msgpack:jackson-dataformat-msgpack:0.9.6=testCompileClasspath,testRuntimeClasspath +org.msgpack:msgpack-core:0.9.6=testCompileClasspath,testRuntimeClasspath +org.objenesis:objenesis:3.3=testCompileClasspath,testRuntimeClasspath +org.openjdk.jmh:jmh-core:1.37=annotationProcessor,compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.openjdk.jmh:jmh-generator-annprocess:1.37=annotationProcessor +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.7.1=testRuntimeClasspath +org.ow2.asm:asm-analysis:9.9=spotbugs +org.ow2.asm:asm-commons:9.9=spotbugs +org.ow2.asm:asm-commons:9.9.1=testRuntimeClasspath +org.ow2.asm:asm-tree:9.9=spotbugs +org.ow2.asm:asm-tree:9.9.1=testRuntimeClasspath +org.ow2.asm:asm-util:9.7.1=testRuntimeClasspath +org.ow2.asm:asm-util:9.9=spotbugs +org.ow2.asm:asm:9.9=spotbugs +org.ow2.asm:asm:9.9.1=testCompileClasspath,testRuntimeClasspath +org.skyscreamer:jsonassert:1.5.1=testCompileClasspath,testRuntimeClasspath +org.slf4j:jcl-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:jul-to-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:log4j-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:1.7.32=testCompileClasspath +org.slf4j:slf4j-api:1.7.36=testRuntimeClasspath +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j +org.snakeyaml:snakeyaml-engine:2.9=testRuntimeClasspath +org.spockframework:spock-bom:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath +org.spockframework:spock-core:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-junit:1.2.1=testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-parser:1.2.0=testCompileClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=spotbugs +org.xmlunit:xmlunit-core:2.10.3=testCompileClasspath,testRuntimeClasspath +empty=spotbugsPlugins,testAnnotationProcessor diff --git a/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java b/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java new file mode 100644 index 00000000000..27393e3de82 --- /dev/null +++ b/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java @@ -0,0 +1,262 @@ +package datadog.smoketest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.config.CiVisibilityConfig; +import datadog.trace.api.config.GeneralConfig; +import datadog.trace.civisibility.CiVisibilitySmokeTest; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class JmhSmokeTest extends CiVisibilitySmokeTest { + + private static final String TEST_SERVICE_NAME = "test-jmh-service"; + private static final int PROCESS_TIMEOUT_SECS = 120; + + private static final String PROJECT_NAME = "test-jmh-benchmark"; + private static final String BENCHMARK_CLASS = "com.example.SmokeTestBenchmark"; + + private static final String JMH_CORE_JAR = + System.getProperty("datadog.smoketest.jmh.core.jar.path"); + private static final String JMH_ANNPROC_JAR = + System.getProperty("datadog.smoketest.jmh.annproc.jar.path"); + + private static final Set CI_VISIBILITY_EVENT_TYPES = + new HashSet<>(Arrays.asList("test", "test_suite_end", "test_module_end", "test_session_end")); + + private static final List BENCHMARK_DYNAMIC_TAGS = + Arrays.asList( + "content.metrics.['benchmark.value']", + "content.metrics.['benchmark.error']", + "content.metrics.['benchmark.p50']", + "content.metrics.['benchmark.p90']", + "content.metrics.['benchmark.p95']", + "content.metrics.['benchmark.p99']", + "content.metrics.['benchmark.min']", + "content.metrics.['benchmark.max']", + "content.metrics.['benchmark.sample_count']"); + + @TempDir Path projectHome; + + static final MockBackend mockBackend = new MockBackend(); + + @BeforeEach + void resetMockBackend() { + mockBackend.reset(); + } + + @AfterAll + static void closeMockBackend() throws Exception { + mockBackend.close(); + } + + @Test + void testBenchmarkSpansAreEmitted() throws Exception { + // only the 4 CI Visibility events are emitted. + assertBenchmarkSpansAreEmitted(0, 4); + } + + @Test + void testBenchmarkSpansAreEmittedWhenForked() throws Exception { + // when forking, the parent records one APM "command_execution" span for the forked java + // process, hence 5 total events (4 CI Visibility + 1 process span) + assertBenchmarkSpansAreEmitted(1, 5); + } + + private void assertBenchmarkSpansAreEmitted(int forks, int expectedEventCount) throws Exception { + givenBenchmarkProject(); + assertEquals(0, compileBenchmark(), "benchmark project should compile"); + + Map agentArgs = new HashMap<>(); + agentArgs.put(CiVisibilityConfig.CIVISIBILITY_BUILD_INSTRUMENTATION_ENABLED, "false"); + agentArgs.put(GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL, mockBackend.getIntakeUrl()); + agentArgs.put(CiVisibilityConfig.CIVISIBILITY_CODE_COVERAGE_ENABLED, "false"); + + int exitCode = runBenchmark(agentArgs, forks); + assertEquals(0, exitCode, "JMH process should exit cleanly"); + + // filter out APM spans in forked-mode + List> events = + ciVisibilityEvents(mockBackend.waitForEvents(expectedEventCount)); + + // validate benchmark.value is a real measure before verifyEventsAndCoverages rewrites it + Map testEvent = findEvent(events, "test"); + assertNotNull(testEvent, "Expected a test span for the benchmark method"); + @SuppressWarnings("unchecked") + Map metrics = + (Map) ((Map) testEvent.get("content")).get("metrics"); + Object benchmarkValue = metrics.get("benchmark.value"); + assertNotNull(benchmarkValue, "benchmark.value should be present"); + assertTrue(((Number) benchmarkValue).doubleValue() > 0, "benchmark.value should be positive"); + + // drop to reuse the same fixture between both test scenarios + metrics.remove("benchmark.run.forks"); + + verifyEventsAndCoverages( + PROJECT_NAME, + "jmh", + "headless", + events, + mockBackend.waitForCoverages(0), + BENCHMARK_DYNAMIC_TAGS); + } + + private void givenBenchmarkProject() throws Exception { + Path projectResources = + Paths.get(getClass().getClassLoader().getResource(PROJECT_NAME).toURI()); + copyFolder(projectResources, projectHome); + // empty .git so the tracer detects projectHome (not the build's repo) as the project root. + Files.createDirectories(projectHome.resolve(".git")); + } + + private int compileBenchmark() throws Exception { + // TODO: extract to common util for JUnitConsole and JMH + assertTrue(new File(JMH_CORE_JAR).isFile(), "JMH core jar not found: " + JMH_CORE_JAR); + assertTrue( + new File(JMH_ANNPROC_JAR).isFile(), + "JMH annotation processor jar not found: " + JMH_ANNPROC_JAR); + + Path classesDir = projectHome.resolve("target/classes"); + Files.createDirectories(classesDir); + + List command = new ArrayList<>(); + command.add(javacPath()); + command.addAll(Arrays.asList("-cp", JMH_CORE_JAR + File.pathSeparator + JMH_ANNPROC_JAR)); + command.addAll( + Arrays.asList("-processorpath", JMH_ANNPROC_JAR + File.pathSeparator + JMH_CORE_JAR)); + command.addAll(Arrays.asList("-processor", "org.openjdk.jmh.generators.BenchmarkProcessor")); + command.addAll(Arrays.asList("-d", classesDir.toString())); + command.addAll(findJavaFiles(projectHome.resolve("src/main/java"))); + + return runProcess(new ProcessBuilder(command), "javac"); + } + + private int runBenchmark(Map additionalAgentArgs, int forks) throws Exception { + String classpath = + projectHome.resolve("target/classes") + + File.pathSeparator + + System.getProperty("java.class.path"); + + List command = new ArrayList<>(); + command.add(javaPath()); + command.addAll( + buildJvmArguments(mockBackend.getIntakeUrl(), TEST_SERVICE_NAME, additionalAgentArgs)); + Collections.addAll(command, "-cp", classpath); + command.add("org.openjdk.jmh.Main"); + command.add(BENCHMARK_CLASS + ".*"); + Collections.addAll(command, "-f", Integer.toString(forks)); // fork count + Collections.addAll(command, "-wi", "1"); // 1 warmup iteration + Collections.addAll(command, "-i", "1"); // 1 measurement iteration + Collections.addAll(command, "-w", "1ms"); // warmup duration + Collections.addAll(command, "-r", "1ms"); // measurement duration + Collections.addAll(command, "-jvmArgs", "-Djmh.ignoreLock=true"); + + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.environment().put("DD_API_KEY", "01234567890abcdef123456789ABCDEF"); + return runProcess(processBuilder, "jmh"); + } + + private int runProcess(ProcessBuilder processBuilder, String name) throws Exception { + processBuilder.directory(projectHome.toFile()); + processBuilder.redirectErrorStream(true); + Process p = processBuilder.start(); + + final InputStream stdout = p.getInputStream(); + Thread outputConsumer = + new Thread() { + @Override + public void run() { + try { + byte[] buf = new byte[1024]; + int n; + while ((n = stdout.read(buf)) != -1) { + System.out.write(buf, 0, n); + } + } catch (Exception ignored) { + } + } + }; + outputConsumer.setDaemon(true); + outputConsumer.start(); + + if (!p.waitFor(PROCESS_TIMEOUT_SECS, TimeUnit.SECONDS)) { + p.destroyForcibly(); + throw new TimeoutException(name + " process timed out after " + PROCESS_TIMEOUT_SECS + "s"); + } + return p.exitValue(); + } + + private static List findJavaFiles(Path directory) throws IOException { + List javaFiles = new ArrayList<>(); + Files.walkFileTree( + directory, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (file.toString().endsWith(".java")) { + javaFiles.add(file.toString()); + } + return FileVisitResult.CONTINUE; + } + }); + return javaFiles; + } + + private static void copyFolder(Path src, Path dest) throws IOException { + Files.walkFileTree( + src, + new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + Files.createDirectories(dest.resolve(src.relativize(dir))); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + Files.copy(file, dest.resolve(src.relativize(file))); + return FileVisitResult.CONTINUE; + } + }); + } + + @SuppressWarnings("unchecked") + private static Map findEvent(List> events, String type) { + return events.stream().filter(e -> type.equals(e.get("type"))).findFirst().orElse(null); + } + + private static List> ciVisibilityEvents(List> events) { + List> filtered = new ArrayList<>(); + for (Map event : events) { + if (CI_VISIBILITY_EVENT_TYPES.contains(event.get("type"))) { + filtered.add(event); + } + } + return filtered; + } +} diff --git a/dd-smoke-tests/jmh/src/test/resources/logback.xml b/dd-smoke-tests/jmh/src/test/resources/logback.xml new file mode 100644 index 00000000000..24d6bcd768a --- /dev/null +++ b/dd-smoke-tests/jmh/src/test/resources/logback.xml @@ -0,0 +1,3 @@ + + + diff --git a/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/coverages.ftl b/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/events.ftl b/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/events.ftl new file mode 100644 index 00000000000..b883a10fb86 --- /dev/null +++ b/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/events.ftl @@ -0,0 +1,188 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.test.is_user_provided_service" : "true", + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "jmh", + "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.framework_version" : "1.37", + "test.itr.tests_skipping.enabled" : "true", + "test.module" : "test-jmh-service", + "test.status" : "pass", + "test.suite" : "com.example.SmokeTestBenchmark", + "test.type" : "benchmark", + "test_session.name" : "test-jmh-service" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "jmh.test_suite", + "resource" : "com.example.SmokeTestBenchmark", + "service" : "test-jmh-service", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.test.is_user_provided_service" : "true", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "jmh", + "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.framework_version" : "1.37", + "test.itr.tests_skipping.enabled" : "true", + "test.module" : "test-jmh-service", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "com.example.SmokeTestBenchmark", + "test.type" : "benchmark", + "test_session.name" : "test-jmh-service" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "benchmark.value" : ${content_metrics_benchmark_value}, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "com.example.SmokeTestBenchmark.measure", + "service" : "test-jmh-service", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "_dd.test.is_user_provided_service" : "true", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "jmh", + "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test_session_end", + "test.command" : "test-jmh-service", + "test.framework" : "jmh", + "test.framework_version" : "1.37", + "test.itr.tests_skipping.enabled" : "true", + "test.itr.tests_skipping.type" : "test", + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "test-jmh-service" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.itr.tests_skipping.count" : 0 + }, + "name" : "jmh.test_session", + "resource" : "test-jmh-service", + "service" : "test-jmh-service", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_4}, + "_dd.test.is_user_provided_service" : "true", + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "jmh", + "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test_module_end", + "test.framework" : "jmh", + "test.framework_version" : "1.37", + "test.itr.tests_skipping.enabled" : "true", + "test.itr.tests_skipping.type" : "test", + "test.module" : "test-jmh-service", + "test.status" : "pass", + "test.type" : "benchmark", + "test_session.name" : "test-jmh-service" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "test.itr.tests_skipping.count" : 0 + }, + "name" : "jmh.test_module", + "resource" : "test-jmh-service", + "service" : "test-jmh-service", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/src/main/java/com/example/SmokeTestBenchmark.java b/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/src/main/java/com/example/SmokeTestBenchmark.java new file mode 100644 index 00000000000..3fc686d6e74 --- /dev/null +++ b/dd-smoke-tests/jmh/src/test/resources/test-jmh-benchmark/src/main/java/com/example/SmokeTestBenchmark.java @@ -0,0 +1,16 @@ +package com.example; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +public class SmokeTestBenchmark { + @Benchmark + public int measure() { + return 42; + } +} diff --git a/docs/design/jmh-ci-visibility.md b/docs/design/jmh-ci-visibility.md new file mode 100644 index 00000000000..dc62f345021 --- /dev/null +++ b/docs/design/jmh-ci-visibility.md @@ -0,0 +1,235 @@ +# Design: JMH Benchmark CI Visibility Instrumentation (SDTEST-930) + +## Problem + +JMH (Java Microbenchmark Harness, `org.openjdk.jmh`, version 1.37) is the dominant Java +benchmarking framework. Benchmark runs are not currently reported to CI Visibility, so +performance regressions are invisible in the Datadog test explorer. + +## Goals + +- Report each JMH benchmark method as a CI Visibility **test span** (`test.type = "benchmark"`) +- Attach aggregated performance metrics (score, error, unit, percentiles) as span tags +- **Zero overhead on the benchmark hot path** — hook only fires once per benchmark method, not per invocation +- Disabled by default (like Renaissance); opt-in via `DD_TRACE_JMH_ENABLED=true` + +## JMH Lifecycle and Hook Point + +JMH execution flow (per benchmark method): + +``` +Runner.run() + └─ for each benchmark method: + OutputFormat.startBenchmark(BenchmarkParams) ← span start + for each fork: + for each warmup iteration: + OutputFormat.iteration(...) + for each measurement iteration: + OutputFormat.iterationResult(...) + OutputFormat.iteration(...) + OutputFormat.endBenchmark(BenchmarkResult) ← span finish + attach metrics + └─ OutputFormat.endRun(Collection) +``` + +`OutputFormat` is an interface that JMH calls for all lifecycle events. Critically: +- `startBenchmark(BenchmarkParams)` fires **once** per benchmark method before any invocations +- `endBenchmark(BenchmarkResult)` fires **once** per benchmark method after all forks and iterations + +These are the only two hooks needed. No hot-path instrumentation is required. + +### Why not instrument `@Benchmark`-annotated methods directly? + +Those methods are called millions of times during warmup and measurement. Advice on the +hot path would perturb the benchmark results and add massive instrumentation overhead. + +## Instrumentation Strategy + +### Hook: `OutputFormat` injection + +JMH's `Runner` is constructed by user code: + +```java +Runner runner = new Runner(options); +runner.run(); +``` + +The `Runner` constructor accepts an `Options` object, which includes an `OutputFormat`. We +instrument `Runner.` to wrap the user-supplied `OutputFormat` with our own +`DDOutputFormat` decorator before the field is stored. + +**Bytecode advice on `Runner.`:** + +```java +@Advice.OnMethodExit +public static void onExit(@Advice.FieldValue(value = "out", readOnly = false) OutputFormat out) { + out = new DDOutputFormat(out); +} +``` + +`Runner.out` is the `OutputFormat` field. Wrapping it at construction time means our +decorator receives all lifecycle callbacks without any per-invocation cost. + +### Alternative: instrument `Runner.run()` return value + +If wrapping the constructor is fragile due to JMH refactors, a fallback is to instrument +`Runner.run()` / `Runner.runBenchmarks()` exit and iterate the returned +`Collection` to emit spans retroactively. This loses wall-clock timing fidelity +(span duration reflects only the post-run callback time) but is simpler and requires no +field access. + +The constructor approach is preferred; the `run()` return approach is the fallback. + +## Data Model + +### Span structure + +Each JMH benchmark method produces **two spans**: + +| Span | Mapping | Notes | +|------|---------|-------| +| Test suite span | Benchmark class (e.g., `com.example.MyBenchmark`) | One per class | +| Test span | Benchmark method (e.g., `myMethod`) | One per `@Benchmark` method per parameter set | + +`BenchmarkParams.getBenchmark()` returns the fully-qualified name +`"com.example.MyBenchmark.myMethod"` — split on the last `.` to get class and method. + +### Standard CI Visibility tags + +| Tag | Source | Value | +|-----|--------|-------| +| `test.type` | constant | `"benchmark"` | +| `test.framework` | constant | `"jmh"` | +| `test.framework_version` | `Version.getVersion(Runner.class)` | e.g. `"1.37"` | +| `test.name` | `BenchmarkParams.getBenchmark()` last segment | e.g. `"myMethod"` | +| `test.suite` | `BenchmarkParams.getBenchmark()` prefix | e.g. `"com.example.MyBenchmark"` | +| `test.status` | always `"pass"` (JMH throws on error) | `"fail"` if exception from `endBenchmark` | +| `test.parameters` | `BenchmarkParams.getParamsKeys()` + `getParam(key)` | JSON object, omit if empty | +| `test.source.class` | derived from suite name | class name | +| `test.source.method` | derived from test name | method name | + +### Benchmark-specific metric tags + +These are numeric tags added on the test span, not on suite spans: + +| Tag | Source | Notes | +|-----|--------|-------| +| `benchmark.run.iterations` | `BenchmarkParams.getMeasurement().getCount()` | Measurement iteration count | +| `benchmark.run.forks` | `BenchmarkParams.getForks()` | Fork count | +| `benchmark.run.threads` | `BenchmarkParams.getThreads()` | Thread count | +| `benchmark.run.warmup_iterations` | `BenchmarkParams.getWarmup().getCount()` | Warmup iteration count | +| `benchmark.run.time_unit` | `BenchmarkParams.getTimeUnit().name()` | e.g. `"NANOSECONDS"` | +| `benchmark.run.mode` | `BenchmarkParams.getMode().shortLabel()` | e.g. `"thrpt"`, `"avgt"` | +| `benchmark.value` | `Result.getScore()` | Primary metric score | +| `benchmark.error` | `Result.getScoreError()` | 99.9% CI half-width; `NaN` for single-shot | +| `benchmark.unit` | `Result.getScoreUnit()` | e.g. `"ops/ms"`, `"ns/op"` | +| `benchmark.p50` | `Statistics.getPercentile(50)` | Median | +| `benchmark.p90` | `Statistics.getPercentile(90)` | | +| `benchmark.p95` | `Statistics.getPercentile(95)` | | +| `benchmark.p99` | `Statistics.getPercentile(99)` | | +| `benchmark.min` | `Statistics.getMin()` | | +| `benchmark.max` | `Statistics.getMax()` | | +| `benchmark.sample_count` | `Statistics.getN()` | Total sample count | + +`Statistics` is available via `BenchmarkResult.getPrimaryResult().getStatistics()`. + +For `SingleShotTime` mode, only `benchmark.value` and `benchmark.unit` are populated +(no iterations → no distribution). + +### Access path summary (zero hot-path calls) + +``` +endBenchmark(BenchmarkResult result) +├── result.getParams() → BenchmarkParams (all config) +│ ├── .getBenchmark() → "com.example.MyBenchmark.myMethod" +│ ├── .getMode().shortLabel() → "thrpt" +│ ├── .getThreads() → 4 +│ ├── .getForks() → 5 +│ ├── .getMeasurement().getCount() → 5 +│ ├── .getWarmup().getCount() → 5 +│ ├── .getTimeUnit() → NANOSECONDS +│ └── .getParamsKeys() + .getParam(key) → {"size": "1000"} +└── result.getPrimaryResult() → Result + ├── .getScore() → 1234.56 + ├── .getScoreError() → 12.34 + ├── .getScoreUnit() → "ns/op" + └── .getStatistics() → Statistics + ├── .getPercentile(50/90/95/99) → distribution + ├── .getMin() / .getMax() → bounds + └── .getN() → sample count +``` + +## Module Layout + +``` +dd-java-agent/instrumentation/jmh/ +└── jmh-1.0/ # JMH's OutputFormat API is stable since 1.0 + ├── build.gradle + ├── gradle.lockfile + └── src/ + └── main/java/datadog/trace/instrumentation/jmh/ + ├── JmhInstrumentation.java # InstrumenterModule targeting Runner. + ├── DDOutputFormat.java # OutputFormat decorator + └── JmhUtils.java # Parsing helpers (benchmark name split, etc.) +``` + +### `build.gradle` + +```groovy +apply from: "$rootDir/gradle/java.gradle" + +dependencies { + compileOnly group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.0' + testImplementation group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.37' +} +``` + +## Changes Required Outside the New Module + +| File | Change | +|------|--------| +| `internal-api/.../telemetry/tag/TestFrameworkInstrumentation.java` | Add `JMH` enum constant | +| `internal-api/.../bootstrap/instrumentation/api/Tags.java` | Add `benchmark.*` tag constants | +| `internal-api/.../decorator/TestDecorator.java` | Add `TEST_TYPE_BENCHMARK = "benchmark"` constant | +| `dd-java-agent/agent-ci-visibility/.../decorator/TestDecoratorImpl.java` | Handle benchmark type | + +## Resolved Design Decisions + +### Suite span scope +Flat: each benchmark method (including each `@Param` combination) gets its own suite span +and test span pair. No grouping by class. + +### Parameterized benchmarks +Follow the same pattern as JUnit 5 parameterized tests: set `test.parameters` to +`{"metadata":{"test_name":""}}` where the display name is the parameterized +suffix of the benchmark name. + +JMH encodes `@Param` combinations by appending them after a colon: +`"com.example.MyBenchmark.myMethod:size=1000,threads=4"` + +Parsing: +- `test.suite` = everything before the last `.` before the colon: `"com.example.MyBenchmark"` +- `test.name` = method name segment without params: `"myMethod"` +- `test.parameters` = `{"metadata":{"test_name":"myMethod:size=1000,threads=4"}}` (non-null only when a colon is present) + +The parameterized variant is a distinct test identity (unique `test.name` + `test.parameters` +combination), matching how JUnit 5 handles `@ParameterizedTest`. + +## Open Questions + +1. **CI Visibility opt-out for ITR**: JMH benchmarks should likely be excluded from + Intelligent Test Runner (skip logic) since skipping a benchmark run defeats its purpose. + Mark them as `@ITRUnskippable` equivalent or configure the handler to always-run. + +2. **Forked JVM mode**: When `@Fork(1+)` is used, each fork is a separate JVM process. + The tracer in the forked process needs to propagate the session/module/suite IDs from + the parent. This is the same challenge as Gradle worker forks — check if the existing + IPC mechanism in `ProcessHierarchy` covers it. + +## Implementation Order + +1. Add `benchmark.*` tag constants to `Tags.java` +2. Add `JMH` to `TestFrameworkInstrumentation` +3. Add `TEST_TYPE_BENCHMARK` to `TestDecorator` +4. Implement `DDOutputFormat` + `JmhInstrumentation` + `JmhUtils` +5. Wire module into the agent's instrumentation list +6. Add JUnit 5 integration tests (use JMH `Runner` programmatically in test; verify spans) diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/TestFrameworkInstrumentation.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/TestFrameworkInstrumentation.java index eedbaed7e80..20604f54739 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/TestFrameworkInstrumentation.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/TestFrameworkInstrumentation.java @@ -13,6 +13,7 @@ public enum TestFrameworkInstrumentation implements TagValue { SCALATEST, KARATE, WEAVER, + JMH, OTHER; private final String s; diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java index 14496e8b243..66018e35cec 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java @@ -86,6 +86,23 @@ public class Tags { public static final String TEST_BROWSER_VERSION = "test.browser.version"; public static final String TEST_CALLBACK = "test.callback"; + public static final String BENCHMARK_VALUE = "benchmark.value"; + public static final String BENCHMARK_ERROR = "benchmark.error"; + public static final String BENCHMARK_UNIT = "benchmark.unit"; + public static final String BENCHMARK_MODE = "benchmark.run.mode"; + public static final String BENCHMARK_ITERATIONS = "benchmark.run.iterations"; + public static final String BENCHMARK_WARMUP_ITERATIONS = "benchmark.run.warmup_iterations"; + public static final String BENCHMARK_FORKS = "benchmark.run.forks"; + public static final String BENCHMARK_THREADS = "benchmark.run.threads"; + public static final String BENCHMARK_TIME_UNIT = "benchmark.run.time_unit"; + public static final String BENCHMARK_P50 = "benchmark.p50"; + public static final String BENCHMARK_P90 = "benchmark.p90"; + public static final String BENCHMARK_P95 = "benchmark.p95"; + public static final String BENCHMARK_P99 = "benchmark.p99"; + public static final String BENCHMARK_MIN = "benchmark.min"; + public static final String BENCHMARK_MAX = "benchmark.max"; + public static final String BENCHMARK_SAMPLE_COUNT = "benchmark.sample_count"; + public static final String TEST_SESSION_ID = "test_session_id"; public static final String TEST_MODULE_ID = "test_module_id"; public static final String TEST_SUITE_ID = "test_suite_id"; diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index b70f1b960d3..20b88200625 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -7289,6 +7289,14 @@ "aliases": ["DD_TRACE_INTEGRATION_JETTY_WEBSOCKET_ENABLED", "DD_INTEGRATION_JETTY_WEBSOCKET_ENABLED"] } ], + "DD_TRACE_JMH_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_TRACE_INTEGRATION_JMH_ENABLED", "DD_INTEGRATION_JMH_ENABLED"] + } + ], "DD_TRACE_JMS_1_ENABLED": [ { "version": "A", diff --git a/settings.gradle.kts b/settings.gradle.kts index 487f6275cf1..d3c1d449224 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -201,6 +201,7 @@ include( ":dd-smoke-tests:jersey-2", ":dd-smoke-tests:jersey-3", ":dd-smoke-tests:jboss-modules", + ":dd-smoke-tests:jmh", ":dd-smoke-tests:junit-console", ":dd-smoke-tests:kafka-2", ":dd-smoke-tests:kafka-3", @@ -445,6 +446,7 @@ include( ":dd-java-agent:instrumentation:jetty:jetty-util-9.4.31", ":dd-java-agent:instrumentation:jms:jakarta-jms-3.0", ":dd-java-agent:instrumentation:jms:javax-jms-1.1", + ":dd-java-agent:instrumentation:jmh:jmh-1.0", ":dd-java-agent:instrumentation:jose-jwt-4.0", ":dd-java-agent:instrumentation:jsp-2.3", ":dd-java-agent:instrumentation:junit:junit-4:junit-4-cucumber-5.4",