Skip to content

Commit 9a8bc0e

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
authored
Implement ITR tests skipping for JUnit 5 (#5451)
1 parent 1db6db0 commit 9a8bc0e

File tree

8 files changed

+388
-105
lines changed

8 files changed

+388
-105
lines changed

‎dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/ItrFilter.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package datadog.trace.instrumentation.junit5;
2+
3+
import datadog.trace.api.Config;
4+
import datadog.trace.api.civisibility.config.SkippableTest;
5+
import java.util.Set;
6+
import org.junit.platform.engine.TestDescriptor;
7+
import org.junit.platform.engine.TestSource;
8+
import org.junit.platform.engine.UniqueId;
9+
import org.junit.platform.engine.support.descriptor.MethodSource;
10+
11+
public class ItrFilter {
12+
13+
public static final ItrFilter INSTANCE = new ItrFilter();
14+
15+
private volatile boolean testsSkipped;
16+
17+
private ItrFilter() {}
18+
19+
public boolean skip(TestDescriptor testDescriptor) {
20+
SkippableTest test = toSkippableTest(testDescriptor);
21+
Set<SkippableTest> skippableTests = Config.get().getCiVisibilitySkippableTests();
22+
if (skippableTests.contains(test)) {
23+
testsSkipped = true;
24+
return true;
25+
} else {
26+
return false;
27+
}
28+
}
29+
30+
private SkippableTest toSkippableTest(TestDescriptor testDescriptor) {
31+
TestSource testSource = testDescriptor.getSource().orElse(null);
32+
if (!(testSource instanceof MethodSource)) {
33+
return null;
34+
}
35+
36+
MethodSource methodSource = (MethodSource) testSource;
37+
String testSuitName = methodSource.getClassName();
38+
39+
String displayName = testDescriptor.getDisplayName();
40+
UniqueId uniqueId = testDescriptor.getUniqueId();
41+
String testEngineId = uniqueId.getEngineId().orElse(null);
42+
String testName = TestFrameworkUtils.getTestName(displayName, methodSource, testEngineId);
43+
44+
String testParameters = TestFrameworkUtils.getParameters(methodSource, displayName);
45+
46+
return new SkippableTest(testSuitName, testName, testParameters, null);
47+
}
48+
49+
public boolean testsSkipped() {
50+
return testsSkipped;
51+
}
52+
}

‎dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5Instrumentation.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,10 @@ public ElementMatcher<TypeDescription> hierarchyMatcher() {
3838
@Override
3939
public String[] helperClassNames() {
4040
return new String[] {
41-
packageName + ".TestFrameworkUtils", packageName + ".TracingListener",
41+
packageName + ".JUnit5Utils",
42+
packageName + ".TestFrameworkUtils",
43+
packageName + ".ItrFilter",
44+
packageName + ".TracingListener",
4245
};
4346
}
4447

@@ -59,7 +62,17 @@ public static void addTracingListener(
5962
@Advice.This LauncherConfig config,
6063
@Advice.Return(readOnly = false) Collection<TestExecutionListener> listeners) {
6164

62-
Collection<TestEngine> testEngines = TestFrameworkUtils.getTestEngines(config);
65+
if (JUnit5Utils.isTestInProgress()) {
66+
// a test case that is in progress starts a new JUnit instance.
67+
// It might be done in order to achieve classloader isolation
68+
// (for example, spring-boot uses this technique).
69+
// We are already tracking the active test case,
70+
// and do not want to report the "embedded" JUnit execution
71+
// as a separate module
72+
return;
73+
}
74+
75+
Collection<TestEngine> testEngines = JUnit5Utils.getTestEngines(config);
6376
final TracingListener listener = new TracingListener(testEngines);
6477

6578
Collection<TestExecutionListener> modifiedListeners = new ArrayList<>(listeners);

‎dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5ItrInstrumentation.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package datadog.trace.instrumentation.junit5;
2+
3+
import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.implementsInterface;
4+
import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named;
5+
import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
6+
7+
import com.google.auto.service.AutoService;
8+
import datadog.trace.agent.tooling.Instrumenter;
9+
import datadog.trace.api.Config;
10+
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
11+
import java.util.Set;
12+
import net.bytebuddy.asm.Advice;
13+
import net.bytebuddy.description.type.TypeDescription;
14+
import net.bytebuddy.matcher.ElementMatcher;
15+
import org.junit.platform.engine.TestDescriptor;
16+
import org.junit.platform.engine.support.hierarchical.Node;
17+
import org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService;
18+
19+
@AutoService(Instrumenter.class)
20+
public class JUnit5ItrInstrumentation extends Instrumenter.CiVisibility
21+
implements Instrumenter.ForTypeHierarchy {
22+
23+
public JUnit5ItrInstrumentation() {
24+
super("junit", "junit-5");
25+
}
26+
27+
@Override
28+
public boolean isApplicable(Set<TargetSystem> enabledSystems) {
29+
return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityItrEnabled();
30+
}
31+
32+
@Override
33+
public String hierarchyMarkerType() {
34+
return "org.junit.platform.engine.support.hierarchical.Node";
35+
}
36+
37+
@Override
38+
public ElementMatcher<TypeDescription> hierarchyMatcher() {
39+
return implementsInterface(named(hierarchyMarkerType()))
40+
.and(implementsInterface(named("org.junit.platform.engine.TestDescriptor")));
41+
}
42+
43+
@Override
44+
public String[] helperClassNames() {
45+
return new String[] {
46+
packageName + ".TestFrameworkUtils", packageName + ".ItrFilter",
47+
};
48+
}
49+
50+
@Override
51+
public void adviceTransformations(AdviceTransformation transformation) {
52+
transformation.applyAdvice(
53+
named("shouldBeSkipped").and(takesArguments(1)),
54+
JUnit5ItrInstrumentation.class.getName() + "$JUnit5ItrAdvice");
55+
}
56+
57+
public static class JUnit5ItrAdvice {
58+
59+
@SuppressFBWarnings(
60+
value = "UC_USELESS_OBJECT",
61+
justification = "skipResult is the return value of the instrumented method")
62+
@Advice.OnMethodExit
63+
public static void shouldBeSkipped(
64+
@Advice.This TestDescriptor testDescriptor,
65+
@Advice.Return(readOnly = false) Node.SkipResult skipResult) {
66+
67+
if (!skipResult.isSkipped() && ItrFilter.INSTANCE.skip(testDescriptor)) {
68+
skipResult = Node.SkipResult.skip("Skipped by Datadog Intelligent Test Runner");
69+
}
70+
}
71+
72+
// JUnit 5.3.0 and above
73+
public static void muzzleCheck(final SameThreadHierarchicalTestExecutorService service) {
74+
service.invokeAll(null);
75+
}
76+
}
77+
}

‎dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5Utils.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
package datadog.trace.instrumentation.junit5;
2+
3+
import datadog.trace.bootstrap.instrumentation.api.AgentScope;
4+
import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
5+
import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
6+
import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes;
7+
import java.util.Collection;
8+
import java.util.LinkedHashSet;
9+
import java.util.ServiceLoader;
10+
import java.util.Set;
11+
import org.junit.platform.commons.util.ClassLoaderUtils;
12+
import org.junit.platform.engine.TestEngine;
13+
import org.junit.platform.engine.TestSource;
14+
import org.junit.platform.engine.support.descriptor.ClassSource;
15+
import org.junit.platform.engine.support.descriptor.MethodSource;
16+
import org.junit.platform.launcher.TestIdentifier;
17+
import org.junit.platform.launcher.core.LauncherConfig;
18+
19+
public abstract class JUnit5Utils {
20+
21+
public static Class<?> getJavaClass(TestIdentifier testIdentifier) {
22+
TestSource testSource = testIdentifier.getSource().orElse(null);
23+
if (testSource instanceof ClassSource) {
24+
ClassSource classSource = (ClassSource) testSource;
25+
return classSource.getJavaClass();
26+
27+
} else if (testSource instanceof MethodSource) {
28+
MethodSource methodSource = (MethodSource) testSource;
29+
return TestFrameworkUtils.getTestClass(methodSource);
30+
31+
} else {
32+
return null;
33+
}
34+
}
35+
36+
/*
37+
* JUnit5 considers parameterized or factory test cases as containers.
38+
* We need to differentiate this type of containers from "regular" ones, that are test classes
39+
*/
40+
public static boolean isTestCase(TestIdentifier testIdentifier) {
41+
return testIdentifier.isContainer() && getMethodSourceOrNull(testIdentifier) != null;
42+
}
43+
44+
public static boolean isRootContainer(TestIdentifier testIdentifier) {
45+
return !testIdentifier.getParentId().isPresent();
46+
}
47+
48+
private static MethodSource getMethodSourceOrNull(TestIdentifier testIdentifier) {
49+
return (MethodSource)
50+
testIdentifier.getSource().filter(s -> s instanceof MethodSource).orElse(null);
51+
}
52+
53+
public static boolean isAssumptionFailure(Throwable throwable) {
54+
switch (throwable.getClass().getName()) {
55+
case "org.junit.AssumptionViolatedException":
56+
case "org.junit.internal.AssumptionViolatedException":
57+
case "org.opentest4j.TestAbortedException":
58+
case "org.opentest4j.TestSkippedException":
59+
// If the test assumption fails, one of the following exceptions will be thrown.
60+
// The consensus is to treat "assumptions failure" as skipped tests.
61+
return true;
62+
default:
63+
return false;
64+
}
65+
}
66+
67+
public static Collection<TestEngine> getTestEngines(LauncherConfig config) {
68+
Set<TestEngine> engines = new LinkedHashSet<>();
69+
if (config.isTestEngineAutoRegistrationEnabled()) {
70+
ClassLoader defaultClassLoader = ClassLoaderUtils.getDefaultClassLoader();
71+
ServiceLoader.load(TestEngine.class, defaultClassLoader).forEach(engines::add);
72+
}
73+
engines.addAll(config.getAdditionalTestEngines());
74+
return engines;
75+
}
76+
77+
public static boolean isTestInProgress() {
78+
AgentScope activeScope = AgentTracer.activeScope();
79+
if (activeScope == null) {
80+
return false;
81+
}
82+
AgentSpan span = activeScope.span();
83+
if (span == null) {
84+
return false;
85+
}
86+
return InternalSpanTypes.TEST.toString().equals(span.getSpanType());
87+
}
88+
}

‎dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/TestFrameworkUtils.java‎

Lines changed: 15 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,16 @@
33
import datadog.trace.util.Strings;
44
import java.lang.annotation.Annotation;
55
import java.lang.reflect.Method;
6-
import java.util.Collection;
7-
import java.util.LinkedHashSet;
8-
import java.util.ServiceLoader;
9-
import java.util.Set;
6+
import javax.annotation.Nullable;
107
import org.junit.platform.commons.JUnitException;
11-
import org.junit.platform.commons.util.ClassLoaderUtils;
128
import org.junit.platform.commons.util.ReflectionUtils;
13-
import org.junit.platform.engine.TestEngine;
14-
import org.junit.platform.engine.TestSource;
15-
import org.junit.platform.engine.support.descriptor.ClassSource;
169
import org.junit.platform.engine.support.descriptor.MethodSource;
17-
import org.junit.platform.launcher.TestIdentifier;
18-
import org.junit.platform.launcher.core.LauncherConfig;
1910

2011
public abstract class TestFrameworkUtils {
12+
private static final String SPOCK_ENGINE_ID = "spock";
2113

22-
private static final Method GET_JAVA_CLASS;
23-
private static final Method GET_JAVA_METHOD;
14+
static final Method GET_JAVA_CLASS;
15+
static final Method GET_JAVA_METHOD;
2416

2517
static {
2618
GET_JAVA_CLASS = accessGetJavaClass();
@@ -51,21 +43,6 @@ private static Method accessGetJavaMethod() {
5143
}
5244
}
5345

54-
public static Class<?> getJavaClass(TestIdentifier testIdentifier) {
55-
TestSource testSource = testIdentifier.getSource().orElse(null);
56-
if (testSource instanceof ClassSource) {
57-
ClassSource classSource = (ClassSource) testSource;
58-
return classSource.getJavaClass();
59-
60-
} else if (testSource instanceof MethodSource) {
61-
MethodSource methodSource = (MethodSource) testSource;
62-
return getTestClass(methodSource);
63-
64-
} else {
65-
return null;
66-
}
67-
}
68-
6946
public static Class<?> getTestClass(MethodSource methodSource) {
7047
if (GET_JAVA_CLASS != null && GET_JAVA_CLASS.isAccessible()) {
7148
try {
@@ -111,7 +88,7 @@ public static Method getSpockTestMethod(MethodSource methodSource) {
11188
return null;
11289
}
11390

114-
Class<?> testClass = TestFrameworkUtils.getTestClass(methodSource);
91+
Class<?> testClass = getTestClass(methodSource);
11592
if (testClass == null) {
11693
return null;
11794
}
@@ -145,54 +122,23 @@ public static Method getSpockTestMethod(MethodSource methodSource) {
145122
return null;
146123
}
147124

148-
public static String getParameters(MethodSource methodSource, TestIdentifier testIdentifier) {
125+
public static String getParameters(MethodSource methodSource, String displayName) {
149126
if (methodSource.getMethodParameterTypes() == null
150127
|| methodSource.getMethodParameterTypes().isEmpty()) {
151128
return null;
152129
}
153-
return "{\"metadata\":{\"test_name\":\""
154-
+ Strings.escapeToJson(testIdentifier.getDisplayName())
155-
+ "\"}}";
130+
return "{\"metadata\":{\"test_name\":\"" + Strings.escapeToJson(displayName) + "\"}}";
156131
}
157132

158-
/*
159-
* JUnit5 considers parameterized or factory test cases as containers.
160-
* We need to differentiate this type of containers from "regular" ones, that are test classes
161-
*/
162-
public static boolean isTestCase(TestIdentifier testIdentifier) {
163-
return testIdentifier.isContainer() && getMethodSourceOrNull(testIdentifier) != null;
133+
public static String getTestName(
134+
String displayName, MethodSource methodSource, String testEngineId) {
135+
return SPOCK_ENGINE_ID.equals(testEngineId) ? displayName : methodSource.getMethodName();
164136
}
165137

166-
public static boolean isRootContainer(TestIdentifier testIdentifier) {
167-
return !testIdentifier.getParentId().isPresent();
168-
}
169-
170-
private static MethodSource getMethodSourceOrNull(TestIdentifier testIdentifier) {
171-
return (MethodSource)
172-
testIdentifier.getSource().filter(s -> s instanceof MethodSource).orElse(null);
173-
}
174-
175-
public static boolean isAssumptionFailure(Throwable throwable) {
176-
switch (throwable.getClass().getName()) {
177-
case "org.junit.AssumptionViolatedException":
178-
case "org.junit.internal.AssumptionViolatedException":
179-
case "org.opentest4j.TestAbortedException":
180-
case "org.opentest4j.TestSkippedException":
181-
// If the test assumption fails, one of the following exceptions will be thrown.
182-
// The consensus is to treat "assumptions failure" as skipped tests.
183-
return true;
184-
default:
185-
return false;
186-
}
187-
}
188-
189-
public static Collection<TestEngine> getTestEngines(LauncherConfig config) {
190-
Set<TestEngine> engines = new LinkedHashSet<>();
191-
if (config.isTestEngineAutoRegistrationEnabled()) {
192-
ClassLoader defaultClassLoader = ClassLoaderUtils.getDefaultClassLoader();
193-
ServiceLoader.load(TestEngine.class, defaultClassLoader).forEach(engines::add);
194-
}
195-
engines.addAll(config.getAdditionalTestEngines());
196-
return engines;
138+
@Nullable
139+
public static Method getTestMethod(MethodSource methodSource, String testEngineId) {
140+
return SPOCK_ENGINE_ID.equals(testEngineId)
141+
? getSpockTestMethod(methodSource)
142+
: getTestMethod(methodSource);
197143
}
198144
}

0 commit comments

Comments
 (0)