Skip to content

Commit 943847f

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
committed
Implement fail-fast tests ordering for JUnit 5
1 parent 626485f commit 943847f

File tree

30 files changed

+469
-53
lines changed

30 files changed

+469
-53
lines changed

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package datadog.trace.civisibility.config;
22

33
import datadog.trace.api.Config;
4+
import datadog.trace.api.civisibility.CIConstants;
45
import datadog.trace.api.civisibility.CiVisibilityWellKnownTags;
56
import datadog.trace.api.civisibility.config.TestIdentifier;
67
import datadog.trace.api.civisibility.config.TestMetadata;
@@ -143,7 +144,10 @@ private TracerEnvironment buildTracerEnvironment(
143144
: null;
144145

145146
Map<String, Collection<TestIdentifier>> knownTestsByModule =
146-
earlyFlakeDetectionEnabled ? getKnownTestsByModule(tracerEnvironment) : null;
147+
earlyFlakeDetectionEnabled
148+
|| CIConstants.FAIL_FAST_TEST_ORDER.equals(config.getCiVisibilityTestOrder())
149+
? getKnownTestsByModule(tracerEnvironment)
150+
: null;
147151

148152
Set<String> moduleNames = new HashSet<>(Collections.singleton(DEFAULT_SETTINGS));
149153
moduleNames.addAll(skippableTestIdentifiers.keySet());

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,11 @@ public TestRetryPolicy retryPolicy(TestIdentifier test) {
265265
return testModule.retryPolicy(test);
266266
}
267267

268+
@Override
269+
public boolean isNew(TestIdentifier test) {
270+
return testModule.isNew(test);
271+
}
272+
268273
@Override
269274
public void close() {
270275
testModule.end(null);

‎dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import datadog.communication.serialization.GrowableBuffer
55
import datadog.communication.serialization.msgpack.MsgPackWriter
66
import datadog.trace.agent.test.AgentTestRunner
77
import datadog.trace.api.Config
8+
import datadog.trace.api.civisibility.CIConstants
89
import datadog.trace.api.civisibility.DDTest
910
import datadog.trace.api.civisibility.DDTestSuite
1011
import datadog.trace.api.civisibility.InstrumentationBridge
@@ -124,7 +125,7 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
124125
skippableTestsWithMetadata,
125126
[:],
126127
flakyTests,
127-
earlyFlakinessDetectionEnabled ? knownTests : null)
128+
earlyFlakinessDetectionEnabled || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(Config.get().ciVisibilityTestOrder) ? knownTests : null)
128129
}
129130
}
130131

@@ -229,7 +230,14 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
229230

230231
def givenKnownTests(List<TestIdentifier> tests) {
231232
knownTests.addAll(tests)
232-
earlyFlakinessDetectionEnabled = true
233+
}
234+
235+
def givenEarlyFlakinessDetectionEnabled(boolean earlyFlakinessDetectionEnabled) {
236+
this.earlyFlakinessDetectionEnabled = earlyFlakinessDetectionEnabled
237+
}
238+
239+
def givenTestsOrder(String testsOrder) {
240+
injectSysConfig(CiVisibilityConfig.CIVISIBILITY_TEST_ORDER, testsOrder)
233241
}
234242

235243
@Override
@@ -274,6 +282,33 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
274282
return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements)
275283
}
276284

285+
def assertTestsOrder(List<TestIdentifier> expectedOrder) {
286+
TEST_WRITER.waitForTraces(expectedOrder.size() + 1)
287+
def traces = TEST_WRITER.toList()
288+
def events = getEventsAsJson(traces)
289+
def identifiers = getTestIdentifiers(events)
290+
if (identifiers != expectedOrder) {
291+
throw new AssertionError("Expected order: $expectedOrder, but got: $identifiers")
292+
}
293+
return true
294+
}
295+
296+
def getTestIdentifiers(List<Map> events) {
297+
events.sort(Comparator.comparing { it['content']['start'] as Long })
298+
def testIdentifiers = []
299+
for (Map event : events) {
300+
if (event['content']['meta']['test.name']) {
301+
testIdentifiers.add(test(event['content']['meta']['test.suite'] as String, event['content']['meta']['test.name'] as String))
302+
}
303+
}
304+
return testIdentifiers
305+
}
306+
307+
def test(String suite, String name, String parameters = null) {
308+
309+
return new TestIdentifier(suite, name, parameters)
310+
}
311+
277312
def getEventsAsJson(List<List<DDSpan>> traces) {
278313
return getSpansAsJson(new CiTestCycleMapperV1(Config.get().getCiVisibilityWellKnownTags(), false), traces)
279314
}

‎dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
7171
}
7272

7373
def "test early flakiness detection #testcaseName"() {
74+
givenEarlyFlakinessDetectionEnabled(true)
7475
givenKnownTests(knownTestsList)
7576

7677
runFeatures(features)

‎dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
4545
}
4646

4747
def "test early flakiness detection #testcaseName"() {
48+
givenEarlyFlakinessDetectionEnabled(true)
4849
givenKnownTests(knownTestsList)
4950

5051
runTests(tests)

‎dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ class JUnit4Test extends CiVisibilityInstrumentationTest {
110110
}
111111

112112
def "test early flakiness detection #testcaseName"() {
113+
givenEarlyFlakinessDetectionEnabled(true)
113114
givenKnownTests(knownTestsList)
114115

115116
runTests(tests)

‎dd-java-agent/instrumentation/junit-5.3/build.gradle‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@ dependencies {
3333

3434
// versions used below are not the minimum ones that we support,
3535
// but the tests need to use them in order to be compliant with Spock 2.x
36-
testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.8.2'
37-
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.2'
38-
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.2'
36+
testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.9.2'
37+
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.2'
38+
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.2'
3939

4040
latestDepTestImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '+'
4141
latestDepTestImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '+'
@@ -44,8 +44,8 @@ dependencies {
4444

4545
configurations.matching({ it.name.startsWith('test') }).each({
4646
it.resolutionStrategy {
47-
force group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.8.2'
48-
force group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.2'
49-
force group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.2'
47+
force group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.9.2'
48+
force group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.2'
49+
force group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.2'
5050
}
5151
})

‎dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberInstrumentation.java‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,12 @@
77
import com.google.auto.service.AutoService;
88
import datadog.trace.agent.tooling.Instrumenter;
99
import datadog.trace.agent.tooling.InstrumenterModule;
10-
import datadog.trace.bootstrap.ContextStore;
11-
import datadog.trace.bootstrap.InstrumentationContext;
1210
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
1311
import io.cucumber.junit.platform.engine.CucumberTestEngine;
14-
import java.util.Collections;
15-
import java.util.Map;
1612
import net.bytebuddy.asm.Advice;
1713
import net.bytebuddy.matcher.ElementMatcher;
1814
import org.junit.platform.engine.EngineExecutionListener;
1915
import org.junit.platform.engine.ExecutionRequest;
20-
import org.junit.platform.engine.TestDescriptor;
2116
import org.junit.platform.engine.TestEngine;
2217
import org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService;
2318

@@ -51,11 +46,6 @@ public String[] helperClassNames() {
5146
};
5247
}
5348

54-
@Override
55-
public Map<String, String> contextStore() {
56-
return Collections.singletonMap("org.junit.platform.engine.TestDescriptor", "java.lang.Object");
57-
}
58-
5949
@Override
6050
public void methodAdvice(MethodTransformer transformer) {
6151
transformer.applyAdvice(
@@ -87,12 +77,6 @@ public static void addTracingListener(
8777
return;
8878
}
8979

90-
ContextStore<TestDescriptor, Object> contextStore =
91-
InstrumentationContext.get(TestDescriptor.class, Object.class);
92-
TestEventsHandlerHolder.setContextStores(
93-
(ContextStore) contextStore, (ContextStore) contextStore);
94-
TestEventsHandlerHolder.start();
95-
9680
CucumberTracingListener tracingListener = new CucumberTracingListener(testEngine);
9781
EngineExecutionListener originalListener = executionRequest.getEngineExecutionListener();
9882
EngineExecutionListener compositeListener =

‎dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
7676
}
7777

7878
def "test early flakiness detection #testcaseName"() {
79+
givenEarlyFlakinessDetectionEnabled(true)
7980
givenKnownTests(knownTestsList)
8081

8182
runFeatures(features, false)

‎dd-java-agent/instrumentation/junit-5.3/junit-5.8/build.gradle‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,9 @@ dependencies {
3636

3737
// versions used below are not the minimum ones that we support,
3838
// but the tests need to use them in order to be compliant with Spock 2.x
39-
testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.8.2'
40-
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.2'
41-
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.2'
39+
testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.9.2'
40+
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.2'
41+
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.2'
4242

4343
latestDepTestImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '+'
4444
latestDepTestImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '+'
@@ -47,8 +47,8 @@ dependencies {
4747

4848
configurations.matching({ it.name.startsWith('test') }).each({
4949
it.resolutionStrategy {
50-
force group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.8.2'
51-
force group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.2'
52-
force group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.2'
50+
force group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.9.2'
51+
force group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.2'
52+
force group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.2'
5353
}
5454
})

0 commit comments

Comments
 (0)