Skip to content

Commit 2fda4f6

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
committed
Add instrumentation tests for impacted tests detection
1 parent e767b74 commit 2fda4f6

File tree

22 files changed

+1203
-5
lines changed

22 files changed

+1203
-5
lines changed

‎dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import datadog.trace.civisibility.coverage.file.FileCoverageStore
2424
import datadog.trace.civisibility.coverage.percentage.NoOpCoverageCalculator
2525
import datadog.trace.civisibility.decorator.TestDecorator
2626
import datadog.trace.civisibility.decorator.TestDecoratorImpl
27+
import datadog.trace.civisibility.diff.Diff
2728
import datadog.trace.civisibility.diff.LineDiff
2829
import datadog.trace.civisibility.domain.BuildSystemSession
2930
import datadog.trace.civisibility.domain.TestFrameworkModule
@@ -52,7 +53,6 @@ import java.nio.file.Files
5253
import java.nio.file.Path
5354
import java.nio.file.Paths
5455

55-
// FIXME nikita: add integration/smoke tests
5656
abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
5757

5858
static String dummyModule
@@ -70,9 +70,13 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
7070
private static final List<TestIdentifier> skippableTests = new ArrayList<>()
7171
private static final List<TestIdentifier> flakyTests = new ArrayList<>()
7272
private static final List<TestIdentifier> knownTests = new ArrayList<>()
73+
private static volatile Diff diff = LineDiff.EMPTY
74+
7375
private static volatile boolean itrEnabled = false
7476
private static volatile boolean flakyRetryEnabled = false
7577
private static volatile boolean earlyFlakinessDetectionEnabled = false
78+
private static volatile boolean impactedTestsDetectionEnabled = false
79+
7680
public static final int SLOW_TEST_THRESHOLD_MILLIS = 1000
7781
public static final int VERY_SLOW_TEST_THRESHOLD_MILLIS = 2000
7882

@@ -118,14 +122,14 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
118122
false,
119123
itrEnabled,
120124
flakyRetryEnabled,
121-
false,
125+
impactedTestsDetectionEnabled,
122126
earlyFlakinessDetectionSettings,
123127
itrEnabled ? "itrCorrelationId" : null,
124128
skippableTestsWithMetadata,
125129
[:],
126130
flakyTests,
127131
earlyFlakinessDetectionEnabled || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(Config.get().ciVisibilityTestOrder) ? knownTests : null,
128-
LineDiff.EMPTY)
132+
diff)
129133
}
130134
}
131135

@@ -202,8 +206,8 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
202206
TestFrameworkSession testSession = testFrameworkSessionFactory.startSession(dummyModule, component, null)
203207
TestFrameworkModule testModule = testSession.testModuleStart(dummyModule, null)
204208
new TestEventsHandlerImpl(metricCollector, testSession, testModule,
205-
suiteStore != null ? suiteStore : new ConcurrentHashMapContextStore<>(),
206-
testStore != null ? testStore : new ConcurrentHashMapContextStore<>())
209+
suiteStore != null ? suiteStore : new ConcurrentHashMapContextStore<>(),
210+
testStore != null ? testStore : new ConcurrentHashMapContextStore<>())
207211
}
208212
}
209213

@@ -217,9 +221,11 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
217221
skippableTests.clear()
218222
flakyTests.clear()
219223
knownTests.clear()
224+
diff = LineDiff.EMPTY
220225
itrEnabled = false
221226
flakyRetryEnabled = false
222227
earlyFlakinessDetectionEnabled = false
228+
impactedTestsDetectionEnabled = false
223229
}
224230

225231
def givenSkippableTests(List<TestIdentifier> tests) {
@@ -235,6 +241,10 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
235241
knownTests.addAll(tests)
236242
}
237243

244+
def givenDiff(Diff diff) {
245+
this.diff = diff
246+
}
247+
238248
def givenFlakyRetryEnabled(boolean flakyRetryEnabled) {
239249
this.flakyRetryEnabled = flakyRetryEnabled
240250
}
@@ -243,6 +253,10 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
243253
this.earlyFlakinessDetectionEnabled = earlyFlakinessDetectionEnabled
244254
}
245255

256+
def givenImpactedTestsDetectionEnabled(boolean impactedTestsDetectionEnabled) {
257+
this.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled
258+
}
259+
246260
def givenTestsOrder(String testsOrder) {
247261
injectSysConfig(CiVisibilityConfig.CIVISIBILITY_TEST_ORDER, testsOrder)
248262
}
@@ -351,4 +365,12 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
351365
abstract String instrumentedLibraryName()
352366

353367
abstract String instrumentedLibraryVersion()
368+
369+
BitSet lines(int ... setBits) {
370+
BitSet bitSet = new BitSet()
371+
for (int bit : setBits) {
372+
bitSet.set(bit)
373+
}
374+
return bitSet
375+
}
354376
}

‎dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import datadog.trace.api.DisableTestTrace
22
import datadog.trace.api.civisibility.config.TestIdentifier
33
import datadog.trace.civisibility.CiVisibilityInstrumentationTest
4+
import datadog.trace.civisibility.diff.FileDiff
5+
import datadog.trace.civisibility.diff.LineDiff
46
import datadog.trace.instrumentation.junit4.MUnitTracingListener
57
import datadog.trace.instrumentation.junit4.TestEventsHandlerHolder
68
import org.example.TestFailedAssumptionMUnit
@@ -60,6 +62,23 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
6062
"test-efd-new-slow-test" | [TestSucceedMUnitSlow] | 3 | [] // is executed only twice
6163
}
6264

65+
def "test impacted tests detection #testcaseName"() {
66+
givenImpactedTestsDetectionEnabled(true)
67+
givenDiff(prDiff)
68+
69+
runTests(tests)
70+
71+
assertSpansData(testcaseName, expectedTracesCount)
72+
73+
where:
74+
testcaseName | tests | expectedTracesCount | prDiff
75+
"test-succeed" | [TestSucceedMUnit] | 2 | LineDiff.EMPTY
76+
"test-succeed" | [TestSucceedMUnit] | 2 | new FileDiff(new HashSet())
77+
"test-succeed-impacted" | [TestSucceedMUnit] | 2 | new FileDiff(new HashSet([DUMMY_SOURCE_PATH]))
78+
"test-succeed" | [TestSucceedMUnit] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines()])
79+
"test-succeed-impacted" | [TestSucceedMUnit] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)])
80+
}
81+
6382
private void runTests(Collection<Class<?>> tests) {
6483
TestEventsHandlerHolder.start()
6584
try {

‎dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-succeed-impacted/coverages.ftl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ ]

‎dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-succeed-impacted/events.ftl‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
[ {
2+
"content" : {
3+
"duration" : ${content_duration},
4+
"error" : 0,
5+
"meta" : {
6+
"_dd.p.tid" : ${content_meta__dd_p_tid},
7+
"_dd.profiling.ctx" : "test",
8+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
9+
"component" : "junit",
10+
"dummy_ci_tag" : "dummy_ci_tag_value",
11+
"env" : "none",
12+
"language" : "jvm",
13+
"library_version" : ${content_meta_library_version},
14+
"runtime-id" : ${content_meta_runtime_id},
15+
"span.kind" : "test_session_end",
16+
"test.command" : "munit-junit-4",
17+
"test.framework" : "munit",
18+
"test.framework_version" : ${content_meta_test_framework_version},
19+
"test.status" : "pass",
20+
"test.type" : "test",
21+
"test_session.name" : "session-name"
22+
},
23+
"metrics" : {
24+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count},
25+
"_dd.profiling.enabled" : 0,
26+
"_dd.trace_span_attribute_schema" : 0,
27+
"process_id" : ${content_metrics_process_id}
28+
},
29+
"name" : "junit.test_session",
30+
"resource" : "munit-junit-4",
31+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
32+
"start" : ${content_start},
33+
"test_session_id" : ${content_test_session_id}
34+
},
35+
"type" : "test_session_end",
36+
"version" : 1
37+
}, {
38+
"content" : {
39+
"duration" : ${content_duration_2},
40+
"error" : 0,
41+
"meta" : {
42+
"_dd.p.tid" : ${content_meta__dd_p_tid_2},
43+
"component" : "junit",
44+
"dummy_ci_tag" : "dummy_ci_tag_value",
45+
"env" : "none",
46+
"library_version" : ${content_meta_library_version},
47+
"span.kind" : "test_module_end",
48+
"test.framework" : "munit",
49+
"test.framework_version" : ${content_meta_test_framework_version},
50+
"test.module" : "munit-junit-4",
51+
"test.status" : "pass",
52+
"test.type" : "test",
53+
"test_session.name" : "session-name"
54+
},
55+
"metrics" : {
56+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}
57+
},
58+
"name" : "junit.test_module",
59+
"resource" : "munit-junit-4",
60+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
61+
"start" : ${content_start_2},
62+
"test_module_id" : ${content_test_module_id},
63+
"test_session_id" : ${content_test_session_id}
64+
},
65+
"type" : "test_module_end",
66+
"version" : 1
67+
}, {
68+
"content" : {
69+
"duration" : ${content_duration_3},
70+
"error" : 0,
71+
"meta" : {
72+
"_dd.p.tid" : ${content_meta__dd_p_tid_3},
73+
"component" : "junit",
74+
"dummy_ci_tag" : "dummy_ci_tag_value",
75+
"env" : "none",
76+
"library_version" : ${content_meta_library_version},
77+
"span.kind" : "test_suite_end",
78+
"test.codeowners" : "[\"owner1\",\"owner2\"]",
79+
"test.framework" : "munit",
80+
"test.framework_version" : ${content_meta_test_framework_version},
81+
"test.module" : "munit-junit-4",
82+
"test.source.file" : "dummy_source_path",
83+
"test.status" : "pass",
84+
"test.suite" : "org.example.TestSucceedMUnit",
85+
"test.type" : "test",
86+
"test_session.name" : "session-name"
87+
},
88+
"metrics" : {
89+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3},
90+
"test.source.end" : 19,
91+
"test.source.start" : 11
92+
},
93+
"name" : "junit.test_suite",
94+
"resource" : "org.example.TestSucceedMUnit",
95+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
96+
"start" : ${content_start_3},
97+
"test_module_id" : ${content_test_module_id},
98+
"test_session_id" : ${content_test_session_id},
99+
"test_suite_id" : ${content_test_suite_id}
100+
},
101+
"type" : "test_suite_end",
102+
"version" : 1
103+
}, {
104+
"content" : {
105+
"duration" : ${content_duration_4},
106+
"error" : 0,
107+
"meta" : {
108+
"_dd.profiling.ctx" : "test",
109+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
110+
"component" : "junit",
111+
"dummy_ci_tag" : "dummy_ci_tag_value",
112+
"env" : "none",
113+
"language" : "jvm",
114+
"library_version" : ${content_meta_library_version},
115+
"runtime-id" : ${content_meta_runtime_id},
116+
"span.kind" : "test",
117+
"test.codeowners" : "[\"owner1\",\"owner2\"]",
118+
"test.framework" : "munit",
119+
"test.framework_version" : ${content_meta_test_framework_version},
120+
"test.is_modified" : "true",
121+
"test.module" : "munit-junit-4",
122+
"test.name" : "Calculator.add",
123+
"test.source.file" : "dummy_source_path",
124+
"test.status" : "pass",
125+
"test.suite" : "org.example.TestSucceedMUnit",
126+
"test.traits" : "{\"category\":[\"myTag\"]}",
127+
"test.type" : "test",
128+
"test_session.name" : "session-name"
129+
},
130+
"metrics" : {
131+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4},
132+
"_dd.profiling.enabled" : 0,
133+
"_dd.trace_span_attribute_schema" : 0,
134+
"process_id" : ${content_metrics_process_id}
135+
},
136+
"name" : "junit.test",
137+
"parent_id" : ${content_parent_id},
138+
"resource" : "org.example.TestSucceedMUnit.Calculator.add",
139+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
140+
"span_id" : ${content_span_id},
141+
"start" : ${content_start_4},
142+
"test_module_id" : ${content_test_module_id},
143+
"test_session_id" : ${content_test_session_id},
144+
"test_suite_id" : ${content_test_suite_id},
145+
"trace_id" : ${content_trace_id}
146+
},
147+
"type" : "test",
148+
"version" : 2
149+
} ]

‎dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import datadog.trace.api.DisableTestTrace
22
import datadog.trace.api.civisibility.config.TestIdentifier
33
import datadog.trace.civisibility.CiVisibilityInstrumentationTest
4+
import datadog.trace.civisibility.diff.FileDiff
5+
import datadog.trace.civisibility.diff.LineDiff
46
import datadog.trace.instrumentation.junit4.TestEventsHandlerHolder
57
import junit.runner.Version
68
import org.example.TestAssumption
@@ -133,6 +135,23 @@ class JUnit4Test extends CiVisibilityInstrumentationTest {
133135
"test-efd-faulty-session-threshold" | [TestFailedAndSucceed] | 8 | []
134136
}
135137

138+
def "test impacted tests detection #testcaseName"() {
139+
givenImpactedTestsDetectionEnabled(true)
140+
givenDiff(prDiff)
141+
142+
runTests(tests)
143+
144+
assertSpansData(testcaseName, expectedTracesCount)
145+
146+
where:
147+
testcaseName | tests | expectedTracesCount | prDiff
148+
"test-succeed" | [TestSucceed] | 2 | LineDiff.EMPTY
149+
"test-succeed" | [TestSucceed] | 2 | new FileDiff(new HashSet())
150+
"test-succeed-impacted" | [TestSucceed] | 2 | new FileDiff(new HashSet([DUMMY_SOURCE_PATH]))
151+
"test-succeed" | [TestSucceed] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines()])
152+
"test-succeed-impacted" | [TestSucceed] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)])
153+
}
154+
136155
private void runTests(Collection<Class<?>> tests) {
137156
TestEventsHandlerHolder.start()
138157
try {

‎dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-succeed-impacted/coverages.ftl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ ]

0 commit comments

Comments
 (0)