Skip to content

Commit 4a73a97

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
authored
Implement Intelligent Test Runner skippable tests request (#5413)
1 parent f911e80 commit 4a73a97

File tree

21 files changed

+618
-67
lines changed

21 files changed

+618
-67
lines changed

‎dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -746,15 +746,16 @@ private static void startIast(SubscriptionService ss, Class<?> scoClass, Object
746746
}
747747
}
748748

749-
private static void maybeStartCiVisibility(Class<?> scoClass, Object o) {
749+
private static void maybeStartCiVisibility(Class<?> scoClass, Object sco) {
750750
if (ciVisibilityEnabled) {
751751
StaticEventLogger.begin("CI Visibility");
752752

753753
try {
754754
final Class<?> ciVisibilitySysClass =
755755
AGENT_CLASSLOADER.loadClass("datadog.trace.civisibility.CiVisibilitySystem");
756-
final Method ciVisibilityInstallerMethod = ciVisibilitySysClass.getMethod("start");
757-
ciVisibilityInstallerMethod.invoke(null);
756+
final Method ciVisibilityInstallerMethod =
757+
ciVisibilitySysClass.getMethod("start", scoClass);
758+
ciVisibilityInstallerMethod.invoke(null, sco);
758759
} catch (final Throwable e) {
759760
log.warn("Not starting CI Visibility subsystem", e);
760761
}

‎dd-java-agent/agent-ci-visibility/build.gradle‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,22 @@ excludedClassesCoverage += [
1818
"datadog.trace.civisibility.ci.CIInfo",
1919
"datadog.trace.civisibility.ci.CIInfo.Builder",
2020
"datadog.trace.civisibility.communication.*",
21-
"datadog.trace.civisibility.config.CiVisibilitySettings",
22-
"datadog.trace.civisibility.config.ConfigurationApiImpl",
23-
"datadog.trace.civisibility.config.ConfigurationApiImpl.MultiEnvelopeDto",
24-
"datadog.trace.civisibility.config.ConfigurationsJson.ConfigurationsJsonAdapter",
25-
"datadog.trace.civisibility.config.JvmInfo",
2621
"datadog.trace.civisibility.config.JvmInfoFactory",
27-
"datadog.trace.civisibility.config.JvmInfoFactory.JvmVersionOutputParser",
22+
"datadog.trace.civisibility.config.ConfigurationApi",
2823
"datadog.trace.civisibility.config.ModuleExecutionSettingsFactory",
24+
"datadog.trace.civisibility.config.JvmInfo",
25+
"datadog.trace.civisibility.config.JvmInfoFactory.JvmVersionOutputParser",
26+
"datadog.trace.civisibility.config.CiVisibilitySettings",
27+
"datadog.trace.civisibility.config.ConfigurationApiImpl.MultiEnvelopeDto",
28+
"datadog.trace.civisibility.config.ConfigurationApi.1",
2929
"datadog.trace.civisibility.context.AbstractTestContext",
3030
"datadog.trace.civisibility.context.EmptyTestContext",
3131
"datadog.trace.civisibility.context.ParentProcessTestContext",
3232
"datadog.trace.civisibility.context.SpanTestContext",
33-
"datadog.trace.civisibility.coverage.*",
33+
"datadog.trace.civisibility.coverage.TestProbes.TestProbesFactory",
34+
"datadog.trace.civisibility.coverage.ExecutionDataAdapter",
35+
"datadog.trace.civisibility.coverage.TestProbes",
36+
"datadog.trace.civisibility.coverage.SourceAnalyzer",
3437
"datadog.trace.civisibility.events.BuildEventsHandlerImpl",
3538
"datadog.trace.civisibility.events.CachingTestEventsHandlerFactory.CacheKey",
3639
"datadog.trace.civisibility.events.TestEventsHandlerImpl",

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java‎

Lines changed: 79 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package datadog.trace.civisibility;
22

3+
import datadog.communication.ddagent.SharedCommunicationObjects;
34
import datadog.trace.api.Config;
45
import datadog.trace.api.civisibility.CIVisibility;
56
import datadog.trace.api.civisibility.InstrumentationBridge;
@@ -12,6 +13,12 @@
1213
import datadog.trace.civisibility.ci.CITagsProvider;
1314
import datadog.trace.civisibility.codeowners.Codeowners;
1415
import datadog.trace.civisibility.codeowners.CodeownersProvider;
16+
import datadog.trace.civisibility.communication.BackendApi;
17+
import datadog.trace.civisibility.communication.BackendApiFactory;
18+
import datadog.trace.civisibility.config.ConfigurationApi;
19+
import datadog.trace.civisibility.config.ConfigurationApiImpl;
20+
import datadog.trace.civisibility.config.JvmInfoFactory;
21+
import datadog.trace.civisibility.config.ModuleExecutionSettingsFactory;
1522
import datadog.trace.civisibility.coverage.TestProbes;
1623
import datadog.trace.civisibility.decorator.TestDecorator;
1724
import datadog.trace.civisibility.decorator.TestDecoratorImpl;
@@ -20,6 +27,10 @@
2027
import datadog.trace.civisibility.events.TestEventsHandlerImpl;
2128
import datadog.trace.civisibility.git.CILocalGitInfoBuilder;
2229
import datadog.trace.civisibility.git.CIProviderGitInfoBuilder;
30+
import datadog.trace.civisibility.git.tree.GitClient;
31+
import datadog.trace.civisibility.git.tree.GitDataApi;
32+
import datadog.trace.civisibility.git.tree.GitDataUploader;
33+
import datadog.trace.civisibility.git.tree.GitDataUploaderImpl;
2334
import datadog.trace.civisibility.source.BestEfforSourcePathResolver;
2435
import datadog.trace.civisibility.source.CompilerAidedSourcePathResolver;
2536
import datadog.trace.civisibility.source.MethodLinesResolver;
@@ -28,6 +39,7 @@
2839
import java.nio.file.Path;
2940
import java.nio.file.Paths;
3041
import java.util.Map;
42+
import java.util.concurrent.CompletableFuture;
3143
import org.slf4j.Logger;
3244
import org.slf4j.LoggerFactory;
3345

@@ -37,30 +49,34 @@ public class CiVisibilitySystem {
3749

3850
private static final String GIT_FOLDER_NAME = ".git";
3951

40-
public static void start() {
52+
public static void start(SharedCommunicationObjects sco) {
4153
Config config = Config.get();
4254
if (!config.isCiVisibilityEnabled()) {
4355
LOGGER.debug("CI Visibility is disabled");
4456
return;
4557
}
4658

47-
TestEventsHandler.Factory factory =
48-
new CachingTestEventsHandlerFactory(
49-
CiVisibilitySystem::createTestEventsHandler,
50-
config.getCiVisibilityTestEventsHandlerCacheSize());
51-
52-
InstrumentationBridge.registerTestEventsHandlerFactory(factory);
53-
InstrumentationBridge.registerBuildEventsHandlerFactory(BuildEventsHandlerImpl::new);
54-
5559
GitInfoProvider.INSTANCE.registerGitInfoBuilder(new CIProviderGitInfoBuilder());
5660
GitInfoProvider.INSTANCE.registerGitInfoBuilder(new CILocalGitInfoBuilder(GIT_FOLDER_NAME));
5761

58-
CIVisibility.registerSessionFactory(buildSessionFactory(config));
59-
62+
InstrumentationBridge.registerTestEventsHandlerFactory(buildTestEventsHandlerFactory(config));
63+
InstrumentationBridge.registerBuildEventsHandlerFactory(BuildEventsHandlerImpl::new);
6064
InstrumentationBridge.registerCoverageProbeStoreFactory(new TestProbes.TestProbesFactory());
65+
66+
CIVisibility.registerSessionFactory(buildSessionFactory(config, sco));
67+
}
68+
69+
private static TestEventsHandler.Factory buildTestEventsHandlerFactory(Config config) {
70+
return new CachingTestEventsHandlerFactory(
71+
CiVisibilitySystem::createTestEventsHandler,
72+
config.getCiVisibilityTestEventsHandlerCacheSize());
6173
}
6274

63-
private static CIVisibility.SessionFactory buildSessionFactory(Config config) {
75+
private static CIVisibility.SessionFactory buildSessionFactory(
76+
Config config, SharedCommunicationObjects sco) {
77+
BackendApiFactory backendApiFactory = new BackendApiFactory(config, sco);
78+
BackendApi backendApi = backendApiFactory.createBackendApi();
79+
6480
return (String projectName, Path projectRoot, String component, Long startTime) -> {
6581
CIProviderInfoFactory ciProviderInfoFactory = new CIProviderInfoFactory();
6682
CIProviderInfo ciProviderInfo = ciProviderInfoFactory.createCIProviderInfo(projectRoot);
@@ -73,17 +89,67 @@ private static CIVisibility.SessionFactory buildSessionFactory(Config config) {
7389
Map<String, String> ciTags = new CITagsProvider().getCiTags(ciInfo);
7490
TestDecorator testDecorator = new TestDecoratorImpl(component, null, null, ciTags);
7591

92+
GitDataUploader gitDataUploader = buildGitDataUploader(config, backendApi, repoRoot);
93+
gitDataUploader.startOrObserveGitDataUpload();
94+
95+
ModuleExecutionSettingsFactory moduleExecutionSettingsFactory =
96+
buildModuleExecutionSettingsFactory(config, backendApi, gitDataUploader, repoRoot);
97+
7698
return new DDTestSessionImpl(
7799
projectName,
78100
startTime,
79101
config,
80102
testDecorator,
81103
sourcePathResolver,
82104
codeowners,
83-
methodLinesResolver);
105+
methodLinesResolver,
106+
moduleExecutionSettingsFactory);
84107
};
85108
}
86109

110+
private static GitDataUploader buildGitDataUploader(
111+
Config config, BackendApi backendApi, String repoRoot) {
112+
if (!config.isCiVisibilityGitUploadEnabled()) {
113+
return () -> CompletableFuture.completedFuture(null);
114+
}
115+
116+
if (backendApi == null) {
117+
LOGGER.warn(
118+
"Git tree data upload will be skipped since backend API client could not be created");
119+
return () -> CompletableFuture.completedFuture(null);
120+
}
121+
122+
if (repoRoot == null) {
123+
LOGGER.warn(
124+
"Git tree data upload will be skipped since Git repository path could not be determined");
125+
return () -> CompletableFuture.completedFuture(null);
126+
}
127+
128+
long commandTimeoutMillis = config.getCiVisibilityGitCommandTimeoutMillis();
129+
String remoteName = config.getCiVisibilityGitRemoteName();
130+
131+
GitDataApi gitDataApi = new GitDataApi(backendApi);
132+
GitClient gitClient = new GitClient(repoRoot, "1 month ago", 1000, commandTimeoutMillis);
133+
return new GitDataUploaderImpl(config, gitDataApi, gitClient, remoteName);
134+
}
135+
136+
private static ModuleExecutionSettingsFactory buildModuleExecutionSettingsFactory(
137+
Config config,
138+
BackendApi backendApi,
139+
GitDataUploader gitDataUploader,
140+
String repositoryRoot) {
141+
ConfigurationApi configurationApi;
142+
if (backendApi == null) {
143+
LOGGER.warn(
144+
"Remote config and skippable tests requests will be skipped since backend API client could not be created");
145+
configurationApi = ConfigurationApi.NO_OP;
146+
} else {
147+
configurationApi = new ConfigurationApiImpl(backendApi);
148+
}
149+
return new ModuleExecutionSettingsFactory(
150+
config, configurationApi, new JvmInfoFactory(), gitDataUploader, repositoryRoot);
151+
}
152+
87153
private static TestEventsHandler createTestEventsHandler(
88154
String component, String testFramework, String testFrameworkVersion, Path path) {
89155
CIProviderInfoFactory ciProviderInfoFactory = new CIProviderInfoFactory();
@@ -109,13 +175,6 @@ private static TestEventsHandler createTestEventsHandler(
109175
methodLinesResolver);
110176
}
111177

112-
private static String getRepositoryRoot(Path path) {
113-
CIProviderInfoFactory ciProviderInfoFactory = new CIProviderInfoFactory();
114-
CIProviderInfo ciProviderInfo = ciProviderInfoFactory.createCIProviderInfo(path);
115-
CIInfo ciInfo = ciProviderInfo.buildCIInfo();
116-
return ciInfo.getCiWorkspace();
117-
}
118-
119178
private static SourcePathResolver getSourcePathResolver(String repoRoot) {
120179
if (repoRoot != null) {
121180
return new BestEfforSourcePathResolver(

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/DDTestSessionImpl.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,18 @@
66
import datadog.trace.api.civisibility.CIConstants;
77
import datadog.trace.api.civisibility.DDTestModule;
88
import datadog.trace.api.civisibility.DDTestSession;
9+
import datadog.trace.api.civisibility.config.ModuleExecutionSettings;
910
import datadog.trace.api.civisibility.source.SourcePathResolver;
1011
import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
1112
import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes;
1213
import datadog.trace.bootstrap.instrumentation.api.Tags;
1314
import datadog.trace.civisibility.codeowners.Codeowners;
15+
import datadog.trace.civisibility.config.ModuleExecutionSettingsFactory;
1416
import datadog.trace.civisibility.context.SpanTestContext;
1517
import datadog.trace.civisibility.context.TestContext;
1618
import datadog.trace.civisibility.decorator.TestDecorator;
1719
import datadog.trace.civisibility.source.MethodLinesResolver;
20+
import java.nio.file.Path;
1821
import javax.annotation.Nullable;
1922

2023
public class DDTestSessionImpl implements DDTestSession {
@@ -26,6 +29,7 @@ public class DDTestSessionImpl implements DDTestSession {
2629
private final SourcePathResolver sourcePathResolver;
2730
private final Codeowners codeowners;
2831
private final MethodLinesResolver methodLinesResolver;
32+
private final ModuleExecutionSettingsFactory moduleExecutionSettingsFactory;
2933

3034
public DDTestSessionImpl(
3135
String projectName,
@@ -34,12 +38,14 @@ public DDTestSessionImpl(
3438
TestDecorator testDecorator,
3539
SourcePathResolver sourcePathResolver,
3640
Codeowners codeowners,
37-
MethodLinesResolver methodLinesResolver) {
41+
MethodLinesResolver methodLinesResolver,
42+
ModuleExecutionSettingsFactory moduleExecutionSettingsFactory) {
3843
this.config = config;
3944
this.testDecorator = testDecorator;
4045
this.sourcePathResolver = sourcePathResolver;
4146
this.codeowners = codeowners;
4247
this.methodLinesResolver = methodLinesResolver;
48+
this.moduleExecutionSettingsFactory = moduleExecutionSettingsFactory;
4349

4450
if (startTime != null) {
4551
span = startSpan(testDecorator.component() + ".test_session", startTime);
@@ -103,4 +109,9 @@ public DDTestModule testModuleStart(String moduleName, @Nullable Long startTime)
103109
codeowners,
104110
methodLinesResolver);
105111
}
112+
113+
@Override
114+
public ModuleExecutionSettings getModuleExecutionSettings(@Nullable Path jvmExecutablePath) {
115+
return moduleExecutionSettingsFactory.create(jvmExecutablePath);
116+
}
106117
}

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApi.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,23 @@
33
import datadog.trace.api.civisibility.config.SkippableTest;
44
import java.io.IOException;
55
import java.util.Collection;
6+
import java.util.Collections;
67

78
public interface ConfigurationApi {
89

10+
ConfigurationApi NO_OP =
11+
new ConfigurationApi() {
12+
@Override
13+
public CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) {
14+
return new CiVisibilitySettings(false, false);
15+
}
16+
17+
@Override
18+
public Collection<SkippableTest> getSkippableTests(TracerEnvironment tracerEnvironment) {
19+
return Collections.emptyList();
20+
}
21+
};
22+
923
CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) throws IOException;
1024

1125
Collection<SkippableTest> getSkippableTests(TracerEnvironment tracerEnvironment)

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ public class ConfigurationApiImpl implements ConfigurationApi {
2727

2828
private final JsonAdapter<EnvelopeDto<TracerEnvironment>> requestAdapter;
2929
private final JsonAdapter<EnvelopeDto<CiVisibilitySettings>> settingsResponseAdapter;
30-
private final JsonAdapter<MultiEnvelopeDto<SkippableTest>> skippablTestsResponseAdapter;
30+
private final JsonAdapter<MultiEnvelopeDto<SkippableTest>> skippableTestsResponseAdapter;
3131

3232
public ConfigurationApiImpl(BackendApi backendApi) {
3333
this(backendApi, () -> UUID.randomUUID().toString());
@@ -53,7 +53,7 @@ public ConfigurationApiImpl(BackendApi backendApi) {
5353
ParameterizedType skippableTestsResponseType =
5454
Types.newParameterizedTypeWithOwner(
5555
ConfigurationApiImpl.class, MultiEnvelopeDto.class, SkippableTest.class);
56-
skippablTestsResponseAdapter = moshi.adapter(skippableTestsResponseType);
56+
skippableTestsResponseAdapter = moshi.adapter(skippableTestsResponseType);
5757
}
5858

5959
@Override
@@ -82,7 +82,7 @@ public Collection<SkippableTest> getSkippableTests(TracerEnvironment tracerEnvir
8282
backendApi.post(
8383
SKIPPABLE_TESTS_URI,
8484
requestBody,
85-
is -> skippablTestsResponseAdapter.fromJson(Okio.buffer(Okio.source(is))).data);
85+
is -> skippableTestsResponseAdapter.fromJson(Okio.buffer(Okio.source(is))).data);
8686
return response.stream().map(DataDto::getAttributes).collect(Collectors.toList());
8787
}
8888

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationsJson.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public ConfigurationsJson(
5555

5656
public static final class ConfigurationsJsonAdapter {
5757
@FromJson
58-
public Configurations eventFromJson(ConfigurationsJson configurationsJson) {
58+
public Configurations fromJson(ConfigurationsJson configurationsJson) {
5959
return new Configurations(
6060
configurationsJson.osPlatform,
6161
configurationsJson.osArchitecture,
@@ -68,7 +68,7 @@ public Configurations eventFromJson(ConfigurationsJson configurationsJson) {
6868
}
6969

7070
@ToJson
71-
public ConfigurationsJson eventToJson(Configurations configurations) {
71+
public ConfigurationsJson toJson(Configurations configurations) {
7272
return new ConfigurationsJson(
7373
configurations.getOsPlatform(),
7474
configurations.getOsArchitecture(),

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/JvmInfo.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package datadog.trace.civisibility.config;
22

3+
import java.util.Objects;
4+
35
public class JvmInfo {
46

57
public static final JvmInfo CURRENT_JVM =
@@ -30,6 +32,25 @@ public String getVendor() {
3032
return vendor;
3133
}
3234

35+
@Override
36+
public boolean equals(Object o) {
37+
if (this == o) {
38+
return true;
39+
}
40+
if (o == null || getClass() != o.getClass()) {
41+
return false;
42+
}
43+
JvmInfo jvmInfo = (JvmInfo) o;
44+
return Objects.equals(name, jvmInfo.name)
45+
&& Objects.equals(version, jvmInfo.version)
46+
&& Objects.equals(vendor, jvmInfo.vendor);
47+
}
48+
49+
@Override
50+
public int hashCode() {
51+
return Objects.hash(name, version, vendor);
52+
}
53+
3354
@Override
3455
public String toString() {
3556
return "JvmInfo{"

0 commit comments

Comments
 (0)