Implement impacted tests detection (#8188)

Author: nikita-tkachenko-datadog

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityRepoServices.java

@@ -8,6 +8,7 @@
 import datadog.trace.civisibility.ci.CIInfo;
 import datadog.trace.civisibility.ci.CIProviderInfo;
 import datadog.trace.civisibility.ci.CITagsProvider;
+import datadog.trace.civisibility.ci.PullRequestInfo;
 import datadog.trace.civisibility.codeowners.Codeowners;
 import datadog.trace.civisibility.codeowners.CodeownersProvider;
 import datadog.trace.civisibility.codeowners.NoCodeowners;
@@ -22,6 +23,7 @@
 import datadog.trace.civisibility.git.tree.GitDataApi;
 import datadog.trace.civisibility.git.tree.GitDataUploader;
 import datadog.trace.civisibility.git.tree.GitDataUploaderImpl;
+import datadog.trace.civisibility.git.tree.GitRepoUnshallow;
 import datadog.trace.civisibility.ipc.ExecutionSettingsRequest;
 import datadog.trace.civisibility.ipc.ExecutionSettingsResponse;
 import datadog.trace.civisibility.ipc.SignalClient;
@@ -31,6 +33,8 @@
 import datadog.trace.civisibility.source.SourcePathResolver;
 import datadog.trace.civisibility.source.index.RepoIndexProvider;
 import datadog.trace.civisibility.source.index.RepoIndexSourcePathResolver;
+import datadog.trace.util.Strings;
+import java.io.File;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Map;
@@ -59,16 +63,26 @@ public class CiVisibilityRepoServices {
     ciProvider = ciProviderInfo.getProvider();
 
     CIInfo ciInfo = ciProviderInfo.buildCIInfo();
-    repoRoot = ciInfo.getNormalizedCiWorkspace();
-    moduleName = getModuleName(services.config, path, ciInfo);
-    ciTags = new CITagsProvider().getCiTags(ciInfo);
+    PullRequestInfo pullRequestInfo = ciProviderInfo.buildPullRequestInfo();
+
+    if (pullRequestInfo.isNotEmpty()) {
+      LOGGER.info("PR detected: {}", pullRequestInfo);
+    }
+
+    repoRoot = appendSlashIfNeeded(getRepoRoot(ciInfo, services.gitClientFactory));
+    moduleName = getModuleName(services.config, repoRoot, path);
+    ciTags = new CITagsProvider().getCiTags(ciInfo, pullRequestInfo);
+
+    GitClient gitClient = services.gitClientFactory.create(repoRoot);
+    GitRepoUnshallow gitRepoUnshallow = new GitRepoUnshallow(services.config, gitClient);
 
     gitDataUploader =
         buildGitDataUploader(
             services.config,
             services.metricCollector,
             services.gitInfoProvider,
-            services.gitClientFactory,
+            gitClient,
+            gitRepoUnshallow,
             services.backendApi,
             repoRoot);
     repoIndexProvider = services.repoIndexProviderFactory.create(repoRoot);
@@ -84,18 +98,49 @@ public class CiVisibilityRepoServices {
               services.config,
               services.metricCollector,
               services.backendApi,
+              gitClient,
+              gitRepoUnshallow,
               gitDataUploader,
+              pullRequestInfo,
               repoRoot);
     }
   }
 
-  static String getModuleName(Config config, Path path, CIInfo ciInfo) {
+  private static String getRepoRoot(CIInfo ciInfo, GitClient.Factory gitClientFactory) {
+    String ciWorkspace = ciInfo.getNormalizedCiWorkspace();
+    if (Strings.isNotBlank(ciWorkspace)) {
+      return ciWorkspace;
+
+    } else {
+      try {
+        return gitClientFactory.create(".").getRepoRoot();
+
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        LOGGER.error("Interrupted while getting repo root", e);
+        return null;
+
+      } catch (Exception e) {
+        LOGGER.error("Error while getting repo root", e);
+        return null;
+      }
+    }
+  }
+
+  private static String appendSlashIfNeeded(String repoRoot) {
+    if (repoRoot != null && !repoRoot.endsWith(File.separator)) {
+      return repoRoot + File.separator;
+    } else {
+      return repoRoot;
+    }
+  }
+
+  static String getModuleName(Config config, String repoRoot, Path path) {
     // if parent process is instrumented, it will provide build system's module name
     String parentModuleName = config.getCiVisibilityModuleName();
     if (parentModuleName != null) {
       return parentModuleName;
     }
-    String repoRoot = ciInfo.getNormalizedCiWorkspace();
     if (repoRoot != null && path.startsWith(repoRoot)) {
       String relativePath = Paths.get(repoRoot).relativize(path).toString();
       if (!relativePath.isEmpty()) {
@@ -126,7 +171,10 @@ private static ExecutionSettingsFactory buildExecutionSettingsFactory(
       Config config,
       CiVisibilityMetricCollector metricCollector,
       BackendApi backendApi,
+      GitClient gitClient,
+      GitRepoUnshallow gitRepoUnshallow,
       GitDataUploader gitDataUploader,
+      PullRequestInfo pullRequestInfo,
       String repoRoot) {
     ConfigurationApi configurationApi;
     if (backendApi == null) {
@@ -138,7 +186,14 @@ private static ExecutionSettingsFactory buildExecutionSettingsFactory(
     }
 
     ExecutionSettingsFactoryImpl factory =
-        new ExecutionSettingsFactoryImpl(config, configurationApi, gitDataUploader, repoRoot);
+        new ExecutionSettingsFactoryImpl(
+            config,
+            configurationApi,
+            gitClient,
+            gitRepoUnshallow,
+            gitDataUploader,
+            pullRequestInfo,
+            repoRoot);
     if (processHierarchy.isHeadless()) {
       return factory;
     } else {
@@ -150,7 +205,8 @@ private static GitDataUploader buildGitDataUploader(
       Config config,
       CiVisibilityMetricCollector metricCollector,
       GitInfoProvider gitInfoProvider,
-      GitClient.Factory gitClientFactory,
+      GitClient gitClient,
+      GitRepoUnshallow gitRepoUnshallow,
       BackendApi backendApi,
       String repoRoot) {
     if (!config.isCiVisibilityGitUploadEnabled()) {
@@ -171,9 +227,15 @@ private static GitDataUploader buildGitDataUploader(
 
     String remoteName = config.getCiVisibilityGitRemoteName();
     GitDataApi gitDataApi = new GitDataApi(backendApi, metricCollector);
-    GitClient gitClient = gitClientFactory.create(repoRoot);
     return new GitDataUploaderImpl(
-        config, metricCollector, gitDataApi, gitClient, gitInfoProvider, repoRoot, remoteName);
+        config,
+        metricCollector,
+        gitDataApi,
+        gitClient,
+        gitRepoUnshallow,
+        gitInfoProvider,
+        repoRoot,
+        remoteName);
   }
 
   private static SourcePathResolver buildSourcePathResolver(

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java

@@ -8,8 +8,11 @@
 import datadog.communication.ddagent.SharedCommunicationObjects;
 import datadog.communication.http.HttpRetryPolicy;
 import datadog.communication.http.OkHttpUtils;
+import datadog.communication.util.IOUtils;
 import datadog.trace.api.Config;
+import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
+import datadog.trace.api.civisibility.telemetry.tag.Command;
 import datadog.trace.api.git.GitInfoProvider;
 import datadog.trace.civisibility.ci.CIProviderInfoFactory;
 import datadog.trace.civisibility.ci.env.CiEnvironment;
@@ -22,24 +25,28 @@
 import datadog.trace.civisibility.git.CIProviderGitInfoBuilder;
 import datadog.trace.civisibility.git.GitClientGitInfoBuilder;
 import datadog.trace.civisibility.git.tree.GitClient;
+import datadog.trace.civisibility.git.tree.NoOpGitClient;
+import datadog.trace.civisibility.git.tree.ShellGitClient;
 import datadog.trace.civisibility.ipc.SignalClient;
 import datadog.trace.civisibility.source.BestEffortLinesResolver;
 import datadog.trace.civisibility.source.ByteCodeLinesResolver;
 import datadog.trace.civisibility.source.CompilerAidedLinesResolver;
 import datadog.trace.civisibility.source.LinesResolver;
 import datadog.trace.civisibility.source.index.*;
+import datadog.trace.civisibility.utils.ShellCommandExecutor;
+import java.io.File;
 import java.lang.reflect.Type;
 import java.net.InetSocketAddress;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.Map;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import okhttp3.HttpUrl;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
-import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -78,7 +85,7 @@ public class CiVisibilityServices {
     this.backendApi =
         new BackendApiFactory(config, sco).createBackendApi(BackendApiFactory.Intake.API);
     this.jvmInfoFactory = new CachingJvmInfoFactory(config, new JvmInfoFactoryImpl());
-    this.gitClientFactory = new GitClient.Factory(config, metricCollector);
+    this.gitClientFactory = buildGitClientFactory(config, metricCollector);
 
     CiEnvironment environment = buildCiEnvironment(config, sco);
     this.ciProviderInfoFactory = new CIProviderInfoFactory(config, environment);
@@ -111,7 +118,30 @@ public class CiVisibilityServices {
     }
   }
 
-  @NotNull
+  private static GitClient.Factory buildGitClientFactory(
+      Config config, CiVisibilityMetricCollector metricCollector) {
+    if (!config.isCiVisibilityGitClientEnabled()) {
+      return r -> NoOpGitClient.INSTANCE;
+    }
+    try {
+      ShellCommandExecutor shellCommandExecutor =
+          new ShellCommandExecutor(new File("."), config.getCiVisibilityGitCommandTimeoutMillis());
+      String gitVersion = shellCommandExecutor.executeCommand(IOUtils::readFully, "git", "version");
+      logger.debug("Detected git executable version {}", gitVersion);
+      return new ShellGitClient.Factory(config, metricCollector);
+
+    } catch (Exception e) {
+      metricCollector.add(
+          CiVisibilityCountMetric.GIT_COMMAND_ERRORS,
+          1,
+          Command.OTHER,
+          ShellCommandExecutor.getExitCode(e));
+      logger.info("No git executable detected, some features will not be available");
+      return r -> NoOpGitClient.INSTANCE;
+    }
+  }
+
+  @Nonnull
   private static CiEnvironment buildCiEnvironment(Config config, SharedCommunicationObjects sco) {
     String remoteEnvVarsProviderUrl = config.getCiVisibilityRemoteEnvVarsProviderUrl();
     if (remoteEnvVarsProviderUrl != null) {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java

@@ -238,7 +238,11 @@ private static TestFrameworkSession.Factory childTestFrameworkSessionFactory(
           new TestDecoratorImpl(component, sessionName, testCommand, repoServices.ciTags);
 
       ExecutionStrategy executionStrategy =
-          new ExecutionStrategy(services.config, executionSettings);
+          new ExecutionStrategy(
+              services.config,
+              executionSettings,
+              repoServices.sourcePathResolver,
+              services.linesResolver);
 
       return new ProxyTestSession(
           services.processHierarchy.parentProcessModuleContext,
@@ -268,7 +272,11 @@ private static TestFrameworkSession.Factory headlessTestFrameworkEssionFactory(
           new TestDecoratorImpl(component, sessionName, projectName, repoServices.ciTags);
 
       ExecutionStrategy executionStrategy =
-          new ExecutionStrategy(services.config, executionSettings);
+          new ExecutionStrategy(
+              services.config,
+              executionSettings,
+              repoServices.sourcePathResolver,
+              services.linesResolver);
       return new HeadlessTestSession(
           projectName,
           startTime,

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/AppVeyorInfo.java

@@ -9,6 +9,7 @@
 import datadog.trace.api.git.GitInfo;
 import datadog.trace.api.git.PersonInfo;
 import datadog.trace.civisibility.ci.env.CiEnvironment;
+import javax.annotation.Nonnull;
 
 class AppVeyorInfo implements CIProviderInfo {
 
@@ -79,6 +80,12 @@ public CIInfo buildCIInfo() {
         .build();
   }
 
+  @Nonnull
+  @Override
+  public PullRequestInfo buildPullRequestInfo() {
+    return PullRequestInfo.EMPTY;
+  }
+
   private String buildGitBranch(final String repoProvider) {
     if ("github".equals(repoProvider)) {
       String branch = environment.get(APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/AwsCodePipelineInfo.java

@@ -3,6 +3,7 @@
 import datadog.trace.api.civisibility.telemetry.tag.Provider;
 import datadog.trace.api.git.GitInfo;
 import datadog.trace.civisibility.ci.env.CiEnvironment;
+import javax.annotation.Nonnull;
 
 class AwsCodePipelineInfo implements CIProviderInfo {
 
@@ -35,6 +36,12 @@ public CIInfo buildCIInfo() {
         .build();
   }
 
+  @Nonnull
+  @Override
+  public PullRequestInfo buildPullRequestInfo() {
+    return PullRequestInfo.EMPTY;
+  }
+
   @Override
   public Provider getProvider() {
     return Provider.AWS;

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/AzurePipelinesInfo.java

@@ -11,6 +11,7 @@
 import datadog.trace.api.git.GitInfo;
 import datadog.trace.api.git.PersonInfo;
 import datadog.trace.civisibility.ci.env.CiEnvironment;
+import javax.annotation.Nonnull;
 
 class AzurePipelinesInfo implements CIProviderInfo {
 
@@ -81,6 +82,12 @@ public CIInfo buildCIInfo() {
         .build();
   }
 
+  @Nonnull
+  @Override
+  public PullRequestInfo buildPullRequestInfo() {
+    return PullRequestInfo.EMPTY;
+  }
+
   private String buildGitBranch() {
     String gitBranchOrTag = getGitBranchOrTag();
     if (!isTagReference(gitBranchOrTag)) {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/BitBucketInfo.java

@@ -10,6 +10,7 @@
 import datadog.trace.api.git.GitInfo;
 import datadog.trace.civisibility.ci.env.CiEnvironment;
 import datadog.trace.util.Strings;
+import javax.annotation.Nonnull;
 
 class BitBucketInfo implements CIProviderInfo {
 
@@ -70,6 +71,12 @@ public CIInfo buildCIInfo() {
         .build();
   }
 
+  @Nonnull
+  @Override
+  public PullRequestInfo buildPullRequestInfo() {
+    return PullRequestInfo.EMPTY;
+  }
+
   private String buildPipelineUrl(final String repo, final String number) {
     return String.format(
         "https://bitbucket.org/%s/addon/pipelines/home#!/results/%s", repo, number);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/BitriseInfo.java

@@ -10,6 +10,7 @@
 import datadog.trace.api.git.GitInfo;
 import datadog.trace.api.git.PersonInfo;
 import datadog.trace.civisibility.ci.env.CiEnvironment;
+import javax.annotation.Nonnull;
 
 class BitriseInfo implements CIProviderInfo {
 
@@ -66,6 +67,12 @@ public CIInfo buildCIInfo() {
         .build();
   }
 
+  @Nonnull
+  @Override
+  public PullRequestInfo buildPullRequestInfo() {
+    return PullRequestInfo.EMPTY;
+  }
+
   private String buildGitCommit() {
     final String fromCommit = environment.get(BITRISE_GIT_PR_COMMIT);
     if (fromCommit != null && !fromCommit.isEmpty()) {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/BuddyInfo.java

@@ -9,6 +9,7 @@
 import datadog.trace.api.git.GitInfo;
 import datadog.trace.api.git.PersonInfo;
 import datadog.trace.civisibility.ci.env.CiEnvironment;
+import javax.annotation.Nonnull;
 
 class BuddyInfo implements CIProviderInfo {
 
@@ -58,6 +59,12 @@ public CIInfo buildCIInfo() {
         .build();
   }
 
+  @Nonnull
+  @Override
+  public PullRequestInfo buildPullRequestInfo() {
+    return PullRequestInfo.EMPTY;
+  }
+
   private String getPipelineId(String pipelineNumber) {
     String pipelineId = environment.get(BUDDY_PIPELINE_ID);
     if (pipelineId == null) {