Always request known tests from the backend (#8268)

Author: nikita-tkachenko-datadog

dd-java-agent/agent-ci-visibility/build.gradle

@@ -43,7 +43,7 @@ dependencies {
   testFixturesApi project(':utils:test-utils')
 
   testFixturesApi group: 'org.skyscreamer', name: 'jsonassert', version: '1.5.1'
-  testFixturesApi group: 'org.freemarker', name: 'freemarker', version: '2.3.30'
+  testFixturesApi group: 'org.freemarker', name: 'freemarker', version: '2.3.31'
   testFixturesApi group: 'com.jayway.jsonpath', name: 'json-path', version: '2.8.0'
   testFixturesApi group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.16.0'
   testFixturesApi group: 'org.msgpack', name: 'jackson-dataformat-msgpack', version: '0.9.6'

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/CiVisibilitySettings.java

@@ -3,35 +3,39 @@
 import com.squareup.moshi.FromJson;
 import java.nio.file.Path;
 import java.util.Map;
+import java.util.Objects;
 
 public class CiVisibilitySettings {
 
   public static final CiVisibilitySettings DEFAULT =
       new CiVisibilitySettings(
-          false, false, false, false, false, false, EarlyFlakeDetectionSettings.DEFAULT);
+          false, false, false, false, false, false, false, EarlyFlakeDetectionSettings.DEFAULT);
 
   private final boolean itrEnabled;
   private final boolean codeCoverage;
   private final boolean testsSkipping;
   private final boolean requireGit;
   private final boolean flakyTestRetriesEnabled;
   private final boolean impactedTestsDetectionEnabled;
+  private final boolean knownTestsEnabled;
   private final EarlyFlakeDetectionSettings earlyFlakeDetectionSettings;
 
-  private CiVisibilitySettings(
+  CiVisibilitySettings(
       boolean itrEnabled,
       boolean codeCoverage,
       boolean testsSkipping,
       boolean requireGit,
       boolean flakyTestRetriesEnabled,
       boolean impactedTestsDetectionEnabled,
+      boolean knownTestsEnabled,
       EarlyFlakeDetectionSettings earlyFlakeDetectionSettings) {
     this.itrEnabled = itrEnabled;
     this.codeCoverage = codeCoverage;
     this.testsSkipping = testsSkipping;
     this.requireGit = requireGit;
     this.flakyTestRetriesEnabled = flakyTestRetriesEnabled;
     this.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled;
+    this.knownTestsEnabled = knownTestsEnabled;
     this.earlyFlakeDetectionSettings = earlyFlakeDetectionSettings;
   }
 
@@ -59,10 +63,46 @@ public boolean isImpactedTestsDetectionEnabled() {
     return impactedTestsDetectionEnabled;
   }
 
+  public boolean isKnownTestsEnabled() {
+    return knownTestsEnabled;
+  }
+
   public EarlyFlakeDetectionSettings getEarlyFlakeDetectionSettings() {
     return earlyFlakeDetectionSettings;
   }
 
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    CiVisibilitySettings that = (CiVisibilitySettings) o;
+    return itrEnabled == that.itrEnabled
+        && codeCoverage == that.codeCoverage
+        && testsSkipping == that.testsSkipping
+        && requireGit == that.requireGit
+        && flakyTestRetriesEnabled == that.flakyTestRetriesEnabled
+        && impactedTestsDetectionEnabled == that.impactedTestsDetectionEnabled
+        && knownTestsEnabled == that.knownTestsEnabled
+        && Objects.equals(earlyFlakeDetectionSettings, that.earlyFlakeDetectionSettings);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(
+        itrEnabled,
+        codeCoverage,
+        testsSkipping,
+        requireGit,
+        flakyTestRetriesEnabled,
+        impactedTestsDetectionEnabled,
+        knownTestsEnabled,
+        earlyFlakeDetectionSettings);
+  }
+
   public interface Factory {
     CiVisibilitySettings create(Path path);
   }
@@ -84,6 +124,7 @@ public CiVisibilitySettings fromJson(Map<String, Object> json) {
           getBoolean(json, "require_git", false),
           getBoolean(json, "flaky_test_retries_enabled", false),
           getBoolean(json, "impacted_tests_enabled", false),
+          getBoolean(json, "known_tests_enabled", false),
           EarlyFlakeDetectionSettingsJsonAdapter.INSTANCE.fromJson(
               (Map<String, Object>) json.get("early_flake_detection")));
     }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApi.java

@@ -5,6 +5,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
+import javax.annotation.Nullable;
 
 public interface ConfigurationApi {
 
@@ -45,6 +46,7 @@ public ChangedFiles getChangedFiles(TracerEnvironment tracerEnvironment) {
   Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(TracerEnvironment tracerEnvironment)
       throws IOException;
 
+  @Nullable
   Map<String, Collection<TestIdentifier>> getKnownTestsByModule(TracerEnvironment tracerEnvironment)
       throws IOException;
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java

@@ -16,8 +16,10 @@
 import datadog.trace.api.civisibility.telemetry.tag.CoverageEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.FlakyTestRetriesEnabled;
+import datadog.trace.api.civisibility.telemetry.tag.ImpactedTestsDetectionEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.ItrEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.ItrSkipEnabled;
+import datadog.trace.api.civisibility.telemetry.tag.KnownTestsEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.RequireGit;
 import datadog.trace.civisibility.communication.TelemetryListener;
 import java.io.File;
@@ -141,6 +143,8 @@ public CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) thr
             ? EarlyFlakeDetectionEnabled.TRUE
             : null,
         settings.isFlakyTestRetriesEnabled() ? FlakyTestRetriesEnabled.TRUE : null,
+        settings.isKnownTestsEnabled() ? KnownTestsEnabled.TRUE : null,
+        settings.isImpactedTestsDetectionEnabled() ? ImpactedTestsDetectionEnabled.TRUE : null,
         settings.isGitUploadRequired() ? RequireGit.TRUE : null);
 
     return settings;
@@ -238,15 +242,16 @@ public Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(
     return testIdentifiers;
   }
 
+  @Nullable
   @Override
   public Map<String, Collection<TestIdentifier>> getKnownTestsByModule(
       TracerEnvironment tracerEnvironment) throws IOException {
     OkHttpUtils.CustomListener telemetryListener =
         new TelemetryListener.Builder(metricCollector)
-            .requestCount(CiVisibilityCountMetric.EFD_REQUEST)
-            .requestErrors(CiVisibilityCountMetric.EFD_REQUEST_ERRORS)
-            .requestDuration(CiVisibilityDistributionMetric.EFD_REQUEST_MS)
-            .responseBytes(CiVisibilityDistributionMetric.EFD_RESPONSE_BYTES)
+            .requestCount(CiVisibilityCountMetric.KNOWN_TESTS_REQUEST)
+            .requestErrors(CiVisibilityCountMetric.KNOWN_TESTS_REQUEST_ERRORS)
+            .requestDuration(CiVisibilityDistributionMetric.KNOWN_TESTS_REQUEST_MS)
+            .responseBytes(CiVisibilityDistributionMetric.KNOWN_TESTS_RESPONSE_BYTES)
             .build();
 
     String uuid = uuidGenerator.get();
@@ -288,8 +293,15 @@ private Map<String, Collection<TestIdentifier>> parseTestIdentifiers(KnownTestsD
     }
 
     LOGGER.debug("Received {} known tests in total", knownTestsCount);
-    metricCollector.add(CiVisibilityDistributionMetric.EFD_RESPONSE_TESTS, knownTestsCount);
-    return testIdentifiers;
+    metricCollector.add(CiVisibilityDistributionMetric.KNOWN_TESTS_RESPONSE_TESTS, knownTestsCount);
+    return knownTestsCount > 0
+        ? testIdentifiers
+        // returning null if there are no known tests:
+        // this will disable the features that are reliant on known tests
+        // and is done on purpose:
+        // if no tests are known, this is likely the first execution for this repository,
+        // and we want to fill the backend with the initial set of tests
+        : null;
   }
 
   @Override

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/EarlyFlakeDetectionSettings.java

@@ -71,6 +71,14 @@ public ExecutionsByDuration(long durationMillis, int executions) {
       this.executions = executions;
     }
 
+    public long getDurationMillis() {
+      return durationMillis;
+    }
+
+    public int getExecutions() {
+      return executions;
+    }
+
     @Override
     public boolean equals(Object o) {
       if (this == o) {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java

@@ -24,6 +24,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import org.slf4j.Logger;
@@ -113,14 +114,41 @@ private TracerEnvironment buildTracerEnvironment(
   }
 
   private @Nonnull Map<String, ExecutionSettings> create(TracerEnvironment tracerEnvironment) {
-    CiVisibilitySettings ciVisibilitySettings = getCiVisibilitySettings(tracerEnvironment);
-
-    boolean itrEnabled = isItrEnabled(ciVisibilitySettings);
-    boolean codeCoverageEnabled = isCodeCoverageEnabled(ciVisibilitySettings);
-    boolean testSkippingEnabled = isTestSkippingEnabled(ciVisibilitySettings);
-    boolean flakyTestRetriesEnabled = isFlakyTestRetriesEnabled(ciVisibilitySettings);
-    boolean impactedTestsDetectionEnabled = isImpactedTestsDetectionEnabled(ciVisibilitySettings);
-    boolean earlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled(ciVisibilitySettings);
+    CiVisibilitySettings settings = getCiVisibilitySettings(tracerEnvironment);
+
+    boolean itrEnabled =
+        isFeatureEnabled(
+            settings, CiVisibilitySettings::isItrEnabled, Config::isCiVisibilityItrEnabled);
+    boolean codeCoverageEnabled =
+        isFeatureEnabled(
+            settings,
+            CiVisibilitySettings::isCodeCoverageEnabled,
+            Config::isCiVisibilityCodeCoverageEnabled);
+    boolean testSkippingEnabled =
+        isFeatureEnabled(
+            settings,
+            CiVisibilitySettings::isTestsSkippingEnabled,
+            Config::isCiVisibilityTestSkippingEnabled);
+    boolean flakyTestRetriesEnabled =
+        isFeatureEnabled(
+            settings,
+            CiVisibilitySettings::isFlakyTestRetriesEnabled,
+            Config::isCiVisibilityFlakyRetryEnabled);
+    boolean impactedTestsDetectionEnabled =
+        isFeatureEnabled(
+            settings,
+            CiVisibilitySettings::isImpactedTestsDetectionEnabled,
+            Config::isCiVisibilityImpactedTestsDetectionEnabled);
+    boolean earlyFlakeDetectionEnabled =
+        isFeatureEnabled(
+            settings,
+            s -> s.getEarlyFlakeDetectionSettings().isEnabled(),
+            Config::isCiVisibilityEarlyFlakeDetectionEnabled);
+    boolean knownTestsRequest =
+        isFeatureEnabled(
+            settings,
+            CiVisibilitySettings::isKnownTestsEnabled,
+            Config::isCiVisibilityKnownTestsRequestEnabled);
 
     LOGGER.info(
         "CI Visibility settings ({}, {}/{}/{}):\n"
@@ -129,6 +157,7 @@ private TracerEnvironment buildTracerEnvironment(
             + "Tests skipping - {},\n"
             + "Early flakiness detection - {},\n"
             + "Impacted tests detection - {},\n"
+            + "Known tests marking - {},\n"
             + "Auto test retries - {}",
         repositoryRoot,
         tracerEnvironment.getConfigurations().getRuntimeName(),
@@ -139,6 +168,7 @@ private TracerEnvironment buildTracerEnvironment(
         testSkippingEnabled,
         earlyFlakeDetectionEnabled,
         impactedTestsDetectionEnabled,
+        knownTestsRequest,
         flakyTestRetriesEnabled);
 
     String itrCorrelationId = null;
@@ -163,11 +193,7 @@ private TracerEnvironment buildTracerEnvironment(
             : null;
 
     Map<String, Collection<TestIdentifier>> knownTestsByModule =
-        earlyFlakeDetectionEnabled
-                || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(
-                    config.getCiVisibilityTestOrder())
-            ? getKnownTestsByModule(tracerEnvironment)
-            : null;
+        knownTestsRequest ? getKnownTestsByModule(tracerEnvironment) : null;
 
     Set<String> moduleNames = new HashSet<>(Collections.singleton(DEFAULT_SETTINGS));
     moduleNames.addAll(skippableTestIdentifiers.keySet());
@@ -191,7 +217,7 @@ private TracerEnvironment buildTracerEnvironment(
               flakyTestRetriesEnabled,
               impactedTestsDetectionEnabled,
               earlyFlakeDetectionEnabled
-                  ? ciVisibilitySettings.getEarlyFlakeDetectionSettings()
+                  ? settings.getEarlyFlakeDetectionSettings()
                   : EarlyFlakeDetectionSettings.DEFAULT,
               itrCorrelationId,
               skippableTestIdentifiers.getOrDefault(moduleName, Collections.emptyMap()),
@@ -225,33 +251,11 @@ private CiVisibilitySettings getCiVisibilitySettings(TracerEnvironment tracerEnv
     }
   }
 
-  private boolean isItrEnabled(CiVisibilitySettings ciVisibilitySettings) {
-    return ciVisibilitySettings.isItrEnabled() && config.isCiVisibilityItrEnabled();
-  }
-
-  private boolean isTestSkippingEnabled(CiVisibilitySettings ciVisibilitySettings) {
-    return ciVisibilitySettings.isTestsSkippingEnabled()
-        && config.isCiVisibilityTestSkippingEnabled();
-  }
-
-  private boolean isCodeCoverageEnabled(CiVisibilitySettings ciVisibilitySettings) {
-    return ciVisibilitySettings.isCodeCoverageEnabled()
-        && config.isCiVisibilityCodeCoverageEnabled();
-  }
-
-  private boolean isFlakyTestRetriesEnabled(CiVisibilitySettings ciVisibilitySettings) {
-    return ciVisibilitySettings.isFlakyTestRetriesEnabled()
-        && config.isCiVisibilityFlakyRetryEnabled();
-  }
-
-  private boolean isImpactedTestsDetectionEnabled(CiVisibilitySettings ciVisibilitySettings) {
-    return ciVisibilitySettings.isImpactedTestsDetectionEnabled()
-        && config.isCiVisibilityImpactedTestsDetectionEnabled();
-  }
-
-  private boolean isEarlyFlakeDetectionEnabled(CiVisibilitySettings ciVisibilitySettings) {
-    return ciVisibilitySettings.getEarlyFlakeDetectionSettings().isEnabled()
-        && config.isCiVisibilityEarlyFlakeDetectionEnabled();
+  private boolean isFeatureEnabled(
+      CiVisibilitySettings ciVisibilitySettings,
+      Function<CiVisibilitySettings, Boolean> remoteSetting,
+      Function<Config, Boolean> killSwitch) {
+    return remoteSetting.apply(ciVisibilitySettings) && killSwitch.apply(config);
   }
 
   @Nullable

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TracerEnvironment.java

@@ -40,6 +40,26 @@ public String getSha() {
     return sha;
   }
 
+  public String getService() {
+    return service;
+  }
+
+  public String getEnv() {
+    return env;
+  }
+
+  public String getRepositoryUrl() {
+    return repositoryUrl;
+  }
+
+  public String getBranch() {
+    return branch;
+  }
+
+  public String getTestLevel() {
+    return testLevel;
+  }
+
   public Configurations getConfigurations() {
     return configurations;
   }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java

@@ -22,6 +22,7 @@
 import datadog.trace.api.civisibility.telemetry.tag.IsNew;
 import datadog.trace.api.civisibility.telemetry.tag.IsRetry;
 import datadog.trace.api.civisibility.telemetry.tag.IsRum;
+import datadog.trace.api.civisibility.telemetry.tag.RetryReason;
 import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
 import datadog.trace.api.gateway.RequestContextSlot;
 import datadog.trace.bootstrap.instrumentation.api.AgentScope;
@@ -265,12 +266,25 @@ public void end(@Nullable Long endTime) {
         span.getTag(Tags.TEST_IS_NEW) != null ? IsNew.TRUE : null,
         span.getTag(Tags.TEST_IS_MODIFIED) != null ? IsModified.TRUE : null,
         span.getTag(Tags.TEST_IS_RETRY) != null ? IsRetry.TRUE : null,
+        getRetryReason(),
         span.getTag(Tags.TEST_IS_RUM_ACTIVE) != null ? IsRum.TRUE : null,
         CIConstants.SELENIUM_BROWSER_DRIVER.equals(span.getTag(Tags.TEST_BROWSER_DRIVER))
             ? BrowserDriver.SELENIUM
             : null);
   }
 
+  private RetryReason getRetryReason() {
+    String retryReason = (String) span.getTag(Tags.TEST_RETRY_REASON);
+    if (retryReason != null) {
+      try {
+        return RetryReason.valueOf(retryReason.toUpperCase());
+      } catch (IllegalArgumentException e) {
+        log.debug("Non-standard retry-reason: {}", retryReason);
+      }
+    }
+    return null;
+  }
+
   /**
    * Tests often perform operations that involve APM instrumentations: sending an HTTP request,
    * executing a database query, etc. APM instrumentations create spans that correspond to those