Implement quarantined tests tagging

Author: nikita-tkachenko-datadog

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java

@@ -31,6 +31,8 @@ TestSuiteImpl testSuiteStart(
 
   boolean isModified(TestSourceData testSourceData);
 
+  boolean isQuarantined(TestIdentifier test);
+
   /**
    * Returns the reason for skipping a test, IF it can be skipped.
    *

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java

@@ -18,8 +18,10 @@
 import datadog.trace.api.civisibility.telemetry.TagValue;
 import datadog.trace.api.civisibility.telemetry.tag.BrowserDriver;
 import datadog.trace.api.civisibility.telemetry.tag.EventType;
+import datadog.trace.api.civisibility.telemetry.tag.HasFailedAllRetries;
 import datadog.trace.api.civisibility.telemetry.tag.IsModified;
 import datadog.trace.api.civisibility.telemetry.tag.IsNew;
+import datadog.trace.api.civisibility.telemetry.tag.IsQuarantined;
 import datadog.trace.api.civisibility.telemetry.tag.IsRetry;
 import datadog.trace.api.civisibility.telemetry.tag.IsRum;
 import datadog.trace.api.civisibility.telemetry.tag.SkipReason;
@@ -265,13 +267,15 @@ public void end(@Nullable Long endTime) {
 
     Object retryReason = span.getTag(Tags.TEST_RETRY_REASON);
     metricCollector.add(
-        CiVisibilityCountMetric.EVENT_FINISHED,
+        CiVisibilityCountMetric.TEST_EVENT_FINISHED,
         1,
         instrumentation,
         EventType.TEST,
         span.getTag(Tags.TEST_IS_NEW) != null ? IsNew.TRUE : null,
         span.getTag(Tags.TEST_IS_MODIFIED) != null ? IsModified.TRUE : null,
+        span.getTag(Tags.TEST_MANAGEMENT_IS_QUARANTINED) != null ? IsQuarantined.TRUE : null,
         span.getTag(Tags.TEST_IS_RETRY) != null ? IsRetry.TRUE : null,
+        span.getTag(Tags.TEST_HAS_FAILED_ALL_RETRIES) != null ? HasFailedAllRetries.TRUE : null,
         retryReason instanceof TagValue ? (TagValue) retryReason : null,
         span.getTag(Tags.TEST_IS_RUM_ACTIVE) != null ? IsRum.TRUE : null,
         CIConstants.SELENIUM_BROWSER_DRIVER.equals(span.getTag(Tags.TEST_BROWSER_DRIVER))

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java

@@ -103,6 +103,11 @@ public boolean isModified(TestSourceData testSourceData) {
     return executionStrategy.isModified(testSourceData);
   }
 
+  @Override
+  public boolean isQuarantined(TestIdentifier test) {
+    return executionStrategy.isQuarantined(test);
+  }
+
   @Nullable
   @Override
   public SkipReason skipReason(TestIdentifier test) {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java

@@ -88,6 +88,11 @@ public boolean isModified(TestSourceData testSourceData) {
     return executionStrategy.isModified(testSourceData);
   }
 
+  @Override
+  public boolean isQuarantined(TestIdentifier test) {
+    return executionStrategy.isQuarantined(test);
+  }
+
   @Nullable
   @Override
   public SkipReason skipReason(TestIdentifier test) {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/NoOpTestEventsHandler.java

@@ -59,6 +59,7 @@ public void onTestStart(
       @Nullable Collection<String> categories,
       @Nonnull TestSourceData testSourceData,
       RetryReason retryReason,
+      boolean hasFailedAllRetries,
       @Nullable Long startTime) {
     // do nothing
   }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java

@@ -140,6 +140,7 @@ public void onTestStart(
       final @Nullable Collection<String> categories,
       final @Nonnull TestSourceData testSourceData,
       final @Nullable RetryReason retryReason,
+      final boolean hasFailedAllRetries,
       final @Nullable Long startTime) {
     if (skipTrace(testSourceData.getTestClass())) {
       return;
@@ -166,6 +167,14 @@ public void onTestStart(
       test.setTag(Tags.TEST_IS_MODIFIED, true);
     }
 
+    if (testModule.isQuarantined(thisTest)) {
+      test.setTag(Tags.TEST_MANAGEMENT_IS_QUARANTINED, true);
+    }
+
+    if (hasFailedAllRetries) {
+      test.setTag(Tags.TEST_HAS_FAILED_ALL_RETRIES, true);
+    }
+
     if (testFramework != null) {
       test.setTag(Tags.TEST_FRAMEWORK, testFramework);
       if (testFrameworkVersion != null) {
@@ -257,6 +266,7 @@ public void onTestIgnore(
         categories,
         testSourceData,
         null,
+        false,
         null);
     onTestSkip(testDescriptor, reason);
     onTestFinish(testDescriptor, null);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/Regular.java

@@ -31,4 +31,9 @@ public boolean retry(boolean successful, long durationMillis) {
   public RetryReason currentExecutionRetryReason() {
     return null;
   }
+
+  @Override
+  public boolean hasFailedAllRetries() {
+    return false;
+  }
 }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RetryUntilSuccessful.java

@@ -11,6 +11,7 @@ public class RetryUntilSuccessful implements TestExecutionPolicy {
   private final int maxExecutions;
   private final boolean suppressFailures;
   private int executions;
+  private boolean successfulExecutionSeen;
 
   /** Total execution counter that is shared by all retry policies */
   private final AtomicInteger totalExecutions;
@@ -41,6 +42,7 @@ private boolean currentExecutionIsNotLast() {
 
   @Override
   public boolean retry(boolean successful, long durationMillis) {
+    successfulExecutionSeen |= successful;
     if (!successful && ++executions < maxExecutions) {
       totalExecutions.incrementAndGet();
       return true;
@@ -58,4 +60,9 @@ public RetryReason currentExecutionRetryReason() {
   private boolean currentExecutionIsRetry() {
     return executions > 0;
   }
+
+  @Override
+  public boolean hasFailedAllRetries() {
+    return executions == maxExecutions && !successfulExecutionSeen;
+  }
 }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunNTimes.java

@@ -12,6 +12,7 @@ public class RunNTimes implements TestExecutionPolicy {
   private final boolean suppressFailures;
   private int executions;
   private int maxExecutions;
+  private boolean successfulExecutionSeen;
 
   public RunNTimes(
       EarlyFlakeDetectionSettings earlyFlakeDetectionSettings, boolean suppressFailures) {
@@ -37,6 +38,7 @@ public boolean suppressFailures() {
 
   @Override
   public boolean retry(boolean successful, long durationMillis) {
+    successfulExecutionSeen |= successful;
     // adjust maximum retries based on the now known test duration
     int maxExecutionsForGivenDuration = earlyFlakeDetectionSettings.getExecutions(durationMillis);
     maxExecutions = Math.min(maxExecutions, maxExecutionsForGivenDuration);
@@ -52,4 +54,9 @@ public RetryReason currentExecutionRetryReason() {
   private boolean currentExecutionIsRetry() {
     return executions > 0;
   }
+
+  @Override
+  public boolean hasFailedAllRetries() {
+    return executions == maxExecutions && !successfulExecutionSeen;
+  }
 }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunOnceIgnoreOutcome.java

@@ -33,4 +33,9 @@ public boolean retry(boolean successful, long durationMillis) {
   public RetryReason currentExecutionRetryReason() {
     return null;
   }
+
+  @Override
+  public boolean hasFailedAllRetries() {
+    return false;
+  }
 }