Implement quarantined tests tagging (#8326)

Author: nikita-tkachenko-datadog

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java

@@ -31,6 +31,8 @@ TestSuiteImpl testSuiteStart(
 
   boolean isModified(TestSourceData testSourceData);
 
+  boolean isQuarantined(TestIdentifier test);
+
   /**
    * Returns the reason for skipping a test, IF it can be skipped.
    *

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java

@@ -18,8 +18,10 @@
 import datadog.trace.api.civisibility.telemetry.TagValue;
 import datadog.trace.api.civisibility.telemetry.tag.BrowserDriver;
 import datadog.trace.api.civisibility.telemetry.tag.EventType;
+import datadog.trace.api.civisibility.telemetry.tag.HasFailedAllRetries;
 import datadog.trace.api.civisibility.telemetry.tag.IsModified;
 import datadog.trace.api.civisibility.telemetry.tag.IsNew;
+import datadog.trace.api.civisibility.telemetry.tag.IsQuarantined;
 import datadog.trace.api.civisibility.telemetry.tag.IsRetry;
 import datadog.trace.api.civisibility.telemetry.tag.IsRum;
 import datadog.trace.api.civisibility.telemetry.tag.SkipReason;
@@ -190,6 +192,10 @@ public TestIdentifier getIdentifier() {
     return identifier;
   }
 
+  public boolean hasFailed() {
+    return span.isError();
+  }
+
   @Override
   public void setTag(String key, Object value) {
     span.setTag(key, value);
@@ -265,13 +271,15 @@ public void end(@Nullable Long endTime) {
 
     Object retryReason = span.getTag(Tags.TEST_RETRY_REASON);
     metricCollector.add(
-        CiVisibilityCountMetric.EVENT_FINISHED,
+        CiVisibilityCountMetric.TEST_EVENT_FINISHED,
         1,
         instrumentation,
         EventType.TEST,
         span.getTag(Tags.TEST_IS_NEW) != null ? IsNew.TRUE : null,
         span.getTag(Tags.TEST_IS_MODIFIED) != null ? IsModified.TRUE : null,
+        span.getTag(Tags.TEST_MANAGEMENT_IS_QUARANTINED) != null ? IsQuarantined.TRUE : null,
         span.getTag(Tags.TEST_IS_RETRY) != null ? IsRetry.TRUE : null,
+        span.getTag(Tags.TEST_HAS_FAILED_ALL_RETRIES) != null ? HasFailedAllRetries.TRUE : null,
         retryReason instanceof TagValue ? (TagValue) retryReason : null,
         span.getTag(Tags.TEST_IS_RUM_ACTIVE) != null ? IsRum.TRUE : null,
         CIConstants.SELENIUM_BROWSER_DRIVER.equals(span.getTag(Tags.TEST_BROWSER_DRIVER))

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java

@@ -103,6 +103,11 @@ public boolean isModified(TestSourceData testSourceData) {
     return executionStrategy.isModified(testSourceData);
   }
 
+  @Override
+  public boolean isQuarantined(TestIdentifier test) {
+    return executionStrategy.isQuarantined(test);
+  }
+
   @Nullable
   @Override
   public SkipReason skipReason(TestIdentifier test) {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java

@@ -88,6 +88,11 @@ public boolean isModified(TestSourceData testSourceData) {
     return executionStrategy.isModified(testSourceData);
   }
 
+  @Override
+  public boolean isQuarantined(TestIdentifier test) {
+    return executionStrategy.isQuarantined(test);
+  }
+
   @Nullable
   @Override
   public SkipReason skipReason(TestIdentifier test) {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/NoOpTestEventsHandler.java

@@ -5,8 +5,8 @@
 import datadog.trace.api.civisibility.config.TestIdentifier;
 import datadog.trace.api.civisibility.config.TestSourceData;
 import datadog.trace.api.civisibility.events.TestEventsHandler;
+import datadog.trace.api.civisibility.execution.TestExecutionHistory;
 import datadog.trace.api.civisibility.execution.TestExecutionPolicy;
-import datadog.trace.api.civisibility.telemetry.tag.RetryReason;
 import datadog.trace.api.civisibility.telemetry.tag.SkipReason;
 import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
 import datadog.trace.bootstrap.ContextStore;
@@ -58,8 +58,8 @@ public void onTestStart(
       @Nullable String testParameters,
       @Nullable Collection<String> categories,
       @Nonnull TestSourceData testSourceData,
-      RetryReason retryReason,
-      @Nullable Long startTime) {
+      @Nullable Long startTime,
+      @Nullable TestExecutionHistory testExecutionHistory) {
     // do nothing
   }
 
@@ -74,7 +74,8 @@ public void onTestFailure(TestKey descriptor, @Nullable Throwable throwable) {
   }
 
   @Override
-  public void onTestFinish(TestKey descriptor, @Nullable Long endTime) {
+  public void onTestFinish(
+      TestKey descriptor, @Nullable Long endTime, @Nullable TestExecutionHistory executionHistory) {
     // do nothing
   }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java

@@ -8,6 +8,7 @@
 import datadog.trace.api.civisibility.config.TestIdentifier;
 import datadog.trace.api.civisibility.config.TestSourceData;
 import datadog.trace.api.civisibility.events.TestEventsHandler;
+import datadog.trace.api.civisibility.execution.TestExecutionHistory;
 import datadog.trace.api.civisibility.execution.TestExecutionPolicy;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
@@ -139,8 +140,8 @@ public void onTestStart(
       final @Nullable String testParameters,
       final @Nullable Collection<String> categories,
       final @Nonnull TestSourceData testSourceData,
-      final @Nullable RetryReason retryReason,
-      final @Nullable Long startTime) {
+      final @Nullable Long startTime,
+      final @Nullable TestExecutionHistory testExecutionHistory) {
     if (skipTrace(testSourceData.getTestClass())) {
       return;
     }
@@ -166,6 +167,18 @@ public void onTestStart(
       test.setTag(Tags.TEST_IS_MODIFIED, true);
     }
 
+    if (testModule.isQuarantined(thisTest)) {
+      test.setTag(Tags.TEST_MANAGEMENT_IS_QUARANTINED, true);
+    }
+
+    if (testExecutionHistory != null) {
+      RetryReason retryReason = testExecutionHistory.currentExecutionRetryReason();
+      if (retryReason != null) {
+        test.setTag(Tags.TEST_IS_RETRY, true);
+        test.setTag(Tags.TEST_RETRY_REASON, retryReason);
+      }
+    }
+
     if (testFramework != null) {
       test.setTag(Tags.TEST_FRAMEWORK, testFramework);
       if (testFrameworkVersion != null) {
@@ -198,11 +211,6 @@ public void onTestStart(
       }
     }
 
-    if (retryReason != null) {
-      test.setTag(Tags.TEST_IS_RETRY, true);
-      test.setTag(Tags.TEST_RETRY_REASON, retryReason);
-    }
-
     inProgressTests.put(descriptor, test);
   }
 
@@ -227,12 +235,22 @@ public void onTestFailure(TestKey descriptor, @Nullable Throwable throwable) {
   }
 
   @Override
-  public void onTestFinish(TestKey descriptor, @Nullable Long endTime) {
+  public void onTestFinish(
+      TestKey descriptor,
+      @Nullable Long endTime,
+      @Nullable TestExecutionHistory testExecutionHistory) {
     TestImpl test = inProgressTests.remove(descriptor);
     if (test == null) {
       log.debug("Ignoring finish event, could not find test {}", descriptor);
       return;
     }
+
+    if (testExecutionHistory != null) {
+      if (test.hasFailed() && testExecutionHistory.hasFailedAllRetries()) {
+        test.setTag(Tags.TEST_HAS_FAILED_ALL_RETRIES, true);
+      }
+    }
+
     test.end(endTime);
   }
 
@@ -259,7 +277,7 @@ public void onTestIgnore(
         null,
         null);
     onTestSkip(testDescriptor, reason);
-    onTestFinish(testDescriptor, null);
+    onTestFinish(testDescriptor, null, null);
   }
 
   @Override

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/Regular.java

@@ -31,4 +31,9 @@ public boolean retry(boolean successful, long durationMillis) {
   public RetryReason currentExecutionRetryReason() {
     return null;
   }
+
+  @Override
+  public boolean hasFailedAllRetries() {
+    return false;
+  }
 }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RetryUntilSuccessful.java

@@ -11,6 +11,7 @@ public class RetryUntilSuccessful implements TestExecutionPolicy {
   private final int maxExecutions;
   private final boolean suppressFailures;
   private int executions;
+  private boolean successfulExecutionSeen;
 
   /** Total execution counter that is shared by all retry policies */
   private final AtomicInteger totalExecutions;
@@ -25,22 +26,23 @@ public RetryUntilSuccessful(
 
   @Override
   public boolean applicable() {
-    return currentExecutionIsNotLast() || suppressFailures;
+    return !currentExecutionIsLast() || suppressFailures;
   }
 
   @Override
   public boolean suppressFailures() {
     // do not suppress failures for last execution
     // (unless flag to suppress all failures is set)
-    return currentExecutionIsNotLast() || suppressFailures;
+    return !currentExecutionIsLast() || suppressFailures;
   }
 
-  private boolean currentExecutionIsNotLast() {
-    return executions < maxExecutions - 1;
+  private boolean currentExecutionIsLast() {
+    return executions == maxExecutions - 1;
   }
 
   @Override
   public boolean retry(boolean successful, long durationMillis) {
+    successfulExecutionSeen |= successful;
     if (!successful && ++executions < maxExecutions) {
       totalExecutions.incrementAndGet();
       return true;
@@ -58,4 +60,9 @@ public RetryReason currentExecutionRetryReason() {
   private boolean currentExecutionIsRetry() {
     return executions > 0;
   }
+
+  @Override
+  public boolean hasFailedAllRetries() {
+    return currentExecutionIsLast() && !successfulExecutionSeen;
+  }
 }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunNTimes.java

@@ -12,6 +12,7 @@ public class RunNTimes implements TestExecutionPolicy {
   private final boolean suppressFailures;
   private int executions;
   private int maxExecutions;
+  private boolean successfulExecutionSeen;
 
   public RunNTimes(
       EarlyFlakeDetectionSettings earlyFlakeDetectionSettings, boolean suppressFailures) {
@@ -23,11 +24,11 @@ public RunNTimes(
 
   @Override
   public boolean applicable() {
-    return currentExecutionIsNotLast() || suppressFailures();
+    return !currentExecutionIsLast() || suppressFailures();
   }
 
-  private boolean currentExecutionIsNotLast() {
-    return executions < maxExecutions - 1;
+  private boolean currentExecutionIsLast() {
+    return executions == maxExecutions - 1;
   }
 
   @Override
@@ -37,6 +38,7 @@ public boolean suppressFailures() {
 
   @Override
   public boolean retry(boolean successful, long durationMillis) {
+    successfulExecutionSeen |= successful;
     // adjust maximum retries based on the now known test duration
     int maxExecutionsForGivenDuration = earlyFlakeDetectionSettings.getExecutions(durationMillis);
     maxExecutions = Math.min(maxExecutions, maxExecutionsForGivenDuration);
@@ -52,4 +54,9 @@ public RetryReason currentExecutionRetryReason() {
   private boolean currentExecutionIsRetry() {
     return executions > 0;
   }
+
+  @Override
+  public boolean hasFailedAllRetries() {
+    return currentExecutionIsLast() && !successfulExecutionSeen;
+  }
 }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunOnceIgnoreOutcome.java

@@ -33,4 +33,9 @@ public boolean retry(boolean successful, long durationMillis) {
   public RetryReason currentExecutionRetryReason() {
     return null;
   }
+
+  @Override
+  public boolean hasFailedAllRetries() {
+    return false;
+  }
 }