test: introduce FTR smoke tests for headfull and headless modes

Author: daniel-mohedano

.github/CODEOWNERS

@@ -87,6 +87,7 @@
 /dd-java-agent/instrumentation/maven-surefire/      @DataDog/ci-app-libraries-java
 /dd-java-agent/instrumentation/weaver/              @DataDog/ci-app-libraries-java
 /dd-smoke-tests/gradle/                             @DataDog/ci-app-libraries-java
+/dd-smoke-tests/junit-console/                      @DataDog/ci-app-libraries-java
 /dd-smoke-tests/maven/                              @DataDog/ci-app-libraries-java
 /internal-api/src/main/java/datadog/trace/api/git/  @DataDog/ci-app-libraries-java
 **/civisibility/                                    @DataDog/ci-app-libraries-java

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java

@@ -102,7 +102,8 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) {
       }
 
       if (executionSettings.isFailedTestReplayEnabled()) {
-        config.setCiVisibilityFailedTestReplayEnabled(true);
+        // only marks the feature as active in child or headless processes
+        config.setCiVisibilityFailedTestReplayActive(true);
       }
 
       CiVisibilityCoverageServices.Child coverageServices =

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl.java

@@ -185,12 +185,13 @@ private Map<String, String> getPropertiesPropagatedToChildProcess(
         Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.TEST_MANAGEMENT_ENABLED),
         Boolean.toString(executionSettings.getTestManagementSettings().isEnabled()));
 
+    propagatedSystemProperties.put(
+        Strings.propertyNameToSystemPropertyName(
+            CiVisibilityConfig.TEST_FAILED_TEST_REPLAY_ENABLED),
+        Boolean.toString(executionSettings.isFailedTestReplayEnabled()));
+
     // enable exception replay if failed test replay is enabled
     if (executionSettings.isFailedTestReplayEnabled()) {
-      propagatedSystemProperties.put(
-          Strings.propertyNameToSystemPropertyName(
-              CiVisibilityConfig.TEST_FAILED_TEST_REPLAY_ENABLED),
-          "true");
       propagatedSystemProperties.put(
           Strings.propertyNameToSystemPropertyName(DebuggerConfig.EXCEPTION_REPLAY_ENABLED),
           "true");

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/DefaultExceptionDebugger.java

@@ -75,7 +75,7 @@ public DefaultExceptionDebugger(
   @Override
   public void handleException(Throwable t, AgentSpan span) {
     // CIVIS Failed Test Replay acts on errors
-    if (t instanceof Error && !Config.get().isCiVisibilityFailedTestReplayEnabled()) {
+    if (t instanceof Error && !Config.get().isCiVisibilityFailedTestReplayActive()) {
       if (LOGGER.isDebugEnabled()) {
         LOGGER.debug("Skip handling error: {}", t.toString());
       }
@@ -114,7 +114,7 @@ public void handleException(Throwable t, AgentSpan span) {
             exceptionProbeManager.createProbesForException(
                 throwable.getStackTrace(), chainedExceptionIdx);
         if (creationResult.probesCreated > 0) {
-          if (Config.get().isCiVisibilityFailedTestReplayEnabled()) {
+          if (Config.get().isCiVisibilityFailedTestReplayActive()) {
             // Assume Exception Replay is working under Failed Test Replay logic,
             // instrumentation applied sync for immediate test retries
             applyExceptionConfiguration(fingerprint);

dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy

@@ -40,6 +40,7 @@ class MockBackend implements AutoCloseable {
   private boolean knownTestsEnabled = false
   private boolean testManagementEnabled = false
   private int attemptToFixRetries = 0
+  private boolean failedTestReplayEnabled = false
 
   void reset() {
     receivedTraces.clear()
@@ -61,6 +62,7 @@ class MockBackend implements AutoCloseable {
     knownTestsEnabled = false
     testManagementEnabled = false
     attemptToFixRetries = 0
+    failedTestReplayEnabled = false
   }
 
   @Override
@@ -131,6 +133,10 @@ class MockBackend implements AutoCloseable {
     ])
   }
 
+  void givenFailedTestReplay(boolean failedTestReplayEnabled) {
+    this.failedTestReplayEnabled = failedTestReplayEnabled
+  }
+
   String getIntakeUrl() {
     return intakeServer.address.toString()
   }
@@ -177,6 +183,7 @@ class MockBackend implements AutoCloseable {
               "flaky_test_retries_enabled": $flakyRetriesEnabled,
               "impacted_tests_enabled": $impactedTestsDetectionEnabled,
               "known_tests_enabled": $knownTestsEnabled,
+              "di_enabled": $failedTestReplayEnabled,
               "test_management": {
                 "enabled": $testManagementEnabled,
                 "attempt_to_fix_retries": $attemptToFixRetries

dd-smoke-tests/junit-console/build.gradle

@@ -0,0 +1,20 @@
+import java.time.Duration
+import java.time.temporal.ChronoUnit
+
+apply from: "$rootDir/gradle/java.gradle"
+description = 'JUnit Console Smoke Tests.'
+
+dependencies {
+  implementation group: 'org.junit.platform', name: 'junit-platform-console-standalone', version: '1.13.4'
+
+  testImplementation project(':dd-smoke-tests:backend-mock')
+}
+
+tasks.withType(Test).configureEach {
+  jvmArgs "-Ddatadog.smoketest.junit.console.jar.path=${configurations.runtimeClasspath.find { it.name.contains('junit-platform-console-standalone') }}"
+
+  if (project.hasProperty("mavenRepositoryProxy")) {
+    // propagate proxy URL to tests, to then propagate it to nested Gradle builds
+    environment "MAVEN_REPOSITORY_PROXY", project.property("mavenRepositoryProxy")
+  }
+}

dd-smoke-tests/junit-console/gradle.lockfile

@@ -0,0 +1,166 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+cafe.cryptography:curve25519-elisabeth:0.1.0=testRuntimeClasspath
+cafe.cryptography:ed25519-elisabeth:0.1.0=testRuntimeClasspath
+ch.qos.logback:logback-classic:1.2.3=testCompileClasspath,testRuntimeClasspath
+ch.qos.logback:logback-core:1.2.3=testCompileClasspath,testRuntimeClasspath
+com.beust:jcommander:1.78=testRuntimeClasspath
+com.blogspot.mydailyjava:weak-lock-free:0.17=testCompileClasspath,testRuntimeClasspath
+com.datadoghq.okhttp3:okhttp:3.12.15=testCompileClasspath,testRuntimeClasspath
+com.datadoghq.okio:okio:1.17.6=testCompileClasspath,testRuntimeClasspath
+com.datadoghq:dd-javac-plugin-client:0.2.2=testCompileClasspath,testRuntimeClasspath
+com.datadoghq:java-dogstatsd-client:4.4.3=testRuntimeClasspath
+com.datadoghq:sketches-java:0.8.3=testRuntimeClasspath
+com.fasterxml.jackson.core:jackson-annotations:2.16.0=testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson.core:jackson-core:2.16.0=testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson.core:jackson-databind:2.16.0=testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson:jackson-bom:2.16.0=testCompileClasspath,testRuntimeClasspath
+com.github.javaparser:javaparser-core:3.25.6=testCompileClasspath,testRuntimeClasspath
+com.github.jnr:jffi:1.3.13=testRuntimeClasspath
+com.github.jnr:jnr-a64asm:1.0.0=testRuntimeClasspath
+com.github.jnr:jnr-constants:0.10.4=testRuntimeClasspath
+com.github.jnr:jnr-enxio:0.32.17=testRuntimeClasspath
+com.github.jnr:jnr-ffi:2.2.16=testRuntimeClasspath
+com.github.jnr:jnr-posix:3.1.19=testRuntimeClasspath
+com.github.jnr:jnr-unixsocket:0.38.22=testRuntimeClasspath
+com.github.jnr:jnr-x86asm:1.0.2=testRuntimeClasspath
+com.github.spotbugs:spotbugs-annotations:4.2.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.github.spotbugs:spotbugs-annotations:4.7.3=spotbugs
+com.github.spotbugs:spotbugs:4.7.3=spotbugs
+com.github.stefanbirkner:system-rules:1.19.0=testCompileClasspath,testRuntimeClasspath
+com.google.code.findbugs:jsr305:3.0.2=compileClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath
+com.google.code.gson:gson:2.9.1=spotbugs
+com.google.guava:guava:20.0=testCompileClasspath,testRuntimeClasspath
+com.google.re2j:re2j:1.7=testRuntimeClasspath
+com.jayway.jsonpath:json-path:2.8.0=testCompileClasspath,testRuntimeClasspath
+com.squareup.moshi:moshi:1.11.0=testCompileClasspath,testRuntimeClasspath
+com.squareup.okhttp3:logging-interceptor:3.12.12=testCompileClasspath,testRuntimeClasspath
+com.squareup.okhttp3:okhttp:3.12.12=testCompileClasspath,testRuntimeClasspath
+com.squareup.okio:okio:1.17.5=testCompileClasspath,testRuntimeClasspath
+com.thoughtworks.qdox:qdox:1.12.1=testRuntimeClasspath
+com.vaadin.external.google:android-json:0.0.20131108.vaadin1=testCompileClasspath,testRuntimeClasspath
+commons-codec:commons-codec:1.15=spotbugs
+commons-fileupload:commons-fileupload:1.5=testCompileClasspath,testRuntimeClasspath
+commons-io:commons-io:2.11.0=testCompileClasspath,testRuntimeClasspath
+de.thetaphi:forbiddenapis:3.8=compileClasspath
+info.picocli:picocli:4.6.3=testRuntimeClasspath
+io.sqreen:libsqreen:15.0.1=testRuntimeClasspath
+javax.servlet:javax.servlet-api:3.1.0=testCompileClasspath,testRuntimeClasspath
+jaxen:jaxen:1.2.0=spotbugs
+jline:jline:2.14.6=testRuntimeClasspath
+junit:junit-dep:4.11=testCompileClasspath,testRuntimeClasspath
+junit:junit:4.13.2=testCompileClasspath,testRuntimeClasspath
+net.bytebuddy:byte-buddy-agent:1.17.5=testCompileClasspath,testRuntimeClasspath
+net.bytebuddy:byte-buddy:1.17.5=testCompileClasspath,testRuntimeClasspath
+net.java.dev.jna:jna-platform:5.8.0=testRuntimeClasspath
+net.java.dev.jna:jna:5.8.0=testRuntimeClasspath
+net.jcip:jcip-annotations:1.0=compileClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath
+net.minidev:accessors-smart:2.4.9=testRuntimeClasspath
+net.minidev:json-smart:2.4.10=testRuntimeClasspath
+net.sf.saxon:Saxon-HE:11.4=spotbugs
+org.apache.ant:ant-antlr:1.10.15=testRuntimeClasspath
+org.apache.ant:ant-antlr:1.9.15=codenarc
+org.apache.ant:ant-junit:1.10.15=testRuntimeClasspath
+org.apache.ant:ant-junit:1.9.15=codenarc
+org.apache.ant:ant-launcher:1.10.15=testRuntimeClasspath
+org.apache.ant:ant:1.10.15=testCompileClasspath,testRuntimeClasspath
+org.apache.bcel:bcel:6.5.0=spotbugs
+org.apache.commons:commons-lang3:3.12.0=spotbugs
+org.apache.commons:commons-text:1.10.0=spotbugs
+org.apache.httpcomponents.client5:httpclient5:5.1.3=spotbugs
+org.apache.httpcomponents.core5:httpcore5-h2:5.1.3=spotbugs
+org.apache.httpcomponents.core5:httpcore5:5.1.3=spotbugs
+org.apache.logging.log4j:log4j-api:2.19.0=spotbugs
+org.apache.logging.log4j:log4j-core:2.19.0=spotbugs
+org.apache.maven.wrapper:maven-wrapper:3.2.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath
+org.codehaus.groovy:groovy-all:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-ant:2.5.14=codenarc
+org.codehaus.groovy:groovy-ant:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-astbuilder:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-cli-picocli:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-console:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-datetime:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-docgenerator:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-groovydoc:2.5.14=codenarc
+org.codehaus.groovy:groovy-groovydoc:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-groovysh:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-jmx:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-json:2.5.14=codenarc
+org.codehaus.groovy:groovy-json:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-jsr223:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-macro:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-nio:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-servlet:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-sql:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-swing:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-templates:2.5.14=codenarc
+org.codehaus.groovy:groovy-templates:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-test-junit5:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-test:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-testng:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-xml:2.5.14=codenarc
+org.codehaus.groovy:groovy-xml:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy:2.5.14=codenarc
+org.codehaus.groovy:groovy:3.0.24=testCompileClasspath,testRuntimeClasspath
+org.codenarc:CodeNarc:2.2.0=codenarc
+org.dom4j:dom4j:2.1.3=spotbugs
+org.eclipse.jetty:jetty-http:9.4.56.v20240826=testCompileClasspath,testRuntimeClasspath
+org.eclipse.jetty:jetty-io:9.4.56.v20240826=testCompileClasspath,testRuntimeClasspath
+org.eclipse.jetty:jetty-server:9.4.56.v20240826=testCompileClasspath,testRuntimeClasspath
+org.eclipse.jetty:jetty-util:9.4.56.v20240826=testCompileClasspath,testRuntimeClasspath
+org.freemarker:freemarker:2.3.31=testCompileClasspath,testRuntimeClasspath
+org.gmetrics:GMetrics:1.1=codenarc
+org.hamcrest:hamcrest-core:1.3=testCompileClasspath,testRuntimeClasspath
+org.hamcrest:hamcrest:2.2=testCompileClasspath,testRuntimeClasspath
+org.jacoco:org.jacoco.core:0.8.12=testRuntimeClasspath
+org.jacoco:org.jacoco.report:0.8.12=testRuntimeClasspath
+org.jctools:jctools-core:3.3.0=testRuntimeClasspath
+org.jetbrains.kotlin:kotlin-stdlib-common:1.6.21=testCompileClasspath,testRuntimeClasspath
+org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.6.21=testCompileClasspath,testRuntimeClasspath
+org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.6.21=testCompileClasspath,testRuntimeClasspath
+org.jetbrains.kotlin:kotlin-stdlib:1.6.21=testCompileClasspath,testRuntimeClasspath
+org.jetbrains:annotations:13.0=testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-api:5.12.0=testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-engine:5.12.0=testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-params:5.12.0=testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter:5.12.0=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-commons:1.12.0=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-engine:1.12.0=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-launcher:1.12.0=testRuntimeClasspath
+org.junit.platform:junit-platform-runner:1.12.0=testRuntimeClasspath
+org.junit.platform:junit-platform-suite-api:1.12.0=testRuntimeClasspath
+org.junit.platform:junit-platform-suite-commons:1.12.0=testRuntimeClasspath
+org.junit:junit-bom:5.12.0=testCompileClasspath,testRuntimeClasspath
+org.junit:junit-bom:5.9.1=spotbugs
+org.msgpack:jackson-dataformat-msgpack:0.9.6=testCompileClasspath,testRuntimeClasspath
+org.msgpack:msgpack-core:0.9.6=testCompileClasspath,testRuntimeClasspath
+org.objenesis:objenesis:3.3=testCompileClasspath,testRuntimeClasspath
+org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath
+org.ow2.asm:asm-analysis:9.2=testRuntimeClasspath
+org.ow2.asm:asm-analysis:9.4=spotbugs
+org.ow2.asm:asm-commons:9.4=spotbugs
+org.ow2.asm:asm-commons:9.8=testRuntimeClasspath
+org.ow2.asm:asm-tree:9.4=spotbugs
+org.ow2.asm:asm-tree:9.8=testRuntimeClasspath
+org.ow2.asm:asm-util:9.2=testRuntimeClasspath
+org.ow2.asm:asm-util:9.4=spotbugs
+org.ow2.asm:asm:9.4=spotbugs
+org.ow2.asm:asm:9.8=testRuntimeClasspath
+org.skyscreamer:jsonassert:1.5.1=testCompileClasspath,testRuntimeClasspath
+org.slf4j:jcl-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath
+org.slf4j:jul-to-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath
+org.slf4j:log4j-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath
+org.slf4j:slf4j-api:1.7.30=testCompileClasspath
+org.slf4j:slf4j-api:1.7.36=testRuntimeClasspath
+org.slf4j:slf4j-api:2.0.0=spotbugs,spotbugsSlf4j
+org.slf4j:slf4j-simple:2.0.0=spotbugsSlf4j
+org.snakeyaml:snakeyaml-engine:2.9=testRuntimeClasspath
+org.spockframework:spock-core:2.3-groovy-3.0=testCompileClasspath,testRuntimeClasspath
+org.spockframework:spock-junit4:2.3-groovy-3.0=testCompileClasspath,testRuntimeClasspath
+org.testng:testng:7.5.1=testRuntimeClasspath
+org.webjars:jquery:3.5.1=testRuntimeClasspath
+org.xmlresolver:xmlresolver:4.4.3=spotbugs
+xml-apis:xml-apis:1.4.01=spotbugs
+empty=annotationProcessor,shadow,spotbugsPlugins,testAnnotationProcessor