Merge branch 'master' into bdu/spotbugs-bump-need-jdk11-min

Author: AlexeyKuznetsov-DD

.github/CODEOWNERS

@@ -115,6 +115,9 @@
 **/datastreams/                                                              @DataDog/data-streams-monitoring
 **/DataStreams*                                                              @DataDog/data-streams-monitoring
 
+# @DataDog/feature-flagging-and-experimentation-sdk
+/products/feature-flagging/                           @DataDog/feature-flagging-and-experimentation-sdk
+
 # @DataDog/profiling-java
 /dd-java-agent/agent-profiling/                                                             @DataDog/profiling-java
 /dd-java-agent/agent-crashtracking/                                                         @DataDog/profiling-java

.gitignore

@@ -46,6 +46,10 @@ out/
 ######################
 .vscode
 
+# Cursor #
+##########
+.cursor
+
 # Others #
 ##########
 /logs/*

.gitlab-ci.yml

@@ -171,6 +171,13 @@ default:
     - export ORG_GRADLE_PROJECT_akkaRepositoryToken=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.akka_repo_token --with-decryption --query "Parameter.Value" --out text)
     - export ORG_GRADLE_PROJECT_mavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY
     - export ORG_GRADLE_PROJECT_gradlePluginProxy=$GRADLE_PLUGIN_PROXY
+    - |
+      JAVA_HOMES=$(env | grep -E '^JAVA_[A-Z0-9_]+_HOME=' | sed 's/=.*//' | paste -sd,)
+      cat >> gradle.properties <<EOF
+      org.gradle.java.installations.auto-detect=false
+      org.gradle.java.installations.auto-download=false
+      org.gradle.java.installations.fromEnv=$JAVA_HOMES
+      EOF
     - mkdir -p .gradle
     - export GRADLE_USER_HOME=$(pwd)/.gradle
     # replace maven central part by MAVEN_REPOSITORY_PROXY in .mvn/wrapper/maven-wrapper.properties
@@ -982,13 +989,14 @@ create_key:
     paths:
       - pubkeys
 
-validate_supported_configurations_local_file:
-  extends: .validate_supported_configurations_local_file
+validate_supported_configurations_v2_local_file:
+  extends: .validate_supported_configurations_v2_local_file
   variables:
     LOCAL_JSON_PATH: "metadata/supported-configurations.json"
+    BACKFILLED: "false"
 
-update_central_configurations_version_range:
-  extends: .update_central_configurations_version_range
+update_central_configurations_version_range_v2:
+  extends: .update_central_configurations_version_range_v2
   variables:
     LOCAL_REPO_NAME: "dd-trace-java"
     LOCAL_JSON_PATH: "metadata/supported-configurations.json"

.gitlab/benchmarks.yml

@@ -1,6 +1,5 @@
 .benchmarks:
   stage: benchmarks
-  interruptible: true
   timeout: 1h
   tags: ["runner:apm-k8s-tweaked-metal"]
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-java-benchmarks
@@ -11,7 +10,11 @@
     - if: '$CI_COMMIT_TAG =~ /^v?[0-9]+\.[0-9]+\.[0-9]+$/'
       when: manual
       allow_failure: true
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: on_success
+      interruptible: false
     - when: on_success
+      interruptible: true
   script:
     - export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
     - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"

.gitlab/macrobenchmarks.yml

@@ -11,11 +11,12 @@ include:
       when: never
     - if: ($NIGHTLY_BENCHMARKS || $CI_PIPELINE_SOURCE != "schedule") && $CI_COMMIT_REF_NAME == "master"
       when: always
+      interruptible: false
     - when: manual
+      interruptible: true
       allow_failure: true
   tags: ["runner:apm-k8s-same-cpu"]
   needs: ["build"]
-  interruptible: true
   timeout: 1h
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-java-petclinic
   script:

buildSrc/build.gradle.kts

@@ -57,7 +57,7 @@ dependencies {
   implementation(gradleApi())
   implementation(localGroovy())
 
-  implementation("net.bytebuddy", "byte-buddy-gradle-plugin", "1.17.7")
+  implementation("net.bytebuddy", "byte-buddy-gradle-plugin", "1.18.1")
 
   implementation("org.eclipse.aether", "aether-connector-basic", "1.1.0")
   implementation("org.eclipse.aether", "aether-transport-http", "1.1.0")

buildSrc/src/main/kotlin/datadog.configure-tests.gradle.kts

@@ -101,6 +101,9 @@ tasks.withType<Test>().configureEach {
   develocity.testRetry {
     if (providers.environmentVariable("CI").isPresent()) {
       maxRetries = 3
+      filter {
+        excludeAnnotationClasses.add("*NonRetryable") // allow to mark classes non retryable
+      }
     }
   }
 }

buildSrc/src/main/kotlin/datadog/gradle/plugin/CallSiteInstrumentationPlugin.kt

@@ -228,16 +228,14 @@ abstract class CallSiteInstrumentationPlugin : Plugin<Project>{
 
   private fun getProgramClasspath(project: Project): List<File> {
     val classpath = ArrayList<File>()
-    // 1. Compilation outputs
+    // 1. Compilation outputs - exclude latestDep and forked test variants
     project.tasks.withType(AbstractCompile::class.java)
+      .filter { task -> !task.name.contains("LatestDep", ignoreCase = true) && !task.name.contains("Forked", ignoreCase = true) }
       .map { it.destinationDirectory.asFile.get() }
       .forEach(classpath::add)
-    // 2. Compile time dependencies
+    // 2. Compile time dependencies - exclude latestDep and forked test variants
     project.tasks.withType(AbstractCompile::class.java)
-      .flatMap { it.classpath }
-      .forEach(classpath::add)
-    // 3. Test time dependencies
-    project.tasks.withType(Test::class.java)
+      .filter { task -> !task.name.contains("LatestDep", ignoreCase = true) && !task.name.contains("Forked", ignoreCase = true) }
       .flatMap { it.classpath }
       .forEach(classpath::add)
     return classpath

buildSrc/src/main/kotlin/datadog/gradle/plugin/testJvmConstraints/TestJvmSpec.kt

@@ -1,88 +1,162 @@
 package datadog.gradle.plugin.testJvmConstraints
 
-import org.gradle.kotlin.dsl.support.serviceOf
 import org.gradle.api.GradleException
 import org.gradle.api.Project
+import org.gradle.api.internal.provider.PropertyFactory
+import org.gradle.api.provider.Provider
+import org.gradle.jvm.toolchain.JavaLanguageVersion
 import org.gradle.jvm.toolchain.JavaLauncher
 import org.gradle.jvm.toolchain.JavaToolchainService
+import org.gradle.jvm.toolchain.JavaToolchainSpec
+import org.gradle.jvm.toolchain.JvmImplementation
+import org.gradle.jvm.toolchain.JvmVendorSpec
+import org.gradle.jvm.toolchain.internal.DefaultToolchainSpec
+import org.gradle.jvm.toolchain.internal.SpecificInstallationToolchainSpec
+import org.gradle.kotlin.dsl.support.serviceOf
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
 
-
+/**
+ * Handles the `testJvm` property to resolve a Java launcher for testing.
+ *
+ * The `testJvm` property can be set via command line or environment variable to specify
+ * which JVM to use for running tests. E.g.
+ *
+ * ```shell
+ * ./gradlew test -PtestJvm=ZULU11
+ * ```
+ *
+ * This handles local setup, and CI environment, where the environment variables are defined here:
+ * * https://github.com/DataDog/dd-trace-java-docker-build/blob/a4f4bfa9d7fe0708858e595697dc67970a2a458f/Dockerfile#L182-L188
+ * * https://github.com/DataDog/dd-trace-java-docker-build/blob/a4f4bfa9d7fe0708858e595697dc67970a2a458f/Dockerfile#L222-L241
+ */
 class TestJvmSpec(val project: Project) {
   companion object {
     const val TEST_JVM = "testJvm"
   }
 
   private val currentJavaHomePath = project.providers.systemProperty("java.home").map { it.normalizeToJDKJavaHome() }
-  
-  val testJvmProperty = project.providers.gradleProperty(TEST_JVM)
 
-  val normalizedTestJvm = testJvmProperty.map { testJvm ->
+  /**
+   * The raw `testJvm` property as passed via command line or environment variable.
+   */
+  val testJvmProperty: Provider<String> = project.providers.gradleProperty(TEST_JVM)
+
+  /**
+   * Normalized `stable` string to the highest JAVA_X_HOME found in environment variables.
+   */
+  val normalizedTestJvm: Provider<String> = testJvmProperty.map { testJvm ->
     if (testJvm.isBlank()) {
       throw GradleException("testJvm property is blank")
     }
 
     // "stable" is calculated as the largest X found in JAVA_X_HOME
-    if (testJvm == "stable") {
-      val javaVersions = project.providers.environmentVariablesPrefixedBy("JAVA_").map { javaHomes ->
-        javaHomes
-          .filter { it.key.matches(Regex("^JAVA_[0-9]+_HOME$")) }
-          .map { Regex("^JAVA_(\\d+)_HOME$").find(it.key)!!.groupValues[1].toInt() }
-      }.get()
-
-      if (javaVersions.isEmpty()) {
-        throw GradleException("No valid JAVA_X_HOME environment variables found.")
+    when (testJvm) {
+      "stable" -> {
+        val javaVersions = project.providers.environmentVariablesPrefixedBy("JAVA_").map { javaHomes ->
+          javaHomes
+            .filter { it.key.matches(Regex("^JAVA_[0-9]+_HOME$")) }
+            .map { Regex("^JAVA_(\\d+)_HOME$").find(it.key)!!.groupValues[1].toInt() }
+        }.get()
+
+        if (javaVersions.isEmpty()) {
+          throw GradleException("No valid JAVA_X_HOME environment variables found.")
+        }
+
+        javaVersions.max().toString()
       }
 
-      javaVersions.max().toString()
-    } else {
-      testJvm
+      else -> testJvm
     }
   }.map { project.logger.info("normalized testJvm: $it"); it }
 
-  val testJvmHomePath = normalizedTestJvm.map {
-    if (Files.exists(Paths.get(it))) {
-      it.normalizeToJDKJavaHome()
-    } else {
-      val matcher = Regex("([a-zA-Z]*)([0-9]+)").find(it)
-      if (matcher == null) {
-        throw GradleException("Unable to find launcher for Java '$it'. It needs to match '([a-zA-Z]*)([0-9]+)'.")
-      }
-      val testJvmEnv = "JAVA_${it}_HOME"
-      val testJvmHome = project.providers.environmentVariable(testJvmEnv).orNull
-      if (testJvmHome == null) {
-        throw GradleException("Unable to find launcher for Java '$it'. Have you set '$testJvmEnv'?")
+  /**
+   * The home path of the test JVM.
+   *
+   * The `<testJvm>` string (`8`, `11`, `ZULU8`, `GRAALVM25`, etc.) is interpreted in that order:
+   * 1. Lookup for a valid path,
+   * 2. Look JVM via Gradle toolchains
+   *
+   * Holds the resolved JavaToolchainSpec for the test JVM.
+   */
+  private val testJvmSpec = normalizedTestJvm.map {
+    val (distribution, version) = Regex("([a-zA-Z]*)([0-9]+)").matchEntire(it)?.groupValues?.drop(1) ?: listOf("", "")
+
+    when {
+      Files.exists(Paths.get(it)) -> it.normalizeToJDKJavaHome().toToolchainSpec()
+
+      version.isNotBlank() -> {
+        // Best effort to make a spec for the passed testJvm
+        // `8`, `11`, `ZULU8`, `GRAALVM25`, etc.
+        // if it is an integer, we assume it's a Java version
+        // also we can handle on macOs oracle, zulu, semeru, graalvm prefixes
+
+        // This is using internal APIs
+        DefaultToolchainSpec(project.serviceOf<PropertyFactory>()).apply {
+          languageVersion.set(JavaLanguageVersion.of(version.toInt()))
+          when (distribution.lowercase()) {
+            "oracle" -> {
+              vendor.set(JvmVendorSpec.ORACLE)
+            }
+
+            "zulu" -> {
+              vendor.set(JvmVendorSpec.AZUL)
+            }
+
+            "semeru" -> {
+              vendor.set(JvmVendorSpec.IBM)
+              implementation.set(JvmImplementation.J9)
+            }
+
+            "graalvm" -> {
+              vendor.set(JvmVendorSpec.GRAAL_VM)
+              nativeImageCapable.set(true)
+            }
+          }
+        }
       }
 
-      testJvmHome.normalizeToJDKJavaHome()
+      else -> throw GradleException(
+        """
+        Unable to find launcher for Java '$it'. It needs to be:
+        1. A valid path to a JDK home, or
+        2. An environment variable named 'JAVA_<testJvm>_HOME' or '<testJvm>' pointing to a JDK home, or
+        3. A Java version or a known distribution+version combination (e.g. '11', 'zulu8', 'graalvm11', etc.) that can be resolved via Gradle toolchains.
+        4. If using Gradle toolchains, ensure that the requested JDK is installed and configured correctly.
+        """.trimIndent()
+      )
     }
   }.map { project.logger.info("testJvm home path: $it"); it }
 
-  val javaTestLauncher = project.providers.zip(testJvmHomePath, normalizedTestJvm) { testJvmHome, testJvm ->
-    // Only change test JVM if it's not the one we are running the gradle build with
-    if (currentJavaHomePath.get() == testJvmHome) {
-      project.providers.provider<JavaLauncher?> { null }
-    } else {
-      // This is using internal APIs
-      val jvmSpec = org.gradle.jvm.toolchain.internal.SpecificInstallationToolchainSpec(
-        project.serviceOf<org.gradle.api.internal.provider.PropertyFactory>(),
-        project.file(testJvmHome)
-      )
-
-      // The provider always says that a value is present so we need to wrap it for proper error messages
-      project.javaToolchains.launcherFor(jvmSpec).orElse(project.providers.provider {
-        throw GradleException("Unable to find launcher for Java $testJvm. Does '$testJvmHome' point to a JDK?")
-      })
-    }
-  }.flatMap { it }.map { project.logger.info("testJvm launcher: ${it.executablePath}"); it }
+  /**
+   * The Java launcher for the test JVM.
+   *
+   * Current JVM or a launcher specified via the testJvm.
+   */
+  val javaTestLauncher: Provider<JavaLauncher> =
+    project.providers.zip(testJvmSpec, normalizedTestJvm) { jvmSpec, testJvm ->
+      // Only change test JVM if it's not the one we are running the gradle build with
+      if ((jvmSpec as? SpecificInstallationToolchainSpec)?.javaHome == currentJavaHomePath.get()) {
+        project.providers.provider<JavaLauncher?> { null }
+      } else {
+        // The provider always says that a value is present so we need to wrap it for proper error messages
+        project.javaToolchains.launcherFor(jvmSpec).orElse(project.providers.provider {
+          throw GradleException("Unable to find launcher for Java '$testJvm'. Does $TEST_JVM point to a JDK?")
+        })
+      }
+    }.flatMap { it }.map { project.logger.info("testJvm launcher: ${it.executablePath}"); it }
 
   private fun String.normalizeToJDKJavaHome(): Path {
     val javaHome = project.file(this).toPath().toRealPath()
     return if (javaHome.endsWith("jre")) javaHome.parent else javaHome
   }
 
-  private val Project.javaToolchains: JavaToolchainService get() =
-    extensions.getByName("javaToolchains") as JavaToolchainService
+  private fun Path.toToolchainSpec(): JavaToolchainSpec =
+    // This is using internal APIs
+    SpecificInstallationToolchainSpec(project.serviceOf<PropertyFactory>(), project.file(this))
+
+  private val Project.javaToolchains: JavaToolchainService
+    get() =
+      extensions.getByName("javaToolchains") as JavaToolchainService
 }

buildSrc/src/test/groovy/CallSiteInstrumentationPluginTest.groovy

@@ -27,7 +27,7 @@ class CallSiteInstrumentationPluginTest extends Specification {
     }
     
     dependencies {
-      implementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.17.7'
+      implementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.18.1'
       implementation group: 'com.google.auto.service', name: 'auto-service-annotations', version: '1.1.1'
     }
   '''