Merge remote-tracking branch 'origin/master' into bdu/pr-gate

Author: bric3

.gitlab-ci.yml

@@ -20,6 +20,10 @@ stages:
   - generate-signing-key
 
 variables:
+  # Gitlab runner features; see https://docs.gitlab.com/runner/configuration/feature-flags.html
+  # Fold and time all script sections
+  FF_SCRIPT_SECTIONS: 1
+
   REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com
   BUILD_JOB_NAME: "build"
   DEPENDENCY_CACHE_POLICY: pull
@@ -133,8 +137,14 @@ default:
   before_script:
     - source .gitlab/gitlab-utils.sh
     - export GRADLE_USER_HOME=`pwd`/.gradle
+    - |
+      # Don't put jvm args here as it will be picked up by child gradle processes used in tests
+      cat << EOF > $GRADLE_USER_HOME/gradle.properties
+      mavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY
+      gradlePluginProxy=$GRADLE_PLUGIN_PROXY
+      EOF
     - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx$GRADLE_MEM -Xms$GRADLE_MEM -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
-    - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS -PmavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY -PgradlePluginProxy=$GRADLE_PLUGIN_PROXY"
+    - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
     - *normalize_node_index
     # for weird reasons, gradle will always "chmod 700" the .gradle folder
     # with Gitlab caching, .gradle is always owned by root and thus gradle's chmod invocation fails

.gitlab/one-pipeline.locked.yml

@@ -1,4 +1,4 @@
 # DO NOT EDIT THIS FILE MANUALLY
 # This file is auto-generated by automation.
 include:
-  - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/05e116339b9780a138a474d0348e97debfca97f27bbc4ca489cf4e4c90d9cc94/one-pipeline.yml
+  - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/f2050f53c1f5aed62a24e6b406c746e7d593230ce02b5d56d2a2296db763ebf4/one-pipeline.yml

benchmark/Dockerfile

@@ -1,14 +1,20 @@
 # Petclinic download and compilation stage
 FROM eclipse-temurin:17-jammy as petclinic
 
+ARG SPRING_PETCLINIC_COMMIT=cefaf55dd124d0635abfe857c3c99a3d3ea62017
+
 RUN apt-get update \
   && apt-get -y install git  \
   && apt-get -y clean \
   && rm -rf /var/lib/apt/lists/*
 
-RUN git clone --depth 1 --branch main --single-branch https://github.com/spring-projects/spring-petclinic.git \
-  && cd spring-petclinic \
-  && ./mvnw dependency:go-offline
+RUN set -eux;\
+  git init spring-petclinic;\
+  cd spring-petclinic;\
+  git remote add origin https://github.com/spring-projects/spring-petclinic.git;\
+  git fetch --depth 1 origin ${SPRING_PETCLINIC_COMMIT};\
+  git checkout ${SPRING_PETCLINIC_COMMIT};\
+  ./mvnw dependency:go-offline
 
 RUN cd spring-petclinic \
   && ./mvnw package -Dmaven.test.skip=true \

dd-java-agent/src/main/java/datadog/trace/bootstrap/AgentBootstrap.java

@@ -46,6 +46,7 @@
 public final class AgentBootstrap {
   static final String LIB_INJECTION_ENABLED_ENV_VAR = "DD_INJECTION_ENABLED";
   static final String LIB_INJECTION_FORCE_SYS_PROP = "dd.inject.force";
+  static final String LIB_INSTRUMENTATION_SOURCE_SYS_PROP = "dd.instrumentation.source";
 
   private static final Class<?> thisClass = AgentBootstrap.class;
   private static final int MAX_EXCEPTION_CHAIN_LENGTH = 99;
@@ -134,6 +135,12 @@ private static void agentmainImpl(
       return;
     }
 
+    if (getConfig(LIB_INJECTION_ENABLED_ENV_VAR)) {
+      recordInstrumentationSource("ssi");
+    } else {
+      recordInstrumentationSource("cmd_line");
+    }
+
     final URL agentJarURL = installAgentJar(inst);
     final Class<?> agentClass;
     try {
@@ -164,6 +171,10 @@ static boolean getConfig(String configName) {
     }
   }
 
+  private static void recordInstrumentationSource(String source) {
+    SystemUtils.trySetProperty(LIB_INSTRUMENTATION_SOURCE_SYS_PROP, source);
+  }
+
   static boolean exceptionCauseChainContains(Throwable ex, String exClassName) {
     Set<Throwable> stack = Collections.newSetFromMap(new IdentityHashMap<>());
     Throwable t = ex;

dd-java-agent/src/main/java/datadog/trace/bootstrap/SystemUtils.java

@@ -23,6 +23,14 @@ public static String tryGetProperty(String property) {
     }
   }
 
+  public static String trySetProperty(String property, String value) {
+    try {
+      return System.setProperty(property, value);
+    } catch (SecurityException e) {
+      return null;
+    }
+  }
+
   public static String getPropertyOrDefault(String property, String defaultValue) {
     try {
       return System.getProperty(property, defaultValue);

dd-java-agent/src/test/groovy/datadog/trace/agent/InstrumenterUnloadTest.groovy

@@ -26,16 +26,26 @@ class InstrumenterUnloadTest extends Specification {
       , ["DD_API_KEY": API_KEY]
       , new PrintStream(testOutput))
 
-    int unloadCount = 0
+    boolean canaryUnloaded = false
+    int unloadedInstrumentationCount = 0
     new ByteArrayInputStream((testOutput.toByteArray())).eachLine {
       System.out.println(it)
+      if (it =~ /(?i)unload.*Canary/) {
+        canaryUnloaded = true
+      }
       if (it =~ /(?i)unload.* datadog.trace.instrumentation./) {
-        unloadCount++
+        unloadedInstrumentationCount++
       }
     }
 
+    if (!canaryUnloaded) {
+      System.out.println("WARNING: Canary class was not unloaded!")
+    }
+
     then:
     returnCode == 0
-    unloadCount > 0
+    // skip check if we couldn't even unload our Canary class, as that
+    // indicates full GC didn't happen enough to trigger any unloading
+    !canaryUnloaded || unloadedInstrumentationCount > 0
   }
 }

dd-java-agent/src/test/java/jvmbootstraptest/UnloadingChecker.java

@@ -1,13 +1,32 @@
 package jvmbootstraptest;
 
+import static java.util.concurrent.TimeUnit.MINUTES;
+
 import datadog.trace.test.util.GCUtils;
+import java.lang.management.ClassLoadingMXBean;
+import java.lang.management.ManagementFactory;
 
 public class UnloadingChecker {
-  public static void main(final String[] args) {
-    try {
-      GCUtils.awaitGC();
-    } catch (InterruptedException e) {
-      e.printStackTrace();
+  static class Canary {}
+
+  public static void main(final String[] args) throws Exception {
+    ClassLoadingMXBean classLoadingMXBean = ManagementFactory.getClassLoadingMXBean();
+    long initialUnloadCount = classLoadingMXBean.getUnloadedClassCount();
+
+    // load an isolated class which we know can be unloaded after a full GC
+    new IsolatingClassLoader().loadClass("jvmbootstraptest.UnloadingChecker$Canary");
+
+    long waitNanos = MINUTES.toNanos(2);
+    long startNanos = System.nanoTime();
+
+    while (System.nanoTime() - startNanos < waitNanos) {
+      try {
+        GCUtils.awaitGC();
+      } catch (Throwable ignore) {
+      }
+      if (initialUnloadCount < classLoadingMXBean.getUnloadedClassCount()) {
+        break; // some class unloading has taken place, stop and check results
+      }
     }
   }
 }

dd-smoke-tests/gradle/build.gradle

@@ -18,6 +18,11 @@ test {
     events "passed", "skipped", "failed", "standardOut", "standardError"
   }
 
+  if (project.hasProperty("mavenRepositoryProxy")) {
+    // propagate proxy URL to tests, to then propagate it to nested Gradle builds
+    environment "MAVEN_REPOSITORY_PROXY", project.property("mavenRepositoryProxy")
+  }
+
   // overriding the default timeout of 9 minutes set in configure_tests.gradle,
   // as Gradle smoke tests might run for a longer duration
   timeout = Duration.of(15, ChronoUnit.MINUTES)

dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy

@@ -98,7 +98,7 @@ class GradleDaemonSmokeTest extends AbstractGradleTest {
 
     where:
     gradleVersion         | projectName                           | flakyTests | expectedOrder | eventsNumber
-    "5.1"                 | "test-succeed-junit-4-class-ordering" | [
+    "7.6.4"               | "test-succeed-junit-4-class-ordering" | [
       test("datadog.smoke.TestSucceedB", "test_succeed"),
       test("datadog.smoke.TestSucceedB", "test_succeed_another"),
       test("datadog.smoke.TestSucceedA", "test_succeed")
@@ -239,12 +239,19 @@ class GradleDaemonSmokeTest extends AbstractGradleTest {
   }
 
   private runGradle(String gradleVersion, List<String> arguments, boolean successExpected) {
+    def buildEnv = ["GRADLE_VERSION": gradleVersion]
+
+    def mavenRepositoryProxy = System.getenv("MAVEN_REPOSITORY_PROXY")
+    if (mavenRepositoryProxy != null) {
+      buildEnv += ["MAVEN_REPOSITORY_PROXY": System.getenv("MAVEN_REPOSITORY_PROXY")]
+    }
+
     GradleRunner gradleRunner = GradleRunner.create()
       .withTestKitDir(testKitFolder.toFile())
       .withProjectDir(projectFolder.toFile())
       .withGradleVersion(gradleVersion)
       .withArguments(arguments)
-      .withEnvironment(["GRADLE_VERSION": gradleVersion])
+      .withEnvironment(buildEnv)
       .forwardOutput()
 
     println "${new Date()}: $specificationContext.currentIteration.displayName - Starting Gradle run"

dd-smoke-tests/gradle/src/test/resources/test-corrupted-config-legacy-instrumentation/build.gradleTest

@@ -2,6 +2,16 @@ apply plugin: 'java'
 
 repositories {
   mavenLocal()
+
+  def proxyUrl = System.getenv("MAVEN_REPOSITORY_PROXY")
+  if (proxyUrl) {
+    println "Using proxy repository: $proxyUrl"
+    maven {
+      url = proxyUrl
+      allowInsecureProtocol = true
+    }
+  }
+
   mavenCentral()
 }