Merge branch 'master' into kr-igor/kafka-lag-spark-streaming

Author: kr-igor

.circleci/collect_results.sh

@@ -10,8 +10,17 @@ shopt -s globstar
 TEST_RESULTS_DIR=./results
 mkdir -p $TEST_RESULTS_DIR >/dev/null 2>&1
 
-echo "saving test results"
 mkdir -p $TEST_RESULTS_DIR
-find workspace/**/build/test-results -name \*.xml -exec sh -c '
+
+mkdir -p workspace
+mapfile -t test_result_dirs < <(find workspace -name test-results -type d)
+
+if [[ ${#test_result_dirs[@]} -eq 0 ]]; then
+  echo "No test results found"
+  exit 0
+fi
+
+echo "saving test results"
+find "${test_result_dirs[@]}" -name \*.xml -exec sh -c '
   file=$(echo "$0" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_")
   cp "$0" "$1/$file"' {} $TEST_RESULTS_DIR \;

.circleci/config.continue.yml.j2

@@ -32,11 +32,11 @@ system_test_matrix: &system_test_matrix
 
 agent_integration_tests_modules: &agent_integration_tests_modules "dd-trace-core|communication|internal-api|utils"
 core_modules: &core_modules "dd-java-agent|dd-trace-core|communication|internal-api|telemetry|utils|dd-java-agent/agent-bootstrap|dd-java-agent/agent-installer|dd-java-agent/agent-tooling|dd-java-agent/agent-builder|dd-java-agent/appsec|dd-java-agent/agent-crashtracking|dd-trace-api|dd-trace-ot"
-instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication"
+instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-iast|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication"
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 06cbfad5f9facbc6e258d9c8208c466cd46efeea
+default_system_tests_commit: &default_system_tests_commit 20b61e9be1e8c922804f8e5ddc80ba4a1f4cdd5b
 
 parameters:
   nightly:
@@ -99,6 +99,11 @@ commands:
   setup_code:
     steps:
       - checkout
+{% if use_git_changes %}
+      - run:
+          name: Fetch base branch
+          command: git fetch origin {{ pr_base_ref }}
+{% endif %}
       - run:
           name: Checkout merge commit
           command: .circleci/checkout_merge_commit.sh
@@ -312,6 +317,9 @@ jobs:
             ./gradlew clean
             << parameters.gradleTarget >>
             -PskipTests
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             << pipeline.parameters.gradle_flags >>
             --max-workers=8
             --rerun-tasks
@@ -411,6 +419,9 @@ jobs:
             ./gradlew
             << parameters.gradleTarget >>
             -PskipTests
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             -PrunBuildSrcTests
             -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}
             << pipeline.parameters.gradle_flags >>
@@ -556,6 +567,9 @@ jobs:
             ./gradlew
             << parameters.gradleTarget >>
             << parameters.gradleParameters >>
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}
             <<# parameters.testJvm >>-PtestJvm=<< parameters.testJvm >><</ parameters.testJvm >>
             << pipeline.parameters.gradle_flags >>
@@ -916,7 +930,7 @@ jobs:
   parametric-tests:
     machine:
       image: ubuntu-2404:current
-    resource_class: large
+    resource_class: xlarge
     steps:
       - setup_system_tests
 
@@ -933,6 +947,7 @@ jobs:
           name: Build runner
           command: |
             cd system-tests
+            export TEST_LIBRARY=java
             ./build.sh -i runner
 
       - run:
@@ -941,12 +956,12 @@ jobs:
             set -e
             cd system-tests
             export TEST_LIBRARY=java
-            export PYTEST_WORKER_COUNT=8
+            export PYTEST_XDIST_AUTO_NUM_WORKERS=6
             set +e
             RUN_ATTEMPTS=1
             while [ $RUN_ATTEMPTS -le 3 ]; do
               echo "Running parametric test attempt $RUN_ATTEMPTS"
-              timeout 780s ./run.sh PARAMETRIC --log-cli-level=DEBUG --durations=30 -vv
+              timeout 12m ./run.sh PARAMETRIC --log-cli-level=DEBUG --durations=30 -vv
               status=$?
               #timneout returns 124 if it times out
               #if the return code is not 124, then we exit with the status

.circleci/render_config.py

@@ -28,7 +28,7 @@
 }
 # Version to use for all the base Docker images, see
 # https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build
-DOCKER_IMAGE_VERSION="v24.04"
+DOCKER_IMAGE_VERSION="v24.08"
 
 # Get labels from pull requests to override some defaults for jobs to run.
 # `run-tests: all` will run all tests.
@@ -52,7 +52,7 @@
             )
             resp.raise_for_status()
         except Exception as e:
-            print(f"Request filed: {e}")
+            print(f"Request failed: {e}")
             time.sleep(1)
             continue
         data = resp.json()
@@ -63,12 +63,18 @@
     labels = {
         l.replace("run-tests: ", "") for l in labels if l.startswith("run-tests: ")
     }
+    # get the base reference (e.g. `master`), commit hash is also available at the `sha` field.
+    pr_base_ref = data.get("base", {}).get("ref")
 else:
     labels = set()
+    pr_base_ref = ""
 
 
 branch = os.environ.get("CIRCLE_BRANCH", "")
-if branch == "master" or branch.startswith("release/v") or "all" in labels:
+run_all = "all" in labels
+is_master_or_release = branch == "master" or branch.startswith("release/v")
+
+if is_master_or_release or run_all:
     all_jdks = ALWAYS_ON_JDKS | MASTER_ONLY_JDKS
 else:
     all_jdks = ALWAYS_ON_JDKS | (MASTER_ONLY_JDKS & labels)
@@ -83,6 +89,9 @@
 is_weekly = os.environ.get("CIRCLE_IS_WEEKLY", "false") == "true"
 is_regular = not is_nightly and not is_weekly
 
+# Use git changes detection on PRs
+use_git_changes = not run_all and not is_master_or_release and is_regular
+
 vars = {
     "is_nightly": is_nightly,
     "is_weekly": is_weekly,
@@ -92,12 +101,14 @@
     "nocov_jdks": nocov_jdks,
     "flaky": branch == "master" or "flaky" in labels or "all" in labels,
     "docker_image_prefix": "" if is_nightly else f"{DOCKER_IMAGE_VERSION}-",
+    "use_git_changes": use_git_changes,
+    "pr_base_ref": pr_base_ref,
 }
 
 print(f"Variables for this build: {vars}")
 
 loader = jinja2.FileSystemLoader(searchpath=SCRIPT_DIR)
-env = jinja2.Environment(loader=loader)
+env = jinja2.Environment(loader=loader, trim_blocks=True)
 tpl = env.get_template(TPL_FILENAME)
 out = tpl.render(**vars)
 

.circleci/upload_ciapp.sh

@@ -15,6 +15,7 @@ java_prop () {
 
 # based on tracer implementation: https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/TestDecorator.java#L55-L77
 datadog-ci junit upload --service $SERVICE_NAME \
+    --logs \
     --tags "test.traits:{\"marker\":[\"$1\"]}" \
     --tags "runtime.name:$(java_prop java.runtime.name)" \
     --tags "runtime.vendor:$(java_prop java.vendor)" \

.github/CODEOWNERS

@@ -43,6 +43,7 @@ dd-java-agent/agent-iast/              @DataDog/asm-java
 dd-java-agent/instrumentation/*iast*   @DataDog/asm-java
 dd-java-agent/instrumentation/*appsec* @DataDog/asm-java
 dd-java-agent/instrumentation/json/    @DataDog/asm-java
+dd-java-agent/instrumentation/snakeyaml/    @DataDog/asm-java
 dd-smoke-tests/iast-util/              @DataDog/asm-java
 dd-smoke-tests/spring-security/        @DataDog/asm-java
 dd-java-agent/instrumentation/commons-fileupload/    @DataDog/asm-java
@@ -54,4 +55,4 @@ dd-java-agent/instrumentation/spring-security-5/ @DataDog/asm-java
 
 # @DataDog/data-jobs-monitoring
 dd-java-agent/instrumentation/spark/            @DataDog/data-jobs-monitoring
-dd-java-agent/instrumentation/spark-executor/   @DataDog/data-jobs-monitoring
+dd-java-agent/instrumentation/spark-executor/   @DataDog/data-jobs-monitoring

.github/workflows/add-release-to-cloudfoundry.yaml

@@ -26,7 +26,7 @@ jobs:
           MAX_TRIES=60 # Wait up to 30 minutes
           DELAY=30
           while [ $TRY -lt $MAX_TRIES ]; do
-            if curl -s -I ${{ steps.get-release-url.outputs.URL }} | grep -q "200 OK"; then
+            if curl -s -I ${{ steps.get-release-url.outputs.URL }} | grep -q "^HTTP/.* 200"; then
               break
             fi
             echo "Waiting for the release to be available..."

build.gradle

@@ -151,57 +151,4 @@ allprojects {
   }
 }
 
-allprojects { project ->
-  project.ext {
-    activePartition = true
-  }
-  final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition")
-  if (shouldUseTaskPartitions) {
-    final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int
-    final int taskPartition = project.rootProject.property("taskPartition") as int
-    final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount)
-    project.setProperty("activePartition", currentTaskPartition == taskPartition)
-  }
-}
-
-
-def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) {
-  def createRootTask = { rootTaskName, subProjTaskName ->
-    def coverage = forceCoverage || rootProject.hasProperty("checkCoverage")
-    tasks.register(rootTaskName) { aggTest ->
-      subprojects { subproject ->
-        if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) {
-          def testTask = subproject.tasks.findByName(subProjTaskName)
-          if (testTask != null) {
-            aggTest.dependsOn(testTask)
-          }
-          if (coverage) {
-            def coverageTask = subproject.tasks.findByName("jacocoTestReport")
-            if (coverageTask != null) {
-              aggTest.dependsOn(coverageTask)
-            }
-            coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification")
-            if (coverageTask != null) {
-              aggTest.dependsOn(coverageTask)
-            }
-          }
-        }
-      }
-    }
-  }
-
-  createRootTask "${baseTaskName}Test", 'allTests'
-  createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests'
-  createRootTask "${baseTaskName}Check", 'check'
-}
-
-testAggregate("smoke", [":dd-smoke-tests"], [])
-testAggregate("instrumentation", [":dd-java-agent:instrumentation"], [])
-testAggregate("profiling", [":dd-java-agent:agent-profiling"], [])
-testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true)
-testAggregate("base", [":"], [
-  ":dd-java-agent:instrumentation",
-  ":dd-smoke-tests",
-  ":dd-java-agent:agent-profiling",
-  ":dd-java-agent:agent-debugger"
-])
+apply from: "$rootDir/gradle/ci_jobs.gradle"

buildSrc/call-site-instrumentation-plugin/src/main/java/datadog/trace/plugin/csi/impl/AdviceGeneratorImpl.java

@@ -55,6 +55,7 @@
 import datadog.trace.plugin.csi.util.ErrorCode;
 import datadog.trace.plugin.csi.util.MethodType;
 import java.io.File;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -156,9 +157,11 @@ private static ClassOrInterfaceDeclaration callSitesType(
     type.setModifier(PUBLIC, true);
     type.setName(getClassName(advice));
     type.addImplementedType(CALL_SITES_CLASS);
-    if (!CALL_SITES_FQCN.equals(callSite.getSpi().getClassName())) {
-      javaClass.addImport(callSite.getSpi().getClassName());
-      type.addImplementedType(getClassName(callSite.getSpi(), false));
+    for (final Type spi : callSite.getSpi()) {
+      if (!CALL_SITES_FQCN.equals(spi.getClassName())) {
+        javaClass.addImport(spi.getClassName());
+        type.addImplementedType(getClassName(spi, false));
+      }
     }
     javaClass.addType(type);
     return type;
@@ -168,9 +171,12 @@ private static void addAutoServiceAnnotation(
       final ClassOrInterfaceDeclaration javaClass, final CallSiteSpecification callSite) {
     final NormalAnnotationExpr autoService = new NormalAnnotationExpr();
     autoService.setName(AUTO_SERVICE_FQDN);
-    autoService.addPair(
-        "value",
-        new ClassExpr(new ClassOrInterfaceType().setName(getClassName(callSite.getSpi(), false))));
+    final Type[] spiTypes = callSite.getSpi();
+    final List<Expression> spiExprs = new ArrayList<>(spiTypes.length);
+    for (final Type spi : spiTypes) {
+      spiExprs.add(new ClassExpr(new ClassOrInterfaceType().setName(getClassName(spi, false))));
+    }
+    autoService.addPair("value", new ArrayInitializerExpr().setValues(new NodeList<>(spiExprs)));
     javaClass.addAnnotation(autoService);
   }
 

buildSrc/call-site-instrumentation-plugin/src/main/java/datadog/trace/plugin/csi/impl/AsmSpecificationBuilder.java

@@ -76,7 +76,7 @@ private static class SpecificationVisitor extends ClassVisitor {
     private boolean isCallSite;
     private final List<AdviceSpecification> advices = new ArrayList<>();
     private final Set<Type> helpers = new HashSet<>();
-    private Type spi;
+    private final Set<Type> spi = new HashSet<>();
     private List<String> enabled = new ArrayList<>();
     private CallSiteSpecification result;
 
@@ -101,16 +101,17 @@ public AnnotationVisitor visitAnnotation(final String descriptor, final boolean
       if (isCallSite) {
         helpers.add(clazz);
         return new AnnotationVisitor(ASM_API_VERSION) {
-          @Override
-          public void visit(final String key, final Object value) {
-            if ("spi".equals(key)) {
-              spi = (Type) value;
-            }
-          }
 
           @Override
           public AnnotationVisitor visitArray(final String name) {
-            if ("helpers".equals(name)) {
+            if ("spi".equals(name)) {
+              return new AnnotationVisitor(ASM_API_VERSION) {
+                @Override
+                public void visit(final String name, final Object value) {
+                  spi.add((Type) value);
+                }
+              };
+            } else if ("helpers".equals(name)) {
               return new AnnotationVisitor(ASM_API_VERSION) {
                 @Override
                 public void visit(final String name, final Object value) {

buildSrc/call-site-instrumentation-plugin/src/main/java/datadog/trace/plugin/csi/impl/CallSiteSpecification.java

@@ -32,19 +32,19 @@ public class CallSiteSpecification implements Validatable {
 
   private final Type clazz;
   private final List<AdviceSpecification> advices;
-  private final Type spi;
+  private final Type[] spi;
   private final Enabled enabled;
   private final Type[] helpers;
 
   public CallSiteSpecification(
       @Nonnull final Type clazz,
       @Nonnull final List<AdviceSpecification> advices,
-      @Nonnull final Type spi,
+      @Nonnull final Set<Type> spi,
       @Nonnull final List<String> enabled,
       @Nonnull final Set<Type> helpers) {
     this.clazz = clazz;
     this.advices = advices;
-    this.spi = spi;
+    this.spi = spi.toArray(new Type[0]);
     this.enabled = enabled.isEmpty() ? null : new Enabled(enabled);
     this.helpers = helpers.toArray(new Type[0]);
   }
@@ -53,12 +53,14 @@ public CallSiteSpecification(
   public void validate(@Nonnull final ValidationContext context) {
     final TypeResolver typeResolver = context.getContextProperty(TYPE_RESOLVER);
     try {
-      Class<?> spiClass = typeResolver.resolveType(spi);
-      if (!spiClass.isInterface()) {
-        context.addError(ErrorCode.CALL_SITE_SPI_SHOULD_BE_AN_INTERFACE, spiClass);
-      } else {
-        if (spiClass.getDeclaredMethods().length > 0) {
-          context.addError(ErrorCode.CALL_SITE_SPI_SHOULD_BE_EMPTY, spiClass);
+      for (Type spiType : spi) {
+        Class<?> spiClass = typeResolver.resolveType(spiType);
+        if (!spiClass.isInterface()) {
+          context.addError(ErrorCode.CALL_SITE_SPI_SHOULD_BE_AN_INTERFACE, spiClass);
+        } else {
+          if (spiClass.getDeclaredMethods().length > 0) {
+            context.addError(ErrorCode.CALL_SITE_SPI_SHOULD_BE_EMPTY, spiClass);
+          }
         }
       }
     } catch (ResolutionException e) {
@@ -84,7 +86,7 @@ public Type getClazz() {
     return clazz;
   }
 
-  public Type getSpi() {
+  public Type[] getSpi() {
     return spi;
   }