Merge branch 'master' into dougqh/guardrails-telemetry

Author: dougqh

.circleci/collect_results.sh

@@ -10,8 +10,17 @@ shopt -s globstar
 TEST_RESULTS_DIR=./results
 mkdir -p $TEST_RESULTS_DIR >/dev/null 2>&1
 
-echo "saving test results"
 mkdir -p $TEST_RESULTS_DIR
-find workspace/**/build/test-results -name \*.xml -exec sh -c '
+
+mkdir -p workspace
+mapfile -t test_result_dirs < <(find workspace -name test-results -type d)
+
+if [[ ${#test_result_dirs[@]} -eq 0 ]]; then
+  echo "No test results found"
+  exit 0
+fi
+
+echo "saving test results"
+find "${test_result_dirs[@]}" -name \*.xml -exec sh -c '
   file=$(echo "$0" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_")
   cp "$0" "$1/$file"' {} $TEST_RESULTS_DIR \;

.circleci/config.continue.yml.j2

@@ -32,7 +32,7 @@ system_test_matrix: &system_test_matrix
 
 agent_integration_tests_modules: &agent_integration_tests_modules "dd-trace-core|communication|internal-api|utils"
 core_modules: &core_modules "dd-java-agent|dd-trace-core|communication|internal-api|telemetry|utils|dd-java-agent/agent-bootstrap|dd-java-agent/agent-installer|dd-java-agent/agent-tooling|dd-java-agent/agent-builder|dd-java-agent/appsec|dd-java-agent/agent-crashtracking|dd-trace-api|dd-trace-ot"
-instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication"
+instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-iast|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication"
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
@@ -99,6 +99,11 @@ commands:
   setup_code:
     steps:
       - checkout
+{% if use_git_changes %}
+      - run:
+          name: Fetch base branch
+          command: git fetch origin {{ pr_base_ref }}
+{% endif %}
       - run:
           name: Checkout merge commit
           command: .circleci/checkout_merge_commit.sh
@@ -312,6 +317,9 @@ jobs:
             ./gradlew clean
             << parameters.gradleTarget >>
             -PskipTests
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             << pipeline.parameters.gradle_flags >>
             --max-workers=8
             --rerun-tasks
@@ -411,6 +419,9 @@ jobs:
             ./gradlew
             << parameters.gradleTarget >>
             -PskipTests
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             -PrunBuildSrcTests
             -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}
             << pipeline.parameters.gradle_flags >>
@@ -556,6 +567,9 @@ jobs:
             ./gradlew
             << parameters.gradleTarget >>
             << parameters.gradleParameters >>
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}
             <<# parameters.testJvm >>-PtestJvm=<< parameters.testJvm >><</ parameters.testJvm >>
             << pipeline.parameters.gradle_flags >>
@@ -1057,7 +1071,7 @@ build_test_jobs: &build_test_jobs
       name: z_test_<< matrix.testJvm >>_base
       triggeredBy: *core_modules
       gradleTarget: ":baseTest"
-      gradleParameters: "-PskipFlakyTests -PskipInstTests -PskipSmokeTests -PskipProfilingTests -PskipDebuggerTests"
+      gradleParameters: "-PskipFlakyTests"
       stage: core
       cacheType: base
       parallelism: 4
@@ -1070,8 +1084,8 @@ build_test_jobs: &build_test_jobs
         - ok_to_test
       name: z_test_8_base
       triggeredBy: *core_modules
-      gradleTarget: :baseTest jacocoTestReport jacocoTestCoverageVerification
-      gradleParameters: "-PskipFlakyTests -PskipInstTests -PskipSmokeTests -PskipProfilingTests -PskipDebuggerTests"
+      gradleTarget: :baseTest
+      gradleParameters: "-PskipFlakyTests -PcheckCoverage"
       stage: core
       cacheType: base
       parallelism: 4

.circleci/render_config.py

@@ -28,7 +28,7 @@
 }
 # Version to use for all the base Docker images, see
 # https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build
-DOCKER_IMAGE_VERSION="v24.04"
+DOCKER_IMAGE_VERSION="v24.08"
 
 # Get labels from pull requests to override some defaults for jobs to run.
 # `run-tests: all` will run all tests.
@@ -52,7 +52,7 @@
             )
             resp.raise_for_status()
         except Exception as e:
-            print(f"Request filed: {e}")
+            print(f"Request failed: {e}")
             time.sleep(1)
             continue
         data = resp.json()
@@ -63,12 +63,18 @@
     labels = {
         l.replace("run-tests: ", "") for l in labels if l.startswith("run-tests: ")
     }
+    # get the base reference (e.g. `master`), commit hash is also available at the `sha` field.
+    pr_base_ref = data.get("base", {}).get("ref")
 else:
     labels = set()
+    pr_base_ref = ""
 
 
 branch = os.environ.get("CIRCLE_BRANCH", "")
-if branch == "master" or branch.startswith("release/v") or "all" in labels:
+run_all = "all" in labels
+is_master_or_release = branch == "master" or branch.startswith("release/v")
+
+if is_master_or_release or run_all:
     all_jdks = ALWAYS_ON_JDKS | MASTER_ONLY_JDKS
 else:
     all_jdks = ALWAYS_ON_JDKS | (MASTER_ONLY_JDKS & labels)
@@ -83,6 +89,9 @@
 is_weekly = os.environ.get("CIRCLE_IS_WEEKLY", "false") == "true"
 is_regular = not is_nightly and not is_weekly
 
+# Use git changes detection on PRs
+use_git_changes = not run_all and not is_master_or_release and is_regular
+
 vars = {
     "is_nightly": is_nightly,
     "is_weekly": is_weekly,
@@ -92,12 +101,14 @@
     "nocov_jdks": nocov_jdks,
     "flaky": branch == "master" or "flaky" in labels or "all" in labels,
     "docker_image_prefix": "" if is_nightly else f"{DOCKER_IMAGE_VERSION}-",
+    "use_git_changes": use_git_changes,
+    "pr_base_ref": pr_base_ref,
 }
 
 print(f"Variables for this build: {vars}")
 
 loader = jinja2.FileSystemLoader(searchpath=SCRIPT_DIR)
-env = jinja2.Environment(loader=loader)
+env = jinja2.Environment(loader=loader, trim_blocks=True)
 tpl = env.get_template(TPL_FILENAME)
 out = tpl.render(**vars)
 

.circleci/upload_ciapp.sh

@@ -15,6 +15,7 @@ java_prop () {
 
 # based on tracer implementation: https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/TestDecorator.java#L55-L77
 datadog-ci junit upload --service $SERVICE_NAME \
+    --logs \
     --tags "test.traits:{\"marker\":[\"$1\"]}" \
     --tags "runtime.name:$(java_prop java.runtime.name)" \
     --tags "runtime.vendor:$(java_prop java.vendor)" \

.github/workflows/README.md

@@ -98,7 +98,7 @@ _Action:_ Run GitHub CodeQL action, upload result to GitHub security tab and Dat
 
 _Trigger:_ Every week or manually.
 
-_Action:_ Update the Grade dependencies and their locking files.
+_Action:_ Create a PR updating the Grade dependencies and their locking files.
 
 _Recovery:_ Manually trigger the action again.
 

.github/workflows/add-milestone-to-pull-requests.yaml

@@ -57,13 +57,14 @@ jobs:
                   return a.version.patch - b.version.patch
                 })
                 .pop()?.number
-            } else if (base.startsWith('release/v')) {
-              // Pick the milestone with the same title as the base branch
-              const version = base.substring(9)
+            } else if (base.startsWith('release/v') && base.endsWith('.x')) {
+              // Extract the minor version related to the base branch (e.g. "release/v1.2.x" -> "1.2.")
+              const version = base.substring(9, base.length - 1)
+              // Pick the milestone with title matching the extracted version
               const versionMilestone = response.data
-                .find(milestone => milestone.title == version)
+                .find(milestone => milestone.title.startsWith(version))
               if (!versionMilestone) {
-                core.setFailed(`Version milestone not found: ${version}`)
+                core.setFailed(`Milestone not found for minor version: ${version}`)
               } else {
                 milestoneNumber = versionMilestone.number
               }

.github/workflows/add-release-to-cloudfoundry.yaml

@@ -26,7 +26,7 @@ jobs:
           MAX_TRIES=60 # Wait up to 30 minutes
           DELAY=30
           while [ $TRY -lt $MAX_TRIES ]; do
-            if curl -s -I ${{ steps.get-release-url.outputs.URL }} | grep -q "200 OK"; then
+            if curl -s -I ${{ steps.get-release-url.outputs.URL }} | grep -q "^HTTP/.* 200"; then
               break
             fi
             echo "Waiting for the release to be available..."

.github/workflows/update-gradle-dependencies.yaml

@@ -70,7 +70,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: |
           gh pr create --title "Update Gradle dependencies" \
-            --body "This PR updates the Gradle dependencies.  \n:warning: Don't forget to squash commits before merging :warning:" \
+            --body "This PR updates the Gradle dependencies. ⚠️ Don't forget to squash commits before merging. ⚠️" \
             --base master \
             --head $BRANCH_NAME \
             --label "tag: dependencies" \

build.gradle

@@ -151,56 +151,4 @@ allprojects {
   }
 }
 
-allprojects { project ->
-  project.ext {
-    activePartition = true
-  }
-  final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition")
-  if (shouldUseTaskPartitions) {
-    final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int
-    final int taskPartition = project.rootProject.property("taskPartition") as int
-    final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount)
-    project.setProperty("activePartition", currentTaskPartition == taskPartition)
-  }
-}
-
-
-def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean coverage = false) {
-  def createRootTask = { rootTaskName, subProjTaskName ->
-    tasks.register(rootTaskName) { aggTest ->
-      subprojects { subproject ->
-        if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) {
-          def testTask = subproject.tasks.findByName(subProjTaskName)
-          if (testTask != null) {
-            aggTest.dependsOn(testTask)
-          }
-          if (coverage) {
-            def coverageTask = subproject.tasks.findByName("jacocoTestReport")
-            if (coverageTask != null) {
-              aggTest.dependsOn(coverageTask)
-            }
-            coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification")
-            if (coverageTask != null) {
-              aggTest.dependsOn(coverageTask)
-            }
-          }
-        }
-      }
-    }
-  }
-
-  createRootTask "${baseTaskName}Test", 'allTests'
-  createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests'
-  createRootTask "${baseTaskName}Check", 'check'
-}
-
-testAggregate("smoke", [":dd-smoke-tests"], [])
-testAggregate("instrumentation", [":dd-java-agent:instrumentation"], [])
-testAggregate("profiling", [":dd-java-agent:agent-profiling"], [])
-testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true)
-testAggregate("base", [":"], [
-  ":dd-java-agent:instrumentation",
-  ":dd-smoke-tests",
-  ":dd-java-agent:agent-profiling",
-  ":dd-java-agent:agent-debugger"
-])
+apply from: "$rootDir/gradle/ci_jobs.gradle"

buildSrc/call-site-instrumentation-plugin/src/main/java/datadog/trace/plugin/csi/impl/AdviceGeneratorImpl.java

@@ -55,6 +55,7 @@
 import datadog.trace.plugin.csi.util.ErrorCode;
 import datadog.trace.plugin.csi.util.MethodType;
 import java.io.File;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -156,9 +157,11 @@ private static ClassOrInterfaceDeclaration callSitesType(
     type.setModifier(PUBLIC, true);
     type.setName(getClassName(advice));
     type.addImplementedType(CALL_SITES_CLASS);
-    if (!CALL_SITES_FQCN.equals(callSite.getSpi().getClassName())) {
-      javaClass.addImport(callSite.getSpi().getClassName());
-      type.addImplementedType(getClassName(callSite.getSpi(), false));
+    for (final Type spi : callSite.getSpi()) {
+      if (!CALL_SITES_FQCN.equals(spi.getClassName())) {
+        javaClass.addImport(spi.getClassName());
+        type.addImplementedType(getClassName(spi, false));
+      }
     }
     javaClass.addType(type);
     return type;
@@ -168,9 +171,12 @@ private static void addAutoServiceAnnotation(
       final ClassOrInterfaceDeclaration javaClass, final CallSiteSpecification callSite) {
     final NormalAnnotationExpr autoService = new NormalAnnotationExpr();
     autoService.setName(AUTO_SERVICE_FQDN);
-    autoService.addPair(
-        "value",
-        new ClassExpr(new ClassOrInterfaceType().setName(getClassName(callSite.getSpi(), false))));
+    final Type[] spiTypes = callSite.getSpi();
+    final List<Expression> spiExprs = new ArrayList<>(spiTypes.length);
+    for (final Type spi : spiTypes) {
+      spiExprs.add(new ClassExpr(new ClassOrInterfaceType().setName(getClassName(spi, false))));
+    }
+    autoService.addPair("value", new ArrayInitializerExpr().setValues(new NodeList<>(spiExprs)));
     javaClass.addAnnotation(autoService);
   }