Merge branch 'master' into andrea.marziali/remove-jctools-queues

Author: amarziali

.github/CODEOWNERS

@@ -51,9 +51,9 @@
 /dd-java-agent/appsec/appsec-test-fixtures/                     @DataDog/asm-java
 /dd-java-agent/instrumentation/*iast*                           @DataDog/asm-java
 /dd-java-agent/instrumentation/*appsec*                         @DataDog/asm-java
-/dd-java-agent/instrumentation/json/                            @DataDog/asm-java
-/dd-java-agent/instrumentation/snakeyaml/                       @DataDog/asm-java
-/dd-java-agent/instrumentation/velocity/                        @DataDog/asm-java
+/dd-java-agent/instrumentation/org-json-20230227/               @DataDog/asm-java
+/dd-java-agent/instrumentation/snakeyaml-1.33/                  @DataDog/asm-java
+/dd-java-agent/instrumentation/velocity-1.5/                    @DataDog/asm-java
 /dd-java-agent/instrumentation/freemarker/                      @DataDog/asm-java
 /dd-smoke-tests/iast-util/                                      @DataDog/asm-java
 /dd-smoke-tests/spring-security/                                @DataDog/asm-java
@@ -77,18 +77,17 @@
 
 # @DataDog/ci-app-libraries-java
 /dd-java-agent/agent-ci-visibility/                 @DataDog/ci-app-libraries-java
-/dd-java-agent/instrumentation/cucumber/            @DataDog/ci-app-libraries-java
-/dd-java-agent/instrumentation/jacoco/              @DataDog/ci-app-libraries-java
+/dd-java-agent/instrumentation/cucumber-5.4/        @DataDog/ci-app-libraries-java
+/dd-java-agent/instrumentation/jacoco-0.8.9/        @DataDog/ci-app-libraries-java
 /dd-java-agent/instrumentation/junit                @DataDog/ci-app-libraries-java
 /dd-java-agent/instrumentation/karate/              @DataDog/ci-app-libraries-java
-/dd-java-agent/instrumentation/scalatest/           @DataDog/ci-app-libraries-java
+/dd-java-agent/instrumentation/scalatest-3.0.8/     @DataDog/ci-app-libraries-java
 /dd-java-agent/instrumentation/selenium/            @DataDog/ci-app-libraries-java
 /dd-java-agent/instrumentation/testng/              @DataDog/ci-app-libraries-java
 /dd-java-agent/instrumentation/gradle/              @DataDog/ci-app-libraries-java
 /dd-java-agent/instrumentation/gradle-testing/      @DataDog/ci-app-libraries-java
 /dd-java-agent/instrumentation/maven                @DataDog/ci-app-libraries-java
-/dd-java-agent/instrumentation/maven-surefire/      @DataDog/ci-app-libraries-java
-/dd-java-agent/instrumentation/weaver/              @DataDog/ci-app-libraries-java
+/dd-java-agent/instrumentation/weaver-0.9/          @DataDog/ci-app-libraries-java
 /dd-smoke-tests/gradle/                             @DataDog/ci-app-libraries-java
 /dd-smoke-tests/junit-console/                      @DataDog/ci-app-libraries-java
 /dd-smoke-tests/maven/                              @DataDog/ci-app-libraries-java
@@ -114,6 +113,23 @@
 /internal-api/src/test/groovy/datadog/trace/api/datastreams                  @DataDog/data-streams-monitoring
 **/datastreams/                                                              @DataDog/data-streams-monitoring
 **/DataStreams*                                                              @DataDog/data-streams-monitoring
+**/dsmTest/**                                                                @DataDog/data-streams-monitoring
+
+/dd-java-agent/instrumentation/confluent-schema-registry/       @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/avro/                            @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/protobuf-3.0/                    @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/               @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/google-pubsub/                   @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/        @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/aws-java/aws-java-sns-1.0/       @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/aws-java/aws-java-sns-2.0/       @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/        @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/         @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/        @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/aws-java/aws-java-sqs-1.0/       @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/aws-java/aws-java-sqs-2.0/       @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/grpc-1.5/                        @DataDog/data-streams-monitoring @DataDog/apm-idm-java
+/dd-java-agent/instrumentation/armeria/armeria-grpc-0.84/       @DataDog/data-streams-monitoring @DataDog/apm-idm-java
 
 # @DataDog/feature-flagging-and-experimentation-sdk
 /dd-smoke-tests/openfeature/                                    @DataDog/feature-flagging-and-experimentation-sdk

.github/chainguard/self.pin-system-tests.create-pr.sts.yaml

@@ -0,0 +1,13 @@
+issuer: https://token.actions.githubusercontent.com
+
+subject_pattern: repo:DataDog/dd-trace-java:ref:refs/heads/(master|release/v.+)
+
+claim_pattern:
+  event_name: (create|workflow_dispatch)
+  ref: refs/heads/(master|release/v.+)
+  job_workflow_ref: DataDog/dd-trace-java/\.github/workflows/pin-system-tests\.yaml@refs/heads/(master|release/v.+)
+
+permissions:
+  contents: write
+  pull_requests: write
+  workflows: write

.github/workflows/pin-system-tests.yaml

@@ -0,0 +1,104 @@
+name: Pin system tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      release-branch-name:
+        description: 'The minor release branch name (e.g. release/v1.54.x)'
+        required: true
+        type: string
+  # run workflow when a release branch is created
+  create:
+
+jobs:
+  pin-system-tests:
+    name: "Pin system tests"
+    if: github.event_name != 'create' || startsWith(github.ref, 'refs/heads/release/v')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write # required for OIDC token federation
+    steps:
+      - uses: DataDog/dd-octo-sts-action@acaa02eee7e3bb0839e4272dacb37b8f3b58ba80 # v1.0.3
+        id: octo-sts
+        with:
+          scope: DataDog/dd-trace-java
+          policy: self.pin-system-tests.create-pr
+      
+      - name: Define base branch
+        id: define-base-branch
+        run: |
+          if [[ -n "${{ github.event.inputs.release-branch-name }}" ]]; then
+            BASE_BRANCH=${{ github.event.inputs.release-branch-name }}
+          else
+            BASE_BRANCH=${GITHUB_REF#refs/heads/}
+          fi
+          echo "base_branch=${BASE_BRANCH}" >> $GITHUB_OUTPUT
+      
+      - name: Checkout the repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          ref: ${{ steps.define-base-branch.outputs.base_branch }}
+      
+      - name: Get latest commit SHA of base branch
+        id: get-latest-commit-sha
+        run: |
+          echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Define branch name
+        id: define-branch
+        run: echo "branch=ci/pin-system-tests-$(date +'%Y%m%d')" >> $GITHUB_OUTPUT
+
+      - name: Check if branch already exists
+        id: check-branch
+        run: |
+          BRANCH=${{ steps.define-branch.outputs.branch }}
+          if git ls-remote --heads origin "$BRANCH" | grep -q "$BRANCH"; then
+            echo "ERROR: Branch $BRANCH already exists - please delete it and re-run the workflow."
+            exit 1
+          else
+            echo "Branch $BRANCH does not exist - creating it now."
+          fi
+      
+      - name: Update system-tests references to latest commit SHA on main
+        run: ./tooling/update_system_test_reference.sh
+      
+      - name: Check if changes should be committed
+        id: check-changes
+        run: |
+          if [[ -z "$(git status -s)" ]]; then
+            echo "ERROR: No changes to commit - the system-tests reference was not updated."
+            exit 1
+          else
+            echo "Changes to commit:"
+            git status -s
+          fi
+      
+      - name: Commit changes
+        id: create-commit
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git commit -m "chore: Pin system-tests for release branch" .github/workflows/run-system-tests.yaml
+          echo "commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+      
+      - name: Push changes
+        uses: DataDog/commit-headless@5a0f3876e0fbdd3a86b3e008acf4ec562db59eee # action/v2.0.1
+        with:
+          token: "${{ steps.octo-sts.outputs.token }}"
+          branch: "${{ steps.define-branch.outputs.branch }}"
+          head-sha: "${{ steps.get-latest-commit-sha.outputs.sha }}"
+          create-branch: true
+          command: push
+          commits: "${{ steps.create-commit.outputs.commit }}"
+
+      - name: Create pull request
+        env:
+          GH_TOKEN: ${{ steps.octo-sts.outputs.token }}
+        run: |
+          gh pr create --title "Pin system tests for release branch" \
+            --base ${{ steps.define-base-branch.outputs.base_branch }} \
+            --head ${{ steps.define-branch.outputs.branch }} \
+            --label "tag: dependencies" \
+            --label "tag: no release notes" \
+            --body "This PR pins the system-tests reference for the release branch." \

.gitlab-ci.yml

@@ -109,17 +109,29 @@ default:
 .normalize_node_index: &normalize_node_index
   - if [ "$CI_NO_SPLIT" == "true" ] ; then CI_NODE_INDEX=1; CI_NODE_TOTAL=1; fi # A job uses parallel but doesn't intend to split by index
   - if [ -n "$CI_SPLIT" ]; then CI_NODE_INDEX="${CI_SPLIT%%/*}"; CI_NODE_TOTAL="${CI_SPLIT##*/}"; fi
-  - echo "CI_NODE_TOTAL=${CI_NODE_TOTAL}, CI_NODE_INDEX=$CI_NODE_INDEX"
-  - export NORMALIZED_NODE_TOTAL=${CI_NODE_TOTAL:-1}
-  - ONE_INDEXED_NODE_INDEX=${CI_NODE_INDEX:-1}; export NORMALIZED_NODE_INDEX=$((ONE_INDEXED_NODE_INDEX - 1))
-  - echo "NORMALIZED_NODE_TOTAL=${NORMALIZED_NODE_TOTAL}, NORMALIZED_NODE_INDEX=$NORMALIZED_NODE_INDEX"
+  - echo "CI_NODE_INDEX=$CI_NODE_INDEX, CI_NODE_TOTAL=${CI_NODE_TOTAL}"
 
 .cgroup_info: &cgroup_info
   - source .gitlab/gitlab-utils.sh
   - gitlab_section_start "cgroup-info" "cgroup info"
   - .gitlab/cgroup-info.sh
   - gitlab_section_end "cgroup-info"
 
+.container_info: &container_info
+  - |
+    # Containers and processes are limited to a 1-hour window, and expire after 36 hours
+    if [ -n "$CI_JOB_STARTED_AT" ]; then
+      FROM_EPOCH_MS=$(date -d "$CI_JOB_STARTED_AT" +%s)000
+      TO_EPOCH_MS=$((FROM_EPOCH_MS + 3600000))
+      TIME_PARAMS="from_ts=${FROM_EPOCH_MS}&to_ts=${TO_EPOCH_MS}&"
+    else
+      TIME_PARAMS=""
+    fi
+
+    echo -e "${TEXT_BOLD}${TEXT_YELLOW}Runner dashboard, these are live (limited to a 1-hour window, and expire after 36 hours)${TEXT_CLEAR}"
+    echo -e "${TEXT_BOLD}${TEXT_YELLOW}   Containers:${TEXT_CLEAR} https://app.datadoghq.com/containers?${TIME_PARAMS}query=image_name%3A%2A%2Fdatadog%2Fdd-trace-java-docker-build%20AND%20pod_name%3A${POD_NAME}&live=false"
+    echo -e "${TEXT_BOLD}${TEXT_YELLOW}   Processes:${TEXT_CLEAR} https://app.datadoghq.com/process?${TIME_PARAMS}query=image_name%3A%2A%2Fdatadog%2Fdd-trace-java-docker-build%20AND%20pod_name%3A${POD_NAME}&live=false"
+
 .gitlab_base_ref_params: &gitlab_base_ref_params
   - |
     export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh)
@@ -168,6 +180,7 @@ default:
       unprotect: true
   before_script:
     - source .gitlab/gitlab-utils.sh
+    - *container_info
     # Akka token added to SSM from https://account.akka.io/token
     - export ORG_GRADLE_PROJECT_akkaRepositoryToken=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.akka_repo_token --with-decryption --query "Parameter.Value" --out text)
     - export ORG_GRADLE_PROJECT_mavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY
@@ -198,6 +211,7 @@ default:
     - gitlab_section_end "gradle-dance"
   after_script:
     - *cgroup_info
+    - *container_info
 
 # Check and fail early if maven central credentials are incorrect. When a new token is generated
 # on the central publisher portal, it invalidates the old one. This check prevents going further.
@@ -420,8 +434,9 @@ test_published_artifacts:
   script:
     - *gitlab_base_ref_params
     - ./gradlew --version
-    - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS
+    - ./gradlew $GRADLE_TARGET -x spotlessCheck $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS
   after_script:
+    - *container_info
     - *cgroup_info
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
@@ -488,8 +503,9 @@ muzzle:
   script:
     - export SKIP_BUILDSCAN="true"
     - ./gradlew --version
-    - ./gradlew :runMuzzle -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS
+    - ./gradlew :runMuzzle -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS
   after_script:
+    - *container_info
     - *cgroup_info
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
@@ -512,6 +528,7 @@ muzzle-dep-report:
     - ./gradlew --version
     - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS
   after_script:
+    - *container_info
     - *cgroup_info
     - .gitlab/collect_muzzle_deps.sh
   artifacts:
@@ -542,8 +559,8 @@ muzzle-dep-report:
   needs: [ build_tests ]
   stage: tests
   variables:
-    KUBERNETES_MEMORY_REQUEST: 17Gi
-    KUBERNETES_MEMORY_LIMIT: 17Gi
+    KUBERNETES_MEMORY_REQUEST: 20Gi
+    KUBERNETES_MEMORY_LIMIT: 20Gi
     KUBERNETES_CPU_REQUEST: 10
     GRADLE_WORKERS: 4
     GRADLE_MEM: 3G
@@ -568,12 +585,13 @@ muzzle-dep-report:
       export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true";
       fi
     - *prepare_test_env
-    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEM -Xmx$GRADLE_MEM $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=1024M -Ddatadog.forkedMinHeapSize=128M"
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEM -Xmx$GRADLE_MEM $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djava.util.prefs.userRoot=/tmp/.java/.userPrefs-${CI_JOB_ID}' -Ddatadog.forkedMaxHeapSize=1024M -Ddatadog.forkedMinHeapSize=128M"
     - ./gradlew --version
-    - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE
+    - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE
   after_script:
     - *restore_pretest_env
     - *set_datadog_api_keys
+    - *container_info
     - *cgroup_info
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
@@ -728,7 +746,7 @@ test_smoke:
     GRADLE_PARAMS: "-PskipFlakyTests"
     CACHE_TYPE: "smoke"
   parallel:
-    matrix: *test_matrix_6
+    matrix: *test_matrix_8
 
 test_ssi_smoke:
   extends: .test_job
@@ -739,7 +757,7 @@ test_ssi_smoke:
     DD_INJECT_FORCE: "true"
     DD_INJECTION_ENABLED: "tracer"
   parallel:
-    matrix: *test_matrix_6
+    matrix: *test_matrix_8
 
 test_smoke_graalvm:
   extends: .test_job

.gitlab/macrobenchmarks.yml

@@ -143,4 +143,4 @@ notify-slo-breaches:
       when: never
     - when: always
   variables:
-    CHANNEL: "apm-java"
+    CHANNEL: "apm-java-ops-bot"

.gitlab/one-pipeline.locked.yml

@@ -1,4 +1,4 @@
 # DO NOT EDIT THIS FILE MANUALLY
 # This file is auto-generated by automation.
 include:
-  - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/04f6a88e3db67cb88821632d138a2a5c3105ba59760bd3dfc60b54733501ecc3/one-pipeline.yml
+  - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/58b2e8d06c714848c8577c8ac9b460b7413823d75ee96d068ebff547d109f5d0/one-pipeline.yml

build.gradle.kts

@@ -2,6 +2,8 @@ import com.diffplug.gradle.spotless.SpotlessExtension
 import datadog.gradle.plugin.ci.testAggregate
 
 plugins {
+  kotlin("jvm") version libs.versions.kotlin.plugin apply false
+
   id("dd-trace-java.gradle-debug")
   id("dd-trace-java.dependency-locking")
   id("dd-trace-java.tracer-version")
@@ -10,13 +12,13 @@ plugins {
   id("dd-trace-java.ci-jobs")
 
   id("com.diffplug.spotless") version "8.1.0"
+  id("me.champeau.gradle.japicmp") version "0.4.3"
   id("com.github.spotbugs") version "6.4.7"
   id("de.thetaphi.forbiddenapis") version "3.10"
   id("io.github.gradle-nexus.publish-plugin") version "2.0.0"
   id("com.gradleup.shadow") version "8.3.6" apply false
   id("me.champeau.jmh") version "0.7.3" apply false
   id("org.gradle.playframework") version "0.13" apply false
-  kotlin("jvm") version libs.versions.kotlin.plugin apply false
 }
 
 description = "dd-trace-java"
@@ -155,3 +157,29 @@ testAggregate(
     ":dd-java-agent:agent-debugger"
   )
 )
+
+// JApiCmp configuration example
+// Usage: ./gradlew japicmp -Partifact=groupId:artifactId -Pbaseline=1.0.0 -Ptarget=2.0.0
+tasks.register<me.champeau.gradle.japicmp.JapicmpTask>("japicmp") {
+  val artifact = providers.gradleProperty("artifact").orNull
+  val baseline = providers.gradleProperty("baseline").orNull
+  val target = providers.gradleProperty("target").orNull
+
+  if (artifact != null && baseline != null && target != null) {
+    oldClasspath.from(
+      configurations.detachedConfiguration(
+        dependencies.create("$artifact:$baseline")
+      )
+    )
+    newClasspath.from(
+      configurations.detachedConfiguration(
+        dependencies.create("$artifact:$target")
+      )
+    )
+    onlyModified.set(true)
+    failOnModification.set(false)
+    ignoreMissingClasses.set(true)
+    txtOutputFile.set(layout.buildDirectory.file("reports/japicmp.txt"))
+    htmlOutputFile.set(layout.buildDirectory.file("reports/japicmp.html"))
+  }
+}