Merge branch 'master' into dougqh/interceptor-bypass

Author: dougqh

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 761b9e7a82ffb136c4653a4d1623d120d67b005b
+default_system_tests_commit: &default_system_tests_commit b0b2e1f212f8c483b52aa3adc6ffd4132b1ba9b8
 
 parameters:
   nightly:
@@ -167,8 +167,7 @@ commands:
               if [[ "$BRANCH" != "master" ]] && [[ "$BRANCH" != "release/*" ]]; then
                 # We know that we have checked out the PR merge branch, so the HEAD commit is a merge
                 # As a backup, if anything goes wrong with the diff, the build will fail
-                # Get list of changed files directly using git diff-tree to avoid issues with large binary files
-                CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r HEAD)
+                CHANGED_FILES=$(git show HEAD | grep -e "^Merge:" | cut -d ' ' -f 2- | sed 's/ /.../' | xargs git diff --name-only)
                 # Count the number of matches, and ignore if the grep doesn't match anything
                 MATCH_COUNT=$(echo "$CHANGED_FILES" | grep -c -E "<< pipeline.parameters.global_pattern >>|<< parameters.pattern >>") || true
                 if [[ "$MATCH_COUNT" -eq "0" ]]; then
@@ -851,8 +850,6 @@ jobs:
             (
             echo "
             DEFAULT
-            APM_TRACING_E2E
-            APM_TRACING_E2E_SINGLE_SPAN
             TRACING_CONFIG_NONDEFAULT
             TRACING_CONFIG_NONDEFAULT_2
             TRACING_CONFIG_NONDEFAULT_3
@@ -876,14 +873,8 @@ jobs:
               "
             fi
             ) | circleci tests split > scenarios.list
+            export DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY
             for scenario in $(<scenarios.list); do
-              if [[ $scenario =~ .*_E2E.* ]]; then
-                export DD_SITE=datadoghq.com
-                export DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY
-                export DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY
-              else
-                export DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY
-              fi
               echo "Running scenario $scenario"
               ./run.sh $scenario
             done
@@ -923,27 +914,11 @@ jobs:
 
       - run:
           name: Run APM Integrations tests
-          environment:
-            - AWS_ACCESS_KEY_ID: $SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID
-            - AWS_SECRET_ACCESS_KEY: $SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY
-            - AWS_REGION: us-east-1
-            - AWS_DEFAULT_REGION: us-east-1  # AWS services should use `AWS_REGION`, but some still use the older `AWS_DEFAULT_REGION`
           # Stop the job after 5m to avoid excessive overhead. Will need adjustment as more tests are added.
           no_output_timeout: 5m
           command: |
             cd system-tests
-            DD_SITE=datadoghq.com DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY ./run.sh INTEGRATIONS
-
-      - run:
-          name: Run IDM Crossed Tracing Libraries propagation tests for messaging
-          environment:
-            - AWS_ACCESS_KEY_ID: $SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID
-            - AWS_SECRET_ACCESS_KEY: $SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY
-            - AWS_REGION: us-east-1
-            - AWS_DEFAULT_REGION: us-east-1  # AWS services should use `AWS_REGION`, but some still use the older `AWS_DEFAULT_REGION`
-          command: |
-            cd system-tests
-            DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY ./run.sh CROSSED_TRACING_LIBRARIES
+            DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY ./run.sh INTEGRATIONS
 
       - store_test_results:
           path: system-tests/logs_integrations
@@ -976,7 +951,7 @@ jobs:
           no_output_timeout: 5m
           command: |
             cd system-tests
-            export DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY
+            export DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY
             ./run.sh DEBUGGER_SCENARIOS
 
       - run:

.circleci/upload_ciapp.sh

@@ -33,10 +33,5 @@ junit_upload() {
         ./results
 }
 
-# Make sure we do not use DATADOG_API_KEY from the environment
-unset DATADOG_API_KEY
-
 # Upload test results to production environment like all other CI jobs
 junit_upload "$DATADOG_API_KEY_PROD"
-# And also upload to staging environment to benefit from the new features not yet released
-junit_upload "$DATADOG_API_KEY_DDSTAGING"

.github/workflows/README.md

@@ -110,7 +110,6 @@ _Trigger:_ When pushing commits to `master` or any pull request targeting `maste
 
 _Action:_
 
-* Run [DataDog Static Analysis](https://docs.datadoghq.com/static_analysis/) and upload result to DataDog Code Analysis,
 * Run [GitHub CodeQL](https://codeql.github.com/) action, upload result to GitHub security tab -- do not apply to pull request, only when pushing to `master`,
 * Run [Trivy security scanner](https://github.com/aquasecurity/trivy) on built artifacts and upload result to GitHub security tab and Datadog Code Analysis.
 
@@ -130,6 +129,21 @@ _Action:_ Create a PR updating the Grade dependencies and their locking files.
 
 _Recovery:_ Manually trigger the action again.
 
+### run-system-tests [🔗](run-system-tests.yaml)
+
+_Trigger:_ When pushing commits to `master` or manually.
+
+_Action:_ Build the Java Client Library and runs [the system tests](https://github.com/DataDog/system-tests) against.
+
+_Recovery:_ Manually trigger the action on the desired branch.
+
+### all-green [🔗](all-green.yaml)
+
+_Trigger:_ Any pull request.
+
+_Action:_ This action will check all other jobs (Github action, Gitlab, CircleCi), and will fail if any of them fails. This action got an `ignored` paraemters to exclude some jobs if they are temprorary failing. The purpose of this job is to be required for merges, achieving Green CI Policy.
+
+_Recovery:_ Manually trigger the action on the desired branch.
 
 ## Maintenance
 

.github/workflows/add-release-to-cloudfoundry.yaml

@@ -43,7 +43,7 @@ jobs:
         run: |
           echo "${{ steps.get-release-version.outputs.VERSION }}: ${{ steps.get-release-url.outputs.URL }}" >> index.yml
       - name: Commit and push changes
-        uses: planetscale/ghcommit-action@9400254a26464337cbe5af17c5f25075134e0089 # v0.2.7
+        uses: planetscale/ghcommit-action@5b20c92facae8dbf8a3836dc65b8503dda378573 # v0.2.13
         with:
           commit_message: "chore: Add version ${{ steps.get-release-version.outputs.VERSION }} to Cloud Foundry"
           repo: ${{ github.repository }}

.github/workflows/all-green.yml

@@ -0,0 +1,36 @@
+name: Check Pull Request CI Status
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  checks: read
+  statuses: read
+
+jobs:
+  all-jobs-are-green:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run Ensure CI Success
+        uses: DataDog/ensure-ci-success@f40e6ffd8e60280d478b9b92209aaa30d3d56895
+        with:
+          initial-delay-seconds: "1000"
+          max-retries: "60"
+          ignored-name-patterns: |
+            dd-gitlab/default-pipeline
+            dd-gitlab/check_inst 4/4
+            dd-gitlab/muzzle .*
+
+# ignored jobs : 
+#
+# * dd-gitlab/default-pipeline => success rate of 70% (needs an owner)
+# * dd-gitlab/check_inst 4/4   => success rate of 78% (needs an owner)
+# * dd-gitlab/muzzle .*        => success rate of ~85% (needs an owner)

.github/workflows/analyze-changes.yaml

@@ -13,25 +13,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  datadog-static-analyzer:
-    name: Analyze changes with DataDog Static Analyzer
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
-        with:
-          submodules: 'recursive'
-      # Run the static analysis on the staging environment to benefit from the new features not yet released
-      - name: Check code meets quality standards (staging)
-        id: datadog-static-analysis-staging
-        uses: DataDog/datadog-static-analyzer-github-action@1297a546e6bb268e2ac5bc98a1477d22be335822 # v1
-        with:
-          dd_app_key: ${{ secrets.DATADOG_APP_KEY_STAGING }}
-          dd_api_key: ${{ secrets.DATADOG_API_KEY_STAGING }}
-          dd_site: "datad0g.com"
-          cpu_count: 2
-          enable_performance_statistics: false
-
   codeql:
     name: Analyze changes with GitHub CodeQL
     # Don’t run on PR, only when pushing to master
@@ -49,7 +30,7 @@ jobs:
           submodules: 'recursive'
 
       - name: Cache Gradle dependencies
-        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: |
             ~/.gradle/caches
@@ -59,7 +40,7 @@ jobs:
             ${{ runner.os }}-gradle-
         
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
+        uses: github/codeql-action/init@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16
         with:
           languages: 'java'
           build-mode: 'manual'
@@ -76,25 +57,7 @@ jobs:
             --build-cache --parallel --stacktrace --no-daemon --max-workers=4
 
       - name: Perform CodeQL Analysis and upload results to GitHub Security tab
-        uses: github/codeql-action/analyze@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
-
-      # For now, CodeQL SARIF results are not supported by Datadog CI
-      # - name: Upload results to Datadog CI Static Analysis
-      #   run: |
-      #     wget --no-verbose https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64 -O datadog-ci
-      #     chmod +x datadog-ci
-      #     ./datadog-ci sarif upload /home/runner/work/dd-trace-java/results/java.sarif --service dd-trace-java --env ci
-      #   env:
-      #     DD_API_KEY: ${{ secrets.DATADOG_APP_KEY_PROD }}
-      #     DD_SITE: datadoghq.com
-
-      # For now, CodeQL SARIF results are not supported by Datadog CI
-      # - name: Upload results to Datadog Staging CI Static Analysis
-      #   run: |
-      #     ./datadog-ci sarif upload /home/runner/work/dd-trace-java/results/java.sarif --service dd-trace-java --env ci
-      #   env:
-      #     DD_API_KEY: ${{ secrets.DATADOG_API_KEY_STAGING }}
-      #     DD_SITE: datad0g.com
+        uses: github/codeql-action/analyze@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16
 
   trivy:
     name: Analyze changes with Trivy
@@ -111,7 +74,7 @@ jobs:
           submodules: 'recursive'
 
       - name: Cache Gradle dependencies
-        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: |
             ~/.gradle/caches
@@ -144,7 +107,7 @@ jobs:
           ls -laR "./workspace/.trivy"
 
       - name: Run Trivy security scanner
-        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # v0.29.0
+        uses: aquasecurity/trivy-action@6c175e9c4083a92bbca2f9724c8a5e33bc2d97a5 # v0.30.0
         with:
           scan-type: rootfs
           scan-ref: './workspace/.trivy/'
@@ -157,7 +120,7 @@ jobs:
           TRIVY_JAVA_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-java-db,public.ecr.aws/aquasecurity/trivy-java-db
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
+        uses: github/codeql-action/upload-sarif@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16
         if: always()
         with:
           sarif_file: 'trivy-results.sarif'
@@ -170,10 +133,3 @@ jobs:
         env:
           DD_API_KEY: ${{ secrets.DATADOG_API_KEY_PROD }}
           DD_SITE: datadoghq.com
-
-      - name: Upload results to Datadog Staging CI Static Analysis
-        run: |
-          ./datadog-ci sarif upload trivy-results.sarif --service dd-trace-java --env ci
-        env:
-          DD_API_KEY: ${{ secrets.DATADOG_API_KEY_STAGING }}
-          DD_SITE: datad0g.com

.github/workflows/draft-release-notes-on-tag.yaml

@@ -125,7 +125,12 @@ jobs:
             }
             function cleanUpTitle(title) {
               // Remove tags between brackets
-              return title.replace(/\[[^\]]+\]/g, '')
+              title = title.replace(/\[[^\]]+\]/g, '')
+              // Remove cherry-pick prefix
+              if (title.startsWith('🍒 ') && title.includes(' - ')) {
+                title = title.substring(title.indexOf(' - ') + 3)
+              }
+              return title
             }
             function format(pullRequest) {
               var line = `${decorate(pullRequest)}${cleanUpTitle(pullRequest.title)} (#${pullRequest.number} - @${pullRequest.user.login}`

.github/workflows/system-tests.yaml .github/workflows/run-system-tests.yaml.github/workflows/system-tests.yaml renamed to .github/workflows/run-system-tests.yaml

@@ -1,4 +1,4 @@
-name: system-tests
+name: Run system tests
 
 on:
   pull_request:
@@ -21,18 +21,18 @@ jobs:
           fetch-depth: 0
 
       - name: Cache Gradle dependencies
-        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
           path: |
             ~/.gradle/caches
             ~/.gradle/wrapper
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
           restore-keys: |
             ${{ runner.os }}-gradle-
-      
+
       - name: Build dd-trace-java
         run: |
-          GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G'" \
+          GRADLE_OPTS="-Xms2g -Xmx4g -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC" \
           JAVA_HOME=$JAVA_HOME_8_X64 \
           JAVA_8_HOME=$JAVA_HOME_8_X64 \
           JAVA_11_HOME=$JAVA_HOME_11_X64 \
@@ -60,5 +60,5 @@ jobs:
       binaries_artifact: binaries
       desired_execution_time: 900  # 15 minutes
       scenarios_groups: tracer-release
-      excluded_scenarios: CROSSED_TRACING_LIBRARIES,INTEGRATIONS_AWS  # require AWS credentials
+      excluded_scenarios: CROSSED_TRACING_LIBRARIES,INTEGRATIONS_AWS,APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,PROFILING  # require AWS and datadog credentials 
       skip_empty_scenarios: true

.gitlab-ci.yml

@@ -42,7 +42,6 @@ default:
 
 .set_datadog_api_keys: &set_datadog_api_keys
   - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text)
-  - export DATADOG_API_KEY_DDSTAGING=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.dd_api_key --with-decryption --query "Parameter.Value" --out text)
 
 # CI_NODE_INDEX and CI_NODE_TOTAL are 1-indexed and not always set. These steps normalize the numbers for jobs
 .normalize_node_index: &normalize_node_index

build.gradle

@@ -22,7 +22,7 @@ plugins {
   id 'pl.allegro.tech.build.axion-release' version '1.14.4'
   id 'io.github.gradle-nexus.publish-plugin' version '1.3.0'
 
-  id "com.github.johnrengelman.shadow" version "7.1.2" apply false
+  id "com.github.johnrengelman.shadow" version "8.1.1" apply false
   id "me.champeau.jmh" version "0.7.0" apply false
   id 'org.gradle.playframework' version '0.13' apply false
   id 'info.solidsoft.pitest' version '1.9.11'  apply false