Merge branch 'master' into kr-igor/kafka-lag-spark-streaming

Author: kr-igor

.circleci/config.continue.yml.j2

@@ -28,15 +28,15 @@ debugger_test_matrix: &debugger_test_matrix
 
 system_test_matrix: &system_test_matrix
   parameters:
-    weblog-variant: [ 'spring-boot', 'spring-boot-jetty', 'spring-boot-openliberty', 'spring-boot-3-native', 'jersey-grizzly2', 'resteasy-netty3','ratpack', 'vertx3' ]
+    weblog-variant: ['akka-http', 'jersey-grizzly2', 'play', 'resteasy-netty3', 'ratpack', 'spring-boot', 'spring-boot-jetty', 'spring-boot-openliberty', 'spring-boot-payara', 'spring-boot-undertow', 'spring-boot-wildfly', 'spring-boot-3-native', 'uds-spring-boot', 'vertx3', 'vertx4']
 
 agent_integration_tests_modules: &agent_integration_tests_modules "dd-trace-core|communication|internal-api|utils"
 core_modules: &core_modules "dd-java-agent|dd-trace-core|communication|internal-api|telemetry|utils|dd-java-agent/agent-bootstrap|dd-java-agent/agent-installer|dd-java-agent/agent-tooling|dd-java-agent/agent-builder|dd-java-agent/appsec|dd-java-agent/agent-crashtracking|dd-trace-api|dd-trace-ot"
 instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-iast|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication"
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 1c9542783eedfed1b995d976963fbeb14ba772b9
+default_system_tests_commit: &default_system_tests_commit c87bd359aad64a29f280fc5c70a879f7c7f4846e
 
 parameters:
   nightly:
@@ -273,13 +273,6 @@ commands:
             git fetch origin << parameters.systemTestsCommit >>
             git reset --hard FETCH_HEAD
 
-      - run:
-          name: Install python 3.12
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y python3.12-full python3.12-dev python3.12-venv
-            echo 'export PATH="$HOME/.local/bin:$PATH"' >>"$BASH_ENV"
-
 jobs:
   build:
     <<: *defaults
@@ -310,7 +303,7 @@ jobs:
           name: Build Project
           command: >-
 {% if is_nightly %}
-            ./gradlew resolveAndLockAll --write-locks
+            ./gradlew resolveAndLockAll --write-locks &&
 {% endif %}
             MAVEN_OPTS="-Xms64M -Xmx256M"
             GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
@@ -1313,6 +1306,8 @@ build_test_jobs: &build_test_jobs
       gradleParameters: "-PskipFlakyTests"
       stage: smoke
       cacheType: smoke
+      parallelism: 4
+      maxWorkers: 3
       testJvm: "8"
 
   - fan_in:

.github/CODEOWNERS

@@ -45,6 +45,7 @@ dd-java-agent/instrumentation/*iast*   @DataDog/asm-java
 dd-java-agent/instrumentation/*appsec* @DataDog/asm-java
 dd-java-agent/instrumentation/json/    @DataDog/asm-java
 dd-java-agent/instrumentation/snakeyaml/    @DataDog/asm-java
+dd-java-agent/instrumentation/freemarker/    @DataDog/asm-java
 dd-smoke-tests/iast-util/              @DataDog/asm-java
 dd-smoke-tests/spring-security/        @DataDog/asm-java
 dd-java-agent/instrumentation/commons-fileupload/    @DataDog/asm-java

.github/workflows/README.md

@@ -1,6 +1,6 @@
 # GitHub Actions Documentation
 
-This lists and describes the repository GitHub actions.
+This lists and describes the repository GitHub actions, how to maintain and test them.
 
 ## Release Management
 
@@ -18,15 +18,16 @@ _Trigger:_ When a release is published.
 
 _Action:_ Append the new release to the Cloud Foundry repository.
 
-_Recovery:_ Manually edit and push the `index.yml` file from [the cloudfoundry branch](https://github.com/DataDog/dd-trace-java/tree/cloudfoundry).
+_Recovery:_ Manually edit and push the `index.yml` file from [the cloudfoundry branch](https://github.com/DataDog/dd-trace-java/tree/cloudfoundry).
 
 ### create-next-milestone [🔗](create-next-milestone.yaml)
 
 _Trigger:_ When closing a milestone.
 
 _Action:_ Create a new milestone by incrementing minor version.
 
-_Comment:_ Already done when closing a tag. To delete?
+_Comment:_ Disabled as also covered by increment-milestone-on-tag.
+This will be removed after some testing.  
 
 ### draft-release-notes-on-tag [🔗](draft-release-notes-on-tag.yaml)
 
@@ -40,18 +41,17 @@ _Actions:_
 
 _Recovery:_ Manually trigger the action again on the relevant tag.
 
-### increment-milestones-on-tag [🔗](increment-milestones-on-tag.yaml)
+### increment-milestone-on-tag [🔗](increment-milestone-on-tag.yaml)
 
-_Trigger:_ When creating a tag. Release Candidate tags containing "-RC" or "-rc" will skip this.
+_Trigger:_ When creating a minor or major version tag.
 
 _Actions:_
 * Close the milestone related to the tag,
 * Create a new milestone by incrementing minor version.
 
-_Recovery:_ Manually close the related milestone and create a new one.
+_Recovery:_ Manually [close the related milestone and create a new one](https://github.com/DataDog/dd-trace-java/milestones).
 
-_Notes:_ This actions will handle _minor_ releases only.
-As there is no milestone for _patch_ releases, it won't close and create _patch_ releated milestone.
+_Notes:_ This action will not apply to release candidate versions using `-RC` tags.
 
 ### update-download-releases [🔗](update-download-releases.yaml)
 
@@ -74,52 +74,55 @@ _Action:_
 
 _Recovery:_ Check at the milestone for the related issues and update them manually.
 
+### prune-github-container-registry [🔗](prune-github-container-registry.yaml)
+
+_Trigger:_ Every week or manually.
+
+_Action:_ Clean up old lib-injection OCI images from GitHub Container Registry.
+
+_Recovery:_ Manually trigger the action again.
+
 ## Code Quality and Security
 
-### ci-static-analysis [🔗](ci-static-analysis.yml)
+### analyze-changes [🔗](analyze-changes-with-github-codeql.yaml)
 
-_Trigger:_ When pushing commits to `master` or any pull request to `master`.
+_Trigger:_ When pushing commits to `master` or any pull request targeting `master`.
 
-_Actions:_ Run [DataDog Static Analysis](https://docs.datadoghq.com/static_analysis/) and upload result to DataDog Code Analysis.
+_Action:_ 
+* Run [DataDog Static Analysis](https://docs.datadoghq.com/static_analysis/) and upload result to DataDog Code Analysis,
+* Run [GitHub CodeQL](https://codeql.github.com/) action, upload result to GitHub security tab and DataDog Code Analysis -- do not apply to pull request, only when pushing to `master`,
+* Run [Trivy security scanner](https://github.com/aquasecurity/trivy) on built artifacts and upload result to GitHub security tab.
 
 ### comment-on-submodule-update [🔗](comment-on-submodule-update.yaml)
 
 _Trigger:_ When creating a PR commits to `master` or a `release/*` branch with a Git Submodule update.
 
 _Action:_ Notify the PR author through comments that about the Git Submodule update.
 
-### codeql-analysis [🔗](codeql-analysis.yml)
-
-_Trigger:_ When pushing commits to `master`.
-
-_Action:_ Run GitHub CodeQL action, upload result to GitHub security tab and DataDog Code Analysis.
-
-### update-gradle-dependencies [🔗](trivy-analysis.yml)
+### update-gradle-dependencies [🔗](update-gradle-dependencies.yml)
 
 _Trigger:_ Every week or manually.
 
 _Action:_ Create a PR updating the Grade dependencies and their locking files.
 
 _Recovery:_ Manually trigger the action again.
 
-### trivy-analysis [🔗](trivy-analysis.yml)
-
-_Trigger:_ When pushing commits to `master` or any pull request to `master`.
 
-_Action:_ Run Trivy security scanner on built artifacts and upload result to GitHub security tab.
+## Maintenance
 
-### gradle-wrapper-validation [🔗](gradle-wrapper-validation.yaml.disabled)
+GitHub actions should be part of the [repository allowed actions to run](https://github.com/DataDog/dd-trace-java/settings/actions).
+While GitHub owned actions are allowed by default, the other ones must be declared.
 
-**DISABLED** - GitHub provides a way to disable actions rather than changing their extensions.
+Run the following script to get the list of actions to declare according the state of your working copy:
+```bash
+find .github/workflows -name "*.yaml" -exec  awk '/uses:/{print $2 ","}' {} \; | grep -vE '^(actions|github)/' | sort | uniq
+```
 
-_Comment:_ To delete?
+## Testing
 
-## Lib Injection
+Workflows can be locally tested using the [`act` CLI](https://github.com/nektos/act/).
+The [.github/workflows/tests/](./tests) folder contains test scripts and event payloads to locally trigger workflows.
 
-### lib-injection-prune-registry [🔗](lib-injection-prune-registry.yaml)
-
-_Trigger:_ Every week or manually.
-
-_Action:_ Clean up old lib-injection Docker images from GHCR.
-
-_Recovery:_ Manually trigger the action again.
+> [!WARNING]
+> Locally running workflows will still query GitHub backend and will update the GitHub project accordingly.
+> Pay extra attention to the workflow jobs you trigger to not create development disruption.

.github/workflows/add-milestone-to-pull-requests.yaml

@@ -5,11 +5,13 @@ on:
     branches:
       - master
       - release/v*
-
 jobs:
   add_milestone_to_merged:
-    if: github.event.pull_request.merged && github.event.pull_request.milestone == null
     name: Add milestone to merged pull requests
+    permissions:
+      issues: write # Required to update a pull request using the issues API
+      pull-requests: write # Required to update the milestone of a pull request
+    if: github.event.pull_request.merged && github.event.pull_request.milestone == null
     runs-on: ubuntu-latest
     steps:
       - name: Add milestone to merged pull requests

.github/workflows/add-release-to-cloudfoundry.yaml

@@ -5,6 +5,8 @@ on:
       - released
 jobs:
   update-releases:
+    permissions:
+      contents: write # Required to commit and push changes to the repository
     runs-on: ubuntu-latest
     steps:
       - name: Checkout "cloudfoundry" branch

.github/workflows/analyze-changes.yaml

@@ -0,0 +1,155 @@
+name: Analyze changes
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+
+jobs:
+  datadog-static-analyzer:
+    name: Analyze changes with DataDog Static Analyzer
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
+        with:
+          submodules: 'recursive'
+      - name: Check code meets quality standards
+        id: datadog-static-analysis
+        uses: DataDog/datadog-static-analyzer-github-action@c74aff158c8cc1c3e285660713bcaa5f9c6d696e # v1
+        with:
+          dd_app_key: ${{ secrets.DD_APP_KEY }}
+          dd_api_key: ${{ secrets.DD_API_KEY }}
+          dd_site: datad0g.com
+          dd_service: "dd-trace-java"
+          dd_env: "ci"
+          cpu_count: 2
+          enable_performance_statistics: false
+
+  codeql:
+    name: Analyze changes with GitHub CodeQL
+    # Don’t run on PR, only when pushing to master
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write # Required to upload the results to the Security tab
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
+        with:
+          submodules: 'recursive'
+
+      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+        
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
+        with:
+          languages: 'java'
+          build-mode: 'manual'
+      
+      - name: Build dd-trace-java for creating the CodeQL database
+        run: |
+          GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G'" \
+          JAVA_HOME=$JAVA_HOME_8_X64 \
+          JAVA_8_HOME=$JAVA_HOME_8_X64 \
+          JAVA_11_HOME=$JAVA_HOME_11_X64 \
+          JAVA_17_HOME=$JAVA_HOME_17_X64 \
+          JAVA_21_HOME=$JAVA_HOME_21_X64 \
+          ./gradlew clean :dd-java-agent:shadowJar \
+            --build-cache --parallel --stacktrace --no-daemon --max-workers=4
+
+      - name: Perform CodeQL Analysis and upload results to GitHub Security tab
+        uses: github/codeql-action/analyze@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
+
+      # For now, CodeQL SARIF results are not supported by Datadog CI
+      # - name: Upload results to Datadog CI Static Analysis
+      #   run: |
+      #     wget --no-verbose https://github.com/DataDog/datadog-ci/releases/download/v2.42.0/datadog-ci_linux-x64 -O datadog-ci
+      #     chmod +x datadog-ci
+      #     ./datadog-ci sarif upload /home/runner/work/dd-trace-java/results/java.sarif --service dd-trace-java --env ci
+      #   env:
+      #     DD_API_KEY: ${{ secrets.DD_API_KEY }}
+      #     DD_SITE: datad0g.com
+
+  trivy:
+    name: Analyze changes with Trivy
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write # Required to upload the results to the Security tab
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
+        with:
+          submodules: 'recursive'
+      
+      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+
+      - name: Remove old artifacts
+        run: |
+          MVN_LOCAL_REPO=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
+          echo "MVN_LOCAL_REPO=${MVN_LOCAL_REPO}" >> "$GITHUB_ENV"
+          rm -rf "${MVN_LOCAL_REPO}/com/datadoghq"
+
+      - name: Build and publish artifacts locally
+        run: |
+          GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G'" \
+          JAVA_HOME=$JAVA_HOME_8_X64 \
+          JAVA_8_HOME=$JAVA_HOME_8_X64 \
+          JAVA_11_HOME=$JAVA_HOME_11_X64 \
+          JAVA_17_HOME=$JAVA_HOME_17_X64 \
+          JAVA_21_HOME=$JAVA_HOME_21_X64 \
+          ./gradlew clean publishToMavenLocal \
+            --build-cache --parallel --stacktrace --no-daemon --max-workers=4
+
+      - name: Copy published artifacts
+        run: |
+          mkdir -p ./workspace/.trivy
+          cp -RP "${MVN_LOCAL_REPO}/com/datadoghq" ./workspace/.trivy/
+          ls -laR "./workspace/.trivy"
+
+      - name: Run Trivy security scanner
+        uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0
+        with:
+          scan-type: rootfs
+          scan-ref: './workspace/.trivy/'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          severity: 'CRITICAL,HIGH'
+          limit-severities-for-sarif: true
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
+        if: always()
+        with:
+          sarif_file: 'trivy-results.sarif'
+
+      - name: Upload results to Datadog CI Static Analysis
+        run: |
+          wget --no-verbose https://github.com/DataDog/datadog-ci/releases/download/v2.42.0/datadog-ci_linux-x64 -O datadog-ci
+          chmod +x datadog-ci
+          ./datadog-ci sarif upload trivy-results.sarif --service dd-trace-java --env ci
+        env:
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+          DD_SITE: datad0g.com