Merge branch 'master' into mhlidd/backfilling_v2_format

Author: mhlidd

.github/chainguard/self.pin-system-tests.create-pr.sts.yaml

@@ -1,12 +1,13 @@
 issuer: https://token.actions.githubusercontent.com
 
-subject_pattern: ^repo:DataDog/dd-trace-java:ref:refs/heads/.+$
+subject_pattern: repo:DataDog/dd-trace-java:ref:refs/heads/(master|test/v.+)
 
 claim_pattern:
-  event_name: (push|workflow_dispatch)
-  ref: refs/heads/.+
-  job_workflow_ref: DataDog/dd-trace-java/\.github/workflows/pin-system-tests\.yaml@refs/heads/.+
+  event_name: (create|workflow_dispatch)
+  ref: refs/heads/(master|test/v.+)
+  job_workflow_ref: DataDog/dd-trace-java/\.github/workflows/pin-system-tests\.yaml@refs/heads/(master|test/v.+)
 
 permissions:
   contents: write
   pull_requests: write
+  workflows: write

.github/workflows/pin-system-tests.yaml

@@ -0,0 +1,112 @@
+name: Pin system tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      release-branch-name:
+        description: 'The minor release branch name (e.g. release/v1.54.x)'
+        required: true
+        type: string
+  # run workflow when a release branch is created
+  create:
+
+jobs:
+  pin-system-tests:
+    name: "Pin system tests"
+    # CHANGE BACK TO release/v*
+    if: github.event_name != 'create' || startsWith(github.ref, 'refs/heads/test/v')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # may not be needed
+      id-token: write # Required for OIDC token federation
+    steps:
+      - uses: DataDog/dd-octo-sts-action@acaa02eee7e3bb0839e4272dacb37b8f3b58ba80 # v1.0.3
+        id: octo-sts
+        with:
+          scope: DataDog/dd-trace-java
+          policy: self.pin-system-tests.create-pr
+      
+      - name: Define base branch
+        id: define-base-branch
+        run: |
+          if [[ -n "${{ github.event.inputs.release-branch-name }}" ]]; then
+            BASE_BRANCH=${{ github.event.inputs.release-branch-name }}
+          else
+            BASE_BRANCH=${GITHUB_REF#refs/heads/}
+          fi
+          echo "base_branch=${BASE_BRANCH}" >> $GITHUB_OUTPUT
+      
+      - name: Checkout the repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          ref: ${{ steps.define-base-branch.outputs.base_branch }}
+      
+      - name: Get latest commit SHA of base branch
+        id: get-latest-commit-sha
+        run: |
+          echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Define branch name
+        id: define-branch
+        run: echo "branch=ci/pin-system-tests-$(date +'%Y%m%d')" >> $GITHUB_OUTPUT
+
+      - name: Check if branch already exists
+        id: check-branch
+        run: |
+          BRANCH=${{ steps.define-branch.outputs.branch }}
+          if git ls-remote --heads origin "$BRANCH" | grep -q "$BRANCH"; then
+            echo "creating_new_branch=false" >> "$GITHUB_OUTPUT"
+            echo "Branch $BRANCH already exists - please delete it and re-run the workflow."
+          else
+            echo "creating_new_branch=true" >> "$GITHUB_OUTPUT"
+            echo "Branch $BRANCH does not exist - creating it now"
+          fi
+      
+      - name: Update system-tests references to latest commit SHA on main
+        run: ./tooling/update_system_test_reference.sh
+      
+      - name: Check if changes should be committed
+        id: check-changes
+        run: |
+          if [[ -z "$(git status -s)" ]]; then
+            echo "No changes to commit, exiting."
+            echo "commit_changes=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "commit_changes=true" >> "$GITHUB_OUTPUT"
+            echo "Changes to commit:"
+            git status -s
+          fi
+      
+      - name: Commit changes
+        if: steps.check-changes.outputs.commit_changes == 'true'
+        id: create-commit
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git commit -m "chore: Pin system-tests for release branch" .github/workflows/run-system-tests.yaml
+          echo "commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+      
+      - name: Push changes
+        uses: DataDog/commit-headless@5a0f3876e0fbdd3a86b3e008acf4ec562db59eee # action/v2.0.1
+        if: steps.check-changes.outputs.commit_changes == 'true' && steps.check-branch.outputs.creating_new_branch == 'true'
+        with:
+          token: "${{ steps.octo-sts.outputs.token }}"
+          branch: "${{ steps.define-branch.outputs.branch }}"
+          head-sha: "${{ steps.get-latest-commit-sha.outputs.sha }}"
+          create-branch: true
+          command: push
+          commits: "${{ steps.create-commit.outputs.commit }}"
+
+      - name: Create pull request
+        if: steps.check-changes.outputs.commit_changes == 'true' && steps.check-branch.outputs.creating_new_branch == 'true'
+        env:
+          GH_TOKEN: ${{ steps.octo-sts.outputs.token }}
+        # REMOVE DRAFT
+        run: |
+          gh pr create --title "Pin system tests for release branch" \
+            --base ${{ steps.define-base-branch.outputs.base_branch }} \
+            --head ${{ steps.define-branch.outputs.branch }} \
+            --label "tag: dependencies" \
+            --label "tag: no release notes" \
+            --body "This PR pins the system-tests reference for the release branch." \
+            --draft

.gitlab-ci.yml

@@ -542,8 +542,8 @@ muzzle-dep-report:
   needs: [ build_tests ]
   stage: tests
   variables:
-    KUBERNETES_MEMORY_REQUEST: 17Gi
-    KUBERNETES_MEMORY_LIMIT: 17Gi
+    KUBERNETES_MEMORY_REQUEST: 20Gi
+    KUBERNETES_MEMORY_LIMIT: 20Gi
     KUBERNETES_CPU_REQUEST: 10
     GRADLE_WORKERS: 4
     GRADLE_MEM: 3G
@@ -728,7 +728,7 @@ test_smoke:
     GRADLE_PARAMS: "-PskipFlakyTests"
     CACHE_TYPE: "smoke"
   parallel:
-    matrix: *test_matrix_6
+    matrix: *test_matrix_8
 
 test_ssi_smoke:
   extends: .test_job
@@ -739,7 +739,7 @@ test_ssi_smoke:
     DD_INJECT_FORCE: "true"
     DD_INJECTION_ENABLED: "tracer"
   parallel:
-    matrix: *test_matrix_6
+    matrix: *test_matrix_8
 
 test_smoke_graalvm:
   extends: .test_job

.gitlab/one-pipeline.locked.yml

@@ -1,4 +1,4 @@
 # DO NOT EDIT THIS FILE MANUALLY
 # This file is auto-generated by automation.
 include:
-  - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/04f6a88e3db67cb88821632d138a2a5c3105ba59760bd3dfc60b54733501ecc3/one-pipeline.yml
+  - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/58b2e8d06c714848c8577c8ac9b460b7413823d75ee96d068ebff547d109f5d0/one-pipeline.yml

dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/propagation/CodecModuleTest.groovy

@@ -290,10 +290,10 @@ class CodecModuleTest extends IastModuleImplTestBase {
     then:
     final helloTainted = to.get(hello)
     helloTainted.ranges.length == 1
-    helloTainted.ranges.first().with {
-      assert it.source.origin == (byte) 0
-      assert it.source.name == 'name1'
-      assert it.source.value == 'Hello'
+    with(helloTainted.ranges.first()) {
+      it.source.origin == (byte) 0
+      it.source.name == 'name1'
+      it.source.value == 'Hello'
     }
 
     when:
@@ -303,10 +303,10 @@ class CodecModuleTest extends IastModuleImplTestBase {
     then:
     final worldTainted = to.get(world)
     worldTainted.ranges.length == 1
-    worldTainted.ranges.first().with {
-      assert it.source.origin == (byte) 1
-      assert it.source.name == 'name2'
-      assert it.source.value == 'World!'
+    with(worldTainted.ranges.first()) {
+      it.source.origin == (byte) 1
+      it.source.name == 'name2'
+      it.source.value == 'World!'
     }
   }
 

dd-java-agent/agent-tooling/src/test/groovy/datadog/trace/agent/tooling/iast/IastPostProcessorFactoryTest.groovy

@@ -76,10 +76,10 @@ class IastPostProcessorFactoryTest extends DDSpecification {
     final metrics = collector.drain()
     assert metrics.size() == 1
     // one method has ben instrumented
-    metrics.first().with {
-      assert it.metric == IastMetric.INSTRUMENTED_SINK
-      assert it.tags == ['vulnerability_type:SQL_INJECTION']
-      assert it.value == 1L
+    with(metrics.first()) {
+      it.metric == IastMetric.INSTRUMENTED_SINK
+      it.tags == ['vulnerability_type:SQL_INJECTION']
+      it.value == 1L
     }
 
     when: 'the advice is used'

dd-java-agent/appsec/src/main/java/com/datadog/appsec/api/security/ApiSecuritySamplerImpl.java

@@ -4,6 +4,8 @@
 import datadog.trace.api.Config;
 import datadog.trace.api.time.SystemTimeSource;
 import datadog.trace.api.time.TimeSource;
+import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
+import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
 import java.util.Deque;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedDeque;
@@ -73,8 +75,15 @@ public boolean preSampleRequest(final @Nonnull AppSecRequestContext ctx) {
       return false;
     }
     if (counter.tryAcquire()) {
-      log.debug("API security sampling is required for this request (presampled)");
       ctx.setKeepOpenForApiSecurityPostProcessing(true);
+      if (!Config.get().isApmTracingEnabled()) {
+        boolean sampled = updateApiAccessIfExpired(hash);
+        if (sampled) {
+          logSamplingDecision("preSampleRequest", hash);
+        }
+        return sampled;
+      }
+      logSamplingDecision("preSampleRequest", hash);
       return true;
     }
     return false;
@@ -91,7 +100,11 @@ public boolean sampleRequest(AppSecRequestContext ctx) {
       // This should never happen, it should have been short-circuited before.
       return false;
     }
-    return updateApiAccessIfExpired(hash);
+    boolean sampled = Config.get().isApmTracingEnabled() ? updateApiAccessIfExpired(hash) : true;
+    if (sampled) {
+      logSamplingDecision("sampleRequest", hash);
+    }
+    return sampled;
   }
 
   @Override
@@ -168,4 +181,23 @@ private long computeApiHash(final String route, final String method, final int s
     result = 31 * result + statusCode;
     return result;
   }
+
+  private void logSamplingDecision(final String method, final long hash) {
+    if (!log.isDebugEnabled()) {
+      return;
+    }
+    AgentSpan activeSpan = AgentTracer.get().activeSpan();
+
+    if (activeSpan != null) {
+      log.debug(
+          "API security sampling decision in {}: hash={}, traceId={}, spanId={}",
+          method,
+          hash,
+          activeSpan.getTraceId(),
+          activeSpan.getSpanId());
+    } else {
+      log.debug(
+          "API security sampling decision in {}: hash={}, traceId=null, spanId=null", method, hash);
+    }
+  }
 }