Merge branch 'dougqh/crash-tracking-in-smoke-tests' of https://github.com/DataDog/dd-trace-java into dougqh/crash-tracking-in-smoke-tests

Author: dougqh

.circleci/config.continue.yml.j2

@@ -816,6 +816,7 @@ jobs:
     parameters:
       weblog-variant:
         type: string
+    parallelism: 3
     steps:
       - setup_system_tests
 
@@ -833,32 +834,38 @@ jobs:
 
       - run:
           name: Run
-          command: |
-            cd system-tests
-            DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY ./run.sh
-
-      - run:
-          name: Run APM E2E default tests
           # Stop the job after 5m to avoid excessive overhead. Will need adjustment as more tests are added.
           no_output_timeout: 5m
           command: |
             cd system-tests
-            DD_SITE=datadoghq.com DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY ./run.sh APM_TRACING_E2E
+            echo "
+            DEFAULT
+            APM_TRACING_E2E
+            APM_TRACING_E2E_SINGLE_SPAN
+            " | circleci tests split > scenarios.list
+            for scenario in $(<scenarios.list); do
+              if [[ $scenario =~ .*_E2E.* ]]; then
+                export DD_SITE=datadoghq.com
+                export DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY
+                export DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY
+              else
+                export DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY
+              fi
+              echo "Running scenario $scenario"
+              ./run.sh $scenario
+            done
 
       - run:
-          name: Run APM E2E Single Span tests
-          # Stop the job after 5m to avoid excessive overhead. Will need adjustment as more tests are added.
-          no_output_timeout: 5m
+          name: Collect artifacts
           command: |
+            mkdir -p artifacts
             cd system-tests
-            DD_SITE=datadoghq.com DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY ./run.sh APM_TRACING_E2E_SINGLE_SPAN
-
-      - run:
-          name: Collect artifacts
-          command: tar -cvzf logs_java_<< parameters.weblog-variant >>_dev.tar.gz -C system-tests logs logs_apm_tracing_e2e logs_apm_tracing_e2e_single_span
+            for log_dir in logs*; do
+              tar -cvzf ../artifacts/${log_dir}_<< parameters.weblog-variant >>.tar.gz $log_dir
+            done
 
       - store_artifacts:
-          path: logs_java_<< parameters.weblog-variant >>_dev.tar.gz
+          path: artifacts
 
   integrations-system-tests:
     machine:
@@ -1363,6 +1370,14 @@ build_test_jobs: &build_test_jobs
       stage: tracing
       testJvm: "8"
 
+  - fan_in:
+      requires:
+        - test_8_inst_latest
+        - test_17_inst_latest
+        - test_21_inst_latest
+      name: test_inst_latest
+      stage: tracing
+
   - agent_integration_tests:
       requires:
         - ok_to_test
@@ -1435,6 +1450,7 @@ build_test_jobs: &build_test_jobs
 {% for jdk in all_jdks %}
         - "test_{{ jdk }}"
 {% endfor %}
+        - test_inst_latest
         - muzzle
         - profiling
         - debugger

.circleci/render_config.py

@@ -28,7 +28,7 @@
 }
 # Version to use for all the base Docker images, see
 # https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build
-DOCKER_IMAGE_VERSION="v24.08"
+DOCKER_IMAGE_VERSION="v24.10"
 
 # Get labels from pull requests to override some defaults for jobs to run.
 # `run-tests: all` will run all tests.

.circleci/upload_ciapp.sh

@@ -1,26 +1,42 @@
 #!/usr/bin/env bash
 SERVICE_NAME="dd-trace-java"
+PIPELINE_STAGE=$1
+TEST_JVM=$2
 
 # JAVA_???_HOME are set in the base image for each used JDK https://github.com/DataDog/dd-trace-java-docker-build/blob/master/Dockerfile#L86
-java_home="JAVA_$2_HOME"
-java_bin="${!java_home}/bin/java"
-if [ ! -x $java_bin ]; then
-    java_bin=$(which java)
+JAVA_HOME="JAVA_${TEST_JVM}_HOME"
+JAVA_BIN="${!JAVA_HOME}/bin/java"
+if [ ! -x "$JAVA_BIN" ]; then
+    JAVA_BIN=$(which java)
 fi
 
-java_props=$($java_bin -XshowSettings:properties -version 2>&1)
-java_prop () {
-    echo "$(echo "$java_props" | grep $1 | head -n1 | cut -d'=' -f2 | xargs)"
+# Extract Java properties from the JVM used to run the tests
+JAVA_PROPS=$($JAVA_BIN -XshowSettings:properties -version 2>&1)
+java_prop() {
+    local PROP_NAME=$1
+    echo "$JAVA_PROPS" | grep "$PROP_NAME" | head -n1 | cut -d'=' -f2 | xargs
 }
 
-# based on tracer implementation: https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/TestDecorator.java#L55-L77
-datadog-ci junit upload --service $SERVICE_NAME \
-    --logs \
-    --tags "test.traits:{\"marker\":[\"$1\"]}" \
-    --tags "runtime.name:$(java_prop java.runtime.name)" \
-    --tags "runtime.vendor:$(java_prop java.vendor)" \
-    --tags "runtime.version:$(java_prop java.version)" \
-    --tags "os.architecture:$(java_prop os.arch)" \
-    --tags "os.platform:$(java_prop os.name)" \
-    --tags "os.version:$(java_prop os.version)" \
-./results
+# Upload test results to CI Visibility
+junit_upload() {
+    # based on tracer implementation: https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/TestDecorator.java#L55-L77
+    DD_API_KEY=$1 \
+        datadog-ci junit upload --service $SERVICE_NAME \
+        --logs \
+        --tags "test.traits:{\"marker\":[\"$PIPELINE_STAGE\"]}" \
+        --tags "runtime.name:$(java_prop java.runtime.name)" \
+        --tags "runtime.vendor:$(java_prop java.vendor)" \
+        --tags "runtime.version:$(java_prop java.version)" \
+        --tags "os.architecture:$(java_prop os.arch)" \
+        --tags "os.platform:$(java_prop os.name)" \
+        --tags "os.version:$(java_prop os.version)" \
+        ./results
+}
+
+# Make sure we do not use DATADOG_API_KEY from the environment
+unset DATADOG_API_KEY
+
+# Upload test results to production environment like all other CI jobs
+junit_upload "$DATADOG_API_KEY_PROD"
+# And also upload to staging environment to benefit from the new features not yet released
+junit_upload "$DATADOG_API_KEY_DDSTAGING"

.github/workflows/README.md

@@ -54,6 +54,7 @@ _Recovery:_ Manually trigger the action again on the relevant tag.
 _Trigger:_ When creating a minor or major version tag.
 
 _Actions:_
+
 * Close the milestone related to the tag,
 * Create a new milestone by incrementing minor version.
 
@@ -76,6 +77,7 @@ _Notes:_ _Download releases_ are special GitHub releases with fixed URL and tags
 _Trigger:_ When a release is published. Releases of type `prereleased` should skip this.
 
 _Action:_
+
 * Find all issues related to the release by checking the related milestone,
 * Add a comment to let know the issue was addressed by the newly published release,
 * Close all those issues.
@@ -105,10 +107,13 @@ _Recovery:_ Manually trigger the action again.
 
 _Trigger:_ When pushing commits to `master` or any pull request targeting `master`.
 
-_Action:_ 
+_Action:_
+
 * Run [DataDog Static Analysis](https://docs.datadoghq.com/static_analysis/) and upload result to DataDog Code Analysis,
-* Run [GitHub CodeQL](https://codeql.github.com/) action, upload result to GitHub security tab and DataDog Code Analysis -- do not apply to pull request, only when pushing to `master`,
-* Run [Trivy security scanner](https://github.com/aquasecurity/trivy) on built artifacts and upload result to GitHub security tab.
+* Run [GitHub CodeQL](https://codeql.github.com/) action, upload result to GitHub security tab -- do not apply to pull request, only when pushing to `master`,
+* Run [Trivy security scanner](https://github.com/aquasecurity/trivy) on built artifacts and upload result to GitHub security tab and Datadog Code Analysis.
+
+_Notes:_ Results are sent on both production and staging environments.
 
 ### comment-on-submodule-update [🔗](comment-on-submodule-update.yaml)
 

.github/workflows/analyze-changes.yaml

@@ -16,13 +16,14 @@ jobs:
         uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
         with:
           submodules: 'recursive'
-      - name: Check code meets quality standards
-        id: datadog-static-analysis
+      # Run the static analysis on the staging environment to benefit from the new features not yet released
+      - name: Check code meets quality standards (staging)
+        id: datadog-static-analysis-staging
         uses: DataDog/datadog-static-analyzer-github-action@c74aff158c8cc1c3e285660713bcaa5f9c6d696e # v1
         with:
-          dd_app_key: ${{ secrets.DD_APP_KEY }}
-          dd_api_key: ${{ secrets.DD_API_KEY }}
-          dd_site: datad0g.com
+          dd_app_key: ${{ secrets.DATADOG_APP_KEY_STAGING }}
+          dd_api_key: ${{ secrets.DATADOG_API_KEY_STAGING }}
+          dd_site: "datad0g.com"
           dd_service: "dd-trace-java"
           dd_env: "ci"
           cpu_count: 2
@@ -77,11 +78,19 @@ jobs:
       # For now, CodeQL SARIF results are not supported by Datadog CI
       # - name: Upload results to Datadog CI Static Analysis
       #   run: |
-      #     wget --no-verbose https://github.com/DataDog/datadog-ci/releases/download/v2.42.0/datadog-ci_linux-x64 -O datadog-ci
+      #     wget --no-verbose https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64 -O datadog-ci
       #     chmod +x datadog-ci
       #     ./datadog-ci sarif upload /home/runner/work/dd-trace-java/results/java.sarif --service dd-trace-java --env ci
       #   env:
-      #     DD_API_KEY: ${{ secrets.DD_API_KEY }}
+      #     DD_API_KEY: ${{ secrets.DATADOG_APP_KEY_PROD }}
+      #     DD_SITE: datadoghq.com
+
+      # For now, CodeQL SARIF results are not supported by Datadog CI
+      # - name: Upload results to Datadog Staging CI Static Analysis
+      #   run: |
+      #     ./datadog-ci sarif upload /home/runner/work/dd-trace-java/results/java.sarif --service dd-trace-java --env ci
+      #   env:
+      #     DD_API_KEY: ${{ secrets.DATADOG_API_KEY_STAGING }}
       #     DD_SITE: datad0g.com
 
   trivy:
@@ -152,9 +161,16 @@ jobs:
 
       - name: Upload results to Datadog CI Static Analysis
         run: |
-          wget --no-verbose https://github.com/DataDog/datadog-ci/releases/download/v2.42.0/datadog-ci_linux-x64 -O datadog-ci
+          wget --no-verbose https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64 -O datadog-ci
           chmod +x datadog-ci
           ./datadog-ci sarif upload trivy-results.sarif --service dd-trace-java --env ci
         env:
-          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+          DD_API_KEY: ${{ secrets.DATADOG_API_KEY_PROD }}
+          DD_SITE: datadoghq.com
+
+      - name: Upload results to Datadog Staging CI Static Analysis
+        run: |
+          ./datadog-ci sarif upload trivy-results.sarif --service dd-trace-java --env ci
+        env:
+          DD_API_KEY: ${{ secrets.DATADOG_API_KEY_STAGING }}
           DD_SITE: datad0g.com

.github/workflows/check-pull-requests.yaml

@@ -1,7 +1,7 @@
 name: Check pull requests
 on:
   pull_request:
-    types: [opened, reopened, edited, labeled, unlabeled]
+    types: [opened, edited, labeled, unlabeled]
     branches:
     - master
     - release/v*
@@ -40,14 +40,36 @@ jobs:
             }
             // Add comment to the pull request
             if (labelsCheckFailed || titleCheckFailed) {
-              await github.rest.issues.createComment({
-                issue_number: context.payload.pull_request.number,
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                body: 'Hi! 👋 Thanks for your pull request! 🎉\n\n' +
+              // Define comment body
+              const commentMarker = '<!-- dd-trace-java-check-pull-requests-workflow -->'
+              const commentBody = 'Hi! 👋 Thanks for your pull request! 🎉\n\n' +
                   'To help us review it, please make sure to:\n\n' +
                   (labelsCheckFailed ? '* Add at least one type, and one component or instrumentation label to the pull request\n' : '') +
                   (titleCheckFailed ? '* Remove the tag from the pull request title\n' : '') +
-                  '\nIf you need help, please check our [contributing guidelines](https://github.com/DataDog/dd-trace-java/blob/master/CONTRIBUTING.md).'
+                  '\nIf you need help, please check our [contributing guidelines](https://github.com/DataDog/dd-trace-java/blob/master/CONTRIBUTING.md).' +
+                  '\n\n' + commentMarker
+              // Look for previous comment
+              const comments = await github.rest.issues.listComments({
+                issue_number: context.payload.pull_request.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo
               })
+              const previousComment = comments.data.find(comment => comment.body.includes(commentMarker))
+              if (previousComment) {
+                // Update previous comment
+                await github.rest.issues.updateComment({
+                  comment_id: previousComment.id,
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  body: commentBody
+                })
+              } else {
+                // Create new comment
+                await github.rest.issues.createComment({
+                  issue_number: context.payload.pull_request.number,
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  body: commentBody
+                })
+              }
             }

dd-java-agent/agent-bootstrap/src/test/groovy/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecoratorTest.groovy

@@ -422,7 +422,8 @@ class HttpServerDecoratorTest extends ServerDecoratorTest {
     def ig = new InstrumentationGateway()
     def ss = ig.getSubscriptionService(RequestContextSlot.APPSEC)
     def cbpAppSec = ig.getCallbackProvider(RequestContextSlot.APPSEC)
-    def callbacks = new IGCallBacks(reqData)
+    def data = reqData ? new Object() : null
+    def callbacks = new IGCallBacks(data)
     if (reqStarted) {
       ss.registerCallback(EVENTS.requestStarted(), callbacks)
     }
@@ -434,7 +435,7 @@ class HttpServerDecoratorTest extends ServerDecoratorTest {
     }
     Map<String, String> headers = ["foo": "bar", "some": "thing", "another": "value"]
     def reqCtxt = Mock(RequestContext) {
-      getData(RequestContextSlot.APPSEC) >> reqData
+      getData(RequestContextSlot.APPSEC) >> data
     }
     def mSpan = Mock(AgentSpan) {
       getRequestContext() >> reqCtxt
@@ -461,13 +462,13 @@ class HttpServerDecoratorTest extends ServerDecoratorTest {
 
     where:
     // spotless:off
-    reqStarted | reqData      | reqHeader | reqHeaderDone | reqStartedCount | reqHeaderCount | reqHeaderDoneCount
-    false      | null         | false     | false         | 0               | 0              | 0
-    false      | new Object() | false     | false         | 0               | 0              | 0
-    true       | null         | false     | false         | 1               | 0              | 0
-    true       | new Object() | false     | false         | 1               | 0              | 0
-    true       | new Object() | true      | false         | 1               | 3              | 0
-    true       | new Object() | true      | true          | 1               | 3              | 1
+    reqStarted | reqData | reqHeader | reqHeaderDone | reqStartedCount | reqHeaderCount | reqHeaderDoneCount
+    false      | false   | false     | false         | 0               | 0              | 0
+    false      | true    | false     | false         | 0               | 0              | 0
+    true       | false   | false     | false         | 1               | 0              | 0
+    true       | true    | false     | false         | 1               | 0              | 0
+    true       | true    | true      | false         | 1               | 3              | 0
+    true       | true    | true      | true          | 1               | 3              | 1
     // spotless:on
   }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java

@@ -23,10 +23,10 @@
 import datadog.trace.civisibility.git.GitClientGitInfoBuilder;
 import datadog.trace.civisibility.git.tree.GitClient;
 import datadog.trace.civisibility.ipc.SignalClient;
-import datadog.trace.civisibility.source.BestEffortMethodLinesResolver;
-import datadog.trace.civisibility.source.ByteCodeMethodLinesResolver;
-import datadog.trace.civisibility.source.CompilerAidedMethodLinesResolver;
-import datadog.trace.civisibility.source.MethodLinesResolver;
+import datadog.trace.civisibility.source.BestEffortLinesResolver;
+import datadog.trace.civisibility.source.ByteCodeLinesResolver;
+import datadog.trace.civisibility.source.CompilerAidedLinesResolver;
+import datadog.trace.civisibility.source.LinesResolver;
 import datadog.trace.civisibility.source.index.*;
 import java.lang.reflect.Type;
 import java.net.InetSocketAddress;
@@ -63,7 +63,7 @@ public class CiVisibilityServices {
   final CIProviderInfoFactory ciProviderInfoFactory;
   final GitClient.Factory gitClientFactory;
   final GitInfoProvider gitInfoProvider;
-  final MethodLinesResolver methodLinesResolver;
+  final LinesResolver linesResolver;
   final RepoIndexProvider.Factory repoIndexProviderFactory;
   @Nullable final SignalClient.Factory signalClientFactory;
 
@@ -82,9 +82,8 @@ public class CiVisibilityServices {
 
     CiEnvironment environment = buildCiEnvironment(config, sco);
     this.ciProviderInfoFactory = new CIProviderInfoFactory(config, environment);
-    this.methodLinesResolver =
-        new BestEffortMethodLinesResolver(
-            new CompilerAidedMethodLinesResolver(), new ByteCodeMethodLinesResolver());
+    this.linesResolver =
+        new BestEffortLinesResolver(new CompilerAidedLinesResolver(), new ByteCodeLinesResolver());
 
     this.gitInfoProvider = gitInfoProvider;
     gitInfoProvider.registerGitInfoBuilder(new CIProviderGitInfoBuilder(config, environment));