Merge branch 'master' into kr-igor/kafka-lag-spark-streaming

Author: kr-igor

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit c87bd359aad64a29f280fc5c70a879f7c7f4846e
+default_system_tests_commit: &default_system_tests_commit 53d9b2c43876a39e5c0426f4df4dd5fd79d062e8
 
 parameters:
   nightly:
@@ -660,7 +660,8 @@ jobs:
           - DD_POOL_TRACE_CHECK_FAILURES=true
           - DD_DISABLE_ERROR_RESPONSES=true
           - ENABLED_CHECKS=trace_content_length,trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service,trace_dd_service
-    resource_class: xlarge
+    # TODO: merge xlarge_tests and tests? or rename this?
+    resource_class: large
 
 
   # The only way to do fan-in in CircleCI seems to have a proper job, so let's have one that
@@ -781,6 +782,7 @@ jobs:
           name: Gather muzzle tasks
           command: >-
             SKIP_BUILDSCAN="true"
+            GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
             ./gradlew writeMuzzleTasksToFile
             << pipeline.parameters.gradle_flags >>
             --max-workers=3
@@ -879,11 +881,27 @@ jobs:
 
       - run:
           name: Run APM Integrations tests
+          environment:
+            - AWS_ACCESS_KEY_ID: $SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID
+            - AWS_SECRET_ACCESS_KEY: $SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY
+            - AWS_REGION: us-east-1
+            - AWS_DEFAULT_REGION: us-east-1  # AWS services should use `AWS_REGION`, but some still use the older `AWS_DEFAULT_REGION`
           # Stop the job after 5m to avoid excessive overhead. Will need adjustment as more tests are added.
           no_output_timeout: 5m
           command: |
             cd system-tests
             DD_SITE=datadoghq.com DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY ./run.sh INTEGRATIONS
+        
+      - run:
+          name: Run IDM Crossed Tracing Libraries propagation tests for messaging
+          environment:
+            - AWS_ACCESS_KEY_ID: $SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID
+            - AWS_SECRET_ACCESS_KEY: $SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY
+            - AWS_REGION: us-east-1
+            - AWS_DEFAULT_REGION: us-east-1  # AWS services should use `AWS_REGION`, but some still use the older `AWS_DEFAULT_REGION`
+          command: |
+            cd system-tests
+            DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY ./run.sh CROSSED_TRACING_LIBRARIES
 
       - store_test_results:
           path: system-tests/logs_integrations
@@ -944,21 +962,18 @@ jobs:
           name: Build runner
           command: |
             cd system-tests
-            export TEST_LIBRARY=java
             ./build.sh -i runner
 
       - run:
           name: Run
           command: |
             set -e
             cd system-tests
-            export TEST_LIBRARY=java
-            export PYTEST_XDIST_AUTO_NUM_WORKERS=6
             set +e
             RUN_ATTEMPTS=1
             while [ $RUN_ATTEMPTS -le 3 ]; do
               echo "Running parametric test attempt $RUN_ATTEMPTS"
-              timeout 12m ./run.sh PARAMETRIC --log-cli-level=DEBUG --durations=30 -vv
+              timeout 20m ./run.sh PARAMETRIC -L java --log-cli-level=DEBUG --durations=30 -vv
               status=$?
               #timneout returns 124 if it times out
               #if the return code is not 124, then we exit with the status
@@ -1099,8 +1114,8 @@ build_test_jobs: &build_test_jobs
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: inst
-      parallelism: 4
-      maxWorkers: 4
+      parallelism: 12
+      maxWorkers: 3
       matrix:
         <<: *test_matrix
 
@@ -1113,8 +1128,8 @@ build_test_jobs: &build_test_jobs
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: inst
-      parallelism: 4
-      maxWorkers: 4
+      parallelism: 12
+      maxWorkers: 3
       testJvm: "8"
 
   - xlarge_tests:
@@ -1127,8 +1142,8 @@ build_test_jobs: &build_test_jobs
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: latestdep
-      parallelism: 4
-      maxWorkers: 4
+      parallelism: 12
+      maxWorkers: 3
       testJvm: "8"
 
   - xlarge_tests:
@@ -1141,8 +1156,8 @@ build_test_jobs: &build_test_jobs
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: latestdep
-      parallelism: 4
-      maxWorkers: 4
+      parallelism: 12
+      maxWorkers: 3
       testJvm: "17"
 
   - xlarge_tests:
@@ -1155,8 +1170,8 @@ build_test_jobs: &build_test_jobs
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: latestdep
-      parallelism: 4
-      maxWorkers: 4
+      parallelism: 12
+      maxWorkers: 3
       testJvm: "21"
 
 {% if flaky %}
@@ -1184,7 +1199,7 @@ build_test_jobs: &build_test_jobs
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: inst
-      parallelism: 2
+      parallelism: 12
       maxWorkers: 4
       testJvm: "8"
 
@@ -1200,6 +1215,20 @@ build_test_jobs: &build_test_jobs
       parallelism: 4
       maxWorkers: 4
       testJvm: "8"
+
+  - tests:
+      requires:
+        - ok_to_test
+      name: z_test_8_flaky_debugger
+      gradleTarget: ":debuggerTest"
+      gradleParameters: "-PrunFlakyTests"
+      continueOnFailure: true
+      triggeredBy: *debugger_modules
+      stage: debugger
+      cacheType: base
+      parallelism: 4
+      maxWorkers: 4
+      testJvm: "8"
 {% endif %}
 
   - tests:
@@ -1403,6 +1432,7 @@ build_test_jobs: &build_test_jobs
 {% for jdk in all_jdks %}
         - "test_{{ jdk }}"
 {% endfor %}
+        - muzzle
         - profiling
         - debugger
         - system-tests

.github/workflows/README.md

@@ -82,6 +82,15 @@ _Action:_ Clean up old lib-injection OCI images from GitHub Container Registry.
 
 _Recovery:_ Manually trigger the action again.
 
+### prune-old-pull-requests [🔗](prune-old-pull-requests.yaml)
+
+_Trigger:_ Every month or manually.
+
+_Action:_ Mark as stale and comment on pull requests with no update during the last quarter.
+Close them if no following update within a week.
+
+_Recovery:_ Manually trigger the action again.
+
 ## Code Quality and Security
 
 ### analyze-changes [🔗](analyze-changes-with-github-codeql.yaml)

.github/workflows/increment-milestone-on-tag.yaml

@@ -55,7 +55,7 @@ jobs:
             const nextMilestoneTitle = versionNumbers.join('.')
             core.info(`Creating next version milestone: ${nextMilestoneTitle}`)
             // Create the next milestone
-            await github.issues.createMilestone({
+            await github.rest.issues.createMilestone({
               owner: context.repo.owner,
               repo: context.repo.repo,
               title: nextMilestoneTitle

.github/workflows/prune-old-pull-requests.yaml

@@ -0,0 +1,23 @@
+name: Prune old pull requests
+on:
+  schedule:
+    - cron: '0 3 1 * *'
+  workflow_dispatch:
+
+jobs:
+  prune-old-pull-requests:
+    name: Prune old pull requests
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+    - name: Prune old pull requests
+      uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
+      with:
+        days-before-stale: -1 # Disable general stale bot
+        days-before-pr-stale: 90 # Only enable stale bot for PRs with no activity for 90 days
+        stale-pr-message: 'This pull request has been marked as stale because it has not had activity over the past quarter. It will be closed in 7 days if no further activity occurs. Feel free to reopen the PR if you are still working on it.'
+        close-pr-message: 'This pull request has been closed because it has not had activity over the past quarter. Feel free to reopen the PR if you are still working on it.'
+        stale-pr-label: 'tag: stale'
+        operations-per-run: 500

.github/workflows/update-gradle-dependencies.yaml

@@ -71,9 +71,11 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          gh pr create --title "Update Gradle dependencies" \
-            --body "This PR updates the Gradle dependencies. ⚠️ Don't forget to squash commits before merging. ⚠️" \
+          # use echo to set a multiline body for the PR
+          echo -e "This PR updates the Gradle dependencies. ⚠️ Don't forget to squash commits before merging. ⚠️\n\n- [ ] Update PR title if a code change is needed to support one of those new dependencies" | \
+            gh pr create --title "Update Gradle dependencies" \
             --base master \
             --head $BRANCH_NAME \
             --label "tag: dependencies" \
             --label "tag: no release notes"
+            --body-file -

.gitlab-ci.yml

@@ -17,13 +17,6 @@ stages:
   - ci-visibility-tests
   - generate-signing-key
 
-workflow:
-  rules:
-    - if: $POPULATE_CACHE == "true"
-      variables:
-        SKIP_SHARED_PIPELINE: "true"
-    - when: always
-
 variables:
   REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com
   BUILD_JOB_NAME: "build"
@@ -210,13 +203,20 @@ deploy_artifacts_to_github:
     max: 2
     when: always
 
+requirements_json_test:
+  rules:
+    - when: on_success
+  variables:
+    REQUIREMENTS_BLOCK_JSON_PATH: "metadata/requirements-block.json"
+    REQUIREMENTS_ALLOW_JSON_PATH: "metadata/requirements-allow.json"
+
 package-oci:
   needs: [ build ]
 
 onboarding_tests_installer:
   parallel:
     matrix:
-      - ONBOARDING_FILTER_WEBLOG: [test-app-java, test-app-java-container, test-app-java-container-jdk15, test-app-java-alpine-libgcc]
+      - ONBOARDING_FILTER_WEBLOG: [test-app-java, test-app-java-container, test-app-java-container-jdk15, test-app-java-alpine]
         SCENARIO: [ SIMPLE_INSTALLER_AUTO_INJECTION, SIMPLE_AUTO_INJECTION_PROFILING ]
       - ONBOARDING_FILTER_WEBLOG: [test-app-java-buildpack]
         SCENARIO: [ SIMPLE_INSTALLER_AUTO_INJECTION ]

.gitlab/exploration-tests.yml

@@ -1,5 +1,5 @@
 variables:
-  EXPLORATION_TESTS_IMAGE: $REGISTRY/ci/dd-trace-java:exploration-tests
+  EXPLORATION_TESTS_IMAGE: registry.ddbuild.io/ci/dd-trace-java:exploration-tests
 
 build-exploration-tests-image:
   stage: exploration-tests
@@ -18,24 +18,33 @@ build-exploration-tests-image:
     - ./gradlew :dd-java-agent:shadowJar --no-scan
     - cp workspace/dd-java-agent/build/libs/*.jar /exploration-tests/dd-java-agent.jar
     - cp dd-java-agent/agent-debugger/exploration-tests/run-exploration-tests.sh /exploration-tests
-    - cp dd-java-agent/agent-debugger/exploration-tests/exclude*.txt /exploration-tests
+    - cp dd-java-agent/agent-debugger/exploration-tests/exclude_*.txt /exploration-tests
+    - cp dd-java-agent/agent-debugger/exploration-tests/include_*.txt /exploration-tests
     - cd /exploration-tests
   after_script:
+    - echo "$PROJECT"
     - cd $CI_PROJECT_DIR
-    - cp /exploration-tests/$PROJECT/agent.log agent.log
-    - gzip agent.log
-    - tar czf surefire-reports.tar.gz /exploration-tests/$PROJECT/target/surefire-reports
-    - tar czf debugger-dumps.tar.gz /tmp/debugger
+    - cp /exploration-tests/$PROJECT/agent.log ${PROJECT}_agent.log
+    - gzip ${PROJECT}_agent.log
+    - tar czf ${PROJECT}_surefire-reports.tar.gz /exploration-tests/${PROJECT}/target/surefire-reports
+    - tar czf ${PROJECT}_debugger-dumps.tar.gz /tmp/debugger
   stage: exploration-tests
   when: manual
   tags: [ "runner:main"]
   needs: []
   image: $EXPLORATION_TESTS_IMAGE
   artifacts:
     paths:
-      - agent.log.gz
-      - surefire-reports.tar.gz
-      - debugger-dumps.tar.gz
+      - ${PROJECT}_agent.log.gz
+      - ${PROJECT}_surefire-reports.tar.gz
+      - ${PROJECT}_debugger-dumps.tar.gz
+
+exploration-tests-jsoup:
+  variables:
+    PROJECT: jsoup
+  <<: *common-exploration-tests
+  script:
+    - ./run-exploration-tests.sh "$PROJECT" "mvn verify" "include_jsoup.txt" "exclude_jsoup.txt"
 
 
 exploration-tests-jackson-core:

.gitlab/prepare-oci-package.sh

@@ -10,3 +10,4 @@ fi
 mkdir -p sources
 cp ../workspace/dd-java-agent/build/libs/*.jar sources/dd-java-agent.jar
 echo -n "$VERSION" > sources/version
+cp ../metadata/requirements.json sources/