v5.44.0 proposal

.github/actions/install/action.yml

@@ -1,8 +1,25 @@
 name: Install dependencies
 description: Install dependencies
+inputs:
+  cache:
+    description: "Whether to enable caching of node_modules."
+    required: false
+    default: 'false'
 runs:
   using: composite
-  steps: # retry in case of server error from registry
+  steps:
+    - id: yarn-cache
+      uses: actions/cache@v4
+      with:
+        key: yarn-cache-${{ github.workflow }}-${{ github.job }}-${{ hashFiles('yarn.lock') }}-v2
+        path: node_modules.tar
+      if: inputs.cache == 'true'
+    - run: tar -xf node_modules.tar
+      shell: bash
+      if: inputs.cache == 'true' && steps.yarn-cache.outputs.cache-hit == 'true'
+      # Retry in case of server error from registry.
     - run: yarn install --frozen-lockfile --ignore-engines || yarn install --frozen-lockfile --ignore-engines
       shell: bash
-
+    - run: tar -cf node_modules.tar node_modules
+      shell: bash
+      if: inputs.cache == 'true' && steps.yarn-cache.outputs.cache-hit != 'true'

.github/actions/node/active-lts/action.yml

@@ -5,5 +5,4 @@ runs:
   steps:
     - uses: actions/setup-node@v4
       with:
-        cache: yarn
         node-version: '22'

.github/actions/node/latest/action.yml

@@ -5,5 +5,4 @@ runs:
   steps:
     - uses: actions/setup-node@v4
       with:
-        cache: yarn
         node-version: 'latest'

.github/actions/node/newest-maintenance-lts/action.yml

@@ -5,5 +5,4 @@ runs:
   steps:
     - uses: actions/setup-node@v4
       with:
-        cache: yarn
         node-version: '20'

.github/actions/node/oldest-maintenance-lts/action.yml

@@ -5,5 +5,4 @@ runs:
   steps:
     - uses: actions/setup-node@v4
       with:
-        cache: yarn
         node-version: '18'

.github/actions/testagent/start/action.yml

@@ -4,5 +4,5 @@ runs:
   using: composite
   steps:
     - uses: actions/checkout@v4
-    - run: docker compose up -d testagent
+    - run: docker compose up -d testagent || docker compose up -d testagent
       shell: bash

.github/workflows/appsec.yml

@@ -44,6 +44,8 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/node/active-lts
       - uses: ./.github/actions/install
+        with:
+          cache: 'true'
       - run: yarn test:appsec:ci
       - uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1
 
@@ -230,7 +232,6 @@ jobs:
       - uses: ./.github/actions/testagent/start
       - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0
         with:
-          cache: yarn
           node-version: ${{ matrix.version }}
       - uses: ./.github/actions/install
       - run: yarn test:appsec:plugins:ci
@@ -257,7 +258,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - run: yarn install
+      - uses: ./.github/actions/install
       - uses: ./.github/actions/node/oldest-maintenance-lts
       - run: yarn test:integration:appsec
       - uses: ./.github/actions/node/active-lts

.github/workflows/llmobs.yml

@@ -15,23 +15,22 @@ concurrency:
 
 jobs:
   sdk:
+    strategy:
+      matrix:
+        version: [18, 20, 22, latest]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/testagent/start
-      - uses: ./.github/actions/node/oldest-maintenance-lts
+      - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0
+        with:
+          node-version: ${{ matrix.version }}
       - uses: ./.github/actions/install
       - run: yarn test:llmobs:sdk:ci
-      - uses: ./.github/actions/node/newest-maintenance-lts
-      - run: yarn test:llmobs:sdk:ci
-      - uses: ./.github/actions/node/active-lts
-      - run: yarn test:llmobs:sdk:ci
-      - uses: ./.github/actions/node/latest
-      - run: yarn test:llmobs:sdk:ci
       - if: always()
         uses: ./.github/actions/testagent/logs
         with:
-          suffix: llmobs-${{ github.job }}
+          suffix: llmobs-${{ github.job }}-${{ matrix.version }}
       - uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1
 
   openai:
@@ -74,7 +73,7 @@ jobs:
         with:
           suffix: llmobs-${{ github.job }}
 
-  aws-sdk:
+  bedrock:
     runs-on: ubuntu-latest
     env:
       PLUGINS: aws-sdk
@@ -93,3 +92,23 @@ jobs:
         uses: ./.github/actions/testagent/logs
         with:
           suffix: llmobs-${{ github.job }}
+
+  vertex-ai:
+    runs-on: ubuntu-latest
+    env:
+      PLUGINS: google-cloud-vertexai
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: ./.github/actions/testagent/start
+      - uses: ./.github/actions/node/oldest-maintenance-lts
+      - uses: ./.github/actions/install
+      - run: yarn test:llmobs:plugins:ci
+        shell: bash
+      - uses: ./.github/actions/node/active-lts
+      - run: yarn test:llmobs:plugins:ci
+        shell: bash
+      - uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1
+      - if: always()
+        uses: ./.github/actions/testagent/logs
+        with:
+          suffix: llmobs-${{ github.job }}

.github/workflows/plugins.yml

@@ -853,7 +853,7 @@ jobs:
           - 1521:1521
           - 5500:5500
       testagent:
-        image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.16.0
+        image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.21.1
         env:
           LOG_LEVEL: DEBUG
           TRACE_LANGUAGE: javascript
@@ -877,7 +877,6 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0
         with:
-          cache: yarn
           node-version: '16'
       - uses: ./.github/actions/install
       - run: yarn config set ignore-engines true

.github/workflows/profiling.yml

@@ -49,6 +49,8 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/node/active-lts
       - uses: ./.github/actions/install
+        with:
+          cache: 'true'
       - run: yarn test:profiler:ci
       - run: yarn test:integration:profiler
       - uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1

.github/workflows/project.yml

@@ -61,11 +61,32 @@ jobs:
         env:
           DD_INJECTION_ENABLED: 'true'
 
+  integration-playwright:
+    strategy:
+      matrix:
+        version: [18, latest]
+    runs-on: ubuntu-latest
+    env:
+      DD_SERVICE: dd-trace-js-integration-tests
+      DD_CIVISIBILITY_AGENTLESS_ENABLED: 1
+      DD_API_KEY: ${{ secrets.DD_API_KEY_CI_APP }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0
+        with:
+          node-version: ${{ matrix.version }}
+      - uses: ./.github/actions/install
+      # Install system dependencies for playwright
+      - run: npx playwright install-deps
+      - run: yarn test:integration:playwright
+        env:
+          NODE_OPTIONS: '-r ./ci/init'
+
   integration-ci:
     strategy:
       matrix:
         version: [18, latest]
-        framework: [cucumber, playwright, selenium, jest, mocha]
+        framework: [cucumber, selenium, jest, mocha]
     runs-on: ubuntu-latest
     env:
       DD_SERVICE: dd-trace-js-integration-tests

.github/workflows/tracing.yml

@@ -44,5 +44,7 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/node/active-lts
       - uses: ./.github/actions/install
+        with:
+          cache: 'true'
       - run: yarn test:trace:core:ci
       - uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1

benchmark/sirun/runtime-metrics/README.md

@@ -0,0 +1,5 @@
+This benchmark runs the with runtime metrics and an accelerated flush. While
+this can catch code regressions, it's mostly meant to catch things like memory
+leaks where metrics would start piling up. This can be hard to catch in tests,
+but it would cause the app to become slower and slower over time which would
+be visible in the benchmark.

benchmark/sirun/runtime-metrics/index.js

@@ -0,0 +1,5 @@
+'use strict'
+
+require('../../..').init()
+
+setTimeout(() => {}, 1000)

benchmark/sirun/runtime-metrics/meta.json

@@ -0,0 +1,22 @@
+{
+  "name": "runtime-metrics",
+  "run": "node index.js",
+  "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node index.js\"",
+  "cachegrind": false,
+  "iterations": 5,
+  "instructions": true,
+  "variants": {
+    "control": {
+      "env": {
+        "DD_RUNTIME_METRICS_ENABLED": "false"
+      }
+    },
+    "with-runtime-metrics": {
+      "baseline": "control",
+      "env": {
+        "DD_RUNTIME_METRICS_ENABLED": "true",
+        "DD_RUNTIME_METRICS_FLUSH_INTERVAL": "20"
+      }
+    }
+  }
+}

docker-compose.yml

@@ -155,7 +155,7 @@ services:
       - LDAP_PASSWORDS=password1,password2
 
   testagent:
-    image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.16.0
+    image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.21.1
     ports:
         - "127.0.0.1:9126:9126"
     environment:

integration-tests/ci-visibility/features-test-management/attempt-to-fix.feature

@@ -0,0 +1,4 @@
+Feature: Attempt to fix
+  Scenario: Say attempt to fix
+    When the greeter says attempt to fix
+    Then I should have heard "attempt to fix"

integration-tests/ci-visibility/features-test-management/support/steps.js

@@ -1,6 +1,8 @@
 const assert = require('assert')
 const { When, Then } = require('@cucumber/cucumber')
 
+let numAttempt = 0
+
 Then('I should have heard {string}', function (expectedResponse) {
   if (this.whatIHeard === 'quarantine') {
     assert.equal(this.whatIHeard, 'fail')
@@ -21,3 +23,16 @@ When('the greeter says disabled', function () {
   // expected to fail if not disabled
   this.whatIHeard = 'disabld'
 })
+
+When('the greeter says attempt to fix', function () {
+  // eslint-disable-next-line no-console
+  console.log('I am running') // just to assert whether this is running
+  // expected to fail
+  if (process.env.SHOULD_ALWAYS_PASS) {
+    this.whatIHeard = 'attempt to fix'
+  } else if (process.env.SHOULD_FAIL_SOMETIMES) {
+    this.whatIHeard = numAttempt++ % 2 === 0 ? 'attempt to fix' : 'attempt to fx'
+  } else {
+    this.whatIHeard = 'attempt to fx'
+  }
+})

integration-tests/ci-visibility/playwright-tests-test-capabilities/passing-test.js

@@ -0,0 +1,11 @@
+const { test, expect } = require('@playwright/test')
+
+test.beforeEach(async ({ page }) => {
+  await page.goto(process.env.PW_BASE_URL)
+})
+
+test('should work with passing tests', async ({ page }) => {
+  await expect(page.locator('.hello-world')).toHaveText([
+    'Hello World'
+  ])
+})

integration-tests/ci-visibility/playwright-tests-test-management/attempt-to-fix-test.js

@@ -0,0 +1,27 @@
+const { test, expect } = require('@playwright/test')
+
+test.beforeEach(async ({ page }) => {
+  await page.goto(process.env.PW_BASE_URL)
+})
+
+test.describe('attempt to fix', () => {
+  test('should attempt to fix failed test', async ({ page }) => {
+    let textToAssert
+
+    if (process.env.SHOULD_ALWAYS_PASS) {
+      textToAssert = 'Hello World'
+    } else if (process.env.SHOULD_FAIL_SOMETIMES) {
+      // can't use numAttempt++ because we're running in parallel
+      if (Number(process.env.TEST_WORKER_INDEX) % 2 === 0) {
+        throw new Error('Hello Warld')
+      }
+      textToAssert = 'Hello World'
+    } else {
+      textToAssert = 'Hello Warld'
+    }
+
+    await expect(page.locator('.hello-world')).toHaveText([
+      textToAssert
+    ])
+  })
+})

integration-tests/ci-visibility/test-management/test-attempt-to-fix-1.js

@@ -0,0 +1,21 @@
+const { expect } = require('chai')
+
+let numAttempts = 0
+
+describe('attempt to fix tests', () => {
+  it('can attempt to fix a test', () => {
+    // eslint-disable-next-line no-console
+    console.log('I am running when attempt to fix') // to check if this is being run
+    if (process.env.SHOULD_ALWAYS_PASS) {
+      expect(1 + 2).to.equal(3)
+    } else if (process.env.SHOULD_FAIL_SOMETIMES) {
+      if (numAttempts++ % 2 === 0) {
+        expect(1 + 2).to.equal(3)
+      } else {
+        expect(1 + 2).to.equal(4)
+      }
+    } else {
+      expect(1 + 2).to.equal(4)
+    }
+  })
+})

integration-tests/ci-visibility/test-management/test-attempt-to-fix-2.js

@@ -0,0 +1,9 @@
+const { expect } = require('chai')
+
+describe('attempt to fix tests 2', () => {
+  it('can attempt to fix a test', () => {
+    // eslint-disable-next-line no-console
+    console.log('I am running when attempt to fix 2') // to check if this is being run
+    expect(1 + 2).to.equal(3)
+  })
+})

integration-tests/ci-visibility/vitest-tests/test-attempt-to-fix.mjs

@@ -0,0 +1,22 @@
+import { describe, test, expect } from 'vitest'
+
+let numAttempt = 0
+
+describe('attempt to fix tests', () => {
+  test('can attempt to fix a test', () => {
+    // eslint-disable-next-line no-console
+    console.log('I am running') // to check if this is being run
+    if (process.env.SHOULD_ALWAYS_PASS) {
+      expect(1 + 2).to.equal(3)
+    } else if (process.env.SHOULD_FAIL_SOMETIMES) {
+      // We need the last attempt to fail for the exit code to be 1
+      if (numAttempt++ % 2 === 1) {
+        expect(1 + 2).to.equal(4)
+      } else {
+        expect(1 + 2).to.equal(3)
+      }
+    } else {
+      expect(1 + 2).to.equal(4)
+    }
+  })
+})