v5.64.0 proposal

.github/dependabot.yml

@@ -79,7 +79,8 @@ updates:
           - "@datadog/wasm-js-rewriter"
           - "@opentelemetry/api"
   - package-ecosystem: "npm"
-    directory: "/packages/dd-trace/test/plugins/versions"
+    directories:
+      - "/packages/dd-trace/test/plugins/versions"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 1
@@ -92,3 +93,22 @@ updates:
       test-versions:
         patterns:
           - "*"
+  - package-ecosystem: "npm"
+    directories:
+      - "/integration-tests/esbuild"
+    schedule:
+      interval: "daily"
+    open-pull-requests-limit: 1
+    labels:
+      - dependabot
+      - dependencies
+      - javascript
+      - semver-patch
+    ignore:
+      - dependency-name: "express"
+        # Update express manually for now due to esbuild breaking otherwise
+        update-types: ["version-update:semver-major"]
+    groups:
+      test-versions:
+        patterns:
+          - "*"

.github/workflows/system-tests.yml

@@ -31,7 +31,7 @@ jobs:
   main:
     needs:
       - build-artifacts
-    uses: DataDog/system-tests/.github/workflows/system-tests.yml@b2523d82a7fcffb5ca642ee7b76eb476fbef04fe
+    uses: DataDog/system-tests/.github/workflows/system-tests.yml@main
     secrets: inherit
     permissions:
       contents: read

.github/workflows/test-optimization.yml

@@ -120,8 +120,9 @@ jobs:
         version: [eol, oldest, latest]
         # 6.7.0 is the minimum version we support in <=5
         # 10.2.0 is the minimum version we support in >=6
+        # 14.5.4 is the latest version that supports Node 18
         # The logic to decide whether the tests run lives in integration-tests/cypress/cypress.spec.js
-        cypress-version: [6.7.0, 10.2.0, latest]
+        cypress-version: [6.7.0, 10.2.0, 14.5.4, latest]
         module-type: ['commonJS', 'esm']
     runs-on: ubuntu-latest
     env:

.gitlab-ci.yml

@@ -5,15 +5,15 @@ stages:
   - single-step-instrumentation-tests
   - macrobenchmarks
 
-check_config_inversion_local_file:
+validate_supported_configurations_local_file:
   rules:
     - when: on_success
-  extends: .check_config_inversion_local_file
+  extends: .validate_supported_configurations_local_file
   variables:
     LOCAL_JSON_PATH: "packages/dd-trace/src/supported-configurations.json"
 
-config-inversion-update-supported-range:
-  extends: .config_inversion_update_central
+update_central_configurations_version_range:
+  extends: .update_central_configurations_version_range
   variables:
     LOCAL_REPO_NAME: "dd-trace-js"
     LOCAL_JSON_PATH: "packages/dd-trace/src/supported-configurations.json"

.gitlab/one-pipeline.locked.yml

@@ -1,4 +1,4 @@
 # DO NOT EDIT THIS FILE MANUALLY
 # This file is auto-generated by automation.
 include:
-  - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/50d49f6898ce86e93326856210e8ab6526895273cb6341ac2d7d0e6c1c14e31e/one-pipeline.yml
+  - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/d5116fedde5478649c2f0cc00457f0f658d157e1bd8b3a7468bcaf5972c442a4/one-pipeline.yml

docs/test.ts

@@ -121,6 +121,8 @@ tracer.init({
     },
     apiSecurity: {
       enabled: true,
+      endpointCollectionEnabled: true,
+      endpointCollectionMessageLimit: 300
     },
     rasp: {
       enabled: true,

index.d.ts

@@ -712,6 +712,16 @@ declare namespace tracer {
          * @default true
          */
         enabled?: boolean,
+
+        /** Whether to enable endpoint collection for API Security.
+         * @default true
+         */
+        endpointCollectionEnabled?: boolean,
+
+        /** Maximum number of endpoints that can be serialized per message.
+         * @default 300
+         */
+        endpointCollectionMessageLimit?: number,
       },
       /**
        * Configuration for RASP

integration-tests/appsec/endpoints-collection.spec.js

@@ -0,0 +1,148 @@
+'use strict'
+
+const { createSandbox, FakeAgent, spawnProc } = require('../helpers')
+const path = require('path')
+
+describe('Endpoints collection', () => {
+  let sandbox, cwd
+
+  before(async function () {
+    this.timeout(process.platform === 'win32' ? 90000 : 30000)
+
+    sandbox = await createSandbox(
+      ['fastify'],
+      false
+    )
+
+    cwd = sandbox.folder
+  })
+
+  after(async function () {
+    this.timeout(60000)
+    await sandbox.remove()
+  })
+
+  function getExpectedEndpoints (framework) {
+    const expectedEndpoints = [
+      // Basic routes
+      { method: 'GET', path: '/users' },
+      { method: 'HEAD', path: '/users' },
+      { method: 'POST', path: '/users/' },
+      { method: 'PUT', path: '/users/:id' },
+      { method: 'DELETE', path: '/users/:id' },
+      { method: 'PATCH', path: '/users/:id/:name' },
+      { method: 'OPTIONS', path: '/users/:id?' },
+
+      // Route with regex
+      { method: 'DELETE', path: '/regex/:hour(^\\d{2})h:minute(^\\d{2})m' },
+
+      // Additional methods
+      { method: 'TRACE', path: '/trace-test' },
+      { method: 'HEAD', path: '/head-test' },
+
+      // Custom method
+      { method: 'MKCOL', path: '/example/near/:lat-:lng/radius/:r' },
+
+      // Using app.route()
+      { method: 'POST', path: '/multi-method' },
+      { method: 'PUT', path: '/multi-method' },
+      { method: 'PATCH', path: '/multi-method' },
+
+      // All supported methods route
+      { method: 'GET', path: '/all-methods' },
+      { method: 'HEAD', path: '/all-methods' },
+      { method: 'TRACE', path: '/all-methods' },
+      { method: 'DELETE', path: '/all-methods' },
+      { method: 'OPTIONS', path: '/all-methods' },
+      { method: 'PATCH', path: '/all-methods' },
+      { method: 'PUT', path: '/all-methods' },
+      { method: 'POST', path: '/all-methods' },
+      { method: 'MKCOL', path: '/all-methods' }, // Added with addHttpMethod
+
+      // Nested routes with Router
+      { method: 'PUT', path: '/v1/nested/:id' },
+
+      // Deeply nested routes
+      { method: 'GET', path: '/api/nested' },
+      { method: 'HEAD', path: '/api/nested' },
+      { method: 'GET', path: '/api/sub/deep' },
+      { method: 'HEAD', path: '/api/sub/deep' },
+      { method: 'POST', path: '/api/sub/deep/:id' },
+
+      // Wildcard routes
+      { method: 'GET', path: '/wildcard/*' },
+      { method: 'HEAD', path: '/wildcard/*' },
+      { method: 'GET', path: '*' },
+      { method: 'HEAD', path: '*' },
+
+      { method: 'GET', path: '/later' },
+      { method: 'HEAD', path: '/later' },
+    ]
+
+    return expectedEndpoints
+  }
+
+  async function runEndpointTest (framework) {
+    let agent, proc
+    const appFile = path.join(cwd, 'appsec', 'endpoints-collection', `${framework}.js`)
+
+    try {
+      agent = await new FakeAgent().start()
+
+      const expectedEndpoints = getExpectedEndpoints(framework)
+      const endpointsFound = []
+      const isFirstFlags = []
+
+      const telemetryPromise = agent.assertTelemetryReceived(({ payload }) => {
+        isFirstFlags.push(Boolean(payload.payload.is_first))
+
+        if (payload.payload.endpoints) {
+          payload.payload.endpoints.forEach(endpoint => {
+            endpointsFound.push({
+              method: endpoint.method,
+              path: endpoint.path,
+              type: endpoint.type,
+              operation_name: endpoint.operation_name,
+              resource_name: endpoint.resource_name
+            })
+          })
+        }
+      }, 'app-endpoints', 5_000, 4)
+
+      proc = await spawnProc(appFile, {
+        cwd,
+        env: {
+          DD_TRACE_AGENT_PORT: agent.port,
+          DD_TELEMETRY_HEARTBEAT_INTERVAL: 1,
+          DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT: '10'
+        }
+      })
+
+      await telemetryPromise
+
+      const trueCount = isFirstFlags.filter(v => v === true).length
+      expect(trueCount).to.equal(1)
+
+      // Check that all expected endpoints were found
+      expectedEndpoints.forEach(expected => {
+        const found = endpointsFound.find(e =>
+          e.method === expected.method && e.path === expected.path
+        )
+        expect(found).to.exist
+        expect(found.type).to.equal('REST')
+        expect(found.operation_name).to.equal('http.request')
+        expect(found.resource_name).to.equal(`${expected.method} ${expected.path}`)
+      })
+
+      // check that no additional endpoints were found
+      expect(endpointsFound.length).to.equal(expectedEndpoints.length)
+    } finally {
+      proc?.kill()
+      await agent?.stop()
+    }
+  }
+
+  it('should send fastify endpoints via telemetry', async () => {
+    await runEndpointTest('fastify')
+  })
+})

integration-tests/appsec/endpoints-collection/fastify.js

@@ -0,0 +1,67 @@
+'use strict'
+
+const tracer = require('dd-trace')
+tracer.init({
+  flushInterval: 0
+})
+
+const fastify = require('fastify')
+const app = fastify()
+
+// Basic routes
+app.get('/users', async (_, reply) => reply.send('ok'))
+app.post('/users/', async (_, reply) => reply.send('ok'))
+app.put('/users/:id', async (_, reply) => reply.send('ok'))
+app.delete('/users/:id', async (_, reply) => reply.send('ok'))
+app.patch('/users/:id/:name', async (_, reply) => reply.send('ok'))
+app.options('/users/:id?', async (_, reply) => reply.send('ok'))
+
+// Route with regex
+app.delete('/regex/:hour(^\\d{2})h:minute(^\\d{2})m', async (_, reply) => reply.send('ok'))
+
+// Additional methods
+app.trace('/trace-test', async (_, reply) => reply.send('ok'))
+app.head('/head-test', async (_, reply) => reply.send('ok'))
+
+// Custom method
+app.addHttpMethod('MKCOL', { hasBody: true })
+app.mkcol('/example/near/:lat-:lng/radius/:r', async (_, reply) => reply.send('ok'))
+
+// Using app.route()
+app.route({
+  method: ['POST', 'PUT', 'PATCH'],
+  url: '/multi-method',
+  handler: async (_, reply) => reply.send('ok')
+})
+
+// All supported methods route
+app.all('/all-methods', async (_, reply) => reply.send('ok'))
+
+// Nested routes with Router
+app.register(async function (router) {
+  router.put('/nested/:id', async (_, reply) => reply.send('ok'))
+}, { prefix: '/v1' })
+
+// Deeply nested routes
+app.register(async function (router) {
+  router.get('/nested', async (_, reply) => reply.send('ok'))
+  router.register(async function (subRouter) {
+    subRouter.get('/deep', async (_, reply) => reply.send('ok'))
+    subRouter.post('/deep/:id', async (_, reply) => reply.send('ok'))
+  }, { prefix: '/sub' })
+}, { prefix: '/api' })
+
+// Wildcard routes
+app.get('/wildcard/*', async (_, reply) => reply.send('ok'))
+app.get('*', async (_, reply) => reply.send('ok'))
+
+const start = async () => {
+  await app.listen({ port: 0, host: '127.0.0.1' })
+  const port = app.server.address().port
+  process.send({ port })
+}
+
+setTimeout(() => {
+  app.get('/later', async (_, reply) => reply.send('ok'))
+  start()
+}, 2e3)

integration-tests/cypress/cypress.spec.js

@@ -86,15 +86,20 @@ function shouldTestsRun (type) {
       return version === '6.7.0' && type === 'commonJS'
     }
     if (NODE_MAJOR > 16) {
-      return version === 'latest'
+      // Cypress 15.0.0 has removed support for Node 18
+      return NODE_MAJOR > 18 ? version === 'latest' : version === '14.5.4'
     }
   }
   if (DD_MAJOR === 6) {
     if (NODE_MAJOR <= 16) {
       return false
     }
     if (NODE_MAJOR > 16) {
-      return version === '10.2.0' || version === 'latest'
+      // Cypress 15.0.0 has removed support for Node 18
+      if (NODE_MAJOR <= 18) {
+        return version === '10.2.0' || version === '14.5.4'
+      }
+      return version === '10.2.0' || version === '14.5.4' || version === 'latest'
     }
   }
   return false

integration-tests/esbuild/package.json

@@ -20,14 +20,14 @@
   "author": "Thomas Hunter II <[email protected]>",
   "license": "ISC",
   "dependencies": {
-    "@apollo/server": "*",
-    "@koa/router": "*",
-    "aws-sdk": "*",
-    "axios": "*",
-    "esbuild": "*",
-    "express": "^4.16.2",
-    "knex": "*",
-    "koa": "*",
-    "openai": "*"
+    "@apollo/server": "5.0.0",
+    "@koa/router": "14.0.0",
+    "aws-sdk": "2.1692.0",
+    "axios": "1.11.0",
+    "esbuild": "0.25.9",
+    "express": "4.21.2",
+    "knex": "3.1.0",
+    "koa": "3.0.1",
+    "openai": "5.15.0"
   }
 }