Endpoints discovery for fastify (#6258)

Author: IlyasShabi

docs/test.ts

@@ -121,6 +121,8 @@ tracer.init({
     },
     apiSecurity: {
       enabled: true,
+      endpointCollectionEnabled: true,
+      endpointCollectionMessageLimit: 300
     },
     rasp: {
       enabled: true,

index.d.ts

@@ -712,6 +712,16 @@ declare namespace tracer {
          * @default true
          */
         enabled?: boolean,
+
+        /** Whether to enable endpoint collection for API Security.
+         * @default true
+         */
+        endpointCollectionEnabled?: boolean,
+
+        /** Maximum number of endpoints that can be serialized per message.
+         * @default 300
+         */
+        endpointCollectionMessageLimit?: number,
       },
       /**
        * Configuration for RASP

integration-tests/appsec/endpoints-collection.spec.js

@@ -0,0 +1,148 @@
+'use strict'
+
+const { createSandbox, FakeAgent, spawnProc } = require('../helpers')
+const path = require('path')
+
+describe('Endpoints collection', () => {
+  let sandbox, cwd
+
+  before(async function () {
+    this.timeout(process.platform === 'win32' ? 90000 : 30000)
+
+    sandbox = await createSandbox(
+      ['fastify'],
+      false
+    )
+
+    cwd = sandbox.folder
+  })
+
+  after(async function () {
+    this.timeout(60000)
+    await sandbox.remove()
+  })
+
+  function getExpectedEndpoints (framework) {
+    const expectedEndpoints = [
+      // Basic routes
+      { method: 'GET', path: '/users' },
+      { method: 'HEAD', path: '/users' },
+      { method: 'POST', path: '/users/' },
+      { method: 'PUT', path: '/users/:id' },
+      { method: 'DELETE', path: '/users/:id' },
+      { method: 'PATCH', path: '/users/:id/:name' },
+      { method: 'OPTIONS', path: '/users/:id?' },
+
+      // Route with regex
+      { method: 'DELETE', path: '/regex/:hour(^\\d{2})h:minute(^\\d{2})m' },
+
+      // Additional methods
+      { method: 'TRACE', path: '/trace-test' },
+      { method: 'HEAD', path: '/head-test' },
+
+      // Custom method
+      { method: 'MKCOL', path: '/example/near/:lat-:lng/radius/:r' },
+
+      // Using app.route()
+      { method: 'POST', path: '/multi-method' },
+      { method: 'PUT', path: '/multi-method' },
+      { method: 'PATCH', path: '/multi-method' },
+
+      // All supported methods route
+      { method: 'GET', path: '/all-methods' },
+      { method: 'HEAD', path: '/all-methods' },
+      { method: 'TRACE', path: '/all-methods' },
+      { method: 'DELETE', path: '/all-methods' },
+      { method: 'OPTIONS', path: '/all-methods' },
+      { method: 'PATCH', path: '/all-methods' },
+      { method: 'PUT', path: '/all-methods' },
+      { method: 'POST', path: '/all-methods' },
+      { method: 'MKCOL', path: '/all-methods' }, // Added with addHttpMethod
+
+      // Nested routes with Router
+      { method: 'PUT', path: '/v1/nested/:id' },
+
+      // Deeply nested routes
+      { method: 'GET', path: '/api/nested' },
+      { method: 'HEAD', path: '/api/nested' },
+      { method: 'GET', path: '/api/sub/deep' },
+      { method: 'HEAD', path: '/api/sub/deep' },
+      { method: 'POST', path: '/api/sub/deep/:id' },
+
+      // Wildcard routes
+      { method: 'GET', path: '/wildcard/*' },
+      { method: 'HEAD', path: '/wildcard/*' },
+      { method: 'GET', path: '*' },
+      { method: 'HEAD', path: '*' },
+
+      { method: 'GET', path: '/later' },
+      { method: 'HEAD', path: '/later' },
+    ]
+
+    return expectedEndpoints
+  }
+
+  async function runEndpointTest (framework) {
+    let agent, proc
+    const appFile = path.join(cwd, 'appsec', 'endpoints-collection', `${framework}.js`)
+
+    try {
+      agent = await new FakeAgent().start()
+
+      const expectedEndpoints = getExpectedEndpoints(framework)
+      const endpointsFound = []
+      const isFirstFlags = []
+
+      const telemetryPromise = agent.assertTelemetryReceived(({ payload }) => {
+        isFirstFlags.push(Boolean(payload.payload.is_first))
+
+        if (payload.payload.endpoints) {
+          payload.payload.endpoints.forEach(endpoint => {
+            endpointsFound.push({
+              method: endpoint.method,
+              path: endpoint.path,
+              type: endpoint.type,
+              operation_name: endpoint.operation_name,
+              resource_name: endpoint.resource_name
+            })
+          })
+        }
+      }, 'app-endpoints', 5_000, 4)
+
+      proc = await spawnProc(appFile, {
+        cwd,
+        env: {
+          DD_TRACE_AGENT_PORT: agent.port,
+          DD_TELEMETRY_HEARTBEAT_INTERVAL: 1,
+          DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT: '10'
+        }
+      })
+
+      await telemetryPromise
+
+      const trueCount = isFirstFlags.filter(v => v === true).length
+      expect(trueCount).to.equal(1)
+
+      // Check that all expected endpoints were found
+      expectedEndpoints.forEach(expected => {
+        const found = endpointsFound.find(e =>
+          e.method === expected.method && e.path === expected.path
+        )
+        expect(found).to.exist
+        expect(found.type).to.equal('REST')
+        expect(found.operation_name).to.equal('http.request')
+        expect(found.resource_name).to.equal(`${expected.method} ${expected.path}`)
+      })
+
+      // check that no additional endpoints were found
+      expect(endpointsFound.length).to.equal(expectedEndpoints.length)
+    } finally {
+      proc?.kill()
+      await agent?.stop()
+    }
+  }
+
+  it('should send fastify endpoints via telemetry', async () => {
+    await runEndpointTest('fastify')
+  })
+})

integration-tests/appsec/endpoints-collection/fastify.js

@@ -0,0 +1,67 @@
+'use strict'
+
+const tracer = require('dd-trace')
+tracer.init({
+  flushInterval: 0
+})
+
+const fastify = require('fastify')
+const app = fastify()
+
+// Basic routes
+app.get('/users', async (_, reply) => reply.send('ok'))
+app.post('/users/', async (_, reply) => reply.send('ok'))
+app.put('/users/:id', async (_, reply) => reply.send('ok'))
+app.delete('/users/:id', async (_, reply) => reply.send('ok'))
+app.patch('/users/:id/:name', async (_, reply) => reply.send('ok'))
+app.options('/users/:id?', async (_, reply) => reply.send('ok'))
+
+// Route with regex
+app.delete('/regex/:hour(^\\d{2})h:minute(^\\d{2})m', async (_, reply) => reply.send('ok'))
+
+// Additional methods
+app.trace('/trace-test', async (_, reply) => reply.send('ok'))
+app.head('/head-test', async (_, reply) => reply.send('ok'))
+
+// Custom method
+app.addHttpMethod('MKCOL', { hasBody: true })
+app.mkcol('/example/near/:lat-:lng/radius/:r', async (_, reply) => reply.send('ok'))
+
+// Using app.route()
+app.route({
+  method: ['POST', 'PUT', 'PATCH'],
+  url: '/multi-method',
+  handler: async (_, reply) => reply.send('ok')
+})
+
+// All supported methods route
+app.all('/all-methods', async (_, reply) => reply.send('ok'))
+
+// Nested routes with Router
+app.register(async function (router) {
+  router.put('/nested/:id', async (_, reply) => reply.send('ok'))
+}, { prefix: '/v1' })
+
+// Deeply nested routes
+app.register(async function (router) {
+  router.get('/nested', async (_, reply) => reply.send('ok'))
+  router.register(async function (subRouter) {
+    subRouter.get('/deep', async (_, reply) => reply.send('ok'))
+    subRouter.post('/deep/:id', async (_, reply) => reply.send('ok'))
+  }, { prefix: '/sub' })
+}, { prefix: '/api' })
+
+// Wildcard routes
+app.get('/wildcard/*', async (_, reply) => reply.send('ok'))
+app.get('*', async (_, reply) => reply.send('ok'))
+
+const start = async () => {
+  await app.listen({ port: 0, host: '127.0.0.1' })
+  const port = app.server.address().port
+  process.send({ port })
+}
+
+setTimeout(() => {
+  app.get('/later', async (_, reply) => reply.send('ok'))
+  start()
+}, 2e3)

packages/dd-trace/src/config.js

@@ -493,6 +493,8 @@ class Config {
     defaults.apmTracingEnabled = true
     defaults['appsec.apiSecurity.enabled'] = true
     defaults['appsec.apiSecurity.sampleDelay'] = 30
+    defaults['appsec.apiSecurity.endpointCollectionEnabled'] = true
+    defaults['appsec.apiSecurity.endpointCollectionMessageLimit'] = 300
     defaults['appsec.blockedTemplateGraphql'] = undefined
     defaults['appsec.blockedTemplateHtml'] = undefined
     defaults['appsec.blockedTemplateJson'] = undefined
@@ -690,6 +692,8 @@ class Config {
       DD_AGENT_HOST,
       DD_API_SECURITY_ENABLED,
       DD_API_SECURITY_SAMPLE_DELAY,
+      DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED,
+      DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT,
       DD_APM_TRACING_ENABLED,
       DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE,
       DD_APPSEC_COLLECT_ALL_HEADERS,
@@ -846,6 +850,10 @@ class Config {
     ))
     this._setBoolean(env, 'appsec.apiSecurity.enabled', DD_API_SECURITY_ENABLED && isTrue(DD_API_SECURITY_ENABLED))
     env['appsec.apiSecurity.sampleDelay'] = maybeFloat(DD_API_SECURITY_SAMPLE_DELAY)
+    this._setBoolean(env, 'appsec.apiSecurity.endpointCollectionEnabled',
+      DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED)
+    env['appsec.apiSecurity.endpointCollectionMessageLimit'] =
+      maybeInt(DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT)
     env['appsec.blockedTemplateGraphql'] = maybeFile(DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON)
     env['appsec.blockedTemplateHtml'] = maybeFile(DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML)
     this._envUnprocessed['appsec.blockedTemplateHtml'] = DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML
@@ -1073,6 +1081,10 @@ class Config {
       options.experimental?.appsec?.standalone && !options.experimental.appsec.standalone.enabled
     ))
     this._setBoolean(opts, 'appsec.apiSecurity.enabled', options.appsec?.apiSecurity?.enabled)
+    this._setBoolean(opts, 'appsec.apiSecurity.endpointCollectionEnabled',
+      options.appsec?.apiSecurity?.endpointCollectionEnabled)
+    opts['appsec.apiSecurity.endpointCollectionMessageLimit'] =
+      maybeInt(options.appsec?.apiSecurity?.endpointCollectionMessageLimit)
     opts['appsec.blockedTemplateGraphql'] = maybeFile(options.appsec?.blockedTemplateGraphql)
     opts['appsec.blockedTemplateHtml'] = maybeFile(options.appsec?.blockedTemplateHtml)
     this._optsUnprocessed['appsec.blockedTemplateHtml'] = options.appsec?.blockedTemplateHtml

packages/dd-trace/src/supported-configurations.json

@@ -8,6 +8,8 @@
     "DD_API_KEY": ["A"],
     "DD_API_SECURITY_ENABLED": ["A"],
     "DD_API_SECURITY_SAMPLE_DELAY": ["A"],
+    "DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED": ["A"],
+    "DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT": ["A"],
     "DD_APM_FLUSH_DEADLINE_MILLISECONDS": ["A"],
     "DD_APM_TRACING_ENABLED": ["A"],
     "DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE": ["A"],