diff --git a/docs/test.ts b/docs/test.ts index fd8e3570479..027f4993630 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -121,6 +121,8 @@ tracer.init({ }, apiSecurity: { enabled: true, + endpointCollectionEnabled: true, + endpointCollectionMessageLimit: 300 }, rasp: { enabled: true, diff --git a/index.d.ts b/index.d.ts index 31a0d62ebc1..441c09148d4 100644 --- a/index.d.ts +++ b/index.d.ts @@ -712,6 +712,16 @@ declare namespace tracer { * @default true */ enabled?: boolean, + + /** Whether to enable endpoint collection for API Security. + * @default true + */ + endpointCollectionEnabled?: boolean, + + /** Maximum number of endpoints that can be serialized per message. + * @default 300 + */ + endpointCollectionMessageLimit?: number, }, /** * Configuration for RASP diff --git a/integration-tests/appsec/endpoints-collection.spec.js b/integration-tests/appsec/endpoints-collection.spec.js new file mode 100644 index 00000000000..f5474c2352f --- /dev/null +++ b/integration-tests/appsec/endpoints-collection.spec.js @@ -0,0 +1,148 @@ +'use strict' + +const { createSandbox, FakeAgent, spawnProc } = require('../helpers') +const path = require('path') + +describe('Endpoints collection', () => { + let sandbox, cwd + + before(async function () { + this.timeout(process.platform === 'win32' ? 90000 : 30000) + + sandbox = await createSandbox( + ['fastify'], + false + ) + + cwd = sandbox.folder + }) + + after(async function () { + this.timeout(60000) + await sandbox.remove() + }) + + function getExpectedEndpoints (framework) { + const expectedEndpoints = [ + // Basic routes + { method: 'GET', path: '/users' }, + { method: 'HEAD', path: '/users' }, + { method: 'POST', path: '/users/' }, + { method: 'PUT', path: '/users/:id' }, + { method: 'DELETE', path: '/users/:id' }, + { method: 'PATCH', path: '/users/:id/:name' }, + { method: 'OPTIONS', path: '/users/:id?' }, + + // Route with regex + { method: 'DELETE', path: '/regex/:hour(^\\d{2})h:minute(^\\d{2})m' }, + + // Additional methods + { method: 'TRACE', path: '/trace-test' }, + { method: 'HEAD', path: '/head-test' }, + + // Custom method + { method: 'MKCOL', path: '/example/near/:lat-:lng/radius/:r' }, + + // Using app.route() + { method: 'POST', path: '/multi-method' }, + { method: 'PUT', path: '/multi-method' }, + { method: 'PATCH', path: '/multi-method' }, + + // All supported methods route + { method: 'GET', path: '/all-methods' }, + { method: 'HEAD', path: '/all-methods' }, + { method: 'TRACE', path: '/all-methods' }, + { method: 'DELETE', path: '/all-methods' }, + { method: 'OPTIONS', path: '/all-methods' }, + { method: 'PATCH', path: '/all-methods' }, + { method: 'PUT', path: '/all-methods' }, + { method: 'POST', path: '/all-methods' }, + { method: 'MKCOL', path: '/all-methods' }, // Added with addHttpMethod + + // Nested routes with Router + { method: 'PUT', path: '/v1/nested/:id' }, + + // Deeply nested routes + { method: 'GET', path: '/api/nested' }, + { method: 'HEAD', path: '/api/nested' }, + { method: 'GET', path: '/api/sub/deep' }, + { method: 'HEAD', path: '/api/sub/deep' }, + { method: 'POST', path: '/api/sub/deep/:id' }, + + // Wildcard routes + { method: 'GET', path: '/wildcard/*' }, + { method: 'HEAD', path: '/wildcard/*' }, + { method: 'GET', path: '*' }, + { method: 'HEAD', path: '*' }, + + { method: 'GET', path: '/later' }, + { method: 'HEAD', path: '/later' }, + ] + + return expectedEndpoints + } + + async function runEndpointTest (framework) { + let agent, proc + const appFile = path.join(cwd, 'appsec', 'endpoints-collection', `${framework}.js`) + + try { + agent = await new FakeAgent().start() + + const expectedEndpoints = getExpectedEndpoints(framework) + const endpointsFound = [] + const isFirstFlags = [] + + const telemetryPromise = agent.assertTelemetryReceived(({ payload }) => { + isFirstFlags.push(Boolean(payload.payload.is_first)) + + if (payload.payload.endpoints) { + payload.payload.endpoints.forEach(endpoint => { + endpointsFound.push({ + method: endpoint.method, + path: endpoint.path, + type: endpoint.type, + operation_name: endpoint.operation_name, + resource_name: endpoint.resource_name + }) + }) + } + }, 'app-endpoints', 5_000, 4) + + proc = await spawnProc(appFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + DD_TELEMETRY_HEARTBEAT_INTERVAL: 1, + DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT: '10' + } + }) + + await telemetryPromise + + const trueCount = isFirstFlags.filter(v => v === true).length + expect(trueCount).to.equal(1) + + // Check that all expected endpoints were found + expectedEndpoints.forEach(expected => { + const found = endpointsFound.find(e => + e.method === expected.method && e.path === expected.path + ) + expect(found).to.exist + expect(found.type).to.equal('REST') + expect(found.operation_name).to.equal('http.request') + expect(found.resource_name).to.equal(`${expected.method} ${expected.path}`) + }) + + // check that no additional endpoints were found + expect(endpointsFound.length).to.equal(expectedEndpoints.length) + } finally { + proc?.kill() + await agent?.stop() + } + } + + it('should send fastify endpoints via telemetry', async () => { + await runEndpointTest('fastify') + }) +}) diff --git a/integration-tests/appsec/endpoints-collection/fastify.js b/integration-tests/appsec/endpoints-collection/fastify.js new file mode 100644 index 00000000000..a8431f25002 --- /dev/null +++ b/integration-tests/appsec/endpoints-collection/fastify.js @@ -0,0 +1,67 @@ +'use strict' + +const tracer = require('dd-trace') +tracer.init({ + flushInterval: 0 +}) + +const fastify = require('fastify') +const app = fastify() + +// Basic routes +app.get('/users', async (_, reply) => reply.send('ok')) +app.post('/users/', async (_, reply) => reply.send('ok')) +app.put('/users/:id', async (_, reply) => reply.send('ok')) +app.delete('/users/:id', async (_, reply) => reply.send('ok')) +app.patch('/users/:id/:name', async (_, reply) => reply.send('ok')) +app.options('/users/:id?', async (_, reply) => reply.send('ok')) + +// Route with regex +app.delete('/regex/:hour(^\\d{2})h:minute(^\\d{2})m', async (_, reply) => reply.send('ok')) + +// Additional methods +app.trace('/trace-test', async (_, reply) => reply.send('ok')) +app.head('/head-test', async (_, reply) => reply.send('ok')) + +// Custom method +app.addHttpMethod('MKCOL', { hasBody: true }) +app.mkcol('/example/near/:lat-:lng/radius/:r', async (_, reply) => reply.send('ok')) + +// Using app.route() +app.route({ + method: ['POST', 'PUT', 'PATCH'], + url: '/multi-method', + handler: async (_, reply) => reply.send('ok') +}) + +// All supported methods route +app.all('/all-methods', async (_, reply) => reply.send('ok')) + +// Nested routes with Router +app.register(async function (router) { + router.put('/nested/:id', async (_, reply) => reply.send('ok')) +}, { prefix: '/v1' }) + +// Deeply nested routes +app.register(async function (router) { + router.get('/nested', async (_, reply) => reply.send('ok')) + router.register(async function (subRouter) { + subRouter.get('/deep', async (_, reply) => reply.send('ok')) + subRouter.post('/deep/:id', async (_, reply) => reply.send('ok')) + }, { prefix: '/sub' }) +}, { prefix: '/api' }) + +// Wildcard routes +app.get('/wildcard/*', async (_, reply) => reply.send('ok')) +app.get('*', async (_, reply) => reply.send('ok')) + +const start = async () => { + await app.listen({ port: 0, host: '127.0.0.1' }) + const port = app.server.address().port + process.send({ port }) +} + +setTimeout(() => { + app.get('/later', async (_, reply) => reply.send('ok')) + start() +}, 2e3) diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 2a606430ea1..097659c3dfd 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -493,6 +493,8 @@ class Config { defaults.apmTracingEnabled = true defaults['appsec.apiSecurity.enabled'] = true defaults['appsec.apiSecurity.sampleDelay'] = 30 + defaults['appsec.apiSecurity.endpointCollectionEnabled'] = true + defaults['appsec.apiSecurity.endpointCollectionMessageLimit'] = 300 defaults['appsec.blockedTemplateGraphql'] = undefined defaults['appsec.blockedTemplateHtml'] = undefined defaults['appsec.blockedTemplateJson'] = undefined @@ -690,6 +692,8 @@ class Config { DD_AGENT_HOST, DD_API_SECURITY_ENABLED, DD_API_SECURITY_SAMPLE_DELAY, + DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED, + DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT, DD_APM_TRACING_ENABLED, DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE, DD_APPSEC_COLLECT_ALL_HEADERS, @@ -846,6 +850,10 @@ class Config { )) this._setBoolean(env, 'appsec.apiSecurity.enabled', DD_API_SECURITY_ENABLED && isTrue(DD_API_SECURITY_ENABLED)) env['appsec.apiSecurity.sampleDelay'] = maybeFloat(DD_API_SECURITY_SAMPLE_DELAY) + this._setBoolean(env, 'appsec.apiSecurity.endpointCollectionEnabled', + DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED) + env['appsec.apiSecurity.endpointCollectionMessageLimit'] = + maybeInt(DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT) env['appsec.blockedTemplateGraphql'] = maybeFile(DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON) env['appsec.blockedTemplateHtml'] = maybeFile(DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML) this._envUnprocessed['appsec.blockedTemplateHtml'] = DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML @@ -1073,6 +1081,10 @@ class Config { options.experimental?.appsec?.standalone && !options.experimental.appsec.standalone.enabled )) this._setBoolean(opts, 'appsec.apiSecurity.enabled', options.appsec?.apiSecurity?.enabled) + this._setBoolean(opts, 'appsec.apiSecurity.endpointCollectionEnabled', + options.appsec?.apiSecurity?.endpointCollectionEnabled) + opts['appsec.apiSecurity.endpointCollectionMessageLimit'] = + maybeInt(options.appsec?.apiSecurity?.endpointCollectionMessageLimit) opts['appsec.blockedTemplateGraphql'] = maybeFile(options.appsec?.blockedTemplateGraphql) opts['appsec.blockedTemplateHtml'] = maybeFile(options.appsec?.blockedTemplateHtml) this._optsUnprocessed['appsec.blockedTemplateHtml'] = options.appsec?.blockedTemplateHtml diff --git a/packages/dd-trace/src/supported-configurations.json b/packages/dd-trace/src/supported-configurations.json index 9a43e8fb436..c003595b36b 100644 --- a/packages/dd-trace/src/supported-configurations.json +++ b/packages/dd-trace/src/supported-configurations.json @@ -8,6 +8,8 @@ "DD_API_KEY": ["A"], "DD_API_SECURITY_ENABLED": ["A"], "DD_API_SECURITY_SAMPLE_DELAY": ["A"], + "DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED": ["A"], + "DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT": ["A"], "DD_APM_FLUSH_DEADLINE_MILLISECONDS": ["A"], "DD_APM_TRACING_ENABLED": ["A"], "DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE": ["A"], diff --git a/packages/dd-trace/src/telemetry/endpoints.js b/packages/dd-trace/src/telemetry/endpoints.js new file mode 100644 index 00000000000..27b7c7f26fa --- /dev/null +++ b/packages/dd-trace/src/telemetry/endpoints.js @@ -0,0 +1,124 @@ +'use strict' + +const dc = require('dc-polyfill') +const { sendData } = require('./send-data') + +const fastifyRouteCh = dc.channel('apm:fastify:route:added') + +let config +let application +let host +let getRetryData +let updateRetryData + +/** + * Keep track of endpoints that still need to be sent. + * Map key is `${METHOD} ${PATH}`, value is { method, path } + */ +const pendingEndpoints = new Map() +let flushScheduled = false +let isFirstPayload = true + +function endpointKey (method, path) { + return `${method.toUpperCase()} ${path}` +} + +function scheduleFlush () { + if (flushScheduled) return + flushScheduled = true + setImmediate(flushAndSend).unref() +} + +function recordEndpoint (method, path) { + const key = endpointKey(method, path) + if (pendingEndpoints.has(key)) return + + pendingEndpoints.set(key, { method: method.toUpperCase(), path }) + scheduleFlush() +} + +function onFastifyRoute (routeData) { + const routeOptions = routeData?.routeOptions + if (!routeOptions?.path) return + + const methods = Array.isArray(routeOptions.method) ? routeOptions.method : [routeOptions.method] + + for (const method of methods) { + recordEndpoint(method, routeOptions.path) + } +} + +function buildEndpointObjects (endpoints) { + return endpoints.map(({ method, path }) => { + return { + type: 'REST', + method, + path, + operation_name: 'http.request', + resource_name: endpointKey(method, path) + } + }) +} + +function flushAndSend () { + flushScheduled = false + if (pendingEndpoints.size === 0) return + + const batchEndpoints = [] + for (const [key, endpoint] of pendingEndpoints) { + batchEndpoints.push(endpoint) + pendingEndpoints.delete(key) + if (batchEndpoints.length >= config.appsec?.apiSecurity?.endpointCollectionMessageLimit) break + } + + const payloadObj = { + is_first: isFirstPayload, + endpoints: buildEndpointObjects(batchEndpoints) + } + + let reqType = 'app-endpoints' + let payload = payloadObj + + const retryData = getRetryData() + if (retryData) { + payload = [ + { request_type: 'app-endpoints', payload: payloadObj }, + { request_type: retryData.reqType, payload: retryData.payload } + ] + reqType = 'message-batch' + } + + sendData(config, application, host, reqType, payload, updateRetryData) + + if (isFirstPayload) { + isFirstPayload = false + } + + // If more endpoints accumulated while sending, schedule another flush. + if (pendingEndpoints.size) scheduleFlush() +} + +function start (_config = {}, _application, _host, getRetryDataFunction, updateRetryDataFunction) { + if (!_config?.appsec?.apiSecurity?.endpointCollectionEnabled) return + + config = _config + application = _application + host = _host + getRetryData = getRetryDataFunction + updateRetryData = updateRetryDataFunction + + fastifyRouteCh.subscribe(onFastifyRoute) +} + +function stop () { + fastifyRouteCh.unsubscribe(onFastifyRoute) + + pendingEndpoints.clear() + flushScheduled = false + config = application = host = getRetryData = updateRetryData = null +} + +module.exports = { + start, + stop +} diff --git a/packages/dd-trace/src/telemetry/telemetry.js b/packages/dd-trace/src/telemetry/telemetry.js index eb80303f44c..c21b1f39768 100644 --- a/packages/dd-trace/src/telemetry/telemetry.js +++ b/packages/dd-trace/src/telemetry/telemetry.js @@ -3,6 +3,7 @@ const tracerVersion = require('../../../../package.json').version const dc = require('dc-polyfill') const os = require('os') const dependencies = require('./dependencies') +const endpoints = require('./endpoints') const { sendData } = require('./send-data') const { errors } = require('../startup-log') const { manager: metricsManager } = require('./metrics') @@ -254,6 +255,7 @@ function start (aConfig, thePluginManager) { dependencies.start(config, application, host, getRetryData, updateRetryData) telemetryLogger.start(config) + endpoints.start(config, application, host, getRetryData, updateRetryData) sendData(config, application, host, 'app-started', appStarted(config)) @@ -280,6 +282,7 @@ function stop () { telemetryStopChannel.publish(getTelemetryData()) + endpoints.stop() config = undefined } diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index ee22eab971a..f88f3888b49 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -265,6 +265,8 @@ describe('Config', () => { expect(config).to.have.nested.property('apmTracingEnabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.sampleDelay', 30) + expect(config).to.have.nested.property('appsec.apiSecurity.endpointCollectionEnabled', true) + expect(config).to.have.nested.property('appsec.apiSecurity.endpointCollectionMessageLimit', 300) expect(config).to.have.nested.property('appsec.blockedTemplateHtml', undefined) expect(config).to.have.nested.property('appsec.blockedTemplateJson', undefined) expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', undefined) @@ -355,6 +357,10 @@ describe('Config', () => { expect(updateConfig.getCall(0).args[0]).to.deep.include.members([ { name: 'apmTracingEnabled', value: true, origin: 'default' }, + { name: 'appsec.apiSecurity.enabled', value: true, origin: 'default' }, + { name: 'appsec.apiSecurity.sampleDelay', value: 30, origin: 'default' }, + { name: 'appsec.apiSecurity.endpointCollectionEnabled', value: true, origin: 'default' }, + { name: 'appsec.apiSecurity.endpointCollectionMessageLimit', value: 300, origin: 'default' }, { name: 'appsec.blockedTemplateHtml', value: undefined, origin: 'default' }, { name: 'appsec.blockedTemplateJson', value: undefined, origin: 'default' }, { name: 'appsec.enabled', value: undefined, origin: 'default' }, @@ -530,6 +536,8 @@ describe('Config', () => { it('should initialize from environment variables', () => { process.env.DD_API_SECURITY_ENABLED = 'true' process.env.DD_API_SECURITY_SAMPLE_DELAY = '25' + process.env.DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED = 'false' + process.env.DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT = '500' process.env.DD_APM_TRACING_ENABLED = 'false' process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = 'extended' process.env.DD_APPSEC_COLLECT_ALL_HEADERS = 'true' @@ -647,6 +655,8 @@ describe('Config', () => { expect(config).to.have.nested.property('apmTracingEnabled', false) expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.sampleDelay', 25) + expect(config).to.have.nested.property('appsec.apiSecurity.endpointCollectionEnabled', false) + expect(config).to.have.nested.property('appsec.apiSecurity.endpointCollectionMessageLimit', 500) expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', BLOCKED_TEMPLATE_GRAPHQL) expect(config).to.have.nested.property('appsec.blockedTemplateHtml', BLOCKED_TEMPLATE_HTML) expect(config).to.have.nested.property('appsec.blockedTemplateJson', BLOCKED_TEMPLATE_JSON) @@ -754,6 +764,10 @@ describe('Config', () => { expect(updateConfig.getCall(0).args[0]).to.deep.include.members([ { name: 'apmTracingEnabled', value: false, origin: 'env_var' }, + { name: 'appsec.apiSecurity.enabled', value: true, origin: 'env_var' }, + { name: 'appsec.apiSecurity.sampleDelay', value: 25, origin: 'env_var' }, + { name: 'appsec.apiSecurity.endpointCollectionEnabled', value: false, origin: 'env_var' }, + { name: 'appsec.apiSecurity.endpointCollectionMessageLimit', value: 500, origin: 'env_var' }, { name: 'appsec.blockedTemplateHtml', value: BLOCKED_TEMPLATE_HTML_PATH, origin: 'env_var' }, { name: 'appsec.blockedTemplateJson', value: BLOCKED_TEMPLATE_JSON_PATH, origin: 'env_var' }, { name: 'appsec.enabled', value: true, origin: 'env_var' }, @@ -1370,6 +1384,8 @@ describe('Config', () => { it('should give priority to the options', () => { process.env.DD_API_KEY = '123' process.env.DD_API_SECURITY_ENABLED = 'false' + process.env.DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED = 'false' + process.env.DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT = '42' process.env.DD_APM_TRACING_ENABLED = 'false' process.env.DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE = 'disabled' process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = 'disabled' @@ -1441,7 +1457,9 @@ describe('Config', () => { apmTracingEnabled: true, appsec: { apiSecurity: { - enabled: true + enabled: true, + endpointCollectionEnabled: true, + endpointCollectionMessageLimit: 150 }, blockedTemplateGraphql: BLOCKED_TEMPLATE_GRAPHQL_PATH, blockedTemplateHtml: BLOCKED_TEMPLATE_HTML_PATH, @@ -1548,6 +1566,8 @@ describe('Config', () => { expect(config).to.have.nested.property('apmTracingEnabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) + expect(config).to.have.nested.property('appsec.apiSecurity.endpointCollectionEnabled', true) + expect(config).to.have.nested.property('appsec.apiSecurity.endpointCollectionMessageLimit', 150) expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', BLOCKED_TEMPLATE_GRAPHQL) expect(config).to.have.nested.property('appsec.blockedTemplateHtml', BLOCKED_TEMPLATE_HTML) expect(config).to.have.nested.property('appsec.blockedTemplateJson', BLOCKED_TEMPLATE_JSON) @@ -1624,7 +1644,9 @@ describe('Config', () => { const config = new Config({ appsec: { apiSecurity: { - enabled: true + enabled: true, + endpointCollectionEnabled: true, + endpointCollectionMessageLimit: 500 }, blockedTemplateGraphql: undefined, blockedTemplateHtml: undefined, @@ -1666,7 +1688,9 @@ describe('Config', () => { experimental: { appsec: { apiSecurity: { - enabled: false + enabled: false, + endpointCollectionEnabled: false, + endpointCollectionMessageLimit: 42 }, blockedTemplateGraphql: BLOCKED_TEMPLATE_GRAPHQL_PATH, blockedTemplateHtml: BLOCKED_TEMPLATE_HTML_PATH, @@ -1711,7 +1735,9 @@ describe('Config', () => { expect(config).to.have.deep.property('appsec', { apiSecurity: { enabled: true, - sampleDelay: 30 + sampleDelay: 30, + endpointCollectionEnabled: true, + endpointCollectionMessageLimit: 500 }, blockedTemplateGraphql: undefined, blockedTemplateHtml: undefined, diff --git a/packages/dd-trace/test/telemetry/endpoints.spec.js b/packages/dd-trace/test/telemetry/endpoints.spec.js new file mode 100644 index 00000000000..55ba1d98d09 --- /dev/null +++ b/packages/dd-trace/test/telemetry/endpoints.spec.js @@ -0,0 +1,197 @@ +'use strict' + +require('../setup/tap') + +const proxyquire = require('proxyquire') +const dc = require('dc-polyfill') + +const originalSetImmediate = global.setImmediate + +describe('endpoints telemetry', () => { + const fastifyRouteCh = dc.channel('apm:fastify:route:added') + const application = 'test' + const host = 'host' + + describe('start', () => { + const subscribe = sinon.stub() + const dc = { channel () { return { subscribe } } } + const endpoints = proxyquire('../../src/telemetry/endpoints', { + 'dc-polyfill': dc + }) + + beforeEach(() => { + sinon.reset() + }) + + it('should subscribe', () => { + const config = { appsec: { apiSecurity: { endpointCollectionEnabled: true } } } + endpoints.start(config) + + expect(subscribe).to.have.been.calledOnce + }) + + it('should not subscribe', () => { + const config = { appsec: { apiSecurity: { endpointCollectionEnabled: false } } } + endpoints.start(config) + + expect(subscribe).to.not.have.been.called + }) + }) + + describe('on events', () => { + let endpoints + let sendData + let getRetryData + let updateRetryData + let scheduledCallbacks + + beforeEach(() => { + sendData = sinon.stub() + getRetryData = sinon.stub() + updateRetryData = sinon.stub() + + endpoints = proxyquire('../../src/telemetry/endpoints', { + './send-data': { sendData } + }) + scheduledCallbacks = [] + global.setImmediate = function (callback) { + scheduledCallbacks.push(callback) + return { unref () {} } + } + + const config = { + appsec: { + apiSecurity: { + endpointCollectionEnabled: true, + endpointCollectionMessageLimit: 100 + } + } + } + + endpoints.start(config, application, host, getRetryData, updateRetryData) + }) + + afterEach(() => { + endpoints.stop() + sendData.reset() + getRetryData.reset() + updateRetryData.reset() + global.setImmediate = originalSetImmediate + }) + + it('should not fail with invalid data', () => { + fastifyRouteCh.publish(null) + fastifyRouteCh.publish({}) + fastifyRouteCh.publish({ routeOptions: {} }) + }) + + it('should record fastify methods array', () => { + fastifyRouteCh.publish({ routeOptions: { method: ['GET', 'post'], path: '/api' } }) + fastifyRouteCh.publish({ routeOptions: { method: 'GET', path: '/api' } }) + fastifyRouteCh.publish({ routeOptions: { method: 'POST', path: '/api' } }) + fastifyRouteCh.publish({ routeOptions: { method: 'PUT', path: '/api' } }) + + scheduledCallbacks.forEach(cb => cb()) + + expect(sendData).to.have.been.calledOnce + const payload = sendData.firstCall.args[4] + expect(payload.endpoints).to.have.deep.members([ + { + type: 'REST', + method: 'GET', + path: '/api', + operation_name: 'http.request', + resource_name: 'GET /api' + }, + { + type: 'REST', + method: 'POST', + path: '/api', + operation_name: 'http.request', + resource_name: 'POST /api' + }, + { + type: 'REST', + method: 'PUT', + path: '/api', + operation_name: 'http.request', + resource_name: 'PUT /api' + } + ]) + }) + + it('should set is_first=true only for the first payload', () => { + fastifyRouteCh.publish({ routeOptions: { method: 'GET', path: '/one' } }) + scheduledCallbacks.forEach(cb => cb()) + + fastifyRouteCh.publish({ routeOptions: { method: 'POST', path: '/two' } }) + scheduledCallbacks.forEach(cb => cb()) + + expect(sendData.callCount).to.equal(2) + const firstPayload = sendData.firstCall.args[4] + const secondPayload = sendData.secondCall.args[4] + + expect(firstPayload).to.have.property('is_first', true) + expect(Boolean(secondPayload.is_first)).to.equal(false) + }) + + it('should send large amount of endpoints in small batches', () => { + for (let i = 0; i < 150; i++) { + fastifyRouteCh.publish({ routeOptions: { method: 'GET', path: '/' + i } }) + } + + scheduledCallbacks.forEach(cb => cb()) + scheduledCallbacks.forEach(cb => cb()) + + expect(sendData.callCount).to.equal(2) + const firstPayload = sendData.firstCall.args[4] + const secondPayload = sendData.secondCall.args[4] + + expect(firstPayload.endpoints).to.have.length(100) + expect(secondPayload.endpoints).to.have.length(50) + }) + + describe('on failed request', () => { + let capturedRequestType + + beforeEach(() => { + capturedRequestType = undefined + + sendData.callsFake((config, application, host, reqType, payload, cb = () => {}) => { + capturedRequestType = reqType + cb(new Error('HTTP request error'), { payload, reqType }) + }) + }) + + it('should update retry data', () => { + fastifyRouteCh.publish({ routeOptions: { method: 'GET', path: '/r' } }) + + scheduledCallbacks.forEach(cb => cb()) + + expect(getRetryData).to.have.been.calledOnce + expect(capturedRequestType).to.equal('app-endpoints') + expect(updateRetryData).to.have.been.calledOnce + }) + + it('should create batch request when retry data exists', () => { + fastifyRouteCh.publish({ routeOptions: { method: 'GET', path: '/first' } }) + + scheduledCallbacks.forEach(cb => cb()) + + expect(getRetryData).to.have.been.calledOnce + expect(capturedRequestType).to.equal('app-endpoints') + + getRetryData.returns({ + reqType: 'app-endpoints', + payload: { endpoints: [] } + }) + + fastifyRouteCh.publish({ routeOptions: { method: 'POST', path: '/second' } }) + scheduledCallbacks.forEach(cb => cb()) + expect(getRetryData).to.have.been.calledTwice + expect(capturedRequestType).to.equal('message-batch') + expect(updateRetryData).to.have.been.calledTwice + }) + }) + }) +})