diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b9b15ac9b18..fddfcb60d0a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -79,7 +79,8 @@ updates: - "@datadog/wasm-js-rewriter" - "@opentelemetry/api" - package-ecosystem: "npm" - directory: "/packages/dd-trace/test/plugins/versions" + directories: + - "/packages/dd-trace/test/plugins/versions" schedule: interval: "daily" open-pull-requests-limit: 1 @@ -92,3 +93,22 @@ updates: test-versions: patterns: - "*" + - package-ecosystem: "npm" + directories: + - "/integration-tests/esbuild" + schedule: + interval: "daily" + open-pull-requests-limit: 1 + labels: + - dependabot + - dependencies + - javascript + - semver-patch + ignore: + - dependency-name: "express" + # Update express manually for now due to esbuild breaking otherwise + update-types: ["version-update:semver-major"] + groups: + test-versions: + patterns: + - "*" diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index f1cc093e83e..e8f49df9e23 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -31,7 +31,7 @@ jobs: main: needs: - build-artifacts - uses: DataDog/system-tests/.github/workflows/system-tests.yml@b2523d82a7fcffb5ca642ee7b76eb476fbef04fe + uses: DataDog/system-tests/.github/workflows/system-tests.yml@main secrets: inherit permissions: contents: read diff --git a/.github/workflows/test-optimization.yml b/.github/workflows/test-optimization.yml index 744954db97e..cfd691576ff 100644 --- a/.github/workflows/test-optimization.yml +++ b/.github/workflows/test-optimization.yml @@ -120,8 +120,9 @@ jobs: version: [eol, oldest, latest] # 6.7.0 is the minimum version we support in <=5 # 10.2.0 is the minimum version we support in >=6 + # 14.5.4 is the latest version that supports Node 18 # The logic to decide whether the tests run lives in integration-tests/cypress/cypress.spec.js - cypress-version: [6.7.0, 10.2.0, latest] + cypress-version: [6.7.0, 10.2.0, 14.5.4, latest] module-type: ['commonJS', 'esm'] runs-on: ubuntu-latest env: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f51e47d2f82..ac8620e411d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,15 +5,15 @@ stages: - single-step-instrumentation-tests - macrobenchmarks -check_config_inversion_local_file: +validate_supported_configurations_local_file: rules: - when: on_success - extends: .check_config_inversion_local_file + extends: .validate_supported_configurations_local_file variables: LOCAL_JSON_PATH: "packages/dd-trace/src/supported-configurations.json" -config-inversion-update-supported-range: - extends: .config_inversion_update_central +update_central_configurations_version_range: + extends: .update_central_configurations_version_range variables: LOCAL_REPO_NAME: "dd-trace-js" LOCAL_JSON_PATH: "packages/dd-trace/src/supported-configurations.json" diff --git a/.gitlab/one-pipeline.locked.yml b/.gitlab/one-pipeline.locked.yml index b8554f3ae89..a385ff771db 100644 --- a/.gitlab/one-pipeline.locked.yml +++ b/.gitlab/one-pipeline.locked.yml @@ -1,4 +1,4 @@ # DO NOT EDIT THIS FILE MANUALLY # This file is auto-generated by automation. include: - - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/50d49f6898ce86e93326856210e8ab6526895273cb6341ac2d7d0e6c1c14e31e/one-pipeline.yml + - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/d5116fedde5478649c2f0cc00457f0f658d157e1bd8b3a7468bcaf5972c442a4/one-pipeline.yml diff --git a/docs/test.ts b/docs/test.ts index fd8e3570479..027f4993630 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -121,6 +121,8 @@ tracer.init({ }, apiSecurity: { enabled: true, + endpointCollectionEnabled: true, + endpointCollectionMessageLimit: 300 }, rasp: { enabled: true, diff --git a/index.d.ts b/index.d.ts index 31a0d62ebc1..441c09148d4 100644 --- a/index.d.ts +++ b/index.d.ts @@ -712,6 +712,16 @@ declare namespace tracer { * @default true */ enabled?: boolean, + + /** Whether to enable endpoint collection for API Security. + * @default true + */ + endpointCollectionEnabled?: boolean, + + /** Maximum number of endpoints that can be serialized per message. + * @default 300 + */ + endpointCollectionMessageLimit?: number, }, /** * Configuration for RASP diff --git a/integration-tests/appsec/endpoints-collection.spec.js b/integration-tests/appsec/endpoints-collection.spec.js new file mode 100644 index 00000000000..f5474c2352f --- /dev/null +++ b/integration-tests/appsec/endpoints-collection.spec.js @@ -0,0 +1,148 @@ +'use strict' + +const { createSandbox, FakeAgent, spawnProc } = require('../helpers') +const path = require('path') + +describe('Endpoints collection', () => { + let sandbox, cwd + + before(async function () { + this.timeout(process.platform === 'win32' ? 90000 : 30000) + + sandbox = await createSandbox( + ['fastify'], + false + ) + + cwd = sandbox.folder + }) + + after(async function () { + this.timeout(60000) + await sandbox.remove() + }) + + function getExpectedEndpoints (framework) { + const expectedEndpoints = [ + // Basic routes + { method: 'GET', path: '/users' }, + { method: 'HEAD', path: '/users' }, + { method: 'POST', path: '/users/' }, + { method: 'PUT', path: '/users/:id' }, + { method: 'DELETE', path: '/users/:id' }, + { method: 'PATCH', path: '/users/:id/:name' }, + { method: 'OPTIONS', path: '/users/:id?' }, + + // Route with regex + { method: 'DELETE', path: '/regex/:hour(^\\d{2})h:minute(^\\d{2})m' }, + + // Additional methods + { method: 'TRACE', path: '/trace-test' }, + { method: 'HEAD', path: '/head-test' }, + + // Custom method + { method: 'MKCOL', path: '/example/near/:lat-:lng/radius/:r' }, + + // Using app.route() + { method: 'POST', path: '/multi-method' }, + { method: 'PUT', path: '/multi-method' }, + { method: 'PATCH', path: '/multi-method' }, + + // All supported methods route + { method: 'GET', path: '/all-methods' }, + { method: 'HEAD', path: '/all-methods' }, + { method: 'TRACE', path: '/all-methods' }, + { method: 'DELETE', path: '/all-methods' }, + { method: 'OPTIONS', path: '/all-methods' }, + { method: 'PATCH', path: '/all-methods' }, + { method: 'PUT', path: '/all-methods' }, + { method: 'POST', path: '/all-methods' }, + { method: 'MKCOL', path: '/all-methods' }, // Added with addHttpMethod + + // Nested routes with Router + { method: 'PUT', path: '/v1/nested/:id' }, + + // Deeply nested routes + { method: 'GET', path: '/api/nested' }, + { method: 'HEAD', path: '/api/nested' }, + { method: 'GET', path: '/api/sub/deep' }, + { method: 'HEAD', path: '/api/sub/deep' }, + { method: 'POST', path: '/api/sub/deep/:id' }, + + // Wildcard routes + { method: 'GET', path: '/wildcard/*' }, + { method: 'HEAD', path: '/wildcard/*' }, + { method: 'GET', path: '*' }, + { method: 'HEAD', path: '*' }, + + { method: 'GET', path: '/later' }, + { method: 'HEAD', path: '/later' }, + ] + + return expectedEndpoints + } + + async function runEndpointTest (framework) { + let agent, proc + const appFile = path.join(cwd, 'appsec', 'endpoints-collection', `${framework}.js`) + + try { + agent = await new FakeAgent().start() + + const expectedEndpoints = getExpectedEndpoints(framework) + const endpointsFound = [] + const isFirstFlags = [] + + const telemetryPromise = agent.assertTelemetryReceived(({ payload }) => { + isFirstFlags.push(Boolean(payload.payload.is_first)) + + if (payload.payload.endpoints) { + payload.payload.endpoints.forEach(endpoint => { + endpointsFound.push({ + method: endpoint.method, + path: endpoint.path, + type: endpoint.type, + operation_name: endpoint.operation_name, + resource_name: endpoint.resource_name + }) + }) + } + }, 'app-endpoints', 5_000, 4) + + proc = await spawnProc(appFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + DD_TELEMETRY_HEARTBEAT_INTERVAL: 1, + DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT: '10' + } + }) + + await telemetryPromise + + const trueCount = isFirstFlags.filter(v => v === true).length + expect(trueCount).to.equal(1) + + // Check that all expected endpoints were found + expectedEndpoints.forEach(expected => { + const found = endpointsFound.find(e => + e.method === expected.method && e.path === expected.path + ) + expect(found).to.exist + expect(found.type).to.equal('REST') + expect(found.operation_name).to.equal('http.request') + expect(found.resource_name).to.equal(`${expected.method} ${expected.path}`) + }) + + // check that no additional endpoints were found + expect(endpointsFound.length).to.equal(expectedEndpoints.length) + } finally { + proc?.kill() + await agent?.stop() + } + } + + it('should send fastify endpoints via telemetry', async () => { + await runEndpointTest('fastify') + }) +}) diff --git a/integration-tests/appsec/endpoints-collection/fastify.js b/integration-tests/appsec/endpoints-collection/fastify.js new file mode 100644 index 00000000000..a8431f25002 --- /dev/null +++ b/integration-tests/appsec/endpoints-collection/fastify.js @@ -0,0 +1,67 @@ +'use strict' + +const tracer = require('dd-trace') +tracer.init({ + flushInterval: 0 +}) + +const fastify = require('fastify') +const app = fastify() + +// Basic routes +app.get('/users', async (_, reply) => reply.send('ok')) +app.post('/users/', async (_, reply) => reply.send('ok')) +app.put('/users/:id', async (_, reply) => reply.send('ok')) +app.delete('/users/:id', async (_, reply) => reply.send('ok')) +app.patch('/users/:id/:name', async (_, reply) => reply.send('ok')) +app.options('/users/:id?', async (_, reply) => reply.send('ok')) + +// Route with regex +app.delete('/regex/:hour(^\\d{2})h:minute(^\\d{2})m', async (_, reply) => reply.send('ok')) + +// Additional methods +app.trace('/trace-test', async (_, reply) => reply.send('ok')) +app.head('/head-test', async (_, reply) => reply.send('ok')) + +// Custom method +app.addHttpMethod('MKCOL', { hasBody: true }) +app.mkcol('/example/near/:lat-:lng/radius/:r', async (_, reply) => reply.send('ok')) + +// Using app.route() +app.route({ + method: ['POST', 'PUT', 'PATCH'], + url: '/multi-method', + handler: async (_, reply) => reply.send('ok') +}) + +// All supported methods route +app.all('/all-methods', async (_, reply) => reply.send('ok')) + +// Nested routes with Router +app.register(async function (router) { + router.put('/nested/:id', async (_, reply) => reply.send('ok')) +}, { prefix: '/v1' }) + +// Deeply nested routes +app.register(async function (router) { + router.get('/nested', async (_, reply) => reply.send('ok')) + router.register(async function (subRouter) { + subRouter.get('/deep', async (_, reply) => reply.send('ok')) + subRouter.post('/deep/:id', async (_, reply) => reply.send('ok')) + }, { prefix: '/sub' }) +}, { prefix: '/api' }) + +// Wildcard routes +app.get('/wildcard/*', async (_, reply) => reply.send('ok')) +app.get('*', async (_, reply) => reply.send('ok')) + +const start = async () => { + await app.listen({ port: 0, host: '127.0.0.1' }) + const port = app.server.address().port + process.send({ port }) +} + +setTimeout(() => { + app.get('/later', async (_, reply) => reply.send('ok')) + start() +}, 2e3) diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index 6eca643635b..e80c757ef5f 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -86,7 +86,8 @@ function shouldTestsRun (type) { return version === '6.7.0' && type === 'commonJS' } if (NODE_MAJOR > 16) { - return version === 'latest' + // Cypress 15.0.0 has removed support for Node 18 + return NODE_MAJOR > 18 ? version === 'latest' : version === '14.5.4' } } if (DD_MAJOR === 6) { @@ -94,7 +95,11 @@ function shouldTestsRun (type) { return false } if (NODE_MAJOR > 16) { - return version === '10.2.0' || version === 'latest' + // Cypress 15.0.0 has removed support for Node 18 + if (NODE_MAJOR <= 18) { + return version === '10.2.0' || version === '14.5.4' + } + return version === '10.2.0' || version === '14.5.4' || version === 'latest' } } return false diff --git a/integration-tests/esbuild/package.json b/integration-tests/esbuild/package.json index 3c9c8a9e157..6a3cee65671 100644 --- a/integration-tests/esbuild/package.json +++ b/integration-tests/esbuild/package.json @@ -20,14 +20,14 @@ "author": "Thomas Hunter II ", "license": "ISC", "dependencies": { - "@apollo/server": "*", - "@koa/router": "*", - "aws-sdk": "*", - "axios": "*", - "esbuild": "*", - "express": "^4.16.2", - "knex": "*", - "koa": "*", - "openai": "*" + "@apollo/server": "5.0.0", + "@koa/router": "14.0.0", + "aws-sdk": "2.1692.0", + "axios": "1.11.0", + "esbuild": "0.25.9", + "express": "4.21.2", + "knex": "3.1.0", + "koa": "3.0.1", + "openai": "5.15.0" } } diff --git a/integration-tests/log_injection.spec.js b/integration-tests/log_injection.spec.js new file mode 100644 index 00000000000..5553e00bea7 --- /dev/null +++ b/integration-tests/log_injection.spec.js @@ -0,0 +1,66 @@ +'use strict' + +const { FakeAgent, createSandbox, spawnProc, curlAndAssertMessage, assertObjectContains } = require('./helpers') +const path = require('path') +const { USER_KEEP } = require('../ext/priority') + +describe('Log Injection', () => { + let agent + let proc + let sandbox + let cwd + let app + let env + + before(async () => { + sandbox = await createSandbox(['express', 'winston']) + cwd = sandbox.folder + app = path.join(cwd, 'log_injection/index.js') + }) + + after(async () => { + await sandbox.remove() + }) + + beforeEach(async () => { + agent = await new FakeAgent().start() + }) + + describe('log injection with rule based sampling', () => { + beforeEach(async () => { + agent = await new FakeAgent().start() + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + it('should correctly apply rule based sampling when log injection is enabled', async () => { + env = { + AGENT_PORT: agent.port, + lOG_INJECTION: 'true' + } + proc = await spawnProc(app, { cwd, env, execArgv: [] }) + const url = proc.url + '/sampled' + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + // Bug: previously got USER_REJECT instead of USER_KEEP when log injection is enabled, + // meaning resource rules are not applied & instead global sampling is applied + // Now gets USER_KEEP because resource rules are applied + assertObjectContains(payload, [[{ metrics: { _sampling_priority_v1: USER_KEEP } }]]) + }, 20000, 1) + }) + + it('should correctly apply rule based sampling when log injection is disabled', async () => { + env = { + AGENT_PORT: agent.port, + lOG_INJECTION: 'false' + } + proc = await spawnProc(app, { cwd, env, execArgv: [] }) + const url = proc.url + '/sampled' + return curlAndAssertMessage(agent, url, ({ headers, payload }) => { + assertObjectContains(payload, [[{ metrics: { _sampling_priority_v1: USER_KEEP } }]]) + }, 20000, 1) + }) + }) +}) diff --git a/integration-tests/log_injection/index.js b/integration-tests/log_injection/index.js new file mode 100644 index 00000000000..d9b7b80cb89 --- /dev/null +++ b/integration-tests/log_injection/index.js @@ -0,0 +1,50 @@ +'use strict' + +const options = { + service: 'test-service', + sampleRate: 0.0, + samplingRules: [ + { + resource: 'GET /sampled', + sampleRate: 1.0 + } + ] +} + +if (process.env.AGENT_PORT) { + options.port = process.env.AGENT_PORT +} + +if (process.env.lOG_INJECTION) { + options.logInjection = process.env.lOG_INJECTION +} + +const tracer = require('dd-trace') +tracer.init(options) + +const express = require('express') +const winston = require('winston') + +const app = express() + +// Create winston logger +const logger = winston.createLogger({ + level: 'info', + format: winston.format.json(), + transports: [ + new winston.transports.Console({ silent: true }) + ] +}) + +// Route WITH logging (demonstrates the bug) +app.get('/sampled', (req, res) => { + // BUG: This winston.info() triggers log injection BEFORE resource.name is set + // which causes sampling decision to happen too early, bypassing the resource rule + logger.info('Processing GET /sampled request') + res.json({ message: 'logged request' }) +}) + +const server = app.listen(0, () => { + const port = server.address().port + process.send({ port }) +}) diff --git a/package.json b/package.json index 857aaf9267d..ca40a9f47f8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dd-trace", - "version": "5.63.3", + "version": "5.64.0", "description": "Datadog APM tracing client for JavaScript", "main": "index.js", "typings": "index.d.ts", diff --git a/packages/datadog-instrumentations/src/ai.js b/packages/datadog-instrumentations/src/ai.js index 3aec3d5fcdb..5d6f0750b6d 100644 --- a/packages/datadog-instrumentations/src/ai.js +++ b/packages/datadog-instrumentations/src/ai.js @@ -98,7 +98,18 @@ function wrapWithTracer (fn) { return function () { const options = arguments[0] - options.experimental_telemetry ??= { isEnabled: true, tracer: noopTracer } + const experimentalTelemetry = options.experimental_telemetry + if (experimentalTelemetry?.isEnabled === false) { + return fn.apply(this, arguments) + } + + if (experimentalTelemetry == null) { + options.experimental_telemetry = { isEnabled: true, tracer: noopTracer } + } else { + experimentalTelemetry.isEnabled = true + experimentalTelemetry.tracer ??= noopTracer + } + wrapTracer(options.experimental_telemetry.tracer) return fn.apply(this, arguments) diff --git a/packages/datadog-instrumentations/src/playwright.js b/packages/datadog-instrumentations/src/playwright.js index a54a007d0cb..a5326ddeeac 100644 --- a/packages/datadog-instrumentations/src/playwright.js +++ b/packages/datadog-instrumentations/src/playwright.js @@ -148,8 +148,8 @@ function getPlaywrightConfig (playwrightRunner) { } } -function getRootDir (playwrightRunner) { - const config = getPlaywrightConfig(playwrightRunner) +function getRootDir (playwrightRunner, configArg) { + const config = configArg?.config || getPlaywrightConfig(playwrightRunner) if (config.rootDir) { return config.rootDir } @@ -162,8 +162,8 @@ function getRootDir (playwrightRunner) { return process.cwd() } -function getProjectsFromRunner (runner) { - const config = getPlaywrightConfig(runner) +function getProjectsFromRunner (runner, configArg) { + const config = configArg?.projects ? configArg : getPlaywrightConfig(runner) return config.projects?.map((project) => { if (project.project) { return project.project @@ -509,11 +509,12 @@ function dispatcherHookNew (dispatcherExport, runWrapper) { return dispatcherExport } -function runnerHook (runnerExport, playwrightVersion) { - shimmer.wrap(runnerExport.Runner.prototype, 'runAllTests', runAllTests => async function () { +function runAllTestsWrapper (runAllTests, playwrightVersion) { + // Config parameter is only available from >=1.55.0 + return async function (config) { let onDone - rootDir = getRootDir(this) + rootDir = getRootDir(this, config) const processArgv = process.argv.slice(2).join(' ') const command = `playwright ${processArgv}` @@ -586,7 +587,7 @@ function runnerHook (runnerExport, playwrightVersion) { } } - const projects = getProjectsFromRunner(this) + const projects = getProjectsFromRunner(this, config) const shouldSetRetries = isFlakyTestRetriesEnabled && flakyTestRetriesCount > 0 && @@ -651,6 +652,23 @@ function runnerHook (runnerExport, playwrightVersion) { // TODO: we can trick playwright into thinking the session passed by returning // 'passed' here. We might be able to use this for both EFD and Test Management tests. return runAllTestsReturn + } +} + +function runnerHook (runnerExport, playwrightVersion) { + shimmer.wrap( + runnerExport.Runner.prototype, + 'runAllTests', + runAllTests => runAllTestsWrapper(runAllTests, playwrightVersion) + ) +} + +function runnerHookNew (runnerExport, playwrightVersion) { + runnerExport = shimmer.wrap(runnerExport, 'runAllTestsWithConfig', function (originalGetter) { + const originalFunction = originalGetter.call(this) + return function () { + return runAllTestsWrapper(originalFunction, playwrightVersion) + } }) return runnerExport @@ -694,6 +712,12 @@ addHook({ versions: ['>=1.38.0'] }, runnerHook) +addHook({ + name: 'playwright', + file: 'lib/runner/testRunner.js', + versions: ['>=1.55.0'] +}, runnerHookNew) + addHook({ name: 'playwright', file: 'lib/runner/dispatcher.js', diff --git a/packages/datadog-plugin-ai/test/index.spec.js b/packages/datadog-plugin-ai/test/index.spec.js index 96dea9bc25b..3ab16c92531 100644 --- a/packages/datadog-plugin-ai/test/index.spec.js +++ b/packages/datadog-plugin-ai/test/index.spec.js @@ -15,6 +15,32 @@ function getAiSdkOpenAiPackage (vercelAiVersion) { return semifies(vercelAiVersion, '>=5.0.0') ? '@ai-sdk/openai' : '@ai-sdk/openai@1.3.23' } +// making a different reference from the default no-op tracer in the instrumentation +// attempted to use the DD tracer provider, but it double-traces the request +// in practice, there is no need to pass in the DD OTel tracer provider, so this +// case shouldn't be an issue in practice +const myTracer = { + startActiveSpan () { + const fn = arguments[arguments.length - 1] + + const span = { + spanContext () { return { traceId: '', spanId: '', traceFlags: 0 } }, + setAttribute () { return this }, + setAttributes () { return this }, + addEvent () { return this }, + addLink () { return this }, + addLinks () { return this }, + setStatus () { return this }, + updateName () { return this }, + end () { return this }, + isRecording () { return false }, + recordException () { return this } + } + + return fn(span) + } +} + describe('Plugin', () => { useEnv({ OPENAI_API_KEY: '' @@ -38,6 +64,123 @@ describe('Plugin', () => { }) }) + describe('patching behavior with experimental_telemetry options', () => { + it('should not error when `isEnabled` is false', async () => { + const experimentalTelemetry = { isEnabled: false } + const result = await ai.generateText({ + model: openai('gpt-4o-mini'), + system: 'You are a helpful assistant', + prompt: 'Hello, OpenAI!', + maxTokens: 100, + temperature: 0.5, + experimental_telemetry: experimentalTelemetry + }) + + assert.ok(result.text, 'Expected result to be truthy') + assert.ok(experimentalTelemetry.tracer == null, 'Tracer should not be set by default') + }) + + it('should not error when a `tracer` is not passed in', async () => { + const checkTraces = agent.assertSomeTraces(traces => { + const generateTextSpan = traces[0][0] + const doGenerateSpan = traces[0][1] + + assert.strictEqual(generateTextSpan.name, 'ai.generateText') + assert.strictEqual(generateTextSpan.resource, 'ai.generateText') + assert.strictEqual(generateTextSpan.meta['ai.request.model'], 'gpt-4o-mini') + assert.strictEqual(generateTextSpan.meta['ai.request.model_provider'], 'openai') + + assert.strictEqual(doGenerateSpan.name, 'ai.generateText.doGenerate') + assert.strictEqual(doGenerateSpan.resource, 'ai.generateText.doGenerate') + assert.strictEqual(doGenerateSpan.meta['ai.request.model'], 'gpt-4o-mini') + assert.strictEqual(doGenerateSpan.meta['ai.request.model_provider'], 'openai') + }) + + const experimentalTelemetry = { isEnabled: true } + + const result = await ai.generateText({ + model: openai('gpt-4o-mini'), + system: 'You are a helpful assistant', + prompt: 'Hello, OpenAI!', + maxTokens: 100, + temperature: 0.5, + experimental_telemetry: experimentalTelemetry + }) + + assert.ok(result.text, 'Expected result to be truthy') + assert.ok(experimentalTelemetry.tracer != null, 'Tracer should be set when `isEnabled` is true') + + await checkTraces + }) + + it('should not error when only a `tracer` is not passed in', async () => { + const checkTraces = agent.assertSomeTraces(traces => { + const generateTextSpan = traces[0][0] + const doGenerateSpan = traces[0][1] + + assert.strictEqual(generateTextSpan.name, 'ai.generateText') + assert.strictEqual(generateTextSpan.resource, 'ai.generateText') + assert.strictEqual(generateTextSpan.meta['ai.request.model'], 'gpt-4o-mini') + assert.strictEqual(generateTextSpan.meta['ai.request.model_provider'], 'openai') + + assert.strictEqual(doGenerateSpan.name, 'ai.generateText.doGenerate') + assert.strictEqual(doGenerateSpan.resource, 'ai.generateText.doGenerate') + assert.strictEqual(doGenerateSpan.meta['ai.request.model'], 'gpt-4o-mini') + assert.strictEqual(doGenerateSpan.meta['ai.request.model_provider'], 'openai') + }) + + const experimentalTelemetry = { tracer: myTracer } + + const result = await ai.generateText({ + model: openai('gpt-4o-mini'), + system: 'You are a helpful assistant', + prompt: 'Hello, OpenAI!', + maxTokens: 100, + temperature: 0.5, + experimental_telemetry: experimentalTelemetry + }) + + assert.ok(result.text, 'Expected result to be truthy') + assert.ok(experimentalTelemetry.isEnabled, 'isEnabled should be set to true') + assert.ok(experimentalTelemetry.tracer === myTracer, 'Tracer should be set when `isEnabled` is true') + + await checkTraces + }) + + it('should use the passed in `tracer`', async () => { + const checkTraces = agent.assertSomeTraces(traces => { + const generateTextSpan = traces[0][0] + const doGenerateSpan = traces[0][1] + + assert.strictEqual(generateTextSpan.name, 'ai.generateText') + assert.strictEqual(generateTextSpan.resource, 'ai.generateText') + assert.strictEqual(generateTextSpan.meta['ai.request.model'], 'gpt-4o-mini') + assert.strictEqual(generateTextSpan.meta['ai.request.model_provider'], 'openai') + + assert.strictEqual(doGenerateSpan.name, 'ai.generateText.doGenerate') + assert.strictEqual(doGenerateSpan.resource, 'ai.generateText.doGenerate') + assert.strictEqual(doGenerateSpan.meta['ai.request.model'], 'gpt-4o-mini') + assert.strictEqual(doGenerateSpan.meta['ai.request.model_provider'], 'openai') + }) + + const experimentalTelemetry = { isEnabled: true, tracer: myTracer } + + const result = await ai.generateText({ + model: openai('gpt-4o-mini'), + system: 'You are a helpful assistant', + prompt: 'Hello, OpenAI!', + maxTokens: 100, + temperature: 0.5, + experimental_telemetry: experimentalTelemetry + }) + + assert.ok(result.text, 'Expected result to be truthy') + assert.ok(experimentalTelemetry.tracer === myTracer, 'Tracer should not override provided tracer') + + await checkTraces + }) + }) + it('creates a span for generateText', async () => { const checkTraces = agent.assertSomeTraces(traces => { const generateTextSpan = traces[0][0] diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 2a606430ea1..7ac39126c62 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -323,12 +323,6 @@ class Config { } } - if (typeof options.runtimeMetrics?.gc === 'boolean') { - options.runtimeMetrics.gc = { - enabled: options.runtimeMetrics.gc - } - } - const DD_INSTRUMENTATION_INSTALL_ID = coalesce( getEnvironmentVariable('DD_INSTRUMENTATION_INSTALL_ID'), null @@ -493,6 +487,8 @@ class Config { defaults.apmTracingEnabled = true defaults['appsec.apiSecurity.enabled'] = true defaults['appsec.apiSecurity.sampleDelay'] = 30 + defaults['appsec.apiSecurity.endpointCollectionEnabled'] = true + defaults['appsec.apiSecurity.endpointCollectionMessageLimit'] = 300 defaults['appsec.blockedTemplateGraphql'] = undefined defaults['appsec.blockedTemplateHtml'] = undefined defaults['appsec.blockedTemplateJson'] = undefined @@ -690,6 +686,8 @@ class Config { DD_AGENT_HOST, DD_API_SECURITY_ENABLED, DD_API_SECURITY_SAMPLE_DELAY, + DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED, + DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT, DD_APM_TRACING_ENABLED, DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE, DD_APPSEC_COLLECT_ALL_HEADERS, @@ -846,6 +844,10 @@ class Config { )) this._setBoolean(env, 'appsec.apiSecurity.enabled', DD_API_SECURITY_ENABLED && isTrue(DD_API_SECURITY_ENABLED)) env['appsec.apiSecurity.sampleDelay'] = maybeFloat(DD_API_SECURITY_SAMPLE_DELAY) + this._setBoolean(env, 'appsec.apiSecurity.endpointCollectionEnabled', + DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED) + env['appsec.apiSecurity.endpointCollectionMessageLimit'] = + maybeInt(DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT) env['appsec.blockedTemplateGraphql'] = maybeFile(DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON) env['appsec.blockedTemplateHtml'] = maybeFile(DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML) this._envUnprocessed['appsec.blockedTemplateHtml'] = DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML @@ -1073,6 +1075,10 @@ class Config { options.experimental?.appsec?.standalone && !options.experimental.appsec.standalone.enabled )) this._setBoolean(opts, 'appsec.apiSecurity.enabled', options.appsec?.apiSecurity?.enabled) + this._setBoolean(opts, 'appsec.apiSecurity.endpointCollectionEnabled', + options.appsec?.apiSecurity?.endpointCollectionEnabled) + opts['appsec.apiSecurity.endpointCollectionMessageLimit'] = + maybeInt(options.appsec?.apiSecurity?.endpointCollectionMessageLimit) opts['appsec.blockedTemplateGraphql'] = maybeFile(options.appsec?.blockedTemplateGraphql) opts['appsec.blockedTemplateHtml'] = maybeFile(options.appsec?.blockedTemplateHtml) this._optsUnprocessed['appsec.blockedTemplateHtml'] = options.appsec?.blockedTemplateHtml @@ -1190,7 +1196,7 @@ class Config { this._setBoolean(opts, 'reportHostname', options.reportHostname) this._setBoolean(opts, 'runtimeMetrics.enabled', options.runtimeMetrics?.enabled) this._setBoolean(opts, 'runtimeMetrics.eventLoop', options.runtimeMetrics?.eventLoop) - this._setBoolean(opts, 'runtimeMetrics.gc', options.runtimeMetrics?.gc?.enabled) + this._setBoolean(opts, 'runtimeMetrics.gc', options.runtimeMetrics?.gc) this._setBoolean(opts, 'runtimeMetricsRuntimeId', options.runtimeMetricsRuntimeId) this._setArray(opts, 'sampler.spanSamplingRules', reformatSpanSamplingRules(options.spanSamplingRules)) this._setUnit(opts, 'sampleRate', coalesce(options.sampleRate, options.ingestion.sampleRate)) diff --git a/packages/dd-trace/src/opentracing/tracer.js b/packages/dd-trace/src/opentracing/tracer.js index 87f96d9b45a..9fb8a467a80 100644 --- a/packages/dd-trace/src/opentracing/tracer.js +++ b/packages/dd-trace/src/opentracing/tracer.js @@ -86,7 +86,7 @@ class DatadogTracer { } try { - if (format !== 'text_map_dsm') { + if (format !== 'text_map_dsm' && format !== formats.LOG) { this._prioritySampler.sample(context) } this._propagators[format].inject(context, carrier) diff --git a/packages/dd-trace/src/runtime_metrics/runtime_metrics.js b/packages/dd-trace/src/runtime_metrics/runtime_metrics.js index ab7865093a9..a823e731845 100644 --- a/packages/dd-trace/src/runtime_metrics/runtime_metrics.js +++ b/packages/dd-trace/src/runtime_metrics/runtime_metrics.js @@ -274,9 +274,10 @@ function startGCObserver () { gcObserver = new PerformanceObserver(list => { for (const entry of list.getEntries()) { const type = gcType(entry.detail?.kind || entry.kind) + const duration = entry.duration * 1_000_000 - runtimeMetrics.histogram('runtime.node.gc.pause.by.type', entry.duration, `gc_type:${type}`) - runtimeMetrics.histogram('runtime.node.gc.pause', entry.duration) + runtimeMetrics.histogram('runtime.node.gc.pause.by.type', duration, `gc_type:${type}`) + runtimeMetrics.histogram('runtime.node.gc.pause', duration) } }) diff --git a/packages/dd-trace/src/supported-configurations.json b/packages/dd-trace/src/supported-configurations.json index 9a43e8fb436..c003595b36b 100644 --- a/packages/dd-trace/src/supported-configurations.json +++ b/packages/dd-trace/src/supported-configurations.json @@ -8,6 +8,8 @@ "DD_API_KEY": ["A"], "DD_API_SECURITY_ENABLED": ["A"], "DD_API_SECURITY_SAMPLE_DELAY": ["A"], + "DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED": ["A"], + "DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT": ["A"], "DD_APM_FLUSH_DEADLINE_MILLISECONDS": ["A"], "DD_APM_TRACING_ENABLED": ["A"], "DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE": ["A"], diff --git a/packages/dd-trace/src/telemetry/endpoints.js b/packages/dd-trace/src/telemetry/endpoints.js new file mode 100644 index 00000000000..27b7c7f26fa --- /dev/null +++ b/packages/dd-trace/src/telemetry/endpoints.js @@ -0,0 +1,124 @@ +'use strict' + +const dc = require('dc-polyfill') +const { sendData } = require('./send-data') + +const fastifyRouteCh = dc.channel('apm:fastify:route:added') + +let config +let application +let host +let getRetryData +let updateRetryData + +/** + * Keep track of endpoints that still need to be sent. + * Map key is `${METHOD} ${PATH}`, value is { method, path } + */ +const pendingEndpoints = new Map() +let flushScheduled = false +let isFirstPayload = true + +function endpointKey (method, path) { + return `${method.toUpperCase()} ${path}` +} + +function scheduleFlush () { + if (flushScheduled) return + flushScheduled = true + setImmediate(flushAndSend).unref() +} + +function recordEndpoint (method, path) { + const key = endpointKey(method, path) + if (pendingEndpoints.has(key)) return + + pendingEndpoints.set(key, { method: method.toUpperCase(), path }) + scheduleFlush() +} + +function onFastifyRoute (routeData) { + const routeOptions = routeData?.routeOptions + if (!routeOptions?.path) return + + const methods = Array.isArray(routeOptions.method) ? routeOptions.method : [routeOptions.method] + + for (const method of methods) { + recordEndpoint(method, routeOptions.path) + } +} + +function buildEndpointObjects (endpoints) { + return endpoints.map(({ method, path }) => { + return { + type: 'REST', + method, + path, + operation_name: 'http.request', + resource_name: endpointKey(method, path) + } + }) +} + +function flushAndSend () { + flushScheduled = false + if (pendingEndpoints.size === 0) return + + const batchEndpoints = [] + for (const [key, endpoint] of pendingEndpoints) { + batchEndpoints.push(endpoint) + pendingEndpoints.delete(key) + if (batchEndpoints.length >= config.appsec?.apiSecurity?.endpointCollectionMessageLimit) break + } + + const payloadObj = { + is_first: isFirstPayload, + endpoints: buildEndpointObjects(batchEndpoints) + } + + let reqType = 'app-endpoints' + let payload = payloadObj + + const retryData = getRetryData() + if (retryData) { + payload = [ + { request_type: 'app-endpoints', payload: payloadObj }, + { request_type: retryData.reqType, payload: retryData.payload } + ] + reqType = 'message-batch' + } + + sendData(config, application, host, reqType, payload, updateRetryData) + + if (isFirstPayload) { + isFirstPayload = false + } + + // If more endpoints accumulated while sending, schedule another flush. + if (pendingEndpoints.size) scheduleFlush() +} + +function start (_config = {}, _application, _host, getRetryDataFunction, updateRetryDataFunction) { + if (!_config?.appsec?.apiSecurity?.endpointCollectionEnabled) return + + config = _config + application = _application + host = _host + getRetryData = getRetryDataFunction + updateRetryData = updateRetryDataFunction + + fastifyRouteCh.subscribe(onFastifyRoute) +} + +function stop () { + fastifyRouteCh.unsubscribe(onFastifyRoute) + + pendingEndpoints.clear() + flushScheduled = false + config = application = host = getRetryData = updateRetryData = null +} + +module.exports = { + start, + stop +} diff --git a/packages/dd-trace/src/telemetry/telemetry.js b/packages/dd-trace/src/telemetry/telemetry.js index eb80303f44c..c21b1f39768 100644 --- a/packages/dd-trace/src/telemetry/telemetry.js +++ b/packages/dd-trace/src/telemetry/telemetry.js @@ -3,6 +3,7 @@ const tracerVersion = require('../../../../package.json').version const dc = require('dc-polyfill') const os = require('os') const dependencies = require('./dependencies') +const endpoints = require('./endpoints') const { sendData } = require('./send-data') const { errors } = require('../startup-log') const { manager: metricsManager } = require('./metrics') @@ -254,6 +255,7 @@ function start (aConfig, thePluginManager) { dependencies.start(config, application, host, getRetryData, updateRetryData) telemetryLogger.start(config) + endpoints.start(config, application, host, getRetryData, updateRetryData) sendData(config, application, host, 'app-started', appStarted(config)) @@ -280,6 +282,7 @@ function stop () { telemetryStopChannel.publish(getTelemetryData()) + endpoints.stop() config = undefined } diff --git a/packages/dd-trace/test/appsec/next/app-dir/next.config.js b/packages/dd-trace/test/appsec/next/app-dir/next.config.js index 40ee170113f..20f24204bfc 100644 --- a/packages/dd-trace/test/appsec/next/app-dir/next.config.js +++ b/packages/dd-trace/test/appsec/next/app-dir/next.config.js @@ -6,7 +6,8 @@ const nextConfig = { experimental: { appDir: true }, - output: 'standalone' + output: 'standalone', + outputFileTracingRoot: __dirname } module.exports = nextConfig diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index ee22eab971a..f88f3888b49 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -265,6 +265,8 @@ describe('Config', () => { expect(config).to.have.nested.property('apmTracingEnabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.sampleDelay', 30) + expect(config).to.have.nested.property('appsec.apiSecurity.endpointCollectionEnabled', true) + expect(config).to.have.nested.property('appsec.apiSecurity.endpointCollectionMessageLimit', 300) expect(config).to.have.nested.property('appsec.blockedTemplateHtml', undefined) expect(config).to.have.nested.property('appsec.blockedTemplateJson', undefined) expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', undefined) @@ -355,6 +357,10 @@ describe('Config', () => { expect(updateConfig.getCall(0).args[0]).to.deep.include.members([ { name: 'apmTracingEnabled', value: true, origin: 'default' }, + { name: 'appsec.apiSecurity.enabled', value: true, origin: 'default' }, + { name: 'appsec.apiSecurity.sampleDelay', value: 30, origin: 'default' }, + { name: 'appsec.apiSecurity.endpointCollectionEnabled', value: true, origin: 'default' }, + { name: 'appsec.apiSecurity.endpointCollectionMessageLimit', value: 300, origin: 'default' }, { name: 'appsec.blockedTemplateHtml', value: undefined, origin: 'default' }, { name: 'appsec.blockedTemplateJson', value: undefined, origin: 'default' }, { name: 'appsec.enabled', value: undefined, origin: 'default' }, @@ -530,6 +536,8 @@ describe('Config', () => { it('should initialize from environment variables', () => { process.env.DD_API_SECURITY_ENABLED = 'true' process.env.DD_API_SECURITY_SAMPLE_DELAY = '25' + process.env.DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED = 'false' + process.env.DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT = '500' process.env.DD_APM_TRACING_ENABLED = 'false' process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = 'extended' process.env.DD_APPSEC_COLLECT_ALL_HEADERS = 'true' @@ -647,6 +655,8 @@ describe('Config', () => { expect(config).to.have.nested.property('apmTracingEnabled', false) expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.sampleDelay', 25) + expect(config).to.have.nested.property('appsec.apiSecurity.endpointCollectionEnabled', false) + expect(config).to.have.nested.property('appsec.apiSecurity.endpointCollectionMessageLimit', 500) expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', BLOCKED_TEMPLATE_GRAPHQL) expect(config).to.have.nested.property('appsec.blockedTemplateHtml', BLOCKED_TEMPLATE_HTML) expect(config).to.have.nested.property('appsec.blockedTemplateJson', BLOCKED_TEMPLATE_JSON) @@ -754,6 +764,10 @@ describe('Config', () => { expect(updateConfig.getCall(0).args[0]).to.deep.include.members([ { name: 'apmTracingEnabled', value: false, origin: 'env_var' }, + { name: 'appsec.apiSecurity.enabled', value: true, origin: 'env_var' }, + { name: 'appsec.apiSecurity.sampleDelay', value: 25, origin: 'env_var' }, + { name: 'appsec.apiSecurity.endpointCollectionEnabled', value: false, origin: 'env_var' }, + { name: 'appsec.apiSecurity.endpointCollectionMessageLimit', value: 500, origin: 'env_var' }, { name: 'appsec.blockedTemplateHtml', value: BLOCKED_TEMPLATE_HTML_PATH, origin: 'env_var' }, { name: 'appsec.blockedTemplateJson', value: BLOCKED_TEMPLATE_JSON_PATH, origin: 'env_var' }, { name: 'appsec.enabled', value: true, origin: 'env_var' }, @@ -1370,6 +1384,8 @@ describe('Config', () => { it('should give priority to the options', () => { process.env.DD_API_KEY = '123' process.env.DD_API_SECURITY_ENABLED = 'false' + process.env.DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED = 'false' + process.env.DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT = '42' process.env.DD_APM_TRACING_ENABLED = 'false' process.env.DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE = 'disabled' process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = 'disabled' @@ -1441,7 +1457,9 @@ describe('Config', () => { apmTracingEnabled: true, appsec: { apiSecurity: { - enabled: true + enabled: true, + endpointCollectionEnabled: true, + endpointCollectionMessageLimit: 150 }, blockedTemplateGraphql: BLOCKED_TEMPLATE_GRAPHQL_PATH, blockedTemplateHtml: BLOCKED_TEMPLATE_HTML_PATH, @@ -1548,6 +1566,8 @@ describe('Config', () => { expect(config).to.have.nested.property('apmTracingEnabled', true) expect(config).to.have.nested.property('appsec.apiSecurity.enabled', true) + expect(config).to.have.nested.property('appsec.apiSecurity.endpointCollectionEnabled', true) + expect(config).to.have.nested.property('appsec.apiSecurity.endpointCollectionMessageLimit', 150) expect(config).to.have.nested.property('appsec.blockedTemplateGraphql', BLOCKED_TEMPLATE_GRAPHQL) expect(config).to.have.nested.property('appsec.blockedTemplateHtml', BLOCKED_TEMPLATE_HTML) expect(config).to.have.nested.property('appsec.blockedTemplateJson', BLOCKED_TEMPLATE_JSON) @@ -1624,7 +1644,9 @@ describe('Config', () => { const config = new Config({ appsec: { apiSecurity: { - enabled: true + enabled: true, + endpointCollectionEnabled: true, + endpointCollectionMessageLimit: 500 }, blockedTemplateGraphql: undefined, blockedTemplateHtml: undefined, @@ -1666,7 +1688,9 @@ describe('Config', () => { experimental: { appsec: { apiSecurity: { - enabled: false + enabled: false, + endpointCollectionEnabled: false, + endpointCollectionMessageLimit: 42 }, blockedTemplateGraphql: BLOCKED_TEMPLATE_GRAPHQL_PATH, blockedTemplateHtml: BLOCKED_TEMPLATE_HTML_PATH, @@ -1711,7 +1735,9 @@ describe('Config', () => { expect(config).to.have.deep.property('appsec', { apiSecurity: { enabled: true, - sampleDelay: 30 + sampleDelay: 30, + endpointCollectionEnabled: true, + endpointCollectionMessageLimit: 500 }, blockedTemplateGraphql: undefined, blockedTemplateHtml: undefined, diff --git a/packages/dd-trace/test/opentracing/tracer.spec.js b/packages/dd-trace/test/opentracing/tracer.spec.js index 31e3df79a33..e580e20c16e 100644 --- a/packages/dd-trace/test/opentracing/tracer.spec.js +++ b/packages/dd-trace/test/opentracing/tracer.spec.js @@ -5,6 +5,7 @@ require('../setup/tap') const opentracing = require('opentracing') const os = require('os') const SpanContext = require('../../src/opentracing/span_context') +const formats = require('../../../../ext/formats') const Reference = opentracing.Reference describe('Tracer', () => { @@ -25,6 +26,7 @@ describe('Tracer', () => { let TextMapPropagator let HttpPropagator let BinaryPropagator + let LogPropagator let propagator let config let log @@ -58,6 +60,7 @@ describe('Tracer', () => { TextMapPropagator = sinon.stub() HttpPropagator = sinon.stub() BinaryPropagator = sinon.stub() + LogPropagator = sinon.stub() propagator = { inject: sinon.stub(), extract: sinon.stub() @@ -90,6 +93,7 @@ describe('Tracer', () => { './propagation/text_map': TextMapPropagator, './propagation/http': HttpPropagator, './propagation/binary': BinaryPropagator, + './propagation/log': LogPropagator, '../log': log, '../exporter': exporter }) @@ -368,6 +372,16 @@ describe('Tracer', () => { expect(prioritySampler.sample).to.have.been.calledWith(spanContext) }) + + it('should not generate sampling priority for log injection', () => { + LogPropagator.returns(propagator) + + tracer = new Tracer(config) + tracer.inject(spanContext, formats.LOG, carrier) + + expect(prioritySampler.sample).to.not.have.been.called + expect(propagator.inject).to.have.been.calledWith(spanContext, carrier) + }) }) describe('extract', () => { diff --git a/packages/dd-trace/test/plugins/versions/package.json b/packages/dd-trace/test/plugins/versions/package.json index 4e03b9131b7..4e10ca2180a 100644 --- a/packages/dd-trace/test/plugins/versions/package.json +++ b/packages/dd-trace/test/plugins/versions/package.json @@ -4,24 +4,24 @@ "license": "BSD-3-Clause", "private": true, "dependencies": { - "@ai-sdk/openai": "2.0.15", + "@ai-sdk/openai": "2.0.20", "@apollo/gateway": "2.11.2", "@apollo/server": "5.0.0", "@apollo/subgraph": "2.11.2", - "@aws-sdk/client-bedrock-runtime": "3.864.0", - "@aws-sdk/client-dynamodb": "3.868.0", - "@aws-sdk/client-kinesis": "3.864.0", - "@aws-sdk/client-lambda": "3.865.0", - "@aws-sdk/client-s3": "3.864.0", - "@aws-sdk/client-sfn": "3.864.0", - "@aws-sdk/client-sns": "3.864.0", - "@aws-sdk/client-sqs": "3.864.0", + "@aws-sdk/client-bedrock-runtime": "3.873.0", + "@aws-sdk/client-dynamodb": "3.873.0", + "@aws-sdk/client-kinesis": "3.873.0", + "@aws-sdk/client-lambda": "3.873.0", + "@aws-sdk/client-s3": "3.873.0", + "@aws-sdk/client-sfn": "3.873.0", + "@aws-sdk/client-sns": "3.873.0", + "@aws-sdk/client-sqs": "3.873.0", "@aws-sdk/node-http-handler": "3.374.0", "@aws-sdk/smithy-client": "3.374.0", "@azure/functions": "4.7.3", "@azure/service-bus": "7.9.5", - "@confluentinc/kafka-javascript": "1.4.0", - "@cucumber/cucumber": "12.1.0", + "@confluentinc/kafka-javascript": "1.4.1", + "@cucumber/cucumber": "12.2.0", "@elastic/elasticsearch": "9.1.1", "@elastic/transport": "9.1.1", "@fastify/cookie": "11.0.2", @@ -42,9 +42,9 @@ "@koa/router": "14.0.0", "@langchain/anthropic": "0.3.26", "@langchain/cohere": "0.3.4", - "@langchain/core": "0.3.71", + "@langchain/core": "0.3.72", "@langchain/google-genai": "0.2.16", - "@langchain/openai": "0.6.7", + "@langchain/openai": "0.6.9", "@node-redis/client": "1.0.6", "@opensearch-project/opensearch": "3.5.1", "@opentelemetry/exporter-jaeger": "2.0.1", @@ -52,17 +52,17 @@ "@opentelemetry/instrumentation-express": "0.52.0", "@opentelemetry/instrumentation-http": "0.203.0", "@opentelemetry/sdk-node": "0.203.0", - "@playwright/test": "1.54.2", + "@playwright/test": "1.55.0", "@prisma/client": "6.14.0", - "@redis/client": "5.8.1", + "@redis/client": "5.8.2", "@smithy/smithy-client": "4.4.10", "@vitest/coverage-istanbul": "3.2.4", "@vitest/coverage-v8": "3.2.4", "@vitest/runner": "3.2.4", "aerospike": "6.3.0", - "ai": "5.0.15", + "ai": "5.0.23", "amqp10": "3.6.0", - "amqplib": "0.10.8", + "amqplib": "0.10.9", "apollo-server-core": "3.13.0", "apollo-server-express": "3.13.0", "apollo-server-fastify": "3.13.0", @@ -74,13 +74,13 @@ "bson": "6.10.4", "bunyan": "2.0.5", "cassandra-driver": "4.8.0", - "chai": "5.2.1", + "chai": "6.0.1", "collections": "5.1.13", "connect": "3.7.0", "cookie": "1.0.2", "cookie-parser": "1.4.7", "couchbase": "4.5.0", - "cypress": "14.5.4", + "cypress": "15.0.0", "cypress-fail-fast": "7.1.1", "dd-trace-api": "1.0.0", "ejs": "3.1.10", @@ -99,7 +99,7 @@ "graphql-yoga": "5.15.1", "handlebars": "4.7.8", "hapi": "18.1.0", - "hono": "4.9.2", + "hono": "4.9.4", "ioredis": "5.7.0", "iovalkey": "0.3.3", "jest": "30.0.5", @@ -117,7 +117,7 @@ "koa-router": "14.0.0", "koa-websocket": "7.0.0", "koalas": "1.0.2", - "langchain": "0.3.30", + "langchain": "0.3.31", "ldapjs": "3.0.7", "ldapjs-promise": "3.0.7", "limitd-client": "2.14.1", @@ -132,18 +132,18 @@ "moleculer": "0.14.35", "mongodb": "6.18.0", "mongodb-core": "3.2.7", - "mongoose": "8.17.1", + "mongoose": "8.18.0", "mquery": "5.0.0", "multer": "2.0.2", "mysql": "2.18.1", "mysql2": "3.14.3", - "next": "15.4.6", + "next": "15.5.0", "nock": "14.0.10", "node-serialize": "0.0.4", "npm": "11.5.2", "nyc": "17.1.0", "office-addin-mock": "3.0.3", - "openai": "5.12.2", + "openai": "5.15.0", "oracledb": "6.9.0", "passport": "0.7.0", "passport-http": "0.3.0", @@ -154,9 +154,9 @@ "pg-query-stream": "4.10.3", "pino": "9.9.0", "pino-pretty": "13.1.1", - "playwright": "1.54.2", - "playwright-core": "1.54.2", - "pnpm": "10.14.0", + "playwright": "1.55.0", + "playwright-core": "1.55.0", + "pnpm": "10.15.0", "promise": "8.3.0", "promise-js": "0.0.7", "protobufjs": "7.5.4", @@ -164,7 +164,7 @@ "q": "2.0.3", "react": "19.1.1", "react-dom": "19.1.1", - "redis": "5.8.1", + "redis": "5.8.2", "request": "2.88.2", "restify": "11.1.0", "rhea": "3.0.4", @@ -176,13 +176,13 @@ "sqlite3": "5.1.7", "tedious": "19.0.0", "typescript": "5.9.2", - "undici": "7.14.0", + "undici": "7.15.0", "vitest": "3.2.4", "when": "3.7.8", "winston": "3.17.0", "workerpool": "9.3.3", "ws": "8.18.3", "yarn": "1.22.22", - "zod": "4.0.17" + "zod": "4.1.1" } } diff --git a/packages/dd-trace/test/profiling/profilers/poisson.spec.js b/packages/dd-trace/test/profiling/profilers/poisson.spec.js index 89a5055115c..355694957e8 100644 --- a/packages/dd-trace/test/profiling/profilers/poisson.spec.js +++ b/packages/dd-trace/test/profiling/profilers/poisson.spec.js @@ -132,7 +132,7 @@ describe('PoissonProcessSamplingFilter', () => { assert.ok(filter.currentSamplingInstant >= prevNextSamplingInstant) assert.strictEqual(typeof filter.nextSamplingInstant, 'number') assert.ok(filter.nextSamplingInstant < 500000) - assert.ok(filter.samplingInstantCount < 10) + assert.ok(filter.samplingInstantCount < 30) }) it('should reset nextSamplingInstant if it is too far in the past', () => { diff --git a/packages/dd-trace/test/telemetry/endpoints.spec.js b/packages/dd-trace/test/telemetry/endpoints.spec.js new file mode 100644 index 00000000000..55ba1d98d09 --- /dev/null +++ b/packages/dd-trace/test/telemetry/endpoints.spec.js @@ -0,0 +1,197 @@ +'use strict' + +require('../setup/tap') + +const proxyquire = require('proxyquire') +const dc = require('dc-polyfill') + +const originalSetImmediate = global.setImmediate + +describe('endpoints telemetry', () => { + const fastifyRouteCh = dc.channel('apm:fastify:route:added') + const application = 'test' + const host = 'host' + + describe('start', () => { + const subscribe = sinon.stub() + const dc = { channel () { return { subscribe } } } + const endpoints = proxyquire('../../src/telemetry/endpoints', { + 'dc-polyfill': dc + }) + + beforeEach(() => { + sinon.reset() + }) + + it('should subscribe', () => { + const config = { appsec: { apiSecurity: { endpointCollectionEnabled: true } } } + endpoints.start(config) + + expect(subscribe).to.have.been.calledOnce + }) + + it('should not subscribe', () => { + const config = { appsec: { apiSecurity: { endpointCollectionEnabled: false } } } + endpoints.start(config) + + expect(subscribe).to.not.have.been.called + }) + }) + + describe('on events', () => { + let endpoints + let sendData + let getRetryData + let updateRetryData + let scheduledCallbacks + + beforeEach(() => { + sendData = sinon.stub() + getRetryData = sinon.stub() + updateRetryData = sinon.stub() + + endpoints = proxyquire('../../src/telemetry/endpoints', { + './send-data': { sendData } + }) + scheduledCallbacks = [] + global.setImmediate = function (callback) { + scheduledCallbacks.push(callback) + return { unref () {} } + } + + const config = { + appsec: { + apiSecurity: { + endpointCollectionEnabled: true, + endpointCollectionMessageLimit: 100 + } + } + } + + endpoints.start(config, application, host, getRetryData, updateRetryData) + }) + + afterEach(() => { + endpoints.stop() + sendData.reset() + getRetryData.reset() + updateRetryData.reset() + global.setImmediate = originalSetImmediate + }) + + it('should not fail with invalid data', () => { + fastifyRouteCh.publish(null) + fastifyRouteCh.publish({}) + fastifyRouteCh.publish({ routeOptions: {} }) + }) + + it('should record fastify methods array', () => { + fastifyRouteCh.publish({ routeOptions: { method: ['GET', 'post'], path: '/api' } }) + fastifyRouteCh.publish({ routeOptions: { method: 'GET', path: '/api' } }) + fastifyRouteCh.publish({ routeOptions: { method: 'POST', path: '/api' } }) + fastifyRouteCh.publish({ routeOptions: { method: 'PUT', path: '/api' } }) + + scheduledCallbacks.forEach(cb => cb()) + + expect(sendData).to.have.been.calledOnce + const payload = sendData.firstCall.args[4] + expect(payload.endpoints).to.have.deep.members([ + { + type: 'REST', + method: 'GET', + path: '/api', + operation_name: 'http.request', + resource_name: 'GET /api' + }, + { + type: 'REST', + method: 'POST', + path: '/api', + operation_name: 'http.request', + resource_name: 'POST /api' + }, + { + type: 'REST', + method: 'PUT', + path: '/api', + operation_name: 'http.request', + resource_name: 'PUT /api' + } + ]) + }) + + it('should set is_first=true only for the first payload', () => { + fastifyRouteCh.publish({ routeOptions: { method: 'GET', path: '/one' } }) + scheduledCallbacks.forEach(cb => cb()) + + fastifyRouteCh.publish({ routeOptions: { method: 'POST', path: '/two' } }) + scheduledCallbacks.forEach(cb => cb()) + + expect(sendData.callCount).to.equal(2) + const firstPayload = sendData.firstCall.args[4] + const secondPayload = sendData.secondCall.args[4] + + expect(firstPayload).to.have.property('is_first', true) + expect(Boolean(secondPayload.is_first)).to.equal(false) + }) + + it('should send large amount of endpoints in small batches', () => { + for (let i = 0; i < 150; i++) { + fastifyRouteCh.publish({ routeOptions: { method: 'GET', path: '/' + i } }) + } + + scheduledCallbacks.forEach(cb => cb()) + scheduledCallbacks.forEach(cb => cb()) + + expect(sendData.callCount).to.equal(2) + const firstPayload = sendData.firstCall.args[4] + const secondPayload = sendData.secondCall.args[4] + + expect(firstPayload.endpoints).to.have.length(100) + expect(secondPayload.endpoints).to.have.length(50) + }) + + describe('on failed request', () => { + let capturedRequestType + + beforeEach(() => { + capturedRequestType = undefined + + sendData.callsFake((config, application, host, reqType, payload, cb = () => {}) => { + capturedRequestType = reqType + cb(new Error('HTTP request error'), { payload, reqType }) + }) + }) + + it('should update retry data', () => { + fastifyRouteCh.publish({ routeOptions: { method: 'GET', path: '/r' } }) + + scheduledCallbacks.forEach(cb => cb()) + + expect(getRetryData).to.have.been.calledOnce + expect(capturedRequestType).to.equal('app-endpoints') + expect(updateRetryData).to.have.been.calledOnce + }) + + it('should create batch request when retry data exists', () => { + fastifyRouteCh.publish({ routeOptions: { method: 'GET', path: '/first' } }) + + scheduledCallbacks.forEach(cb => cb()) + + expect(getRetryData).to.have.been.calledOnce + expect(capturedRequestType).to.equal('app-endpoints') + + getRetryData.returns({ + reqType: 'app-endpoints', + payload: { endpoints: [] } + }) + + fastifyRouteCh.publish({ routeOptions: { method: 'POST', path: '/second' } }) + scheduledCallbacks.forEach(cb => cb()) + expect(getRetryData).to.have.been.calledTwice + expect(capturedRequestType).to.equal('message-batch') + expect(updateRetryData).to.have.been.calledTwice + }) + }) + }) +})