diff --git a/.size-limit.js b/.size-limit.js index d53eaae56712..dd65a987d506 100644 --- a/.size-limit.js +++ b/.size-limit.js @@ -233,7 +233,7 @@ module.exports = [ import: createImport('init'), ignore: [...builtinModules, ...nodePrefixedBuiltinModules], gzip: true, - limit: '147 KB', + limit: '148 KB', }, { name: '@sentry/node - without tracing', diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-options.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-options.mjs new file mode 100644 index 000000000000..9344137a4ed3 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-options.mjs @@ -0,0 +1,18 @@ +import * as Sentry from '@sentry/node'; +import { nodeContextIntegration } from '@sentry/node-core'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + sendDefaultPii: false, + transport: loggingTransport, + integrations: [ + Sentry.anthropicAIIntegration({ + recordInputs: true, + recordOutputs: true, + }), + nodeContextIntegration(), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-pii.mjs new file mode 100644 index 000000000000..eb8b02b1cf8b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-pii.mjs @@ -0,0 +1,15 @@ +import * as Sentry from '@sentry/node'; +import { nodeContextIntegration } from '@sentry/node-core'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + sendDefaultPii: true, + transport: loggingTransport, + integrations: [ + Sentry.anthropicAIIntegration(), + nodeContextIntegration(), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument.mjs new file mode 100644 index 000000000000..fa011052c50c --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument.mjs @@ -0,0 +1,16 @@ +import * as Sentry from '@sentry/node'; +import { nodeContextIntegration } from '@sentry/node-core'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + sendDefaultPii: false, + transport: loggingTransport, + // Force include the integration + integrations: [ + Sentry.anthropicAIIntegration(), + nodeContextIntegration(), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario.mjs new file mode 100644 index 000000000000..425d1366879e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario.mjs @@ -0,0 +1,119 @@ +import { instrumentAnthropicAiClient } from '@sentry/core'; +import * as Sentry from '@sentry/node'; + +class MockAnthropic { + constructor(config) { + this.apiKey = config.apiKey; + + // Create messages object with create and countTokens methods + this.messages = { + create: this._messagesCreate.bind(this), + countTokens: this._messagesCountTokens.bind(this) + }; + + this.models = { + retrieve: this._modelsRetrieve.bind(this), + }; + } + + /** + * Create a mock message + */ + async _messagesCreate(params) { + // Simulate processing time + await new Promise(resolve => setTimeout(resolve, 10)); + + if (params.model === 'error-model') { + const error = new Error('Model not found'); + error.status = 404; + error.headers = { 'x-request-id': 'mock-request-123' }; + throw error; + } + + return { + id: 'msg_mock123', + type: 'message', + model: params.model, + role: 'assistant', + content: [ + { + type: 'text', + text: 'Hello from Anthropic mock!', + }, + ], + stop_reason: 'end_turn', + stop_sequence: null, + usage: { + input_tokens: 10, + output_tokens: 15, + }, + }; + } + + async _messagesCountTokens() { + // Simulate processing time + await new Promise(resolve => setTimeout(resolve, 10)); + + // For countTokens, just return input_tokens + return { + input_tokens: 15 + } + } + + async _modelsRetrieve(modelId) { + // Simulate processing time + await new Promise(resolve => setTimeout(resolve, 10)); + + // Match what the actual implementation would return + return { + id: modelId, + name: modelId, + created_at: 1715145600, + model: modelId, // Add model field to match the check in addResponseAttributes + }; + } +} + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const mockClient = new MockAnthropic({ + apiKey: 'mock-api-key', + }); + + const client = instrumentAnthropicAiClient(mockClient); + + // First test: basic message completion + await client.messages.create({ + model: 'claude-3-haiku-20240307', + system: 'You are a helpful assistant.', + messages: [ + { role: 'user', content: 'What is the capital of France?' }, + ], + temperature: 0.7, + max_tokens: 100, + }); + + // Second test: error handling + try { + await client.messages.create({ + model: 'error-model', + messages: [{ role: 'user', content: 'This will fail' }], + }); + } catch { + // Error is expected and handled + } + + // Third test: count tokens with cached tokens + await client.messages.countTokens({ + model: 'claude-3-haiku-20240307', + messages: [ + { role: 'user', content: 'What is the capital of France?' }, + ], + }); + + // Fourth test: models.retrieve + await client.models.retrieve('claude-3-haiku-20240307'); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts new file mode 100644 index 000000000000..0efaabd07e4a --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -0,0 +1,222 @@ +import { afterAll, describe, expect } from 'vitest'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; + +describe('Anthropic integration', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { + transaction: 'main', + spans: expect.arrayContaining([ + // First span - basic message completion without PII + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'messages.create', + 'sentry.op': 'gen_ai.messages.create', + 'sentry.origin': 'auto.ai.anthropic', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-haiku-20240307', + 'gen_ai.request.temperature': 0.7, + 'gen_ai.request.max_tokens': 100, + 'gen_ai.response.model': 'claude-3-haiku-20240307', + 'gen_ai.response.id': 'msg_mock123', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 15, + 'gen_ai.usage.total_tokens': 25, + }, + description: 'messages.create claude-3-haiku-20240307', + op: 'gen_ai.messages.create', + origin: 'auto.ai.anthropic', + status: 'ok', + }), + // Second span - error handling + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'messages.create', + 'sentry.op': 'gen_ai.messages.create', + 'sentry.origin': 'auto.ai.anthropic', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'error-model', + }, + description: 'messages.create error-model', + op: 'gen_ai.messages.create', + origin: 'auto.ai.anthropic', + status: 'unknown_error', + }), + // Third span - token counting (no response.text because recordOutputs=false by default) + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'messages.countTokens', + 'sentry.op': 'gen_ai.messages.countTokens', + 'sentry.origin': 'auto.ai.anthropic', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-haiku-20240307', + }, + description: 'messages.countTokens claude-3-haiku-20240307', + op: 'gen_ai.messages.countTokens', + origin: 'auto.ai.anthropic', + status: 'ok', + }), + // Fourth span - models.retrieve + expect.objectContaining({ + data: { + 'anthropic.response.timestamp': '2024-05-08T05:20:00.000Z', + 'gen_ai.operation.name': 'retrieve', + 'sentry.op': 'gen_ai.retrieve', + 'sentry.origin': 'auto.ai.anthropic', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-haiku-20240307', + 'gen_ai.response.id': 'claude-3-haiku-20240307', + 'gen_ai.response.model': 'claude-3-haiku-20240307', + }, + description: 'retrieve claude-3-haiku-20240307', + op: 'gen_ai.retrieve', + origin: 'auto.ai.anthropic', + status: 'ok', + }), + ]), + }; + + const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { + transaction: 'main', + spans: expect.arrayContaining([ + // First span - basic message completion with PII + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'messages.create', + 'sentry.op': 'gen_ai.messages.create', + 'sentry.origin': 'auto.ai.anthropic', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-haiku-20240307', + 'gen_ai.request.temperature': 0.7, + 'gen_ai.request.max_tokens': 100, + 'gen_ai.request.messages': '[{"role":"user","content":"What is the capital of France?"}]', + 'gen_ai.response.model': 'claude-3-haiku-20240307', + 'gen_ai.response.id': 'msg_mock123', + 'gen_ai.response.text': 'Hello from Anthropic mock!', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 15, + 'gen_ai.usage.total_tokens': 25, + }, + description: 'messages.create claude-3-haiku-20240307', + op: 'gen_ai.messages.create', + origin: 'auto.ai.anthropic', + status: 'ok', + }), + // Second span - error handling with PII + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'messages.create', + 'sentry.op': 'gen_ai.messages.create', + 'sentry.origin': 'auto.ai.anthropic', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'error-model', + 'gen_ai.request.messages': '[{"role":"user","content":"This will fail"}]', + }, + description: 'messages.create error-model', + op: 'gen_ai.messages.create', + + origin: 'auto.ai.anthropic', + status: 'unknown_error', + }), + // Third span - token counting with PII (response.text is present because sendDefaultPii=true enables recordOutputs) + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'messages.countTokens', + 'sentry.op': 'gen_ai.messages.countTokens', + 'sentry.origin': 'auto.ai.anthropic', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-haiku-20240307', + 'gen_ai.request.messages': '[{"role":"user","content":"What is the capital of France?"}]', + 'gen_ai.response.text': '15', // Only present because recordOutputs=true when sendDefaultPii=true + }, + description: 'messages.countTokens claude-3-haiku-20240307', + op: 'gen_ai.messages.countTokens', + origin: 'auto.ai.anthropic', + status: 'ok', + }), + // Fourth span - models.retrieve with PII + expect.objectContaining({ + data: { + 'anthropic.response.timestamp': '2024-05-08T05:20:00.000Z', + 'gen_ai.operation.name': 'retrieve', + 'sentry.op': 'gen_ai.retrieve', + 'sentry.origin': 'auto.ai.anthropic', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-haiku-20240307', + 'gen_ai.response.id': 'claude-3-haiku-20240307', + 'gen_ai.response.model': 'claude-3-haiku-20240307', + }, + description: 'retrieve claude-3-haiku-20240307', + op: 'gen_ai.retrieve', + origin: 'auto.ai.anthropic', + status: 'ok', + }), + ]), + }; + + const EXPECTED_TRANSACTION_WITH_OPTIONS = { + transaction: 'main', + spans: expect.arrayContaining([ + // Check that custom options are respected + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true + 'gen_ai.response.text': expect.any(String), // Should include response text when recordOutputs: true + }), + }), + // Check token counting with options + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'messages.countTokens', + 'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true + 'gen_ai.response.text': '15', // Present because recordOutputs=true is set in options + }), + op: 'gen_ai.messages.countTokens', + }), + // Check models.retrieve with options + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'retrieve', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-haiku-20240307', + 'gen_ai.response.id': 'claude-3-haiku-20240307', + 'gen_ai.response.model': 'claude-3-haiku-20240307', + }), + op: 'gen_ai.retrieve', + description: 'retrieve claude-3-haiku-20240307', + }), + ]), + }; + + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { + test('creates anthropic related spans with sendDefaultPii: false', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }) + .start() + .completed(); + }); + }); + + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('creates anthropic related spans with sendDefaultPii: true', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }) + .start() + .completed(); + }); + }); + + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-options.mjs', (createRunner, test) => { + test('creates anthropic related spans with custom options', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_WITH_OPTIONS }) + .start() + .completed(); + }); + }); +}); diff --git a/packages/astro/src/index.server.ts b/packages/astro/src/index.server.ts index a9e81aee7db5..bce3b2fba87d 100644 --- a/packages/astro/src/index.server.ts +++ b/packages/astro/src/index.server.ts @@ -12,6 +12,7 @@ export { addEventProcessor, addIntegration, amqplibIntegration, + anthropicAIIntegration, // eslint-disable-next-line deprecation/deprecation anrIntegration, // eslint-disable-next-line deprecation/deprecation diff --git a/packages/aws-serverless/src/index.ts b/packages/aws-serverless/src/index.ts index b99c481fd1d3..01ff0a6ac773 100644 --- a/packages/aws-serverless/src/index.ts +++ b/packages/aws-serverless/src/index.ts @@ -121,6 +121,7 @@ export { zodErrorsIntegration, profiler, amqplibIntegration, + anthropicAIIntegration, vercelAIIntegration, logger, consoleLoggingIntegration, diff --git a/packages/bun/src/index.ts b/packages/bun/src/index.ts index b9af910eb0f1..ec092bcdbbba 100644 --- a/packages/bun/src/index.ts +++ b/packages/bun/src/index.ts @@ -140,6 +140,7 @@ export { zodErrorsIntegration, profiler, amqplibIntegration, + anthropicAIIntegration, vercelAIIntegration, logger, consoleLoggingIntegration, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 0747258113a9..adc08a99f976 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -120,7 +120,10 @@ export { consoleLoggingIntegration } from './logs/console-integration'; export { addVercelAiProcessors } from './utils/vercel-ai'; export { instrumentOpenAiClient } from './utils/openai'; export { OPENAI_INTEGRATION_NAME } from './utils/openai/constants'; +export { instrumentAnthropicAiClient } from './utils/anthropic-ai'; +export { ANTHROPIC_AI_INTEGRATION_NAME } from './utils/anthropic-ai/constants'; export type { OpenAiClient, OpenAiOptions, InstrumentedMethod } from './utils/openai/types'; +export type { AnthropicAiClient, AnthropicAiOptions, AnthropicAiInstrumentedMethod } from './utils/anthropic-ai/types'; export type { FeatureFlag } from './utils/featureFlags'; export { diff --git a/packages/core/src/utils/gen-ai-attributes.ts b/packages/core/src/utils/ai/gen-ai-attributes.ts similarity index 91% rename from packages/core/src/utils/gen-ai-attributes.ts rename to packages/core/src/utils/ai/gen-ai-attributes.ts index d1b45532e8a5..9124602644e4 100644 --- a/packages/core/src/utils/gen-ai-attributes.ts +++ b/packages/core/src/utils/ai/gen-ai-attributes.ts @@ -8,6 +8,11 @@ // OPENTELEMETRY SEMANTIC CONVENTIONS FOR GENAI // ============================================================================= +/** + * The input messages sent to the model + */ +export const GEN_AI_PROMPT_ATTRIBUTE = 'gen_ai.prompt'; + /** * The Generative AI system being used * For OpenAI, this should always be "openai" @@ -164,3 +169,12 @@ export const OPENAI_OPERATIONS = { CHAT: 'chat', RESPONSES: 'responses', } as const; + +// ============================================================================= +// ANTHROPIC AI OPERATIONS +// ============================================================================= + +/** + * The response timestamp from Anthropic AI (ISO string) + */ +export const ANTHROPIC_AI_RESPONSE_TIMESTAMP_ATTRIBUTE = 'anthropic.response.timestamp'; diff --git a/packages/core/src/utils/ai/utils.ts b/packages/core/src/utils/ai/utils.ts new file mode 100644 index 000000000000..6ec52c11020a --- /dev/null +++ b/packages/core/src/utils/ai/utils.ts @@ -0,0 +1,89 @@ +/** + * Shared utils for AI integrations (OpenAI, Anthropic, Verce.AI, etc.) + */ +import type { Span } from '../../types-hoist/span'; +import { + GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, +} from './gen-ai-attributes'; +/** + * Maps AI method paths to Sentry operation name + */ +export function getFinalOperationName(methodPath: string): string { + if (methodPath.includes('messages.create')) { + return 'messages.create'; + } + if (methodPath.includes('messages.countTokens')) { + return 'messages.countTokens'; + } + if (methodPath.includes('completions.create')) { + return 'completions.create'; + } + if (methodPath.includes('models.list')) { + return 'models.list'; + } + if (methodPath.includes('models.get')) { + return 'models.get'; + } + return methodPath.split('.').pop() || 'unknown'; +} + +/** + * Get the span operation for AI methods + * Following Sentry's convention: "gen_ai.{operation_name}" + */ +export function getSpanOperation(methodPath: string): string { + return `gen_ai.${getFinalOperationName(methodPath)}`; +} + +/** + * Build method path from current traversal + */ +export function buildMethodPath(currentPath: string, prop: string): string { + return currentPath ? `${currentPath}.${prop}` : prop; +} + +/** + * Set token usage attributes + * @param span - The span to add attributes to + * @param promptTokens - The number of prompt tokens + * @param completionTokens - The number of completion tokens + * @param cachedInputTokens - The number of cached input tokens + * @param cachedOutputTokens - The number of cached output tokens + */ +export function setTokenUsageAttributes( + span: Span, + promptTokens?: number, + completionTokens?: number, + cachedInputTokens?: number, + cachedOutputTokens?: number, +): void { + if (promptTokens !== undefined) { + span.setAttributes({ + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: promptTokens, + }); + } + if (completionTokens !== undefined) { + span.setAttributes({ + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: completionTokens, + }); + } + if ( + promptTokens !== undefined || + completionTokens !== undefined || + cachedInputTokens !== undefined || + cachedOutputTokens !== undefined + ) { + /** + * Total input tokens in a request is the summation of `input_tokens`, + * `cache_creation_input_tokens`, and `cache_read_input_tokens`. + */ + const totalTokens = + (promptTokens ?? 0) + (completionTokens ?? 0) + (cachedInputTokens ?? 0) + (cachedOutputTokens ?? 0); + + span.setAttributes({ + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: totalTokens, + }); + } +} diff --git a/packages/core/src/utils/anthropic-ai/constants.ts b/packages/core/src/utils/anthropic-ai/constants.ts new file mode 100644 index 000000000000..41a227f171e0 --- /dev/null +++ b/packages/core/src/utils/anthropic-ai/constants.ts @@ -0,0 +1,11 @@ +export const ANTHROPIC_AI_INTEGRATION_NAME = 'Anthropic_AI'; + +// https://docs.anthropic.com/en/api/messages +// https://docs.anthropic.com/en/api/models-list +export const ANTHROPIC_AI_INSTRUMENTED_METHODS = [ + 'messages.create', + 'messages.countTokens', + 'models.get', + 'completions.create', + 'models.retrieve', +] as const; diff --git a/packages/core/src/utils/anthropic-ai/index.ts b/packages/core/src/utils/anthropic-ai/index.ts new file mode 100644 index 000000000000..2d7cc3655481 --- /dev/null +++ b/packages/core/src/utils/anthropic-ai/index.ts @@ -0,0 +1,242 @@ +import { getCurrentScope } from '../../currentScopes'; +import { captureException } from '../../exports'; +import { SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../../semanticAttributes'; +import { startSpan } from '../../tracing/trace'; +import type { Span, SpanAttributeValue } from '../../types-hoist/span'; +import { + ANTHROPIC_AI_RESPONSE_TIMESTAMP_ATTRIBUTE, + GEN_AI_OPERATION_NAME_ATTRIBUTE, + GEN_AI_PROMPT_ATTRIBUTE, + GEN_AI_REQUEST_FREQUENCY_PENALTY_ATTRIBUTE, + GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE, + GEN_AI_REQUEST_MESSAGES_ATTRIBUTE, + GEN_AI_REQUEST_MODEL_ATTRIBUTE, + GEN_AI_REQUEST_STREAM_ATTRIBUTE, + GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE, + GEN_AI_REQUEST_TOP_K_ATTRIBUTE, + GEN_AI_REQUEST_TOP_P_ATTRIBUTE, + GEN_AI_RESPONSE_ID_ATTRIBUTE, + GEN_AI_RESPONSE_MODEL_ATTRIBUTE, + GEN_AI_RESPONSE_TEXT_ATTRIBUTE, + GEN_AI_SYSTEM_ATTRIBUTE, +} from '../ai/gen-ai-attributes'; +import { buildMethodPath, getFinalOperationName, getSpanOperation, setTokenUsageAttributes } from '../ai/utils'; +import { ANTHROPIC_AI_INTEGRATION_NAME } from './constants'; +import type { + AnthropicAiClient, + AnthropicAiInstrumentedMethod, + AnthropicAiIntegration, + AnthropicAiOptions, + AnthropicAiResponse, +} from './types'; +import { shouldInstrument } from './utils'; +/** + * Extract request attributes from method arguments + */ +function extractRequestAttributes(args: unknown[], methodPath: string): Record { + const attributes: Record = { + [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: getFinalOperationName(methodPath), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', + }; + + if (args.length > 0 && typeof args[0] === 'object' && args[0] !== null) { + const params = args[0] as Record; + + attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE] = params.model ?? 'unknown'; + if ('temperature' in params) attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE] = params.temperature; + if ('top_p' in params) attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE] = params.top_p; + if ('stream' in params) attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] = params.stream; + if ('top_k' in params) attributes[GEN_AI_REQUEST_TOP_K_ATTRIBUTE] = params.top_k; + if ('frequency_penalty' in params) + attributes[GEN_AI_REQUEST_FREQUENCY_PENALTY_ATTRIBUTE] = params.frequency_penalty; + if ('max_tokens' in params) attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE] = params.max_tokens; + } else { + if (methodPath === 'models.retrieve' || methodPath === 'models.get') { + // models.retrieve(model-id) and models.get(model-id) + attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE] = args[0]; + } else { + attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE] = 'unknown'; + } + } + + return attributes; +} + +/** + * Add private request attributes to spans. + * This is only recorded if recordInputs is true. + */ +function addPrivateRequestAttributes(span: Span, params: Record): void { + if ('messages' in params) { + span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(params.messages) }); + } + if ('input' in params) { + span.setAttributes({ [GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: JSON.stringify(params.input) }); + } + if ('prompt' in params) { + span.setAttributes({ [GEN_AI_PROMPT_ATTRIBUTE]: JSON.stringify(params.prompt) }); + } +} + +/** + * Add response attributes to spans + */ +function addResponseAttributes(span: Span, response: AnthropicAiResponse, recordOutputs?: boolean): void { + if (!response || typeof response !== 'object') return; + + // Private response attributes that are only recorded if recordOutputs is true. + if (recordOutputs) { + // Messages.create + if ('content' in response) { + if (Array.isArray(response.content)) { + span.setAttributes({ + [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: response.content + .map((item: { text: string | undefined }) => item.text) + .filter((text): text is string => text !== undefined) + .join(''), + }); + } + } + // Completions.create + if ('completion' in response) { + span.setAttributes({ [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: response.completion }); + } + // Models.countTokens + if ('input_tokens' in response) { + span.setAttributes({ [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: JSON.stringify(response.input_tokens) }); + } + } + + span.setAttributes({ + [GEN_AI_RESPONSE_ID_ATTRIBUTE]: response.id, + }); + span.setAttributes({ + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: response.model, + }); + if ('created' in response && typeof response.created === 'number') { + span.setAttributes({ + [ANTHROPIC_AI_RESPONSE_TIMESTAMP_ATTRIBUTE]: new Date(response.created * 1000).toISOString(), + }); + } + if ('created_at' in response && typeof response.created_at === 'number') { + span.setAttributes({ + [ANTHROPIC_AI_RESPONSE_TIMESTAMP_ATTRIBUTE]: new Date(response.created_at * 1000).toISOString(), + }); + } + + if (response.usage) { + setTokenUsageAttributes( + span, + response.usage.input_tokens, + response.usage.output_tokens, + response.usage.cache_creation_input_tokens, + response.usage.cache_read_input_tokens, + ); + } +} + +/** + * Get record options from the integration + */ +function getOptionsFromIntegration(): AnthropicAiOptions { + const scope = getCurrentScope(); + const client = scope.getClient(); + const integration = client?.getIntegrationByName(ANTHROPIC_AI_INTEGRATION_NAME) as AnthropicAiIntegration | undefined; + const shouldRecordInputsAndOutputs = integration ? Boolean(client?.getOptions().sendDefaultPii) : false; + + return { + recordInputs: integration?.options?.recordInputs ?? shouldRecordInputsAndOutputs, + recordOutputs: integration?.options?.recordOutputs ?? shouldRecordInputsAndOutputs, + }; +} + +/** + * Instrument a method with Sentry spans + * Following Sentry AI Agents Manual Instrumentation conventions + * @see https://docs.sentry.io/platforms/javascript/guides/node/tracing/instrumentation/ai-agents-module/#manual-instrumentation + */ +function instrumentMethod( + originalMethod: (...args: T) => Promise, + methodPath: AnthropicAiInstrumentedMethod, + context: unknown, + options?: AnthropicAiOptions, +): (...args: T) => Promise { + return async function instrumentedMethod(...args: T): Promise { + const finalOptions = options || getOptionsFromIntegration(); + const requestAttributes = extractRequestAttributes(args, methodPath); + const model = requestAttributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE] ?? 'unknown'; + const operationName = getFinalOperationName(methodPath); + + // TODO: Handle streaming responses + return startSpan( + { + name: `${operationName} ${model}`, + op: getSpanOperation(methodPath), + attributes: requestAttributes as Record, + }, + async (span: Span) => { + try { + if (finalOptions.recordInputs && args[0] && typeof args[0] === 'object') { + addPrivateRequestAttributes(span, args[0] as Record); + } + + const result = await originalMethod.apply(context, args); + addResponseAttributes(span, result, finalOptions.recordOutputs); + return result; + } catch (error) { + captureException(error, { + mechanism: { + handled: false, + type: 'auto.ai.anthropic', + data: { + function: methodPath, + }, + }, + }); + throw error; + } + }, + ); + }; +} + +/** + * Create a deep proxy for Anthropic AI client instrumentation + */ +function createDeepProxy(target: T, currentPath = '', options?: AnthropicAiOptions): T { + return new Proxy(target, { + get(obj: object, prop: string): unknown { + const value = (obj as Record)[prop]; + const methodPath = buildMethodPath(currentPath, String(prop)); + + if (typeof value === 'function' && shouldInstrument(methodPath)) { + return instrumentMethod(value as (...args: unknown[]) => Promise, methodPath, obj, options); + } + + if (typeof value === 'function') { + // Bind non-instrumented functions to preserve the original `this` context, + return value.bind(obj); + } + + if (value && typeof value === 'object') { + return createDeepProxy(value as object, methodPath, options); + } + + return value; + }, + }) as T; +} + +/** + * Instrument an Anthropic AI client with Sentry tracing + * Can be used across Node.js, Cloudflare Workers, and Vercel Edge + * + * @template T - The type of the client that extends AnthropicAiClient + * @param client - The Anthropic AI client to instrument + * @param options - Optional configuration for recording inputs and outputs + * @returns The instrumented client with the same type as the input + */ +export function instrumentAnthropicAiClient(client: T, options?: AnthropicAiOptions): T { + return createDeepProxy(client, '', options); +} diff --git a/packages/core/src/utils/anthropic-ai/types.ts b/packages/core/src/utils/anthropic-ai/types.ts new file mode 100644 index 000000000000..566e9588d56f --- /dev/null +++ b/packages/core/src/utils/anthropic-ai/types.ts @@ -0,0 +1,63 @@ +import type { ANTHROPIC_AI_INSTRUMENTED_METHODS } from './constants'; + +export interface AnthropicAiOptions { + /** + * Enable or disable input recording. + */ + recordInputs?: boolean; + /** + * Enable or disable output recording. + */ + recordOutputs?: boolean; +} + +export type Message = { + role: 'user' | 'assistant'; + content: string | unknown[]; +}; + +export type AnthropicAiResponse = { + [key: string]: unknown; // Allow for additional unknown properties + id: string; + model: string; + created?: number; + created_at?: number; // Available for Models.retrieve + messages?: Array; + content?: string; // Available for Messages.create + completion?: string; // Available for Completions.create + input_tokens?: number; // Available for Models.countTokens + usage?: { + input_tokens: number; + output_tokens: number; + cache_creation_input_tokens: number; + cache_read_input_tokens: number; + }; +}; + +/** + * Basic interface for Anthropic AI client with only the instrumented methods + * This provides type safety while being generic enough to work with different client implementations + */ +export interface AnthropicAiClient { + messages?: { + create: (...args: unknown[]) => Promise; + countTokens: (...args: unknown[]) => Promise; + }; + models?: { + list: (...args: unknown[]) => Promise; + get: (...args: unknown[]) => Promise; + }; + completions?: { + create: (...args: unknown[]) => Promise; + }; +} + +/** + * Anthropic AI Integration interface for type safety + */ +export interface AnthropicAiIntegration { + name: string; + options: AnthropicAiOptions; +} + +export type AnthropicAiInstrumentedMethod = (typeof ANTHROPIC_AI_INSTRUMENTED_METHODS)[number]; diff --git a/packages/core/src/utils/anthropic-ai/utils.ts b/packages/core/src/utils/anthropic-ai/utils.ts new file mode 100644 index 000000000000..299d20170d6c --- /dev/null +++ b/packages/core/src/utils/anthropic-ai/utils.ts @@ -0,0 +1,9 @@ +import { ANTHROPIC_AI_INSTRUMENTED_METHODS } from './constants'; +import type { AnthropicAiInstrumentedMethod } from './types'; + +/** + * Check if a method path should be instrumented + */ +export function shouldInstrument(methodPath: string): methodPath is AnthropicAiInstrumentedMethod { + return ANTHROPIC_AI_INSTRUMENTED_METHODS.includes(methodPath as AnthropicAiInstrumentedMethod); +} diff --git a/packages/core/src/utils/openai/index.ts b/packages/core/src/utils/openai/index.ts index 3fb4f0d16fce..3fb8b1bf8b98 100644 --- a/packages/core/src/utils/openai/index.ts +++ b/packages/core/src/utils/openai/index.ts @@ -17,7 +17,7 @@ import { GEN_AI_RESPONSE_TEXT_ATTRIBUTE, GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE, GEN_AI_SYSTEM_ATTRIBUTE, -} from '../gen-ai-attributes'; +} from '../ai/gen-ai-attributes'; import { OPENAI_INTEGRATION_NAME } from './constants'; import { instrumentStream } from './streaming'; import type { diff --git a/packages/core/src/utils/openai/streaming.ts b/packages/core/src/utils/openai/streaming.ts index 2791e715920e..c79448effb35 100644 --- a/packages/core/src/utils/openai/streaming.ts +++ b/packages/core/src/utils/openai/streaming.ts @@ -6,7 +6,7 @@ import { GEN_AI_RESPONSE_STREAMING_ATTRIBUTE, GEN_AI_RESPONSE_TEXT_ATTRIBUTE, GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE, -} from '../gen-ai-attributes'; +} from '../ai/gen-ai-attributes'; import { RESPONSE_EVENT_TYPES } from './constants'; import type { OpenAIResponseObject } from './types'; import { diff --git a/packages/core/src/utils/openai/utils.ts b/packages/core/src/utils/openai/utils.ts index f76d26de5d6a..17007693e739 100644 --- a/packages/core/src/utils/openai/utils.ts +++ b/packages/core/src/utils/openai/utils.ts @@ -11,7 +11,7 @@ import { OPENAI_RESPONSE_TIMESTAMP_ATTRIBUTE, OPENAI_USAGE_COMPLETION_TOKENS_ATTRIBUTE, OPENAI_USAGE_PROMPT_TOKENS_ATTRIBUTE, -} from '../gen-ai-attributes'; +} from '../ai/gen-ai-attributes'; import { INSTRUMENTED_METHODS } from './constants'; import type { ChatCompletionChunk, diff --git a/packages/google-cloud-serverless/src/index.ts b/packages/google-cloud-serverless/src/index.ts index 8339e95c77a3..32259eec4150 100644 --- a/packages/google-cloud-serverless/src/index.ts +++ b/packages/google-cloud-serverless/src/index.ts @@ -119,6 +119,7 @@ export { zodErrorsIntegration, profiler, amqplibIntegration, + anthropicAIIntegration, childProcessIntegration, createSentryWinstonTransport, vercelAIIntegration, diff --git a/packages/node/src/index.ts b/packages/node/src/index.ts index bba0f98bc75e..568eb064699f 100644 --- a/packages/node/src/index.ts +++ b/packages/node/src/index.ts @@ -24,6 +24,7 @@ export { dataloaderIntegration } from './integrations/tracing/dataloader'; export { amqplibIntegration } from './integrations/tracing/amqplib'; export { vercelAIIntegration } from './integrations/tracing/vercelai'; export { openAIIntegration } from './integrations/tracing/openai'; +export { anthropicAIIntegration } from './integrations/tracing/anthropic-ai'; export { launchDarklyIntegration, buildLaunchDarklyFlagUsedHandler, diff --git a/packages/node/src/integrations/tracing/anthropic-ai/index.ts b/packages/node/src/integrations/tracing/anthropic-ai/index.ts new file mode 100644 index 000000000000..b9ec00013f49 --- /dev/null +++ b/packages/node/src/integrations/tracing/anthropic-ai/index.ts @@ -0,0 +1,74 @@ +import type { AnthropicAiOptions, IntegrationFn } from '@sentry/core'; +import { ANTHROPIC_AI_INTEGRATION_NAME, defineIntegration } from '@sentry/core'; +import { generateInstrumentOnce } from '@sentry/node-core'; +import { SentryAnthropicAiInstrumentation } from './instrumentation'; + +export const instrumentAnthropicAi = generateInstrumentOnce( + ANTHROPIC_AI_INTEGRATION_NAME, + () => new SentryAnthropicAiInstrumentation({}), +); + +const _anthropicAIIntegration = ((options: AnthropicAiOptions = {}) => { + return { + name: ANTHROPIC_AI_INTEGRATION_NAME, + options, + setupOnce() { + instrumentAnthropicAi(); + }, + }; +}) satisfies IntegrationFn; + +/** + * Adds Sentry tracing instrumentation for the Anthropic AI SDK. + * + * This integration is enabled by default. + * + * When configured, this integration automatically instruments Anthropic AI SDK client instances + * to capture telemetry data following OpenTelemetry Semantic Conventions for Generative AI. + * + * @example + * ```javascript + * import * as Sentry from '@sentry/node'; + * + * Sentry.init({ + * integrations: [Sentry.anthropicAIIntegration()], + * }); + * ``` + * + * ## Options + * + * - `recordInputs`: Whether to record prompt messages (default: respects `sendDefaultPii` client option) + * - `recordOutputs`: Whether to record response text (default: respects `sendDefaultPii` client option) + * + * ### Default Behavior + * + * By default, the integration will: + * - Record inputs and outputs ONLY if `sendDefaultPii` is set to `true` in your Sentry client options + * - Otherwise, inputs and outputs are NOT recorded unless explicitly enabled + * + * @example + * ```javascript + * // Record inputs and outputs when sendDefaultPii is false + * Sentry.init({ + * integrations: [ + * Sentry.anthropicAIIntegration({ + * recordInputs: true, + * recordOutputs: true + * }) + * ], + * }); + * + * // Never record inputs/outputs regardless of sendDefaultPii + * Sentry.init({ + * sendDefaultPii: true, + * integrations: [ + * Sentry.anthropicAIIntegration({ + * recordInputs: false, + * recordOutputs: false + * }) + * ], + * }); + * ``` + * + */ +export const anthropicAIIntegration = defineIntegration(_anthropicAIIntegration); diff --git a/packages/node/src/integrations/tracing/anthropic-ai/instrumentation.ts b/packages/node/src/integrations/tracing/anthropic-ai/instrumentation.ts new file mode 100644 index 000000000000..99fd2c546dd2 --- /dev/null +++ b/packages/node/src/integrations/tracing/anthropic-ai/instrumentation.ts @@ -0,0 +1,122 @@ +import { + type InstrumentationConfig, + type InstrumentationModuleDefinition, + InstrumentationBase, + InstrumentationNodeModuleDefinition, +} from '@opentelemetry/instrumentation'; +import type { AnthropicAiClient, AnthropicAiOptions, Integration } from '@sentry/core'; +import { ANTHROPIC_AI_INTEGRATION_NAME, getCurrentScope, instrumentAnthropicAiClient, SDK_VERSION } from '@sentry/core'; + +const supportedVersions = ['>=0.19.2 <1.0.0']; + +export interface AnthropicAiIntegration extends Integration { + options: AnthropicAiOptions; +} + +/** + * Represents the patched shape of the Anthropic AI module export. + */ +interface PatchedModuleExports { + [key: string]: unknown; + Anthropic: abstract new (...args: unknown[]) => AnthropicAiClient; +} + +/** + * Determines telemetry recording settings. + */ +function determineRecordingSettings( + integrationOptions: AnthropicAiOptions | undefined, + defaultEnabled: boolean, +): { recordInputs: boolean; recordOutputs: boolean } { + const recordInputs = integrationOptions?.recordInputs ?? defaultEnabled; + const recordOutputs = integrationOptions?.recordOutputs ?? defaultEnabled; + return { recordInputs, recordOutputs }; +} + +/** + * Sentry Anthropic AI instrumentation using OpenTelemetry. + */ +export class SentryAnthropicAiInstrumentation extends InstrumentationBase { + public constructor(config: InstrumentationConfig = {}) { + super('@sentry/instrumentation-anthropic-ai', SDK_VERSION, config); + } + + /** + * Initializes the instrumentation by defining the modules to be patched. + */ + public init(): InstrumentationModuleDefinition { + const module = new InstrumentationNodeModuleDefinition( + '@anthropic-ai/sdk', + supportedVersions, + this._patch.bind(this), + ); + return module; + } + + /** + * Core patch logic applying instrumentation to the Anthropic AI client constructor. + */ + private _patch(exports: PatchedModuleExports): PatchedModuleExports | void { + const Original = exports.Anthropic; + + const WrappedAnthropic = function (this: unknown, ...args: unknown[]) { + const instance = Reflect.construct(Original, args); + const scopeClient = getCurrentScope().getClient(); + const integration = scopeClient?.getIntegrationByName(ANTHROPIC_AI_INTEGRATION_NAME); + const integrationOpts = integration?.options; + const defaultPii = Boolean(scopeClient?.getOptions().sendDefaultPii); + + const { recordInputs, recordOutputs } = determineRecordingSettings(integrationOpts, defaultPii); + + return instrumentAnthropicAiClient(instance as AnthropicAiClient, { + recordInputs, + recordOutputs, + }); + } as unknown as abstract new (...args: unknown[]) => AnthropicAiClient; + + // Preserve static and prototype chains + Object.setPrototypeOf(WrappedAnthropic, Original); + Object.setPrototypeOf(WrappedAnthropic.prototype, Original.prototype); + + for (const key of Object.getOwnPropertyNames(Original)) { + if (!['length', 'name', 'prototype'].includes(key)) { + const descriptor = Object.getOwnPropertyDescriptor(Original, key); + if (descriptor) { + Object.defineProperty(WrappedAnthropic, key, descriptor); + } + } + } + + // Constructor replacement - handle read-only properties + // The Anthropic property might have only a getter, so use defineProperty + try { + exports.Anthropic = WrappedAnthropic; + } catch (error) { + // If direct assignment fails, override the property descriptor + Object.defineProperty(exports, 'Anthropic', { + value: WrappedAnthropic, + writable: true, + configurable: true, + enumerable: true, + }); + } + + // Wrap the default export if it points to the original constructor + // Constructor replacement - handle read-only properties + // The Anthropic property might have only a getter, so use defineProperty + if (exports.default === Original) { + try { + exports.default = WrappedAnthropic; + } catch (error) { + // If direct assignment fails, override the property descriptor + Object.defineProperty(exports, 'default', { + value: WrappedAnthropic, + writable: true, + configurable: true, + enumerable: true, + }); + } + } + return exports; + } +} diff --git a/packages/node/src/integrations/tracing/index.ts b/packages/node/src/integrations/tracing/index.ts index 6035cf3669f8..2d660670d297 100644 --- a/packages/node/src/integrations/tracing/index.ts +++ b/packages/node/src/integrations/tracing/index.ts @@ -1,6 +1,7 @@ import type { Integration } from '@sentry/core'; import { instrumentOtelHttp } from '../http'; import { amqplibIntegration, instrumentAmqplib } from './amqplib'; +import { anthropicAIIntegration, instrumentAnthropicAi } from './anthropic-ai'; import { connectIntegration, instrumentConnect } from './connect'; import { expressIntegration, instrumentExpress } from './express'; import { fastifyIntegration, instrumentFastify, instrumentFastifyV3 } from './fastify'; @@ -50,6 +51,7 @@ export function getAutoPerformanceIntegrations(): Integration[] { openAIIntegration(), postgresJsIntegration(), firebaseIntegration(), + anthropicAIIntegration(), ]; } @@ -83,5 +85,6 @@ export function getOpenTelemetryInstrumentationToPreload(): (((options?: any) => instrumentOpenAi, instrumentPostgresJs, instrumentFirebase, + instrumentAnthropicAi, ]; }