diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/package.json b/aws-distro-opentelemetry-node-autoinstrumentation/package.json index 28d63edf..38d5b4e9 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/package.json +++ b/aws-distro-opentelemetry-node-autoinstrumentation/package.json @@ -100,6 +100,9 @@ "@opentelemetry/api": "1.9.0", "@opentelemetry/auto-configuration-propagators": "0.3.2", "@opentelemetry/auto-instrumentations-node": "0.56.0", + "@opentelemetry/api-events": "0.57.1", + "@opentelemetry/sdk-events": "0.57.1", + "@opentelemetry/sdk-logs": "0.57.1", "@opentelemetry/core": "1.30.1", "@opentelemetry/exporter-metrics-otlp-grpc": "0.57.1", "@opentelemetry/exporter-metrics-otlp-http": "0.57.1", diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-metric-attributes-span-exporter.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-metric-attributes-span-exporter.ts index 16329f84..2cfd8649 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-metric-attributes-span-exporter.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/aws-metric-attributes-span-exporter.ts @@ -13,6 +13,7 @@ import { MetricAttributeGenerator, SERVICE_METRIC, } from './metric-attribute-generator'; +import { Mutable } from './utils'; /** * This exporter will update a span with metric attributes before exporting. It depends on a @@ -125,9 +126,6 @@ export class AwsMetricAttributesSpanExporter implements SpanExporter { } // Bypass `readonly` restriction of ReadableSpan's attributes. - // Workaround provided from official TypeScript docs: - // https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-8.html#improved-control-over-mapped-type-modifiers - type Mutable = { -readonly [P in keyof T]: T[P] }; const mutableSpan: Mutable = span; mutableSpan.attributes = updateAttributes; diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/llo-handler.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/llo-handler.ts new file mode 100644 index 00000000..1df53a35 --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/llo-handler.ts @@ -0,0 +1,564 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Attributes, HrTime, ROOT_CONTEXT, createContextKey } from '@opentelemetry/api'; +import { LoggerProvider } from '@opentelemetry/sdk-logs'; +import { EventLoggerProvider } from '@opentelemetry/sdk-events'; +import { Event } from '@opentelemetry/api-events'; +import { ReadableSpan, TimedEvent } from '@opentelemetry/sdk-trace-base'; +import { AnyValue } from '@opentelemetry/api-logs'; + +const ROLE_SYSTEM = 'system'; +const ROLE_USER = 'user'; +const ROLE_ASSISTANT = 'assistant'; +const SESSION_ID = 'session.id'; + +// Types of LLO attribute patterns +export enum PatternType { + INDEXED = 'indexed', + DIRECT = 'direct', +} + +// Configuration for an LLO pattern +interface PatternConfig { + type: PatternType; + regex?: string; + roleKey?: string; + role?: string; + defaultRole?: string; + source: string; +} + +interface Message { + content: string; + role: string; + source: string; +} + +export const LLO_PATTERNS: { [key: string]: PatternConfig } = { + 'gen_ai.prompt.{index}.content': { + type: PatternType.INDEXED, + regex: '^gen_ai\\.prompt\\.(\\d+)\\.content$', + roleKey: 'gen_ai.prompt.{index}.role', + defaultRole: 'unknown', + source: 'prompt', + }, + 'gen_ai.completion.{index}.content': { + type: PatternType.INDEXED, + regex: '^gen_ai\\.completion\\.(\\d+)\\.content$', + roleKey: 'gen_ai.completion.{index}.role', + defaultRole: 'unknown', + source: 'completion', + }, + 'llm.input_messages.{index}.message.content': { + type: PatternType.INDEXED, + regex: '^llm\\.input_messages\\.(\\d+)\\.message\\.content$', + roleKey: 'llm.input_messages.{index}.message.role', + defaultRole: ROLE_USER, + source: 'input', + }, + 'llm.output_messages.{index}.message.content': { + type: PatternType.INDEXED, + regex: '^llm\\.output_messages\\.(\\d+)\\.message\\.content$', + roleKey: 'llm.output_messages.{index}.message.role', + defaultRole: ROLE_ASSISTANT, + source: 'output', + }, + 'traceloop.entity.input': { + type: PatternType.DIRECT, + role: ROLE_USER, + source: 'input', + }, + 'traceloop.entity.output': { + type: PatternType.DIRECT, + role: ROLE_ASSISTANT, + source: 'output', + }, + 'crewai.crew.tasks_output': { + type: PatternType.DIRECT, + role: ROLE_ASSISTANT, + source: 'output', + }, + 'crewai.crew.result': { + type: PatternType.DIRECT, + role: ROLE_ASSISTANT, + source: 'result', + }, + 'gen_ai.prompt': { + type: PatternType.DIRECT, + role: ROLE_USER, + source: 'prompt', + }, + 'gen_ai.completion': { + type: PatternType.DIRECT, + role: ROLE_ASSISTANT, + source: 'completion', + }, + 'gen_ai.content.revised_prompt': { + type: PatternType.DIRECT, + role: ROLE_SYSTEM, + source: 'prompt', + }, + 'gen_ai.agent.actual_output': { + type: PatternType.DIRECT, + role: ROLE_ASSISTANT, + source: 'output', + }, + 'gen_ai.agent.human_input': { + type: PatternType.DIRECT, + role: ROLE_USER, + source: 'input', + }, + 'input.value': { + type: PatternType.DIRECT, + role: ROLE_USER, + source: 'input', + }, + 'output.value': { + type: PatternType.DIRECT, + role: ROLE_ASSISTANT, + source: 'output', + }, + system_prompt: { + type: PatternType.DIRECT, + role: ROLE_SYSTEM, + source: 'prompt', + }, + 'tool.result': { + type: PatternType.DIRECT, + role: ROLE_ASSISTANT, + source: 'output', + }, + 'llm.prompts': { + type: PatternType.DIRECT, + role: ROLE_USER, + source: 'prompt', + }, +}; + +/** + * Utility class for handling Large Language Objects (LLO) in OpenTelemetry spans. + * + * LLOHandler performs three primary functions: + * 1. Identifies Large Language Objects (LLO) content in spans + * 2. Extracts and transforms these attributes into OpenTelemetry Gen AI Events + * 3. Filters LLO from spans to maintain privacy and reduce span size + * + * The handler uses a configuration-driven approach with a pattern registry that defines + * all supported LLO attribute patterns and their extraction rules. This makes it easy + * to add support for new frameworks without modifying the core logic. + */ +export class LLOHandler { + private loggerProvider: LoggerProvider; + private eventLoggerProvider: EventLoggerProvider; + private exactMatchPatterns: Set; + private regexPatterns: Array<[RegExp, string, PatternConfig]>; + private patternConfigs: { [key: string]: PatternConfig }; + + /** + * Initialize an LLOHandler with the specified logger provider. + * + * This constructor sets up the event logger provider and compiles patterns + * from the pattern registry for efficient matching. + * + * @param loggerProvider The OpenTelemetry LoggerProvider used for emitting events. + */ + constructor(loggerProvider: LoggerProvider) { + this.loggerProvider = loggerProvider; + this.eventLoggerProvider = new EventLoggerProvider(this.loggerProvider); + + this.exactMatchPatterns = new Set(); + this.regexPatterns = []; + this.patternConfigs = {}; + this.buildPatternMatchers(); + } + + /** + * Build efficient pattern matching structures from the pattern registry. + * + * Creates: + * - Set of exact match patterns for O(1) lookups + * - List of compiled regex patterns for indexed patterns + * - Mapping of patterns to their configurations + */ + private buildPatternMatchers(): void { + for (const [patternKey, config] of Object.entries(LLO_PATTERNS)) { + if (config.type === PatternType.DIRECT) { + this.exactMatchPatterns.add(patternKey); + this.patternConfigs[patternKey] = config; + } else if (config.type === PatternType.INDEXED) { + if (config.regex) { + const compiledRegex = new RegExp(config.regex); + this.regexPatterns.push([compiledRegex, patternKey, config]); + } + } + } + } + + /** + * Processes a sequence of spans to extract and filter LLO attributes. + * + * For each span, this method: + * 1. Collects all LLO attributes from span attributes and all span events + * 2. Emits a single consolidated Gen AI Event with all collected LLO content + * 3. Filters out LLO attributes from the span and its events to maintain privacy + * 4. Preserves non-LLO attributes in the span + * + * Handles LLO attributes from multiple frameworks: + * - Traceloop (indexed prompt/completion patterns and entity input/output) + * - OpenLit (direct prompt/completion patterns, including from span events) + * - OpenInference (input/output values and structured messages) + * - Strands SDK (system prompts and tool results) + * - CrewAI (tasks output and results) + * + * @param spans A list of OpenTelemetry ReadableSpan objects to process + * @returns {ReadableSpan[]} A list of modified spans with LLO attributes removed + */ + public processSpans(spans: ReadableSpan[]): ReadableSpan[] { + const modifiedSpans: ReadableSpan[] = []; + + for (const span of spans) { + // Collect all LLO attributes from both span attributes and events + const allLloAttributes = this.collectLloAttributesFromSpan(span); + + // Emit a single consolidated event if we found any LLO attributes + if (Object.keys(allLloAttributes).length > 0) { + this.emitLloAttributes(span, allLloAttributes); + } + + // Filter and update span attributes + const filteredAttributes = this.filterAttributes(span.attributes); + (span as any).attributes = filteredAttributes; + + // Filter span events + this.filterSpanEvents(span); + + modifiedSpans.push(span); + } + + return modifiedSpans; + } + + /** + * Collect all LLO attributes from a span's attributes and events. + * + * @param span The span to collect LLO attributes from + * @returns all LLO attributes found in the span + */ + private collectLloAttributesFromSpan(span: ReadableSpan): Attributes { + const allLloAttributes: Attributes = {}; + + // Collect from span attributes + if (span.attributes) { + for (const [key, value] of Object.entries(span.attributes)) { + if (this.isLloAttribute(key)) { + allLloAttributes[key] = value; + } + } + } + + // Collect from span events + if (span.events) { + for (const event of span.events) { + if (event.attributes) { + for (const [key, value] of Object.entries(event.attributes)) { + if (this.isLloAttribute(key)) { + allLloAttributes[key] = value; + } + } + } + } + } + + return allLloAttributes; + } + + /** + * Filter LLO attributes from span events. + * + * This method removes LLO attributes from event attributes while preserving + * the event structure and non-LLO attributes. + * + * @param span The ReadableSpan to filter events for + */ + private filterSpanEvents(span: ReadableSpan): void { + if (!span.events) { + return; + } + + const updatedEvents: TimedEvent[] = []; + + for (const event of span.events) { + if (!event.attributes) { + updatedEvents.push(event); + continue; + } + + const updatedEventAttributes = this.filterAttributes(event.attributes); + + if (Object.keys(updatedEventAttributes).length !== Object.keys(event.attributes).length) { + const updatedEvent: TimedEvent = { + name: event.name, + time: event.time, + attributes: updatedEventAttributes, + }; + + updatedEvents.push(updatedEvent); + } else { + updatedEvents.push(event); + } + } + + (span as any).events = updatedEvents; + } + + /** + * Extract LLO attributes and emit them as a single consolidated Gen AI Event. + * + * This method: + * 1. Collects all LLO attributes using the pattern registry + * 2. Groups them into input and output messages + * 3. Emits one event per span containing all LLO content + * + * The event body format: + * { + * "input": { + * "messages": [ + * { + * "role": "system", + * "content": "..." + * }, + * { + * "role": "user", + * "content": "..." + * } + * ] + * }, + * "output": { + * "messages": [ + * { + * "role": "assistant", + * "content": "..." + * } + * ] + * } + * } + * + * @param span The source ReadableSpan containing the attributes + * @param attributes LLO attributes to process + * @param eventTimestamp Optional timestamp to override span timestamps + * @returns + */ + private emitLloAttributes(span: ReadableSpan, attributes: Attributes, eventTimestamp?: HrTime): void { + if (!attributes || Object.keys(attributes).length === 0) { + return; + } + + //[] maybe add sanity check here from Python "has_llo_attrs" + + const allMessages = this.collectAllLloMessages(span, attributes); + if (allMessages.length === 0) { + return; + } + + // Group messages into input/output categories + const groupedMessages = this.groupMessagesByType(allMessages); + + // Build event body + const eventBody: AnyValue = {}; + if (groupedMessages.input.length > 0) { + eventBody.input = { messages: groupedMessages.input }; + } + if (groupedMessages.output.length > 0) { + eventBody.output = { messages: groupedMessages.output }; + } + + if (Object.keys(eventBody).length === 0) { + return; + } + + // Create and emit the event + const timestamp = eventTimestamp || span.endTime; + const eventLogger = this.eventLoggerProvider.getEventLogger(span.instrumentationLibrary.name); + + // Hack - Workaround to add a custom-made Context to an Event so that the emitted event log + // has the correct associated traceId, spanId, flag. This is needed because a ReadableSpan only + // provides its SpanContext, but does not provide access to its associated Context. We can only + // provide a Context to an Event, but not a SpanContext. Here we attach a custom Context that + // is associated to the ReadableSpan to mimic the ReadableSpan's actual Context. + // + // When a log record instance is created from this event, it will use the "custom Context" to + // extract the ReadableSpan from the "custom Context", then extract the SpanContext from the + // RedableSpan. This way, the emitted log event has the correct SpanContext (traceId, spanId, flag). + // - https://github.com/open-telemetry/opentelemetry-js/blob/experimental/v0.57.1/experimental/packages/sdk-logs/src/LogRecord.ts#L101 + // - https://github.com/open-telemetry/opentelemetry-js/blob/experimental/v0.57.1/api/src/trace/context-utils.ts#L78-L85 + // + // We could omit the context field which is optional, but then the OTel EventLogger will assign + // the same `context.active()` and its associated SpanContext to each span from processSpans(), + // which would be incorrect since each span should have their own Context with unique SpanContext. + // - https://github.com/open-telemetry/opentelemetry-js/blob/experimental/v0.57.1/experimental/packages/sdk-events/src/EventLogger.ts#L34 + const customContext = ROOT_CONTEXT.setValue(OTEL_SPAN_KEY, span); + const event: Event = { + name: span.instrumentationLibrary.name, + timestamp: timestamp, + data: eventBody, + context: customContext, + }; + + if (span.attributes[SESSION_ID]) { + event.attributes = { + [SESSION_ID]: span.attributes[SESSION_ID], + }; + } + + eventLogger.emit(event); + } + + /** + * Collect all LLO messages from attributes using the pattern registry. + * + * This is the main collection method that processes all patterns defined + * in the registry and extracts messages accordingly. + * + * @param span The source ReadableSpan containing the attributes + * @param attributes Attributes to process + * @returns {Message[]} LLO messages from attributes using the pattern registry + */ + private collectAllLloMessages(span: ReadableSpan, attributes: Attributes): Message[] { + const messages: Message[] = []; + + if (!attributes) return messages; + + for (const [attrKey, value] of Object.entries(attributes)) { + if (this.exactMatchPatterns.has(attrKey)) { + const config = this.patternConfigs[attrKey]; + messages.push({ + content: value as string, + role: config.role || 'unknown', + source: config.source || 'unknown', + }); + } + } + + messages.push(...this.collectIndexedMessages(attributes)); + + return messages; + } + + /** + * Collect messages from indexed patterns (e.g., gen_ai.prompt.0.content). + * Handles patterns with numeric indices and their associated role attributes. + * + * @param attributes Attributes to process + * @returns {Message[]} + */ + private collectIndexedMessages(attributes: Attributes): Message[] { + const indexedMessages: (Message & { pattern: string; index: number })[] = []; + + for (const [attrKey, value] of Object.entries(attributes)) { + for (const [regex, patternKey, config] of this.regexPatterns) { + const match = attrKey.match(regex); + if (match) { + const index = parseInt(match[1], 10); + + let role = config.defaultRole || 'unknown'; + if (config.roleKey) { + const roleKey = config.roleKey.replace('{index}', index.toString()); + const roleValue = attributes[roleKey]; + if (typeof roleValue === 'string') { + role = roleValue; + } + } + + indexedMessages.push({ + content: value as string, + role, + source: config.source, + pattern: patternKey, + index: index, + }); + break; + } + } + } + + return indexedMessages + .sort((a, b) => (a.pattern !== b.pattern ? a.pattern.localeCompare(b.pattern) : a.index - b.index)) + .map(({ content, role, source }) => ({ content, role, source })); + } + + private groupMessagesByType(messages: Message[]) { + const input: { role: string; content: string }[] = []; + const output: { role: string; content: string }[] = []; + + for (const message of messages) { + const { role, content, source } = message; + const formattedMessage = { role, content }; + + if (role === ROLE_SYSTEM || role === ROLE_USER) { + input.push(formattedMessage); + } else if (role === ROLE_ASSISTANT) { + output.push(formattedMessage); + } else { + // Route based on source for non-standard roles + if (['completion', 'output', 'result'].some(key => source.includes(key))) { + output.push(formattedMessage); + } else { + input.push(formattedMessage); + } + } + } + + return { input, output }; + } + + /** + * Create new attributes with LLO attributes removed. + * + * This method creates an attributes object containing only non-LLO attributes, + * preserving the original values while filtering out sensitive LLO content. + * This helps maintain privacy and reduces the size of spans. + * + * @param attributes Original span or event attributes + * @returns {Attributes} New attributes with LLO attributes removed, or empty object if input is undefined + */ + private filterAttributes(attributes: Attributes | undefined): Attributes { + if (!attributes) { + return {}; + } + + const filteredAttributes: Attributes = {}; + for (const [key, value] of Object.entries(attributes)) { + if (!this.isLloAttribute(key)) { + filteredAttributes[key] = value; + } + } + + return filteredAttributes; + } + + /** + * Determine if an attribute key contains LLO content based on pattern matching. + * Uses the pattern registry to check if a key matches any LLO pattern. + * + * @param key The attribute key to check + * @returns {boolean} true if the key matches any LLO pattern, false otherwise + */ + private isLloAttribute(key: string): boolean { + if (this.exactMatchPatterns.has(key)) { + return true; + } + + for (const [regex] of this.regexPatterns) { + if (regex.test(key)) { + return true; + } + } + + return false; + } +} + +// Defined by OTel in: +// - https://github.com/open-telemetry/opentelemetry-js/blob/v1.9.0/api/src/trace/context-utils.ts#L24-L27 +export const OTEL_SPAN_KEY = createContextKey('OpenTelemetry Context Key SPAN'); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/otlp-aws-span-exporter.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/otlp-aws-span-exporter.ts index 406d0bea..668365c5 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/src/otlp-aws-span-exporter.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/otlp-aws-span-exporter.ts @@ -1,12 +1,15 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 import { OTLPTraceExporter as OTLPProtoTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto'; -import { diag } from '@opentelemetry/api'; import { OTLPExporterNodeConfigBase } from '@opentelemetry/otlp-exporter-base'; import { ProtobufTraceSerializer } from '@opentelemetry/otlp-transformer'; import { ReadableSpan } from '@opentelemetry/sdk-trace-base'; import { ExportResult } from '@opentelemetry/core'; -import { getNodeVersion } from './utils'; +import { LLOHandler } from './llo-handler'; +import { LoggerProvider as APILoggerProvider, logs } from '@opentelemetry/api-logs'; +import { LoggerProvider } from '@opentelemetry/sdk-logs'; +import { getNodeVersion, isAgentObservabilityEnabled } from './utils'; +import { diag } from '@opentelemetry/api'; /** * This exporter extends the functionality of the OTLPProtoTraceExporter to allow spans to be exported @@ -30,11 +33,39 @@ export class OTLPAwsSpanExporter extends OTLPProtoTraceExporter { // If the required dependencies are installed then we enable SigV4 signing. Otherwise skip it private hasRequiredDependencies: boolean = false; - constructor(endpoint: string, config?: OTLPExporterNodeConfigBase) { + private lloHandler: LLOHandler | undefined; + private loggerProvider: APILoggerProvider | undefined; + + constructor(endpoint: string, config?: OTLPExporterNodeConfigBase, loggerProvider?: APILoggerProvider) { super(OTLPAwsSpanExporter.changeUrlConfig(endpoint, config)); this.initDependencies(); this.region = endpoint.split('.')[1]; this.endpoint = endpoint; + + this.lloHandler = undefined; + this.loggerProvider = loggerProvider; + } + + // Lazily initialize LLO handler when needed to avoid initialization order issues + private ensureLloHandler(): boolean { + if (!this.lloHandler && isAgentObservabilityEnabled()) { + // If loggerProvider wasn't provided, try to get the current one + if (!this.loggerProvider) { + try { + this.loggerProvider = logs.getLoggerProvider(); + } catch (e: unknown) { + diag.debug('Failed to get logger provider', e); + return false; + } + } + + if (this.loggerProvider instanceof LoggerProvider) { + this.lloHandler = new LLOHandler(this.loggerProvider); + return true; + } + } + + return !!this.lloHandler; } /** @@ -43,10 +74,16 @@ export class OTLPAwsSpanExporter extends OTLPProtoTraceExporter { * sending it to the endpoint. Otherwise, we will skip signing. */ public override async export(items: ReadableSpan[], resultCallback: (result: ExportResult) => void): Promise { + let itemsToSerialize: ReadableSpan[] = items; + if (isAgentObservabilityEnabled() && this.ensureLloHandler() && this.lloHandler) { + // items to serialize are now the lloProcessedSpans + itemsToSerialize = this.lloHandler.processSpans(items); + } + // Only do SigV4 Signing if the required dependencies are installed. Otherwise default to the regular http/protobuf exporter. if (this.hasRequiredDependencies) { const url = new URL(this.endpoint); - const serializedSpans: Uint8Array | undefined = ProtobufTraceSerializer.serializeRequest(items); + const serializedSpans: Uint8Array | undefined = ProtobufTraceSerializer.serializeRequest(itemsToSerialize); if (serializedSpans === undefined) { return; @@ -92,7 +129,7 @@ export class OTLPAwsSpanExporter extends OTLPProtoTraceExporter { } } - super.export(items, resultCallback); + super.export(itemsToSerialize, resultCallback); } // Removes Sigv4 headers from old headers to avoid accidentally copying them to the new headers diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/utils.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/utils.ts index 0fd74c0d..eb8d6380 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/src/utils.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/utils.ts @@ -1,6 +1,13 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +const AGENT_OBSERVABILITY_ENABLED = 'AGENT_OBSERVABILITY_ENABLED'; + +// Bypass `readonly` restriction of a Type. +// Workaround provided from official TypeScript docs: +// https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-8.html#improved-control-over-mapped-type-modifiers +export type Mutable = { -readonly [P in keyof T]: T[P] }; + export const getNodeVersion = () => { const nodeVersion = process.versions.node; const versionParts = nodeVersion.split('.'); @@ -17,3 +24,12 @@ export const getNodeVersion = () => { return majorVersion; }; + +export const isAgentObservabilityEnabled = () => { + const agentObservabilityEnabled: string | undefined = process.env[AGENT_OBSERVABILITY_ENABLED]; + if (agentObservabilityEnabled === undefined) { + return false; + } + + return agentObservabilityEnabled.toLowerCase() === 'true'; +}; diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.base.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.base.test.ts new file mode 100644 index 00000000..a2612bcc --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.base.test.ts @@ -0,0 +1,67 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { LoggerProvider } from '@opentelemetry/sdk-logs'; +import { EventLogger, EventLoggerProvider } from '@opentelemetry/sdk-events'; +import { ReadableSpan } from '@opentelemetry/sdk-trace-base'; +import { Attributes, SpanContext, SpanKind, TraceFlags } from '@opentelemetry/api'; +import { LLOHandler } from '../src/llo-handler'; +import * as sinon from 'sinon'; +import { Resource } from '@opentelemetry/resources'; +import { EventLoggerOptions } from '@opentelemetry/api-events'; + +// Class with utilities for LLO Handler tests +export class LLOHandlerTestBase { + public loggerProviderMock: LoggerProvider; + public eventLoggerMock: EventLogger; + public eventLoggerProviderMock: EventLoggerProvider; + public lloHandler: LLOHandler; + + constructor() { + this.loggerProviderMock = sinon.createStubInstance(LoggerProvider); + this.lloHandler = new LLOHandler(this.loggerProviderMock); + this.eventLoggerProviderMock = sinon.stub(this.lloHandler['eventLoggerProvider']); + this.eventLoggerMock = sinon.createStubInstance(EventLogger); + this.eventLoggerProviderMock.getEventLogger = ( + name: string, + version?: string | undefined, + options?: EventLoggerOptions | undefined + ) => { + return this.eventLoggerMock; + }; + } + + public createMockSpan(attributes: Attributes, kind: SpanKind = SpanKind.INTERNAL): ReadableSpan { + const spanContext: SpanContext = { + traceId: '00000000000000000000000000000008', + spanId: '0000000000000009', + traceFlags: TraceFlags.SAMPLED, + }; + + const mockSpan: ReadableSpan = { + name: 'spanName', + kind, + spanContext: () => spanContext, + startTime: [1234567890, 0], + endTime: [1234567891, 0], + status: { code: 0 }, + attributes: attributes || {}, + links: [], + events: [], + duration: [0, 1], + ended: true, + instrumentationLibrary: { name: 'mockedLibrary', version: '1.0.0' }, + resource: new Resource({}), + droppedAttributesCount: 0, + droppedEventsCount: 0, + droppedLinksCount: 0, + }; + + return mockSpan; + } + + public updateMockSpanKey(span: ReadableSpan, key: string, value: T) { + // Bypass `readonly` restriction of ReadableSpan's attributes. + (span as any)[key] = value; + } +} diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.collection.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.collection.test.ts new file mode 100644 index 00000000..b5b8eb68 --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.collection.test.ts @@ -0,0 +1,301 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Attributes, HrTime } from '@opentelemetry/api'; +import expect from 'expect'; +import { LLOHandlerTestBase } from './llo-handler.base.test'; +import * as sinon from 'sinon'; + +/** + * Test message collection from various frameworks. + */ +describe('TestLLOHandlerCollection', () => { + let testBase: LLOHandlerTestBase; + + beforeEach(() => { + testBase = new LLOHandlerTestBase(); + }); + + afterEach(() => { + sinon.restore(); + }); + + /** + * Verify indexed prompt messages with system role are collected with correct content, role, and source. + */ + it('should collect gen_ai_prompt_messages with system role', () => { + const attributes: Attributes = { + 'gen_ai.prompt.0.content': 'system instruction', + 'gen_ai.prompt.0.role': 'system', + }; + + const span = testBase.createMockSpan(attributes); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + const message = messages[0]; + expect(message.content).toBe('system instruction'); + expect(message.role).toBe('system'); + expect(message.source).toBe('prompt'); + }); + + /** + * Verify indexed prompt messages with user role are collected with correct content, role, and source. + */ + it('should collect gen_ai_prompt_messages with user role', () => { + const attributes: Attributes = { + 'gen_ai.prompt.0.content': 'user question', + 'gen_ai.prompt.0.role': 'user', + }; + + const span = testBase.createMockSpan(attributes); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + const message = messages[0]; + expect(message.content).toBe('user question'); + expect(message.role).toBe('user'); + expect(message.source).toBe('prompt'); + }); + + /** + * Verify indexed prompt messages with assistant role are collected with correct content, role, and source. + */ + it('should collect gen_ai_prompt_messages with assistant role', () => { + const attributes: Attributes = { + 'gen_ai.prompt.1.content': 'assistant response', + 'gen_ai.prompt.1.role': 'assistant', + }; + + const span = testBase.createMockSpan(attributes); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + const message = messages[0]; + expect(message.content).toBe('assistant response'); + expect(message.role).toBe('assistant'); + expect(message.source).toBe('prompt'); + }); + + /** + * Verify indexed prompt messages with non-standard 'function' role are collected correctly. + */ + it('should collect gen_ai_prompt_messages with function role', () => { + const attributes: Attributes = { + 'gen_ai.prompt.2.content': 'function data', + 'gen_ai.prompt.2.role': 'function', + }; + + const span = testBase.createMockSpan(attributes); + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + const message = messages[0]; + expect(message.content).toBe('function data'); + expect(message.role).toBe('function'); + expect(message.source).toBe('prompt'); + }); + + /** + * Verify indexed prompt messages with unknown role are collected with the role preserved. + */ + it('should collect gen_ai_prompt_messages with unknown role', () => { + const attributes: Attributes = { + 'gen_ai.prompt.3.content': 'unknown type content', + 'gen_ai.prompt.3.role': 'unknown', + }; + + const span = testBase.createMockSpan(attributes); + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + const message = messages[0]; + expect(message.content).toBe('unknown type content'); + expect(message.role).toBe('unknown'); + expect(message.source).toBe('prompt'); + }); + + /** + * Verify indexed completion messages with assistant role are collected with source='completion'. + */ + it('should collect gen_ai_completion_messages with assistant role', () => { + const attributes: Attributes = { + 'gen_ai.completion.0.content': 'assistant completion', + 'gen_ai.completion.0.role': 'assistant', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + const message = messages[0]; + expect(message.content).toBe('assistant completion'); + expect(message.role).toBe('assistant'); + expect(message.source).toBe('completion'); + }); + + /** + * Verify indexed completion messages with custom roles are collected with source='completion'. + */ + it('should collect gen_ai_completion_messages with other role', () => { + const attributes: Attributes = { + 'gen_ai.completion.1.content': 'other completion', + 'gen_ai.completion.1.role': 'other', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + const message = messages[0]; + expect(message.content).toBe('other completion'); + expect(message.role).toBe('other'); + expect(message.source).toBe('completion'); + }); + + /** + * Verify collectAllLloMessages returns empty list when attributes are empty. + */ + it('should return empty list for collectAllLloMessages with empty attributes', () => { + const span = testBase.createMockSpan({}); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, {}); + + expect(messages).toEqual([]); + expect(messages.length).toBe(0); + }); + + /** + * Verify collectIndexedMessages returns empty list when attributes are None. + */ + it('should return empty list for collectIndexedMessages with empty attributes', () => { + const messages = testBase.lloHandler['collectIndexedMessages']({}); + + expect(messages).toEqual([]); + expect(messages.length).toBe(0); + }); + + /** + * Verify indexed messages use default roles when role attributes are missing. + */ + it('should use default roles when role attributes are missing', () => { + const attributes: Attributes = { + 'gen_ai.prompt.0.content': 'prompt without role', + 'gen_ai.completion.0.content': 'completion without role', + }; + + const span = testBase.createMockSpan(attributes); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(2); + + const promptMsg = messages.find(m => m.content === 'prompt without role'); + expect(promptMsg).toBeDefined(); + expect(promptMsg!.role).toBe('unknown'); + expect(promptMsg!.source).toBe('prompt'); + + const completionMsg = messages.find(m => m.content === 'completion without role'); + expect(completionMsg).toBeDefined(); + expect(completionMsg!.role).toBe('unknown'); + expect(completionMsg!.source).toBe('completion'); + }); + + /** + * Test that indexed messages are sorted correctly even with out-of-order indices + */ + it('should sort indexed messages correctly witwith out-of-order indices', () => { + const attributes: Attributes = { + 'gen_ai.prompt.5.content': 'fifth prompt', + 'gen_ai.prompt.5.role': 'user', + 'gen_ai.prompt.1.content': 'first prompt', + 'gen_ai.prompt.1.role': 'system', + 'gen_ai.prompt.3.content': 'third prompt', + 'gen_ai.prompt.3.role': 'user', + 'llm.input_messages.10.message.content': 'tenth message', + 'llm.input_messages.10.message.role': 'assistant', + 'llm.input_messages.2.message.content': 'second message', + 'llm.input_messages.2.message.role': 'user', + }; + + const messages = testBase.lloHandler['collectIndexedMessages'](attributes); + + // Messages should be sorted by pattern key first, then by index + expect(messages.length).toBe(5); + + // Check gen_ai.prompt messages are in order + const genAiMessages = messages.filter(m => m.source === 'prompt'); + expect(genAiMessages[0].content).toBe('first prompt'); + expect(genAiMessages[1].content).toBe('third prompt'); + expect(genAiMessages[2].content).toBe('fifth prompt'); + + // Check llm.input_messages are in order + const llmMessages = messages.filter(m => m.content.includes('message')); + expect(llmMessages[0].content).toBe('second message'); + expect(llmMessages[1].content).toBe('tenth message'); + }); + + /** + * Verify all message collection methods return consistent message format with content, + * role, and source fields. + */ + it('should maintain consistent message format across collection methods', () => { + const attributes: Attributes = { + 'gen_ai.prompt.0.content': 'prompt', + 'gen_ai.prompt.0.role': 'user', + 'gen_ai.completion.0.content': 'response', + 'gen_ai.completion.0.role': 'assistant', + 'traceloop.entity.input': 'input', + 'gen_ai.prompt': 'direct prompt', + 'input.value': 'inference input', + }; + + const span = testBase.createMockSpan(attributes); + + const promptMessages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + // Check that all messages have the required fields and correct types + for (const msg of promptMessages) { + expect(msg).toHaveProperty('content'); + expect(msg).toHaveProperty('role'); + expect(msg).toHaveProperty('source'); + expect(typeof msg.content).toBe('string'); + expect(typeof msg.role).toBe('string'); + expect(typeof msg.source).toBe('string'); + } + + const completionMessages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + for (const msg of completionMessages) { + expect(msg).toHaveProperty('content'); + expect(msg).toHaveProperty('role'); + expect(msg).toHaveProperty('source'); + } + + const traceloopMessages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + for (const msg of traceloopMessages) { + expect(msg).toHaveProperty('content'); + expect(msg).toHaveProperty('role'); + expect(msg).toHaveProperty('source'); + } + + const openlitMessages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + for (const msg of openlitMessages) { + expect(msg).toHaveProperty('content'); + expect(msg).toHaveProperty('role'); + expect(msg).toHaveProperty('source'); + } + + const openinferenceMessages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + for (const msg of openinferenceMessages) { + expect(msg).toHaveProperty('content'); + expect(msg).toHaveProperty('role'); + expect(msg).toHaveProperty('source'); + } + }); +}); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.events.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.events.test.ts new file mode 100644 index 00000000..3ac5b10b --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.events.test.ts @@ -0,0 +1,638 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { LLOHandlerTestBase } from './llo-handler.base.test'; +import { expect } from 'expect'; +import * as sinon from 'sinon'; +import { Event } from '@opentelemetry/api-events'; +import { TimedEvent } from '@opentelemetry/sdk-trace-base'; +import { InstrumentationScope } from '@opentelemetry/core'; +import { OTEL_SPAN_KEY } from '../src/llo-handler'; +import { Attributes, HrTime, trace } from '@opentelemetry/api'; + +/** + * Test event emission and formatting functionality. + */ +describe('TestLLOHandlerEvents', () => { + let testBase: LLOHandlerTestBase; + + beforeEach(() => { + testBase = new LLOHandlerTestBase(); + }); + + afterEach(() => { + sinon.restore(); + }); + + /** + * Verify emitLloAttributes creates a single consolidated event with input/output message groups + * containing all LLO content from various frameworks. + */ + it('should emit consolidated event with input/output message groups', () => { + // Create attributes simulating content from multiple frameworks + const attributes = { + 'gen_ai.prompt.0.content': 'prompt content', + 'gen_ai.prompt.0.role': 'user', + 'gen_ai.completion.0.content': 'completion content', + 'gen_ai.completion.0.role': 'assistant', + 'traceloop.entity.input': 'traceloop input', + 'traceloop.entity.name': 'entity_name', + 'gen_ai.agent.actual_output': 'agent output', + 'crewai.crew.tasks_output': 'tasks output', + 'crewai.crew.result': 'crew result', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + const emittedEvent = (testBase.eventLoggerMock.emit as any).getCall(0).args[0] as Event; + + expect(emittedEvent.name).toBe('test.scope'); + expect(emittedEvent.timestamp).toEqual(span.endTime); + expect(emittedEvent.context?.getValue(OTEL_SPAN_KEY)).toBe(span); + expect(trace.getSpanContext(emittedEvent.context!)).toBe(span.spanContext()); + + expect(emittedEvent.data).toBeDefined(); + + const eventBody = emittedEvent.data as any; + expect(eventBody.input).toBeDefined(); + expect(eventBody.output).toBeDefined(); + expect(eventBody.input.messages).toBeDefined(); + expect(eventBody.output.messages).toBeDefined(); + + const inputMessages = eventBody.input.messages; + expect(inputMessages.length).toBe(2); + + const userPrompt = inputMessages.find((msg: any) => msg.content === 'prompt content'); + expect(userPrompt).toBeDefined(); + expect(userPrompt.role).toBe('user'); + + const traceloopInput = inputMessages.find((msg: any) => msg.content === 'traceloop input'); + expect(traceloopInput).toBeDefined(); + expect(traceloopInput.role).toBe('user'); + + const outputMessages = eventBody.output.messages; + expect(outputMessages.length).toBeGreaterThanOrEqual(3); + + const completion = outputMessages.find((msg: any) => msg.content === 'completion content'); + expect(completion).toBeDefined(); + expect(completion.role).toBe('assistant'); + + const agentOutput = outputMessages.find((msg: any) => msg.content === 'agent output'); + expect(agentOutput).toBeDefined(); + expect(agentOutput.role).toBe('assistant'); + }); + + /** + * Verify a single span containing LLO attributes from multiple frameworks + * (Traceloop, OpenLit, OpenInference, CrewAI) generates one consolidated event. + */ + it('should emit consolidated event from multiple frameworks', () => { + const attributes = { + 'gen_ai.prompt.0.content': 'Tell me about AI', + 'gen_ai.prompt.0.role': 'user', + 'gen_ai.completion.0.content': 'AI is a field of computer science...', + 'gen_ai.completion.0.role': 'assistant', + 'traceloop.entity.input': 'What is machine learning?', + 'traceloop.entity.output': 'Machine learning is a subset of AI...', + 'gen_ai.prompt': 'Explain neural networks', + 'gen_ai.completion': 'Neural networks are computing systems...', + 'input.value': 'How do transformers work?', + 'output.value': 'Transformers are a type of neural network architecture...', + 'crewai.crew.result': 'Task completed successfully', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.multi.framework', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + const emittedEvent = (testBase.eventLoggerMock.emit as any).getCall(0).args[0] as Event; + + expect(emittedEvent.name).toBe('test.multi.framework'); + expect(emittedEvent.timestamp).toEqual(span.endTime); + + const eventBody = emittedEvent.data as any; + expect(eventBody.input).toBeDefined(); + expect(eventBody.output).toBeDefined(); + + const inputMessages = eventBody.input.messages; + const inputContents = inputMessages.map((msg: any) => msg.content); + expect(inputContents).toContain('Tell me about AI'); + expect(inputContents).toContain('What is machine learning?'); + expect(inputContents).toContain('Explain neural networks'); + expect(inputContents).toContain('How do transformers work?'); + + // Verify output messages from all frameworks + const outputMessages = eventBody.output.messages; + const outputContents = outputMessages.map((msg: any) => msg.content); + expect(outputContents).toContain('AI is a field of computer science...'); + expect(outputContents).toContain('Machine learning is a subset of AI...'); + expect(outputContents).toContain('Neural networks are computing systems...'); + expect(outputContents).toContain('Transformers are a type of neural network architecture...'); + expect(outputContents).toContain('Task completed successfully'); + + inputMessages.forEach((msg: any) => { + expect(['user', 'system']).toContain(msg.role); + }); + outputMessages.forEach((msg: any) => { + expect(msg.role).toBe('assistant'); + }); + }); + + /** + * Verify emitLloAttributes does not emit events when span contains only non-LLO attributes. + */ + it('should not emit event when span contains only non-LLO attributes', () => { + const attributes = { + 'normal.attribute': 'value', + 'another.attribute': 123, + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + sinon.assert.notCalled((testBase.eventLoggerMock as any).emit); + }); + + /** + * Verify event generation correctly separates mixed input (system/user) and output (assistant) messages. + */ + it('should separate mixed input/output messages correctly', () => { + const attributes = { + 'gen_ai.prompt.0.content': 'system message', + 'gen_ai.prompt.0.role': 'system', + 'gen_ai.prompt.1.content': 'user message', + 'gen_ai.prompt.1.role': 'user', + 'gen_ai.completion.0.content': 'assistant response', + 'gen_ai.completion.0.role': 'assistant', + 'input.value': 'direct input', + 'output.value': 'direct output', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + const emittedEvent = (testBase.eventLoggerMock.emit as any).getCall(0).args[0] as Event; + + const eventBody = emittedEvent.data as any; + expect(eventBody.input).toBeDefined(); + expect(eventBody.output).toBeDefined(); + + const inputMessages = eventBody.input.messages; + expect(inputMessages.length).toBe(3); + + const inputRoles = inputMessages.map((msg: any) => msg.role); + expect(inputRoles).toContain('system'); + expect(inputRoles).toContain('user'); + + const outputMessages = eventBody.output.messages; + expect(outputMessages.length).toBe(2); + + outputMessages.forEach((msg: any) => { + expect(msg.role).toBe('assistant'); + }); + }); + + /** + * Verify emitLloAttributes uses provided event timestamp instead of span end time. + */ + it('should use provided event timestamp instead of span end time', () => { + const attributes = { + 'gen_ai.prompt': 'test prompt', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + const eventTimestamp: HrTime = [9999999999, 0]; + + testBase.lloHandler['emitLloAttributes'](span, attributes, eventTimestamp); + + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + const emittedEvent = (testBase.eventLoggerMock.emit as any).getCall(0).args[0] as Event; + expect(emittedEvent.timestamp).toEqual(eventTimestamp); + }); + + /** + * Test emitLloAttributes with null attributes - should return early + */ + it('should handle null attributes in emitLloAttributes', () => { + const span = testBase.createMockSpan({}); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, null as any); + + sinon.assert.notCalled(testBase.eventLoggerMock.emit as any); + }); + + /** + * Test role-based routing for non-standard roles + */ + it('should route non-standard roles based on source', () => { + const attributes = { + // Standard roles - should go to their expected places + 'gen_ai.prompt.0.content': 'system prompt', + 'gen_ai.prompt.0.role': 'system', + 'gen_ai.prompt.1.content': 'user prompt', + 'gen_ai.prompt.1.role': 'user', + 'gen_ai.completion.0.content': 'assistant response', + 'gen_ai.completion.0.role': 'assistant', + // Non-standard roles - should be routed based on source + 'gen_ai.prompt.2.content': 'function prompt', + 'gen_ai.prompt.2.role': 'function', + 'gen_ai.completion.1.content': 'tool completion', + 'gen_ai.completion.1.role': 'tool', + 'gen_ai.prompt.3.content': 'unknown prompt', + 'gen_ai.prompt.3.role': 'custom_role', + 'gen_ai.completion.2.content': 'unknown completion', + 'gen_ai.completion.2.role': 'another_custom', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + // Verify event was emitted + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + const emittedEvent = (testBase.eventLoggerMock.emit as any).getCall(0).args[0] as Event; + const eventBody = emittedEvent.data as any; + + // Check input messages + const inputMessages = eventBody.input.messages; + const inputContents = inputMessages.map((msg: any) => msg.content); + + // Standard roles (system, user) should be in input + expect(inputContents).toContain('system prompt'); + expect(inputContents).toContain('user prompt'); + + // Non-standard roles from prompt source should be in input + expect(inputContents).toContain('function prompt'); + expect(inputContents).toContain('unknown prompt'); + + // Check output messages + const outputMessages = eventBody.output.messages; + const outputContents = outputMessages.map((msg: any) => msg.content); + + // Standard role (assistant) should be in output + expect(outputContents).toContain('assistant response'); + + // Non-standard roles from completion source should be in output + expect(outputContents).toContain('tool completion'); + expect(outputContents).toContain('unknown completion'); + }); + + /** + * Test emitLloAttributes when messages list is empty after collection + */ + it('should not emit event when messages list is empty after collection', () => { + // Create a span with attributes that would normally match patterns but with empty content + const attributes = { + 'gen_ai.prompt.0.content': '', + 'gen_ai.prompt.0.role': 'user', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + // Mock collectAllLloMessages to return an empty array + const collectAllLloMessagesSpy = sinon.stub(testBase.lloHandler as any, 'collectAllLloMessages').returns([]); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + // Should not emit event when no messages collected + sinon.assert.notCalled(testBase.eventLoggerMock.emit as any); + + collectAllLloMessagesSpy.restore(); + }); + + /** + * Test event generation when only input messages are present + */ + it('should test emitLloAttributes with only input messages', () => { + const attributes = { + 'gen_ai.prompt.0.content': 'system instruction', + 'gen_ai.prompt.0.role': 'system', + 'gen_ai.prompt.1.content': 'user question', + 'gen_ai.prompt.1.role': 'user', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + const emittedEvent = (testBase.eventLoggerMock.emit as any).getCall(0).args[0] as Event; + const eventBody = emittedEvent.data as any; + + expect(eventBody.input).toBeDefined(); + expect(eventBody).not.toHaveProperty('output'); + + const inputMessages = eventBody.input.messages; + expect(inputMessages.length).toBe(2); + }); + + /** + * Test event generation when only output messages are present + */ + it('should test emitLloAttributes with only output messages', () => { + const attributes = { + 'gen_ai.completion.0.content': 'assistant response', + 'gen_ai.completion.0.role': 'assistant', + 'output.value': 'another output', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + const emittedEvent = (testBase.eventLoggerMock.emit as any).getCall(0).args[0] as Event; + const eventBody = emittedEvent.data as any; + + expect(eventBody).not.toHaveProperty('input'); + expect(eventBody.output).toBeDefined(); + + const outputMessages = eventBody.output.messages; + expect(outputMessages.length).toBe(2); + }); + + /** + * Test that no event is emitted when event body would be empty + */ + it('should test emitLloAttributes with empty event body', () => { + // Create attributes that would result in messages with empty content from collectAllLloMessages + const attributes: Attributes = { + 'gen_ai.prompt.0.content': '', + 'gen_ai.prompt.0.role': 'user', + }; + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + // Event should still be emitted as we have a message (even with empty content) + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + }); + + /** + * Test groupMessagesByType correctly groups messages with standard roles. + */ + it('should test groupMessagesByType with standard roles', () => { + const messages = [ + { role: 'system', content: 'System message', source: 'prompt' }, + { role: 'user', content: 'User message', source: 'prompt' }, + { role: 'assistant', content: 'Assistant message', source: 'completion' }, + ]; + + const result = testBase.lloHandler['groupMessagesByType'](messages); + + expect(result.input).toBeDefined(); + expect(result.output).toBeDefined(); + + // Check input messages + expect(result.input.length).toBe(2); + expect(result.input[0]).toEqual({ role: 'system', content: 'System message' }); + expect(result.input[1]).toEqual({ role: 'user', content: 'User message' }); + + // Check output messages + expect(result.output.length).toBe(1); + expect(result.output[0]).toEqual({ role: 'assistant', content: 'Assistant message' }); + }); + + /** + * Test groupMessagesByType correctly routes non-standard roles based on source. + */ + it('should test groupMessagesByType with non standard roles', () => { + const messages = [ + { role: 'function', content: 'Function call', source: 'prompt' }, + { role: 'tool', content: 'Tool result', source: 'completion' }, + { role: 'custom', content: 'Custom output', source: 'output' }, + { role: 'other', content: 'Other result', source: 'result' }, + ]; + + const result = testBase.lloHandler['groupMessagesByType'](messages); + + // Non-standard roles from prompt source go to input + expect(result.input.length).toBe(1); + expect(result.input[0]).toEqual({ role: 'function', content: 'Function call' }); + + // Non-standard roles from completion/output/result sources go to output + expect(result.output.length).toBe(3); + const outputContents = result.output.map(msg => msg.content); + expect(outputContents).toContain('Tool result'); + expect(outputContents).toContain('Custom output'); + expect(outputContents).toContain('Other result'); + }); + + /** + * Test groupMessagesByType handles empty message list. + */ + it('should test groupMessagesByType handle empty list', () => { + const result = testBase.lloHandler['groupMessagesByType']([]); + + expect(result.input).toEqual([]); + expect(result.output).toEqual([]); + expect(result.input.length).toBe(0); + expect(result.output.length).toBe(0); + }); + + /** + * Test that llm.prompts attribute is properly emitted in the input section. + */ + it('should handle llm.prompts attribute', () => { + const llmPromptsContent = "[{'role': 'system', 'content': [{'text': 'You are helpful.', 'type': 'text'}]}]"; + const attributes = { + 'llm.prompts': llmPromptsContent, + 'gen_ai.completion.0.content': 'I understand.', + 'gen_ai.completion.0.role': 'assistant', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + const emittedEvent = (testBase.eventLoggerMock.emit as any).getCall(0).args[0] as Event; + const eventBody = emittedEvent.data as any; + + // Check that llm.prompts is in input section + expect(eventBody.input).toBeDefined(); + expect(eventBody.output).toBeDefined(); + + const inputMessages = eventBody.input.messages; + expect(inputMessages.length).toBe(1); + expect(inputMessages[0].content).toBe(llmPromptsContent); + expect(inputMessages[0].role).toBe('user'); + + // Check output section has the completion + const outputMessages = eventBody.output.messages; + expect(outputMessages.length).toBe(1); + expect(outputMessages[0].content).toBe('I understand.'); + expect(outputMessages[0].role).toBe('assistant'); + }); + + /** + * Test that LLO attributes from OpenLit-style span events are collected and emitted + * in a single consolidated event, not as separate events. + */ + it('should emit a single consolidated event for OpenLit-style span events', () => { + // This test simulates the OpenLit pattern where prompt and completion are in span events + // The span processor should collect from both and emit a single event + const spanAttributes = { 'normal.attribute': 'value' }; + + // Create events like OpenLit does + const promptEventAttrs = { 'gen_ai.prompt': 'Explain quantum computing' }; + const promptEvent: TimedEvent = { + attributes: promptEventAttrs, + name: 'prompt_event', + time: [1234567890, 0], + }; + + const completionEventAttrs = { 'gen_ai.completion': 'Quantum computing is...' }; + const completionEvent: TimedEvent = { + attributes: completionEventAttrs, + name: 'completion_event', + time: [1234567891, 0], + }; + + const span = testBase.createMockSpan(spanAttributes); + testBase.updateMockSpanKey(span, 'events', [promptEvent, completionEvent]); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { + name: 'openlit.otel.tracing', + version: '1.0.0', + } as InstrumentationScope); + + // Process the span (this would normally be called by processSpans) + const allLloAttrs = testBase.lloHandler['collectLloAttributesFromSpan'](span); + + // Emit consolidated event + testBase.lloHandler['emitLloAttributes'](span, allLloAttrs); + + // Verify single event was emitted with both input and output + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + const emittedEvent = (testBase.eventLoggerMock.emit as any).getCall(0).args[0] as Event; + const eventBody = emittedEvent.data as any; + + // Both input and output should be in the same event + expect(eventBody.input).toBeDefined(); + expect(eventBody.output).toBeDefined(); + + // Check input section + const inputMessages = eventBody.input.messages; + expect(inputMessages.length).toBe(1); + expect(inputMessages[0].content).toBe('Explain quantum computing'); + expect(inputMessages[0].role).toBe('user'); + + // Check output section + const outputMessages = eventBody.output.messages; + expect(outputMessages.length).toBe(1); + expect(outputMessages[0].content).toBe('Quantum computing is...'); + expect(outputMessages[0].role).toBe('assistant'); + }); + + /** + * Verify session.id attribute from span is copied to event attributes when present. + */ + it('emitLloAttributes should copy session.id to event attributes when present', () => { + const attributes = { + 'session.id': 'test-session-123', + 'gen_ai.prompt': 'Hello, AI', + 'gen_ai.completion': 'Hello! How can I help you?', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + const emittedEvent = (testBase.eventLoggerMock.emit as any).getCall(0).args[0] as Event; + + // Verify session.id was copied to event attributes + expect(emittedEvent.attributes).toBeDefined(); + expect(emittedEvent.attributes?.['session.id']).toBe('test-session-123'); + + // Verify event body still contains LLO data + const eventBody = emittedEvent.data as any; + expect(eventBody.input).toBeDefined(); + expect(eventBody.output).toBeDefined(); + }); + + /** + * Verify event attributes do not contain session.id when not present in span attributes. + */ + it('emitLloAttributes should not include session.id in event attributes when not present', () => { + const attributes = { + 'gen_ai.prompt': 'Hello, AI', + 'gen_ai.completion': 'Hello! How can I help you?', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + const emittedEvent = (testBase.eventLoggerMock.emit as any).getCall(0).args[0] as Event; + + // Verify session.id is not in event attributes (because the event doesn't have attributes) + expect(emittedEvent.attributes).toBeUndefined(); + expect(emittedEvent).not.toHaveProperty('attributes'); + }); + + /** + * Verify only session.id is copied from span attributes when mixed with other attributes. + */ + it('emitLloAttributes should only copy session.id when mixed with other attributes', () => { + const attributes = { + 'session.id': 'session-456', + 'user.id': 'user-789', + 'gen_ai.prompt': "What's the weather?", + 'gen_ai.completion': "I can't check the weather.", + 'other.attribute': 'some-value', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { name: 'test.scope', version: '1.0.0' }); + + testBase.lloHandler['emitLloAttributes'](span, attributes); + + sinon.assert.calledOnce(testBase.eventLoggerMock.emit as any); + const emittedEvent = (testBase.eventLoggerMock.emit as any).getCall(0).args[0] as Event; + + // Verify only session.id was copied to event attributes (plus event.name from Event class) + expect(emittedEvent.attributes).toBeDefined(); + expect(emittedEvent.attributes?.['session.id']).toBe('session-456'); + // Verify other span attributes were not copied + expect(emittedEvent.attributes).not.toHaveProperty('user.id'); + expect(emittedEvent.attributes).not.toHaveProperty('other.attribute'); + expect(emittedEvent.attributes).not.toHaveProperty('gen_ai.prompt'); + expect(emittedEvent.attributes).not.toHaveProperty('gen_ai.completion'); + }); +}); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.framework.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.framework.test.ts new file mode 100644 index 00000000..a6376f89 --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.framework.test.ts @@ -0,0 +1,474 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Attributes, HrTime } from '@opentelemetry/api'; +import expect from 'expect'; +import { LLOHandlerTestBase } from './llo-handler.base.test'; +import { InstrumentationLibrary } from '@opentelemetry/core'; +import * as sinon from 'sinon'; + +/** + * Test framework-specific LLO attribute handling. + */ +describe('TestLLOHandlerFrameworks', () => { + let testBase: LLOHandlerTestBase; + + beforeEach(() => { + testBase = new LLOHandlerTestBase(); + }); + + afterEach(() => { + sinon.restore(); + }); + + /** + * Verify Traceloop entity input/output attributes are collected with correct roles + * (input->user, output->assistant). + */ + it('should collect Traceloop messages', () => { + const attributes: Attributes = { + 'traceloop.entity.input': 'input data', + 'traceloop.entity.output': 'output data', + 'traceloop.entity.name': 'my_entity', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + const traceloopMessages = messages.filter(m => ['input', 'output'].includes(m.source)); + + expect(traceloopMessages.length).toBe(2); + + const inputMessage = traceloopMessages[0]; + expect(inputMessage.content).toBe('input data'); + expect(inputMessage.role).toBe('user'); + expect(inputMessage.source).toBe('input'); + + const outputMessage = traceloopMessages[1]; + expect(outputMessage.content).toBe('output data'); + expect(outputMessage.role).toBe('assistant'); + expect(outputMessage.source).toBe('output'); + }); + + /** + * Verify collection of mixed Traceloop and CrewAI attributes, ensuring all are collected + * with appropriate roles and sources. + */ + it('should collect Traceloop messages with all attributes', () => { + const attributes: Attributes = { + 'traceloop.entity.input': 'input data', + 'traceloop.entity.output': 'output data', + 'crewai.crew.tasks_output': "[TaskOutput(description='Task 1', output='Result 1')]", + 'crewai.crew.result': 'Final crew result', + 'traceloop.entity.name': 'crewai_agent', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(4); + + expect(messages[0].content).toBe('input data'); + expect(messages[0].role).toBe('user'); + expect(messages[0].source).toBe('input'); + + expect(messages[1].content).toBe('output data'); + expect(messages[1].role).toBe('assistant'); + expect(messages[1].source).toBe('output'); + + expect(messages[2].content).toBe("[TaskOutput(description='Task 1', output='Result 1')]"); + expect(messages[2].role).toBe('assistant'); + expect(messages[2].source).toBe('output'); + + expect(messages[3].content).toBe('Final crew result'); + expect(messages[3].role).toBe('assistant'); + expect(messages[3].source).toBe('result'); + }); + + /** + * Verify OpenLit's direct gen_ai.prompt attribute is collected with user role and prompt source. + */ + it('should collect OpenLit messages with direct prompt', () => { + const attributes: Attributes = { 'gen_ai.prompt': 'user direct prompt' }; + + const span = testBase.createMockSpan(attributes); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + const message = messages[0]; + expect(message.content).toBe('user direct prompt'); + expect(message.role).toBe('user'); + expect(message.source).toBe('prompt'); + }); + + /** + * Verify OpenLit's direct gen_ai.completion attribute is collected with assistant role and completion source. + */ + it('should collect OpenLit messages with direct completion', () => { + const attributes: Attributes = { 'gen_ai.completion': 'assistant direct completion' }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + const message = messages[0]; + expect(message.content).toBe('assistant direct completion'); + expect(message.role).toBe('assistant'); + expect(message.source).toBe('completion'); + }); + + /** + * Verify all OpenLit framework attributes (prompt, completion, revised_prompt, agent.*) + * are collected with correct roles and sources. + */ + it('should collect OpenLit messages with all attributes', () => { + const attributes: Attributes = { + 'gen_ai.prompt': 'user prompt', + 'gen_ai.completion': 'assistant response', + 'gen_ai.content.revised_prompt': 'revised prompt', + 'gen_ai.agent.actual_output': 'agent output', + 'gen_ai.agent.human_input': 'human input to agent', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(5); + + expect(messages[0].content).toBe('user prompt'); + expect(messages[0].role).toBe('user'); + expect(messages[0].source).toBe('prompt'); + + expect(messages[1].content).toBe('assistant response'); + expect(messages[1].role).toBe('assistant'); + expect(messages[1].source).toBe('completion'); + + expect(messages[2].content).toBe('revised prompt'); + expect(messages[2].role).toBe('system'); + expect(messages[2].source).toBe('prompt'); + + expect(messages[3].content).toBe('agent output'); + expect(messages[3].role).toBe('assistant'); + expect(messages[3].source).toBe('output'); + + expect(messages[4].content).toBe('human input to agent'); + expect(messages[4].role).toBe('user'); + expect(messages[4].source).toBe('input'); + }); + + /** + * Verify OpenLit's gen_ai.content.revised_prompt is collected with system role and prompt source. + */ + it('should collect OpenLit messages with revised prompt', () => { + const attributes: Attributes = { 'gen_ai.content.revised_prompt': 'revised system prompt' }; + + const span = testBase.createMockSpan(attributes); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + const message = messages[0]; + expect(message.content).toBe('revised system prompt'); + expect(message.role).toBe('system'); + expect(message.source).toBe('prompt'); + }); + + /** + * Verify OpenInference's direct input.value and output.value attributes are collected + * with appropriate roles (user/assistant) and sources. + */ + it('should collect OpenInference messages with direct attributes', () => { + const attributes: Attributes = { + 'input.value': 'user prompt', + 'output.value': 'assistant response', + 'llm.model_name': 'gpt-4', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(2); + + const inputMessage = messages[0]; + expect(inputMessage.content).toBe('user prompt'); + expect(inputMessage.role).toBe('user'); + expect(inputMessage.source).toBe('input'); + + const outputMessage = messages[1]; + expect(outputMessage.content).toBe('assistant response'); + expect(outputMessage.role).toBe('assistant'); + expect(outputMessage.source).toBe('output'); + }); + + /** + * Verify OpenInference's indexed llm.input_messages.{n}.message.content attributes + * are collected with roles from corresponding role attributes. + */ + it('should collect OpenInference messages with structured input', () => { + const attributes: Attributes = { + 'llm.input_messages.0.message.content': 'system prompt', + 'llm.input_messages.0.message.role': 'system', + 'llm.input_messages.1.message.content': 'user message', + 'llm.input_messages.1.message.role': 'user', + 'llm.model_name': 'claude-3', + }; + + const span = testBase.createMockSpan(attributes); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(2); + + const systemMessage = messages[0]; + expect(systemMessage.content).toBe('system prompt'); + expect(systemMessage.role).toBe('system'); + expect(systemMessage.source).toBe('input'); + + const userMessage = messages[1]; + expect(userMessage.content).toBe('user message'); + expect(userMessage.role).toBe('user'); + expect(userMessage.source).toBe('input'); + }); + + /** + * Verify OpenInference's indexed llm.output_messages.{n}.message.content attributes + * are collected with source='output' and roles from corresponding attributes. + */ + it('should collect OpenInference messages with structured output', () => { + const attributes: Attributes = { + 'llm.output_messages.0.message.content': 'assistant response', + 'llm.output_messages.0.message.role': 'assistant', + 'llm.model_name': 'llama-3', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + + const outputMessage = messages[0]; + expect(outputMessage.content).toBe('assistant response'); + expect(outputMessage.role).toBe('assistant'); + expect(outputMessage.source).toBe('output'); + }); + + /** + * Verify mixed OpenInference attributes (direct and indexed) are all collected + * and maintain correct roles and counts. + */ + it('should collect OpenInference messages with mixed attributes', () => { + const attributes: Attributes = { + 'input.value': 'direct input', + 'output.value': 'direct output', + 'llm.input_messages.0.message.content': 'message input', + 'llm.input_messages.0.message.role': 'user', + 'llm.output_messages.0.message.content': 'message output', + 'llm.output_messages.0.message.role': 'assistant', + 'llm.model_name': 'bedrock.claude-3', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(4); + + const contents = messages.map(msg => msg.content); + expect(contents).toContain('direct input'); + expect(contents).toContain('direct output'); + expect(contents).toContain('message input'); + expect(contents).toContain('message output'); + + const roles = messages.map(msg => msg.role); + expect(roles.filter(role => role === 'user').length).toBe(2); + expect(roles.filter(role => role === 'assistant').length).toBe(2); + }); + + /** + * Verify OpenLit's gen_ai.agent.actual_output is collected with assistant role and output source. + */ + it('should collect OpenLit messages with agent actual output', () => { + const attributes: Attributes = { 'gen_ai.agent.actual_output': 'Agent task output result' }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + + const message = messages[0]; + expect(message.content).toBe('Agent task output result'); + expect(message.role).toBe('assistant'); + expect(message.source).toBe('output'); + }); + + /** + * Verify OpenLit's gen_ai.agent.human_input is collected with user role and input source. + */ + it('should collect OpenLit messages with agent human input', () => { + const attributes: Attributes = { 'gen_ai.agent.human_input': 'Human input to the agent' }; + + const span = testBase.createMockSpan(attributes); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + const message = messages[0]; + expect(message.content).toBe('Human input to the agent'); + expect(message.role).toBe('user'); + expect(message.source).toBe('input'); + }); + + /** + * Verify CrewAI-specific attributes (tasks_output, result) are collected with assistant role + * and appropriate sources. + */ + it('should collect Traceloop messages with crew outputs', () => { + const attributes: Attributes = { + 'crewai.crew.tasks_output': "[TaskOutput(description='Task description', output='Task result')]", + 'crewai.crew.result': 'Final crew execution result', + 'traceloop.entity.name': 'crewai', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(2); + + const tasksMessage = messages[0]; + expect(tasksMessage.content).toBe("[TaskOutput(description='Task description', output='Task result')]"); + expect(tasksMessage.role).toBe('assistant'); + expect(tasksMessage.source).toBe('output'); + + const resultMessage = messages[1]; + expect(resultMessage.content).toBe('Final crew execution result'); + expect(resultMessage.role).toBe('assistant'); + expect(resultMessage.source).toBe('result'); + }); + + /** + * Verify OpenInference indexed messages use default roles (user for input, assistant for output) + * when role attributes are missing. + */ + it('should handle OpenInference messages with default roles', () => { + const attributes: Attributes = { + 'llm.input_messages.0.message.content': 'input without role', + 'llm.output_messages.0.message.content': 'output without role', + }; + + const span = testBase.createMockSpan(attributes); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(2); + + const inputMsg = messages.find(m => m.content === 'input without role'); + expect(inputMsg).toBeDefined(); + expect(inputMsg!.role).toBe('user'); + expect(inputMsg!.source).toBe('input'); + + const outputMsg = messages.find(m => m.content === 'output without role'); + expect(outputMsg).toBeDefined(); + expect(outputMsg!.role).toBe('assistant'); + expect(outputMsg!.source).toBe('output'); + }); + + /** + * Verify Strands SDK patterns (system_prompt, tool.result) are collected + * with correct roles and sources. + */ + it('should collect Strands SDK messages', () => { + const attributes: Attributes = { + system_prompt: 'You are a helpful assistant', + 'tool.result': 'Tool execution completed successfully', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'endTime', [1234567899, 0]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { + name: 'strands.sdk', + version: '1.0.0', + }); + + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(2); + + const systemMsg = messages.find(m => m.content === 'You are a helpful assistant'); + expect(systemMsg).toBeDefined(); + expect(systemMsg!.role).toBe('system'); + expect(systemMsg!.source).toBe('prompt'); + + const toolMsg = messages.find(m => m.content === 'Tool execution completed successfully'); + expect(toolMsg).toBeDefined(); + expect(toolMsg!.role).toBe('assistant'); + expect(toolMsg!.source).toBe('output'); + }); + + /** + * Verify llm.prompts attribute is collected as a user message with prompt source. + */ + it('should collect llm.prompts messages', () => { + const attributes: Attributes = { + 'llm.prompts': + "[{'role': 'system', 'content': [{'text': 'You are a helpful AI assistant.', 'type': 'text'}]}, " + + "{'role': 'user', 'content': [{'text': 'What are the benefits of using FastAPI?', 'type': 'text'}]}]", + 'other.attribute': 'not collected', + }; + + const span = testBase.createMockSpan(attributes); + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(1); + const message = messages[0]; + expect(message.content).toBe(attributes['llm.prompts']); + expect(message.role).toBe('user'); + expect(message.source).toBe('prompt'); + }); + + /** + * Verify llm.prompts works correctly alongside other LLO attributes. + */ + it('should collect llm.prompts with other messages', () => { + const attributes: Attributes = { + 'llm.prompts': "[{'role': 'system', 'content': 'System prompt'}]", + 'gen_ai.prompt': 'Direct prompt', + 'gen_ai.completion': 'Assistant response', + }; + + const span = testBase.createMockSpan(attributes); + const messages = testBase.lloHandler['collectAllLloMessages'](span, attributes); + + expect(messages.length).toBe(3); + + // Check llm.prompts message + const llmPromptsMsg = messages.find(m => m.content === attributes['llm.prompts']); + expect(llmPromptsMsg).toBeDefined(); + expect(llmPromptsMsg!.role).toBe('user'); + expect(llmPromptsMsg!.source).toBe('prompt'); + + // Check other messages are still collected + const directPromptMsg = messages.find(m => m.content === 'Direct prompt'); + expect(directPromptMsg).toBeDefined(); + + const completionMsg = messages.find(m => m.content === 'Assistant response'); + expect(completionMsg).toBeDefined(); + }); +}); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.patterns.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.patterns.test.ts new file mode 100644 index 00000000..3a42be8a --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.patterns.test.ts @@ -0,0 +1,140 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import expect from 'expect'; +import { LLO_PATTERNS, LLOHandler, PatternType } from '../src/llo-handler'; +import { LLOHandlerTestBase } from './llo-handler.base.test'; +import * as sinon from 'sinon'; + +/** + * Test pattern matching and recognition functionality. + */ +describe('TestLLOHandlerPatterns', () => { + let testBase: LLOHandlerTestBase; + + before(() => { + testBase = new LLOHandlerTestBase(); + }); + + after(() => { + sinon.restore(); + }); + + /** + * Verify LLOHandler initializes correctly with logger provider and creates event logger provider. + */ + it('should test init', () => { + expect(testBase.lloHandler['loggerProvider']).toEqual(testBase.loggerProviderMock); + expect(testBase.lloHandler['eventLoggerProvider']['_loggerProvider']).toEqual( + testBase.eventLoggerProviderMock['_loggerProvider'] + ); + expect(testBase.lloHandler['eventLoggerProvider'].getEventLogger('123')).toEqual(testBase.eventLoggerMock); + }); + + /** + * Verify isLloAttribute correctly identifies indexed Gen AI prompt patterns (gen_ai.prompt.{n}.content). + */ + it('should test isLloAttribute match', () => { + expect(testBase.lloHandler['isLloAttribute']('gen_ai.prompt.0.content')).toBeTruthy(); + expect(testBase.lloHandler['isLloAttribute']('gen_ai.prompt.123.content')).toBeTruthy(); + }); + + /** + * Verify isLloAttribute correctly rejects malformed patterns and non-LLO attributes. + */ + it('should test isLloAttribute no match', () => { + expect(testBase.lloHandler['isLloAttribute']('gen_ai.prompt.content')).toBeFalsy(); + expect(testBase.lloHandler['isLloAttribute']('gen_ai.prompt.abc.content')).toBeFalsy(); + expect(testBase.lloHandler['isLloAttribute']('some.other.attribute')).toBeFalsy(); + }); + + /** + * Verify isLloAttribute recognizes Traceloop framework patterns (traceloop.entity.input/output). + */ + it('should test isLloAttribute traceloop match', () => { + // Test exact matches for Traceloop attributes + expect(testBase.lloHandler['isLloAttribute']('traceloop.entity.input')).toBeTruthy(); + expect(testBase.lloHandler['isLloAttribute']('traceloop.entity.output')).toBeTruthy(); + }); + + /** + * Verify isLloAttribute recognizes OpenLit framework patterns (gen_ai.prompt, gen_ai.completion, etc.). + */ + it('should test isLloAttribute openlit match', () => { + // Test exact matches for direct OpenLit attributes + expect(testBase.lloHandler['isLloAttribute']('gen_ai.prompt')).toBeTruthy(); + expect(testBase.lloHandler['isLloAttribute']('gen_ai.completion')).toBeTruthy(); + expect(testBase.lloHandler['isLloAttribute']('gen_ai.content.revised_prompt')).toBeTruthy(); + }); + + /** + * Verify isLloAttribute recognizes OpenInference patterns including both direct (input/output.value) + * and indexed (llm.input_messages.{n}.message.content) patterns. + */ + it('should test isLloAttribute openinference match', () => { + expect(testBase.lloHandler['isLloAttribute']('input.value')).toBeTruthy(); + expect(testBase.lloHandler['isLloAttribute']('output.value')).toBeTruthy(); + + expect(testBase.lloHandler['isLloAttribute']('llm.input_messages.0.message.content')).toBeTruthy(); + expect(testBase.lloHandler['isLloAttribute']('llm.output_messages.123.message.content')).toBeTruthy(); + }); + + /** + * Verify isLloAttribute recognizes CrewAI framework patterns (gen_ai.agent.*, crewai.crew.*). + */ + it('should test isLloAttribute crewai match', () => { + expect(testBase.lloHandler['isLloAttribute']('gen_ai.agent.actual_output')).toBeTruthy(); + expect(testBase.lloHandler['isLloAttribute']('gen_ai.agent.human_input')).toBeTruthy(); + expect(testBase.lloHandler['isLloAttribute']('crewai.crew.tasks_output')).toBeTruthy(); + expect(testBase.lloHandler['isLloAttribute']('crewai.crew.result')).toBeTruthy(); + }); + + /** + * Verify isLloAttribute recognizes Strands SDK patterns (system_prompt, tool.result). + */ + it('should test isLloAttribute strands sdk match', () => { + expect(testBase.lloHandler['isLloAttribute']('system_prompt')).toBeTruthy(); + expect(testBase.lloHandler['isLloAttribute']('tool.result')).toBeTruthy(); + }); + + /** + * Verify isLloAttribute recognizes llm.prompts pattern. + */ + it('should test isLloAttribute llm_prompts match', () => { + expect(testBase.lloHandler['isLloAttribute']('llm.prompts')).toBeTruthy(); + }); + + /** + * Test buildPatternMatchers handles patterns with missing regex gracefully. + */ + it('should test build pattern matchers with missing regex', () => { + // Temporarily modify LLO_PATTERNS to have a pattern without regex + const originalPatterns = { ...LLO_PATTERNS }; + + // Add a malformed indexed pattern without regex + const testPattern = 'test.bad.pattern'; + (LLO_PATTERNS as any)[testPattern] = { + type: PatternType.INDEXED, + // Missing "regex" key + roleKey: 'test.bad.pattern.role', + defaultRole: 'unknown', + source: 'test', + }; + + try { + // Create a new handler to trigger pattern building + const handler = new LLOHandler(testBase.loggerProviderMock); + + // Should handle gracefully - the bad pattern should be skipped + expect((handler as any).patternConfigs).not.toHaveProperty(testPattern); + + // Other patterns should still work + expect(handler['isLloAttribute']('gen_ai.prompt')).toBeTruthy(); + expect(handler['isLloAttribute']('test.bad.pattern')).toBeFalsy(); + } finally { + // Restore original patterns + Object.keys(LLO_PATTERNS).forEach(key => delete LLO_PATTERNS[key]); + Object.assign(LLO_PATTERNS, originalPatterns); + } + }); +}); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.processing.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.processing.test.ts new file mode 100644 index 00000000..45dbf40b --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/llo-handler.processing.test.ts @@ -0,0 +1,278 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Attributes } from '@opentelemetry/api'; +import { TimedEvent } from '@opentelemetry/sdk-trace-base'; +import expect from 'expect'; +import * as sinon from 'sinon'; +import { LLOHandlerTestBase } from './llo-handler.base.test'; +import { InstrumentationLibrary } from '@opentelemetry/core'; + +/** + * Test span processing and attribute filtering functionality. + */ +describe('TestLLOHandlerProcessing', () => { + let testBase: LLOHandlerTestBase; + + beforeEach(() => { + testBase = new LLOHandlerTestBase(); + }); + + afterEach(() => { + sinon.restore(); + }); + + /** + * Verify filterAttributes removes LLO content attributes while preserving role attributes + * and other non-LLO attributes. + */ + it('should test filterAttributes', () => { + const attributes: Attributes = { + 'gen_ai.prompt.0.content': 'test content', + 'gen_ai.prompt.0.role': 'user', + 'normal.attribute': 'value', + 'another.normal.attribute': 123, + }; + + const filtered = testBase.lloHandler['filterAttributes'](attributes); + + expect(filtered).not.toHaveProperty('gen_ai.prompt.0.content'); + expect(filtered['gen_ai.prompt.0.role']).toBeDefined(); + expect(filtered['normal.attribute']).toBeDefined(); + expect(filtered['another.normal.attribute']).toBeDefined(); + }); + + /** + * Verify filterAttributes returns empty attributes when given empty attributes. + */ + it('should test filterAttributes with empty attributes', () => { + const result = testBase.lloHandler['filterAttributes']({}); + expect(result).toEqual({}); + }); + + /** + * Verify filterAttributes returns original attributes when no LLO attributes are present. + */ + it('should test filterAttributes does no handling', () => { + const attributes = { 'normal.attr': 'value' }; + const result = testBase.lloHandler['filterAttributes'](attributes); + expect(result).toStrictEqual(attributes); + }); + + /** + * Test filterAttributes when there are no LLO attributes - should return original + */ + it('should test filterAttributes no llo attrs', () => { + const attributes = { + 'normal.attr1': 'value1', + 'normal.attr2': 'value2', + 'other.attribute': 'value', // This is not an LLO attribute + }; + + const result = testBase.lloHandler['filterAttributes'](attributes); + + // Should return the same attributes object when no LLO attrs present + expect(result).toStrictEqual(attributes); + expect(result).toEqual(attributes); + }); + + /** + * Verify processSpans extracts LLO attributes, emits events, filters attributes, + * and processes span events correctly. + */ + it('should test processSpans', () => { + const attributes: Attributes = { + 'gen_ai.prompt.0.content': 'prompt content', + 'normal.attribute': 'normal value', + }; + + const span = testBase.createMockSpan(attributes); + testBase.updateMockSpanKey(span, 'events', []); + + const emitStub = sinon.stub(testBase.lloHandler as any, 'emitLloAttributes'); + const filterStub = sinon + .stub(testBase.lloHandler as any, 'filterAttributes') + .returns({ 'normal.attribute': 'normal value' }); + + const result = testBase.lloHandler.processSpans([span]); + + // Now it's called with only the LLO attributes + const expectedLloAttrs = { 'gen_ai.prompt.0.content': 'prompt content' }; + expect(emitStub.calledOnceWith(span, expectedLloAttrs)).toBeTruthy(); + expect(filterStub.calledOnceWith(attributes)).toBeTruthy(); + + expect(result.length).toBe(1); + expect(result[0]).toBe(span); + expect(result[0].attributes).toEqual({ 'normal.attribute': 'normal value' }); + }); + + /** + * Verify processSpans correctly handles spans with empty attributes. + */ + it('should test processSpans with empty attributes', () => { + const span = testBase.createMockSpan({}); + testBase.updateMockSpanKey(span, 'events', []); + + const result = testBase.lloHandler.processSpans([span]); + + expect(result.length).toBe(1); + expect(result[0].attributes).toEqual({}); + }); + + /** + * Verify filterSpanEvents filters LLO attributes from span events correctly. + */ + it('should test filterSpanEvents', () => { + const eventAttributes: Attributes = { + 'gen_ai.prompt': 'event prompt', + 'normal.attribute': 'keep this', + }; + + const event: TimedEvent = { + name: 'test_event', + attributes: eventAttributes, + time: [1234567890, 0], + }; + + const span = testBase.createMockSpan({}); + testBase.updateMockSpanKey(span, 'events', [event]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { + name: 'test.scope', + version: '1.0.0', + }); + + testBase.lloHandler['filterSpanEvents'](span); + + const spanEvents = span.events; + const updatedEvent = spanEvents[0]; + expect(updatedEvent.attributes!['normal.attribute']).toBeDefined(); + expect(updatedEvent.attributes).not.toHaveProperty('gen_ai.prompt'); + }); + + /** + * Verify filterSpanEvents handles spans with no events gracefully. + */ + it('should test filterSpanEvents no events', () => { + const span = testBase.createMockSpan({}); + testBase.updateMockSpanKey(span, 'events', []); + + testBase.lloHandler['filterSpanEvents'](span); + + expect(span.events).toEqual([]); + }); + + /** + * Test filterSpanEvents when event has no attributes + */ + it('should test filterSpanEvents no attributes', () => { + const event: TimedEvent = { + name: 'test_event', + attributes: {}, + time: [1234567890, 0], + }; + + const span = testBase.createMockSpan({}); + testBase.updateMockSpanKey(span, 'events', [event]); + + testBase.lloHandler['filterSpanEvents'](span); + + // Should handle gracefully and keep the original event + const spanEvents = span.events; + expect(spanEvents.length).toBe(1); + expect(spanEvents[0]).toBe(event); + }); + + /** + * Verify processSpans collects LLO attributes from both span attributes and events, + * then emits a single consolidated event. + */ + it('should test processSpans consolidated event emission', () => { + // Span attributes with prompt + const spanAttributes: Attributes = { + 'gen_ai.prompt': 'What is quantum computing?', + 'normal.attribute': 'keep this', + }; + + // Event attributes with completion + const eventAttributes: Attributes = { + 'gen_ai.completion': 'Quantum computing is...', + 'other.attribute': 'also keep this', + }; + + const event: TimedEvent = { + name: 'gen_ai.content.completion', + attributes: eventAttributes, + time: [1234567890, 0], + }; + + const span = testBase.createMockSpan(spanAttributes); + testBase.updateMockSpanKey(span, 'events', [event]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { + name: 'openlit.otel.tracing', + version: '1.0.0', + }); + + const emitStub = sinon.stub(testBase.lloHandler as any, 'emitLloAttributes'); + + testBase.lloHandler.processSpans([span]); + + // Should emit once with combined attributes + expect(emitStub.calledOnce).toBeTruthy(); + const [emittedSpan, emittedAttributes] = emitStub.firstCall.args; + + expect(emittedSpan).toBe(span); + expect(emittedAttributes['gen_ai.prompt']).toBe('What is quantum computing?'); + expect(emittedAttributes['gen_ai.completion']).toBe('Quantum computing is...'); + + // Verify span attributes are filtered + expect(span.attributes).not.toHaveProperty('gen_ai.prompt'); + expect(span.attributes['normal.attribute']).toBe('keep this'); + + // Verify event attributes are filtered + const updatedEvent = span.events[0]; + expect(updatedEvent.attributes).not.toHaveProperty('gen_ai.completion'); + expect(updatedEvent.attributes!['other.attribute']).toBe('also keep this'); + }); + + /** + * Verify processSpans handles multiple events correctly, collecting all LLO attributes + * into a single consolidated event. + */ + it('should test processSpans multiple events consolidated', () => { + const spanAttributes: Attributes = { 'normal.attribute': 'keep this' }; + + // First event with prompt + const event1Attrs: Attributes = { 'gen_ai.prompt': 'First question' }; + const event1: TimedEvent = { + name: 'gen_ai.content.prompt', + attributes: event1Attrs, + time: [1234567890, 0], + }; + + // Second event with completion + const event2Attrs: Attributes = { 'gen_ai.completion': 'First answer' }; + const event2: TimedEvent = { + name: 'gen_ai.content.completion', + attributes: event2Attrs, + time: [1234567891, 0], + }; + + const span = testBase.createMockSpan(spanAttributes); + testBase.updateMockSpanKey(span, 'events', [event1, event2]); + testBase.updateMockSpanKey(span, 'instrumentationLibrary', { + name: 'openlit.otel.tracing', + version: '1.0.0', + }); + + const emitStub = sinon.stub(testBase.lloHandler as any, 'emitLloAttributes'); + + testBase.lloHandler.processSpans([span]); + + // Should emit once with attributes from both events + expect(emitStub.calledOnce).toBeTruthy(); + const emittedAttributes = emitStub.firstCall.args[1]; + + expect(emittedAttributes['gen_ai.prompt']).toBe('First question'); + expect(emittedAttributes['gen_ai.completion']).toBe('First answer'); + }); +}); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/instrumentation-patch.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/instrumentation-patch.test.ts index 502ba84c..1f815c2a 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/instrumentation-patch.test.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/instrumentation-patch.test.ts @@ -13,6 +13,7 @@ import { AttributeValue, TextMapSetter, INVALID_SPAN_CONTEXT, + SpanStatusCode, } from '@opentelemetry/api'; import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; import { Instrumentation } from '@opentelemetry/instrumentation'; @@ -33,7 +34,7 @@ import { Context } from 'aws-lambda'; import { SinonStub } from 'sinon'; import { Lambda } from '@aws-sdk/client-lambda'; import * as nock from 'nock'; -import { ReadableSpan } from '@opentelemetry/sdk-trace-base'; +import { ReadableSpan, Span as SDKSpan } from '@opentelemetry/sdk-trace-base'; import { getTestSpans } from '@opentelemetry/contrib-test-utils'; import { instrumentationConfigs } from '../../src/register'; @@ -340,6 +341,14 @@ describe('InstrumentationPatchTest', () => { return filteredInstrumentations[0] as AwsInstrumentation; } + function extractAwsLambdaInstrumentation(instrumentations: Instrumentation[]): AwsLambdaInstrumentation { + const filteredInstrumentations: Instrumentation[] = instrumentations.filter( + instrumentation => instrumentation.instrumentationName === '@opentelemetry/instrumentation-aws-lambda' + ); + expect(filteredInstrumentations.length).toEqual(1); + return filteredInstrumentations[0] as AwsLambdaInstrumentation; + } + function extractServicesFromAwsSdkInstrumentation(awsSdkInstrumentation: AwsInstrumentation): Map { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore @@ -650,6 +659,19 @@ describe('InstrumentationPatchTest', () => { .traceId.substring(8, 32)};Parent=${spanId};Sampled=1` ); }); + + it('Tests patched AwsLambdaInstrumentation method with error', done => { + const awsLambdaInstrumentation = extractAwsLambdaInstrumentation(PATCHED_INSTRUMENTATIONS); + const mockSpan: Span = sinon.createStubInstance(SDKSpan); + awsLambdaInstrumentation['_endSpan'](mockSpan, 'SomeError', () => { + expect((mockSpan.recordException as any).getCall(0).args[0]).toEqual('SomeError'); + expect((mockSpan.setStatus as any).getCall(0).args[0]).toEqual({ + code: SpanStatusCode.ERROR, + message: 'SomeError', + }); + done(); + }); + }); }); }); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/sampler/aws-xray-remote-sampler.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/sampler/aws-xray-remote-sampler.test.ts index c47981d2..2ae492f8 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/test/sampler/aws-xray-remote-sampler.test.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/sampler/aws-xray-remote-sampler.test.ts @@ -1,9 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { context, diag, DiagConsoleLogger, Span, SpanKind } from '@opentelemetry/api'; +import { context, Span, SpanKind } from '@opentelemetry/api'; import { Resource } from '@opentelemetry/resources'; -import * as opentelemetry from '@opentelemetry/sdk-node'; import { SamplingDecision, Tracer } from '@opentelemetry/sdk-trace-base'; import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; import { SEMRESATTRS_CLOUD_PLATFORM, SEMRESATTRS_SERVICE_NAME } from '@opentelemetry/semantic-conventions'; @@ -16,8 +15,6 @@ const DATA_DIR_SAMPLING_RULES = __dirname + '/data/test-remote-sampler_sampling- const DATA_DIR_SAMPLING_TARGETS = __dirname + '/data/test-remote-sampler_sampling-targets-response-sample.json'; const TEST_URL = 'http://localhost:2000'; -diag.setLogger(new DiagConsoleLogger(), opentelemetry.core.getEnv().OTEL_LOG_LEVEL); - describe('AwsXrayRemoteSampler', () => { it('testCreateRemoteSamplerWithEmptyResource', () => { const sampler: AwsXRayRemoteSampler = new AwsXRayRemoteSampler({ resource: Resource.EMPTY }); @@ -197,7 +194,7 @@ describe('AwsXrayRemoteSampler', () => { }); it('generates valid ClientId', () => { - const clientId: string = (_AwsXRayRemoteSampler as any).generateClientId(); + const clientId: string = _AwsXRayRemoteSampler['generateClientId'](); const match: RegExpMatchArray | null = clientId.match(/[0-9a-z]{24}/g); expect(match).not.toBeNull(); }); diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/utils.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/utils.test.ts new file mode 100644 index 00000000..caa0f67b --- /dev/null +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/utils.test.ts @@ -0,0 +1,42 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import expect from 'expect'; +import { isAgentObservabilityEnabled } from '../src/utils'; + +describe('Utils', function () { + beforeEach(() => { + delete process.env.AGENT_OBSERVABILITY_ENABLED; + delete process.env.AWS_REGION; + delete process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; + delete process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT; + }); + + it('Test isAgentObservabilityEnabled to be True', () => { + process.env.AGENT_OBSERVABILITY_ENABLED = 'true'; + expect(isAgentObservabilityEnabled()).toBeTruthy(); + + process.env.AGENT_OBSERVABILITY_ENABLED = 'True'; + expect(isAgentObservabilityEnabled()).toBeTruthy(); + + process.env.AGENT_OBSERVABILITY_ENABLED = 'TRUE'; + expect(isAgentObservabilityEnabled()).toBeTruthy(); + }); + + it('Test isAgentObservabilityEnabled to be False', () => { + process.env.AGENT_OBSERVABILITY_ENABLED = 'false'; + expect(isAgentObservabilityEnabled()).toBeFalsy(); + + process.env.AGENT_OBSERVABILITY_ENABLED = 'False'; + expect(isAgentObservabilityEnabled()).toBeFalsy(); + + process.env.AGENT_OBSERVABILITY_ENABLED = 'FALSE'; + expect(isAgentObservabilityEnabled()).toBeFalsy(); + + process.env.AGENT_OBSERVABILITY_ENABLED = 'anything else'; + expect(isAgentObservabilityEnabled()).toBeFalsy(); + + delete process.env.AGENT_OBSERVABILITY_ENABLED; + expect(isAgentObservabilityEnabled()).toBeFalsy(); + }); +}); diff --git a/package-lock.json b/package-lock.json index 237f2090..2d9135e9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "license": "Apache-2.0", "dependencies": { "@opentelemetry/api": "1.9.0", + "@opentelemetry/api-events": "0.57.1", "@opentelemetry/auto-configuration-propagators": "0.3.2", "@opentelemetry/auto-instrumentations-node": "0.56.0", "@opentelemetry/core": "1.30.1", @@ -53,6 +54,8 @@ "@opentelemetry/propagator-aws-xray": "1.26.2", "@opentelemetry/resource-detector-aws": "1.12.0", "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-events": "0.57.1", + "@opentelemetry/sdk-logs": "0.57.1", "@opentelemetry/sdk-metrics": "1.30.1", "@opentelemetry/sdk-node": "0.57.1", "@opentelemetry/sdk-trace-base": "1.30.1", @@ -142,6 +145,7 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz", "integrity": "sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -4109,6 +4113,18 @@ "node": ">=8.0.0" } }, + "node_modules/@opentelemetry/api-events": { + "version": "0.57.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-events/-/api-events-0.57.1.tgz", + "integrity": "sha512-Th/89vlzvDiqUgb4Bv0l9uBPbsvwMT6IenHCPDbO7cTlnq/Si+1MvuGsDOhZbQeZu398ILPe3G/KraKxdjSTCg==", + "dependencies": { + "@opentelemetry/api": "^1.3.0", + "@opentelemetry/api-logs": "0.57.1" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/@opentelemetry/api-logs": { "version": "0.57.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.57.1.tgz", @@ -5356,6 +5372,22 @@ "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, + "node_modules/@opentelemetry/sdk-events": { + "version": "0.57.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-events/-/sdk-events-0.57.1.tgz", + "integrity": "sha512-zi9f/EUCGPp2OktqeW5qYEoWNVOhF23t+bus9VxPmER+xtz+/lw/+1xaE+Co/rKhlK5NJycGhb517P/wuqrODQ==", + "dependencies": { + "@opentelemetry/api-events": "0.57.1", + "@opentelemetry/api-logs": "0.57.1", + "@opentelemetry/sdk-logs": "0.57.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, "node_modules/@opentelemetry/sdk-logs": { "version": "0.57.1", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.57.1.tgz", @@ -6239,12 +6271,13 @@ } }, "node_modules/@smithy/protocol-http": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.0.1.tgz", - "integrity": "sha512-TE4cpj49jJNB/oHyh/cRVEgNZaoPaxd4vteJNB0yGidOCVR0jCw/hjPVsT8Q8FRmj8Bd3bFZt8Dh7xGCT+xMBQ==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.1.1.tgz", + "integrity": "sha512-Vsay2mzq05DwNi9jK01yCFtfvu9HimmgC7a4HTs7lhX12Sx8aWsH0mfz6q/02yspSp+lOB+Q2HJwi4IV2GKz7A==", "dev": true, + "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.1.0", + "@smithy/types": "^4.3.0", "tslib": "^2.6.2" }, "engines": { @@ -6252,10 +6285,11 @@ } }, "node_modules/@smithy/protocol-http/node_modules/@smithy/types": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.1.0.tgz", - "integrity": "sha512-enhjdwp4D7CXmwLtD6zbcDMbo6/T6WtuuKCY49Xxc6OMOmUWlBEBDREsxxgV2LIdeQPW756+f97GzcgAwp3iLw==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.3.0.tgz", + "integrity": "sha512-+1iaIQHthDh9yaLhRzaoQxRk+l9xlk+JjMFxGRhNLz+m9vKOkjNeU8QuB4w3xvzHyVR/BVlp/4AXDHjoRIkfgQ==", "dev": true, + "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" }, @@ -6316,16 +6350,17 @@ } }, "node_modules/@smithy/signature-v4": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.0.1.tgz", - "integrity": "sha512-nCe6fQ+ppm1bQuw5iKoeJ0MJfz2os7Ic3GBjOkLOPtavbD1ONoyE3ygjBfz2ythFWm4YnRm6OxW+8p/m9uCoIA==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.1.1.tgz", + "integrity": "sha512-zy8Repr5zvT0ja+Tf5wjV/Ba6vRrhdiDcp/ww6cvqYbSEudIkziDe3uppNRlFoCViyJXdPnLcwyZdDLA4CHzSg==", "dev": true, + "license": "Apache-2.0", "dependencies": { "@smithy/is-array-buffer": "^4.0.0", - "@smithy/protocol-http": "^5.0.1", - "@smithy/types": "^4.1.0", + "@smithy/protocol-http": "^5.1.1", + "@smithy/types": "^4.3.0", "@smithy/util-hex-encoding": "^4.0.0", - "@smithy/util-middleware": "^4.0.1", + "@smithy/util-middleware": "^4.0.3", "@smithy/util-uri-escape": "^4.0.0", "@smithy/util-utf8": "^4.0.0", "tslib": "^2.6.2" @@ -6347,10 +6382,11 @@ } }, "node_modules/@smithy/signature-v4/node_modules/@smithy/types": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.1.0.tgz", - "integrity": "sha512-enhjdwp4D7CXmwLtD6zbcDMbo6/T6WtuuKCY49Xxc6OMOmUWlBEBDREsxxgV2LIdeQPW756+f97GzcgAwp3iLw==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.3.0.tgz", + "integrity": "sha512-+1iaIQHthDh9yaLhRzaoQxRk+l9xlk+JjMFxGRhNLz+m9vKOkjNeU8QuB4w3xvzHyVR/BVlp/4AXDHjoRIkfgQ==", "dev": true, + "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" }, @@ -6384,12 +6420,13 @@ } }, "node_modules/@smithy/signature-v4/node_modules/@smithy/util-middleware": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.0.1.tgz", - "integrity": "sha512-HiLAvlcqhbzhuiOa0Lyct5IIlyIz0PQO5dnMlmQ/ubYM46dPInB+3yQGkfxsk6Q24Y0n3/JmcA1v5iEhmOF5mA==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.0.3.tgz", + "integrity": "sha512-iIsC6qZXxkD7V3BzTw3b1uK8RVC1M8WvwNxK1PKrH9FnxntCd30CSunXjL/8iJBE8Z0J14r2P69njwIpRG4FBQ==", "dev": true, + "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.1.0", + "@smithy/types": "^4.3.0", "tslib": "^2.6.2" }, "engines": {