|
| 1 | +/* |
| 2 | + * Copyright (c) 2024 Scale3 Labs |
| 3 | + * |
| 4 | + * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + * you may not use this file except in compliance with the License. |
| 6 | + * You may obtain a copy of the License at |
| 7 | + * |
| 8 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | + * |
| 10 | + * Unless required by applicable law or agreed to in writing, software |
| 11 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + * See the License for the specific language governing permissions and |
| 14 | + * limitations under the License. |
| 15 | + */ |
| 16 | + |
| 17 | +import { LANGTRACE_ADDITIONAL_SPAN_ATTRIBUTES_KEY } from '@langtrace-constants/common' |
| 18 | +import { addSpanEvent } from '@langtrace-utils/misc' |
| 19 | +import { APIS, LLMSpanAttributes, Event, Vendors } from '@langtrase/trace-attributes' |
| 20 | +import { |
| 21 | + Exception, |
| 22 | + Span, |
| 23 | + SpanKind, |
| 24 | + SpanStatusCode, |
| 25 | + Tracer, |
| 26 | + context, |
| 27 | + trace |
| 28 | +} from '@opentelemetry/api' |
| 29 | +import { LangtraceSdkError } from 'errors/sdk_error' |
| 30 | + |
| 31 | +export function sendCommand ( |
| 32 | + originalMethod: (...args: any[]) => any, |
| 33 | + tracer: Tracer, |
| 34 | + langtraceVersion: string, |
| 35 | + version?: string, |
| 36 | + stream = false |
| 37 | +): (...args: any[]) => any { |
| 38 | + return async function (this: any, ...args: any[]) { |
| 39 | + // eslint-disable-next-line @typescript-eslint/no-this-alias |
| 40 | + const originalContext = this |
| 41 | + const customAttributes = context.active().getValue(LANGTRACE_ADDITIONAL_SPAN_ATTRIBUTES_KEY) ?? {} |
| 42 | + // Determine the service provider |
| 43 | + const serviceProvider: string = Vendors.AWSBEDROCK |
| 44 | + const attributes: LLMSpanAttributes = { |
| 45 | + 'langtrace.sdk.name': '@langtrase/typescript-sdk', |
| 46 | + 'gen_ai.operation.name': 'chat', |
| 47 | + 'langtrace.service.name': serviceProvider, |
| 48 | + 'langtrace.service.type': 'framework', |
| 49 | + 'langtrace.service.version': version, |
| 50 | + 'langtrace.version': langtraceVersion, |
| 51 | + 'gen_ai.request.model': args[0]?.input.modelId, |
| 52 | + 'url.full': originalContext?._client?.baseURL, |
| 53 | + 'url.path': APIS.awsbedrock.CONVERSE.ENDPOINT, |
| 54 | + 'http.max.retries': originalContext?._client?.maxRetries, |
| 55 | + 'http.timeout': originalContext?._client?.timeout, |
| 56 | + 'gen_ai.request.temperature': args[0]?.input?.inferenceConfig?.temperature, |
| 57 | + 'gen_ai.request.top_p': args[0]?.input?.inferenceConfig?.topP, |
| 58 | + 'gen_ai.user': args[0]?.user, |
| 59 | + 'gen_ai.request.max_tokens': args[0]?.input?.inferenceConfig?.maxTokens, |
| 60 | + 'gen_ai.request.tools': JSON.stringify(args[0]?.input?.toolConfig?.tools), |
| 61 | + ...customAttributes |
| 62 | + } |
| 63 | + /* eslint-disable no-console */ |
| 64 | + const spanName = customAttributes['langtrace.span.name' as keyof typeof customAttributes] ?? APIS.awsbedrock.CONVERSE.METHOD |
| 65 | + const span = tracer.startSpan(spanName, { kind: SpanKind.CLIENT, attributes }, context.active()) |
| 66 | + return await context.with( |
| 67 | + trace.setSpan(context.active(), span), |
| 68 | + async () => { |
| 69 | + try { |
| 70 | + const resp = await originalMethod.apply(this, args) |
| 71 | + const message = args[0]?.input?.messages[0] |
| 72 | + const message_content = message?.content?.map((content: any) => ({ content: content.text, role: message.role })) |
| 73 | + addSpanEvent(span, Event.GEN_AI_PROMPT, { 'gen_ai.prompt': JSON.stringify(message_content) }) |
| 74 | + if (resp.stream === undefined) { |
| 75 | + const responses = resp?.output?.message?.content?.map((content: any) => { |
| 76 | + const result = { |
| 77 | + role: resp?.output?.message?.role, |
| 78 | + content: content?.text !== undefined && content?.text !== null |
| 79 | + ? content?.text |
| 80 | + : content?.toolUse !== undefined |
| 81 | + ? JSON.stringify(content?.toolUse) |
| 82 | + : JSON.stringify(content?.toolResult) |
| 83 | + } |
| 84 | + return result |
| 85 | + }) |
| 86 | + addSpanEvent(span, Event.GEN_AI_COMPLETION, { 'gen_ai.completion': JSON.stringify(responses) }) |
| 87 | + const responseAttributes: Partial<LLMSpanAttributes> = { |
| 88 | + 'gen_ai.response.model': args[0]?.input.modelId, |
| 89 | + 'gen_ai.usage.input_tokens': resp.usage.inputTokens, |
| 90 | + 'gen_ai.usage.output_tokens': resp.usage.outputTokens, |
| 91 | + 'gen_ai.usage.total_tokens': resp.usage.totalTokens |
| 92 | + } |
| 93 | + span.setAttributes({ ...attributes, ...responseAttributes }) |
| 94 | + span.setStatus({ code: SpanStatusCode.OK }) |
| 95 | + return resp |
| 96 | + } else { |
| 97 | + await processConverseStream(resp.stream, span, attributes) |
| 98 | + return resp |
| 99 | + } |
| 100 | + } catch (error: any) { |
| 101 | + span.recordException(error as Exception) |
| 102 | + span.setStatus({ code: SpanStatusCode.ERROR }) |
| 103 | + throw new LangtraceSdkError(error.message as string, error.stack as string) |
| 104 | + } finally { |
| 105 | + span.end() |
| 106 | + } |
| 107 | + } |
| 108 | + ) |
| 109 | + } |
| 110 | +} |
| 111 | + |
| 112 | +async function * processConverseStream (stream: any, span: Span, inputAttributes: Partial<LLMSpanAttributes>): any { |
| 113 | + const customAttributes = context.active().getValue(LANGTRACE_ADDITIONAL_SPAN_ATTRIBUTES_KEY) ?? {} |
| 114 | + addSpanEvent(span, Event.STREAM_START) |
| 115 | + |
| 116 | + const result: string[] = [] |
| 117 | + let completionTokens = 0 |
| 118 | + let promptTokens = 0 |
| 119 | + |
| 120 | + try { |
| 121 | + for await (const chunk of stream) { |
| 122 | + const deserializedChunk = await stream.options.deserializer(chunk) |
| 123 | + // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions |
| 124 | + const content = deserializedChunk.contentBlockDelta?.delta?.text || '' |
| 125 | + promptTokens = deserializedChunk.metadata?.usage?.inputTokens ?? 0 |
| 126 | + completionTokens = deserializedChunk.metadata?.usage?.outputTokens ?? 0 |
| 127 | + result.push(content as string) |
| 128 | + |
| 129 | + yield deserializedChunk |
| 130 | + } |
| 131 | + |
| 132 | + addSpanEvent(span, Event.GEN_AI_COMPLETION, { 'gen_ai.completion': result.length > 0 ? JSON.stringify([{ role: 'assistant', content: result.join('') }]) : undefined }) |
| 133 | + span.setStatus({ code: SpanStatusCode.OK }) |
| 134 | + const stream_attributes: Partial<LLMSpanAttributes> = { |
| 135 | + 'gen_ai.usage.output_tokens': promptTokens, |
| 136 | + 'gen_ai.usage.input_tokens': completionTokens, |
| 137 | + 'gen_ai.usage.total_tokens': promptTokens + completionTokens, |
| 138 | + 'gen_ai.request.stream': true, |
| 139 | + ...customAttributes |
| 140 | + } |
| 141 | + span.setAttributes({ ...inputAttributes, ...stream_attributes }) |
| 142 | + addSpanEvent(span, Event.STREAM_END) |
| 143 | + } catch (error: any) { |
| 144 | + span.recordException(error as Exception) |
| 145 | + span.setStatus({ code: SpanStatusCode.ERROR }) |
| 146 | + throw new LangtraceSdkError(error.message as string, error.stack as string) |
| 147 | + } finally { |
| 148 | + span.end() |
| 149 | + } |
| 150 | +} |
0 commit comments