Skip to content

Commit 07ace1f

Browse files
ezhang6811ezhang6811
committed
update claude support to replace legacy model
1 parent d76618e commit 07ace1f

File tree

5 files changed

+65
-49
lines changed

5 files changed

+65
-49
lines changed

‎src/OpenTelemetry.Instrumentation.AWS/Implementation/AWSLlmModelProcessor.cs‎

Lines changed: 35 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ namespace OpenTelemetry.Instrumentation.AWS.Implementation;
1010

1111
internal class AWSLlmModelProcessor
1212
{
13-
internal static void ProcessGenAiAttributes<T>(Activity activity, T message, string model, bool isRequest)
13+
internal static void ProcessGenAiAttributes<T>(Activity activity, T message, string modelName, bool isRequest)
1414
{
1515
// message can be either a request or a response. isRequest is used by the model-specific methods to determine
1616
// whether to extract the request or response attributes.
@@ -37,26 +37,29 @@ internal static void ProcessGenAiAttributes<T>(Activity activity, T message, str
3737
}
3838

3939
// extract model specific attributes based on model name
40-
switch (model)
40+
if (modelName.Contains("amazon.titan"))
4141
{
42-
case "amazon.titan":
43-
ProcessTitanModelAttributes(activity, jsonObject, isRequest);
44-
break;
45-
case "anthropic.claude":
46-
ProcessClaudeModelAttributes(activity, jsonObject, isRequest);
47-
break;
48-
case "meta.llama3":
49-
ProcessLlamaModelAttributes(activity, jsonObject, isRequest);
50-
break;
51-
case "cohere.command":
52-
ProcessCommandModelAttributes(activity, jsonObject, isRequest);
53-
break;
54-
case "ai21.jamba":
55-
ProcessJambaModelAttributes(activity, jsonObject, isRequest);
56-
break;
57-
case "mistral.mistral":
58-
ProcessMistralModelAttributes(activity, jsonObject, isRequest);
59-
break;
42+
ProcessTitanModelAttributes(activity, jsonObject, isRequest);
43+
}
44+
else if (modelName.Contains("anthropic.claude"))
45+
{
46+
ProcessClaudeModelAttributes(activity, jsonObject, isRequest);
47+
}
48+
else if (modelName.Contains("meta.llama3"))
49+
{
50+
ProcessLlamaModelAttributes(activity, jsonObject, isRequest);
51+
}
52+
else if (modelName.Contains("cohere.command"))
53+
{
54+
ProcessCommandModelAttributes(activity, jsonObject, isRequest);
55+
}
56+
else if (modelName.Contains("ai21.jamba"))
57+
{
58+
ProcessJambaModelAttributes(activity, jsonObject, isRequest);
59+
}
60+
else if (modelName.Contains("mistral.mistral"))
61+
{
62+
ProcessMistralModelAttributes(activity, jsonObject, isRequest);
6063
}
6164
}
6265
catch (Exception ex)
@@ -135,28 +138,27 @@ private static void ProcessClaudeModelAttributes(Activity activity, Dictionary<s
135138
activity.SetTag(AWSSemanticConventions.AttributeGenAiTemperature, temperature.GetDouble());
136139
}
137140

138-
if (jsonBody.TryGetValue("max_tokens_to_sample", out var maxTokens))
141+
if (jsonBody.TryGetValue("max_tokens", out var maxTokens))
139142
{
140143
activity.SetTag(AWSSemanticConventions.AttributeGenAiMaxTokens, maxTokens.GetInt32());
141144
}
142-
143-
// input tokens not provided in Claude response body, so we estimate the value based on input length
144-
if (jsonBody.TryGetValue("prompt", out var input))
145-
{
146-
activity.SetTag(AWSSemanticConventions.AttributeGenAiInputTokens, Convert.ToInt32(Math.Ceiling((double) input.GetString().Length / 6)));
147-
}
148145
}
149146
else
150147
{
151-
if (jsonBody.TryGetValue("stop_reason", out var finishReasons))
148+
if (jsonBody.TryGetValue("usage", out var usage))
152149
{
153-
activity.SetTag(AWSSemanticConventions.AttributeGenAiFinishReasons, new string[] { finishReasons.GetString() });
150+
if (usage.TryGetProperty("input_tokens", out var inputTokens))
151+
{
152+
activity.SetTag(AWSSemanticConventions.AttributeGenAiInputTokens, inputTokens.GetInt32());
153+
}
154+
if (usage.TryGetProperty("output_tokens", out var outputTokens))
155+
{
156+
activity.SetTag(AWSSemanticConventions.AttributeGenAiOutputTokens, outputTokens.GetInt32());
157+
}
154158
}
155-
156-
// output tokens not provided in Claude response body, so we estimate the value based on output length
157-
if (jsonBody.TryGetValue("completion", out var output))
159+
if (jsonBody.TryGetValue("stop_reason", out var finishReasons))
158160
{
159-
activity.SetTag(AWSSemanticConventions.AttributeGenAiOutputTokens, Convert.ToInt32(Math.Ceiling((double) output.GetString().Length / 6)));
161+
activity.SetTag(AWSSemanticConventions.AttributeGenAiFinishReasons, new string[] { finishReasons.GetString() });
160162
}
161163
}
162164
}

‎src/OpenTelemetry.Instrumentation.AWS/Implementation/AWSTracingPipelineHandler.cs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -168,8 +168,7 @@ private static void AddRequestSpecificInformation(Activity activity, IRequestCon
168168
var modelString = model.ToString();
169169
if (modelString != null)
170170
{
171-
var modelName = modelString.Split('-')[0];
172-
AWSLlmModelProcessor.ProcessGenAiAttributes(activity, request, modelName, true);
171+
AWSLlmModelProcessor.ProcessGenAiAttributes(activity, request, modelString, true);
173172
}
174173
}
175174
}
@@ -252,8 +251,7 @@ private static void AddResponseSpecificInformation(Activity activity, IResponseC
252251
var modelString = model.ToString();
253252
if (modelString != null)
254253
{
255-
var modelName = modelString.Split('-')[0];
256-
AWSLlmModelProcessor.ProcessGenAiAttributes(activity, responseContext.Response, modelName, false);
254+
AWSLlmModelProcessor.ProcessGenAiAttributes(activity, responseContext.Response, modelString, false);
257255
}
258256
}
259257
}

‎test/contract-tests/images/applications/TestSimpleApp.AWSSDK.Core/BedrockTests.cs‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -83,14 +83,27 @@ public void InvokeModelAnthropicClaude()
8383
{
8484
bedrockRuntime.InvokeModelAsync(new InvokeModelRequest
8585
{
86-
ModelId = "anthropic.claude-v2:1",
86+
ModelId = "us.anthropic.claude-3-5-haiku-20241022-v1:0",
8787
Body = new MemoryStream(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(new
8888
{
89-
// prompt is 72 chars long, input_tokens should be estimated as ceil(72/6) = 12
90-
prompt = "sample input text sample input text sample input text sample input text ",
89+
messages = new object[]
90+
{
91+
new
92+
{
93+
role = "user",
94+
content = new object[]
95+
{
96+
new
97+
{
98+
type = "text",
99+
text = "sample input text",
100+
}
101+
}
102+
},
103+
},
91104
temperature = 0.123,
92105
top_p = 0.456,
93-
max_tokens_to_sample = 123,
106+
max_tokens = 123,
94107
}))),
95108
ContentType = "application/json",
96109
});
@@ -101,8 +114,11 @@ public object InvokeModelAnthropicClaudeResponse()
101114
{
102115
return new
103116
{
104-
// response is 56 chars long, output_tokens should be estimated as ceil(56/6) = 10
105-
completion = "sample output text sample output text sample output text",
117+
usage = new
118+
{
119+
input_tokens = 456,
120+
output_tokens = 789,
121+
},
106122
stop_reason = "finish_reason",
107123
};
108124
}

‎test/contract-tests/images/applications/TestSimpleApp.AWSSDK.Core/Program.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@
174174
app.MapGet("guardrails/test-guardrail", (BedrockTests bedrock) => bedrock.GetGuardrailResponse());
175175
// For invoke model, we have one test case for each of the 6 suppported models.
176176
app.MapPost("model/amazon.titan-text-express-v1/invoke", (BedrockTests bedrock) => bedrock.InvokeModelAmazonTitanResponse());
177-
app.MapPost("model/anthropic.claude-v2:1/invoke", (BedrockTests bedrock) => bedrock.InvokeModelAnthropicClaudeResponse());
177+
app.MapPost("model/us.anthropic.claude-3-5-haiku-20241022-v1:0/invoke", (BedrockTests bedrock) => bedrock.InvokeModelAnthropicClaudeResponse());
178178
app.MapPost("model/meta.llama3-8b-instruct-v1:0/invoke", (BedrockTests bedrock) => bedrock.InvokeModelMetaLlamaResponse());
179179
app.MapPost("model/cohere.command-r-v1:0/invoke", (BedrockTests bedrock) => bedrock.InvokeModelCohereCommandResponse());
180180
app.MapPost("model/ai21.jamba-1-5-large-v1:0/invoke", (BedrockTests bedrock) => bedrock.InvokeModelAi21JambaResponse());

‎test/contract-tests/tests/test/amazon/awssdk/awssdk_test.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -360,15 +360,15 @@ def test_bedrock_runtime_invoke_model_claude(self):
360360
remote_service="AWS::BedrockRuntime",
361361
remote_operation="InvokeModel",
362362
remote_resource_type="AWS::Bedrock::Model",
363-
remote_resource_identifier="anthropic.claude-v2:1",
363+
remote_resource_identifier="us.anthropic.claude-3-5-haiku-20241022-v1:0",
364364
request_response_specific_attributes={
365365
_GEN_AI_SYSTEM: "aws_bedrock",
366-
_GEN_AI_REQUEST_MODEL: "anthropic.claude-v2:1",
366+
_GEN_AI_REQUEST_MODEL: "us.anthropic.claude-3-5-haiku-20241022-v1:0",
367367
_GEN_AI_REQUEST_TEMPERATURE: 0.123,
368368
_GEN_AI_REQUEST_TOP_P: 0.456,
369369
_GEN_AI_REQUEST_MAX_TOKENS: 123,
370-
_GEN_AI_USAGE_INPUT_TOKENS: 12,
371-
_GEN_AI_USAGE_OUTPUT_TOKENS: 10,
370+
_GEN_AI_USAGE_INPUT_TOKENS: 456,
371+
_GEN_AI_USAGE_OUTPUT_TOKENS: 789,
372372
_GEN_AI_RESPONSE_FINISH_REASONS: ["finish_reason"],
373373
},
374374
span_name="Bedrock Runtime.InvokeModel",
@@ -739,7 +739,7 @@ def _filter_bedrock_metrics(self, target_metrics: List[Metric]):
739739
"GET knowledgebases/test-knowledge-base/datasources/test-data-source",
740740
"POST agents/test-agent/agentAliases/test-agent-alias/sessions/test-session/text",
741741
"POST model/amazon.titan-text-express-v1/invoke",
742-
"POST model/anthropic.claude-v2:1/invoke",
742+
"POST model/us.anthropic.claude-3-5-haiku-20241022-v1:0/invoke",
743743
"POST model/meta.llama3-8b-instruct-v1:0/invoke",
744744
"POST model/cohere.command-r-v1:0/invoke",
745745
"POST model/ai21.jamba-1-5-large-v1:0/invoke",

0 commit comments

Comments
 (0)