Skip to content

Add Contract Tests for LLM Attributes and Models #952

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 21, 2024
Merged

Add Contract Tests for LLM Attributes and Models #952

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
61b530e
set up contract tests for gen ai
yiyuan-he Nov 13, 2024
2df7380
apply lint
yiyuan-he Nov 14, 2024
f7c0b9f
add gen ai patch
yiyuan-he Nov 14, 2024
727d4c0
enable bedrock knowledge base test case in v2
yiyuan-he Nov 20, 2024
14634d8
Merge branch 'main' into llm-contract-test-new
yiyuan-he Nov 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2,947 changes: 917 additions & 2,030 deletions .github/patches/opentelemetry-java-instrumentation.patch
View file
Open in desktop

Large diffs are not rendered by default.

374 changes: 369 additions & 5 deletions ...c/test/java/software/amazon/opentelemetry/appsignals/test/awssdk/base/AwsSdkBaseTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1798,8 +1798,8 @@ protected void doTestBedrockAgentDataSourceId() {
0.0);
}

protected void doTestBedrockRuntimeModelId() {
var response = appClient.get("/bedrockruntime/invokeModel").aggregate().join();
protected void doTestBedrockRuntimeAi21Jamba() {
var response = appClient.get("/bedrockruntime/invokeModel/ai21Jamba").aggregate().join();
var traces = mockCollectorClient.getTraces();
var metrics =
mockCollectorClient.getMetrics(
Expand All @@ -1809,9 +1809,9 @@ protected void doTestBedrockRuntimeModelId() {
AppSignalsConstants.LATENCY_METRIC));

var localService = getApplicationOtelServiceName();
var localOperation = "GET /bedrockruntime/invokeModel";
var localOperation = "GET /bedrockruntime/invokeModel/ai21Jamba";
String type = "AWS::Bedrock::Model";
String identifier = "anthropic.claude-v2";
String identifier = "ai21.jamba-1-5-mini-v1:0";
assertSpanClientAttributes(
traces,
bedrockRuntimeSpanName("InvokeModel"),
Expand All @@ -1828,7 +1828,371 @@ protected void doTestBedrockRuntimeModelId() {
200,
List.of(
assertAttribute(
SemanticConventionsConstants.GEN_AI_REQUEST_MODEL, "anthropic.claude-v2")));
SemanticConventionsConstants.GEN_AI_REQUEST_MODEL, "ai21.jamba-1-5-mini-v1:0"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_TEMPERATURE, "0.7"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_TOP_P, "0.8"),
assertAttribute(SemanticConventionsConstants.GEN_AI_RESPONSE_FINISH_REASONS, "[stop]"),
assertAttribute(SemanticConventionsConstants.GEN_AI_USAGE_INPUT_TOKENS, "5"),
assertAttribute(SemanticConventionsConstants.GEN_AI_USAGE_OUTPUT_TOKENS, "42")));
assertMetricClientAttributes(
metrics,
AppSignalsConstants.LATENCY_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
5000.0);
assertMetricClientAttributes(
metrics,
AppSignalsConstants.FAULT_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
0.0);
assertMetricClientAttributes(
metrics,
AppSignalsConstants.ERROR_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
0.0);
}

protected void doTestBedrockRuntimeAmazonTitan() {
var response = appClient.get("/bedrockruntime/invokeModel/amazonTitan").aggregate().join();
var traces = mockCollectorClient.getTraces();
var metrics =
mockCollectorClient.getMetrics(
Set.of(
AppSignalsConstants.ERROR_METRIC,
AppSignalsConstants.FAULT_METRIC,
AppSignalsConstants.LATENCY_METRIC));

var localService = getApplicationOtelServiceName();
var localOperation = "GET /bedrockruntime/invokeModel/amazonTitan";
String type = "AWS::Bedrock::Model";
String identifier = "amazon.titan-text-premier-v1:0";
assertSpanClientAttributes(
traces,
bedrockRuntimeSpanName("InvokeModel"),
getBedrockRuntimeRpcServiceName(),
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
"bedrock.test",
8080,
"http://bedrock.test:8080",
200,
List.of(
assertAttribute(
SemanticConventionsConstants.GEN_AI_REQUEST_MODEL,
"amazon.titan-text-premier-v1:0"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_MAX_TOKENS, "100"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_TEMPERATURE, "0.7"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_TOP_P, "0.9"),
assertAttribute(
SemanticConventionsConstants.GEN_AI_RESPONSE_FINISH_REASONS, "[FINISHED]"),
assertAttribute(SemanticConventionsConstants.GEN_AI_USAGE_INPUT_TOKENS, "10"),
assertAttribute(SemanticConventionsConstants.GEN_AI_USAGE_OUTPUT_TOKENS, "15")));
assertMetricClientAttributes(
metrics,
AppSignalsConstants.LATENCY_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
5000.0);
assertMetricClientAttributes(
metrics,
AppSignalsConstants.FAULT_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
0.0);
assertMetricClientAttributes(
metrics,
AppSignalsConstants.ERROR_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
0.0);
}

protected void doTestBedrockRuntimeAnthropicClaude() {
var response = appClient.get("/bedrockruntime/invokeModel/anthropicClaude").aggregate().join();

var traces = mockCollectorClient.getTraces();
var metrics =
mockCollectorClient.getMetrics(
Set.of(
AppSignalsConstants.ERROR_METRIC,
AppSignalsConstants.FAULT_METRIC,
AppSignalsConstants.LATENCY_METRIC));

var localService = getApplicationOtelServiceName();
var localOperation = "GET /bedrockruntime/invokeModel/anthropicClaude";
String type = "AWS::Bedrock::Model";
String identifier = "anthropic.claude-3-haiku-20240307-v1:0";

assertSpanClientAttributes(
traces,
bedrockRuntimeSpanName("InvokeModel"),
getBedrockRuntimeRpcServiceName(),
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
"bedrock.test",
8080,
"http://bedrock.test:8080",
200,
List.of(
assertAttribute(
SemanticConventionsConstants.GEN_AI_REQUEST_MODEL,
"anthropic.claude-3-haiku-20240307-v1:0"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_MAX_TOKENS, "512"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_TEMPERATURE, "0.6"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_TOP_P, "0.53"),
assertAttribute(
SemanticConventionsConstants.GEN_AI_RESPONSE_FINISH_REASONS, "[end_turn]"),
assertAttribute(SemanticConventionsConstants.GEN_AI_USAGE_INPUT_TOKENS, "2095"),
assertAttribute(SemanticConventionsConstants.GEN_AI_USAGE_OUTPUT_TOKENS, "503")));
assertMetricClientAttributes(
metrics,
AppSignalsConstants.LATENCY_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
5000.0);
assertMetricClientAttributes(
metrics,
AppSignalsConstants.FAULT_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
0.0);
assertMetricClientAttributes(
metrics,
AppSignalsConstants.ERROR_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
0.0);
}

protected void doTestBedrockRuntimeCohereCommandR() {
var response = appClient.get("/bedrockruntime/invokeModel/cohereCommandR").aggregate().join();

var traces = mockCollectorClient.getTraces();
var metrics =
mockCollectorClient.getMetrics(
Set.of(
AppSignalsConstants.ERROR_METRIC,
AppSignalsConstants.FAULT_METRIC,
AppSignalsConstants.LATENCY_METRIC));

var localService = getApplicationOtelServiceName();
var localOperation = "GET /bedrockruntime/invokeModel/cohereCommandR";
String type = "AWS::Bedrock::Model";
String identifier = "cohere.command-r-v1:0";

assertSpanClientAttributes(
traces,
bedrockRuntimeSpanName("InvokeModel"),
getBedrockRuntimeRpcServiceName(),
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
"bedrock.test",
8080,
"http://bedrock.test:8080",
200,
List.of(
assertAttribute(
SemanticConventionsConstants.GEN_AI_REQUEST_MODEL, "cohere.command-r-v1:0"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_MAX_TOKENS, "4096"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_TEMPERATURE, "0.8"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_TOP_P, "0.45"),
assertAttribute(
SemanticConventionsConstants.GEN_AI_RESPONSE_FINISH_REASONS, "[COMPLETE]"),
assertAttribute(SemanticConventionsConstants.GEN_AI_USAGE_INPUT_TOKENS, "9"),
assertAttribute(SemanticConventionsConstants.GEN_AI_USAGE_OUTPUT_TOKENS, "16")));
assertMetricClientAttributes(
metrics,
AppSignalsConstants.LATENCY_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
5000.0);
assertMetricClientAttributes(
metrics,
AppSignalsConstants.FAULT_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
0.0);
assertMetricClientAttributes(
metrics,
AppSignalsConstants.ERROR_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
0.0);
}

protected void doTestBedrockRuntimeMetaLlama() {
var response = appClient.get("/bedrockruntime/invokeModel/metaLlama").aggregate().join();

var traces = mockCollectorClient.getTraces();
var metrics =
mockCollectorClient.getMetrics(
Set.of(
AppSignalsConstants.ERROR_METRIC,
AppSignalsConstants.FAULT_METRIC,
AppSignalsConstants.LATENCY_METRIC));

var localService = getApplicationOtelServiceName();
var localOperation = "GET /bedrockruntime/invokeModel/metaLlama";
String type = "AWS::Bedrock::Model";
String identifier = "meta.llama3-70b-instruct-v1:0";

assertSpanClientAttributes(
traces,
bedrockRuntimeSpanName("InvokeModel"),
getBedrockRuntimeRpcServiceName(),
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
"bedrock.test",
8080,
"http://bedrock.test:8080",
200,
List.of(
assertAttribute(
SemanticConventionsConstants.GEN_AI_REQUEST_MODEL, "meta.llama3-70b-instruct-v1:0"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_MAX_TOKENS, "128"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_TEMPERATURE, "0.1"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_TOP_P, "0.9"),
assertAttribute(SemanticConventionsConstants.GEN_AI_RESPONSE_FINISH_REASONS, "[stop]"),
assertAttribute(SemanticConventionsConstants.GEN_AI_USAGE_INPUT_TOKENS, "2095"),
assertAttribute(SemanticConventionsConstants.GEN_AI_USAGE_OUTPUT_TOKENS, "503")));
assertMetricClientAttributes(
metrics,
AppSignalsConstants.LATENCY_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
5000.0);
assertMetricClientAttributes(
metrics,
AppSignalsConstants.FAULT_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
0.0);
assertMetricClientAttributes(
metrics,
AppSignalsConstants.ERROR_METRIC,
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
0.0);
}

protected void doTestBedrockRuntimeMistral() {
var response = appClient.get("/bedrockruntime/invokeModel/mistralAi").aggregate().join();

var traces = mockCollectorClient.getTraces();
var metrics =
mockCollectorClient.getMetrics(
Set.of(
AppSignalsConstants.ERROR_METRIC,
AppSignalsConstants.FAULT_METRIC,
AppSignalsConstants.LATENCY_METRIC));

var localService = getApplicationOtelServiceName();
var localOperation = "GET /bedrockruntime/invokeModel/mistralAi";
String type = "AWS::Bedrock::Model";
String identifier = "mistral.mistral-large-2402-v1:0";

assertSpanClientAttributes(
traces,
bedrockRuntimeSpanName("InvokeModel"),
getBedrockRuntimeRpcServiceName(),
localService,
localOperation,
getBedrockRuntimeServiceName(),
"InvokeModel",
type,
identifier,
"bedrock.test",
8080,
"http://bedrock.test:8080",
200,
List.of(
assertAttribute(
SemanticConventionsConstants.GEN_AI_REQUEST_MODEL,
"mistral.mistral-large-2402-v1:0"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_MAX_TOKENS, "4096"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_TEMPERATURE, "0.75"),
assertAttribute(SemanticConventionsConstants.GEN_AI_REQUEST_TOP_P, "0.25"),
assertAttribute(SemanticConventionsConstants.GEN_AI_RESPONSE_FINISH_REASONS, "[stop]"),
assertAttribute(SemanticConventionsConstants.GEN_AI_USAGE_INPUT_TOKENS, "15"),
assertAttribute(SemanticConventionsConstants.GEN_AI_USAGE_OUTPUT_TOKENS, "24")));
assertMetricClientAttributes(
metrics,
AppSignalsConstants.LATENCY_METRIC,
Expand Down
Loading
Loading