Skip to content

Commit 08b9430

Browse files
TimPansinoTimPansino
committed
Add tests for converse streaming
1 parent 9f95eda commit 08b9430

File tree

3 files changed

+321
-223
lines changed

3 files changed

+321
-223
lines changed

‎tests/external_botocore/_mock_external_bedrock_server_converse.py‎

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -112,25 +112,7 @@
112112
"000000bd00000051911972ae0b3a6576656e742d7479706507000b6d65737361676553746f700d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a303132333435222c2273746f70526561736f6e223a226d61785f746f6b656e73227d2963d7e1",
113113
"000000f00000004ebc72e3a30b3a6576656e742d747970650700086d657461646174610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226d657472696373223a7b226c6174656e63794d73223a323134397d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778222c227573616765223a7b22696e707574546f6b656e73223a32362c226f7574707574546f6b656e73223a3130302c22736572766572546f6f6c5573616765223a7b7d2c22746f74616c546f6b656e73223a3132367d7dd415e186",
114114
],
115-
],
116-
"Model does not exist.": [
117-
{
118-
"Content-Type": "application/json",
119-
"x-amzn-RequestId": "bf0106fd-1782-4afc-a289-4b591992ecec",
120-
"x-amzn-ErrorType": "ValidationException:http://internal.amazon.com/coral/com.amazon.bedrock/",
121-
},
122-
400,
123-
{"message": "The provided model identifier is invalid."},
124-
],
125-
"Invalid Token": [
126-
{
127-
"Content-Type": "application/json",
128-
"x-amzn-RequestId": "9dd5a736-67bc-41ea-9c87-c9a66fb7287c",
129-
"x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/",
130-
},
131-
403,
132-
{"message": "The security token included in the request is invalid."},
133-
],
115+
]
134116
}
135117

136118
RESPONSES = {

‎tests/external_botocore/_test_bedrock_chat_completion_converse.py‎

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Ignore unicode characters in this file from LLM responses
16+
# ruff: noqa: RUF001
17+
18+
chat_completion_expected_events = [
19+
(
20+
{"type": "LlmChatCompletionSummary"},
21+
{
22+
"id": None, # UUID that varies with each run
23+
"llm.conversation_id": "my-awesome-id",
24+
"llm.foo": "bar",
25+
"span_id": None,
26+
"trace_id": "trace-id",
27+
"request_id": "c20d345e-6878-4778-b674-6b187bae8ecf",
28+
"duration": None, # Response time varies each test run
29+
"request.model": "anthropic.claude-3-sonnet-20240229-v1:0",
30+
"response.model": "anthropic.claude-3-sonnet-20240229-v1:0",
31+
"request.temperature": 0.7,
32+
"request.max_tokens": 100,
33+
"response.choices.finish_reason": "max_tokens",
34+
"vendor": "bedrock",
35+
"ingest_source": "Python",
36+
"response.number_of_messages": 3,
37+
},
38+
),
39+
(
40+
{"type": "LlmChatCompletionMessage"},
41+
{
42+
"id": None, # UUID that varies with each run
43+
"llm.conversation_id": "my-awesome-id",
44+
"llm.foo": "bar",
45+
"request_id": "c20d345e-6878-4778-b674-6b187bae8ecf",
46+
"span_id": None,
47+
"trace_id": "trace-id",
48+
"content": "You are a scientist.",
49+
"role": "system",
50+
"completion_id": None,
51+
"sequence": 0,
52+
"response.model": "anthropic.claude-3-sonnet-20240229-v1:0",
53+
"vendor": "bedrock",
54+
"ingest_source": "Python",
55+
},
56+
),
57+
(
58+
{"type": "LlmChatCompletionMessage"},
59+
{
60+
"id": None, # UUID that varies with each run
61+
"llm.conversation_id": "my-awesome-id",
62+
"llm.foo": "bar",
63+
"request_id": "c20d345e-6878-4778-b674-6b187bae8ecf",
64+
"span_id": None,
65+
"trace_id": "trace-id",
66+
"content": "What is 212 degrees Fahrenheit converted to Celsius?",
67+
"role": "user",
68+
"completion_id": None,
69+
"sequence": 1,
70+
"response.model": "anthropic.claude-3-sonnet-20240229-v1:0",
71+
"vendor": "bedrock",
72+
"ingest_source": "Python",
73+
},
74+
),
75+
(
76+
{"type": "LlmChatCompletionMessage"},
77+
{
78+
"id": None, # UUID that varies with each run
79+
"llm.conversation_id": "my-awesome-id",
80+
"llm.foo": "bar",
81+
"request_id": "c20d345e-6878-4778-b674-6b187bae8ecf",
82+
"span_id": None,
83+
"trace_id": "trace-id",
84+
"content": "To convert 212°F to Celsius, we can use the formula:\n\nC = (F - 32) × 5/9\n\nWhere:\nC is the temperature in Celsius\nF is the temperature in Fahrenheit\n\nPlugging in 212°F, we get:\n\nC = (212 - 32) × 5/9\nC = 180 × 5/9\nC = 100\n\nTherefore, 212°",
85+
"role": "assistant",
86+
"completion_id": None,
87+
"sequence": 2,
88+
"response.model": "anthropic.claude-3-sonnet-20240229-v1:0",
89+
"vendor": "bedrock",
90+
"ingest_source": "Python",
91+
"is_response": True,
92+
},
93+
),
94+
]
95+
96+
chat_completion_expected_streaming_events = [
97+
(
98+
{"type": "LlmChatCompletionSummary"},
99+
{
100+
"id": None, # UUID that varies with each run
101+
"llm.conversation_id": "my-awesome-id",
102+
"llm.foo": "bar",
103+
"span_id": None,
104+
"trace_id": "trace-id",
105+
"request_id": "f070b880-e0fb-4537-8093-796671c39239",
106+
"duration": None, # Response time varies each test run
107+
"request.model": "anthropic.claude-3-sonnet-20240229-v1:0",
108+
"response.model": "anthropic.claude-3-sonnet-20240229-v1:0",
109+
"request.temperature": 0.7,
110+
"request.max_tokens": 100,
111+
"response.choices.finish_reason": "max_tokens",
112+
"vendor": "bedrock",
113+
"ingest_source": "Python",
114+
"response.number_of_messages": 3,
115+
},
116+
),
117+
(
118+
{"type": "LlmChatCompletionMessage"},
119+
{
120+
"id": None, # UUID that varies with each run
121+
"llm.conversation_id": "my-awesome-id",
122+
"llm.foo": "bar",
123+
"request_id": "f070b880-e0fb-4537-8093-796671c39239",
124+
"span_id": None,
125+
"trace_id": "trace-id",
126+
"content": "You are a scientist.",
127+
"role": "system",
128+
"completion_id": None,
129+
"sequence": 0,
130+
"response.model": "anthropic.claude-3-sonnet-20240229-v1:0",
131+
"vendor": "bedrock",
132+
"ingest_source": "Python",
133+
},
134+
),
135+
(
136+
{"type": "LlmChatCompletionMessage"},
137+
{
138+
"id": None, # UUID that varies with each run
139+
"llm.conversation_id": "my-awesome-id",
140+
"llm.foo": "bar",
141+
"request_id": "f070b880-e0fb-4537-8093-796671c39239",
142+
"span_id": None,
143+
"trace_id": "trace-id",
144+
"content": "What is 212 degrees Fahrenheit converted to Celsius?",
145+
"role": "user",
146+
"completion_id": None,
147+
"sequence": 1,
148+
"response.model": "anthropic.claude-3-sonnet-20240229-v1:0",
149+
"vendor": "bedrock",
150+
"ingest_source": "Python",
151+
},
152+
),
153+
(
154+
{"type": "LlmChatCompletionMessage"},
155+
{
156+
"id": None, # UUID that varies with each run
157+
"llm.conversation_id": "my-awesome-id",
158+
"llm.foo": "bar",
159+
"request_id": "f070b880-e0fb-4537-8093-796671c39239",
160+
"span_id": None,
161+
"trace_id": "trace-id",
162+
"content": "To convert Fahrenheit to Celsius, you use the formula:\n\nC = (F - 32) * 5/9\n\nWhere:\nC is the temperature in Celsius\nF is the temperature in Fahrenheit\n\nPlugging in 212°F for F:\n\nC = (212 - 32) * 5/9\nC = 180 * 5/9\nC = 100\n\nTherefore, 212 degrees Fahren",
163+
"role": "assistant",
164+
"completion_id": None,
165+
"sequence": 2,
166+
"response.model": "anthropic.claude-3-sonnet-20240229-v1:0",
167+
"vendor": "bedrock",
168+
"ingest_source": "Python",
169+
"is_response": True,
170+
},
171+
),
172+
]
173+
174+
chat_completion_invalid_access_key_error_events = [
175+
(
176+
{"type": "LlmChatCompletionSummary"},
177+
{
178+
"id": None, # UUID that varies with each run
179+
"llm.conversation_id": "my-awesome-id",
180+
"llm.foo": "bar",
181+
"span_id": None,
182+
"trace_id": "trace-id",
183+
"request_id": "e1206e19-2318-4a9d-be98-017c73f06118",
184+
"duration": None, # Response time varies each test run
185+
"request.model": "anthropic.claude-3-sonnet-20240229-v1:0",
186+
"response.model": "anthropic.claude-3-sonnet-20240229-v1:0",
187+
"request.temperature": 0.7,
188+
"request.max_tokens": 100,
189+
"vendor": "bedrock",
190+
"ingest_source": "Python",
191+
"response.number_of_messages": 1,
192+
"error": True,
193+
},
194+
),
195+
(
196+
{"type": "LlmChatCompletionMessage"},
197+
{
198+
"id": None, # UUID that varies with each run
199+
"llm.conversation_id": "my-awesome-id",
200+
"llm.foo": "bar",
201+
"request_id": "e1206e19-2318-4a9d-be98-017c73f06118",
202+
"span_id": None,
203+
"trace_id": "trace-id",
204+
"content": "Invalid Token",
205+
"role": "user",
206+
"completion_id": None,
207+
"sequence": 0,
208+
"response.model": "anthropic.claude-3-sonnet-20240229-v1:0",
209+
"vendor": "bedrock",
210+
"ingest_source": "Python",
211+
},
212+
),
213+
]
214+
chat_completion_invalid_model_error_events = [
215+
(
216+
{"type": "LlmChatCompletionSummary"},
217+
{
218+
"id": None, # UUID that varies with each run
219+
"llm.conversation_id": "my-awesome-id",
220+
"llm.foo": "bar",
221+
"request_id": "f4908827-3db9-4742-9103-2bbc34578b03",
222+
"span_id": None,
223+
"trace_id": "trace-id",
224+
"duration": None, # Response time varies each test run
225+
"request.model": "does-not-exist",
226+
"response.model": "does-not-exist",
227+
"request.temperature": 0.7,
228+
"request.max_tokens": 100,
229+
"response.number_of_messages": 1,
230+
"vendor": "bedrock",
231+
"ingest_source": "Python",
232+
"error": True,
233+
},
234+
),
235+
(
236+
{"type": "LlmChatCompletionMessage"},
237+
{
238+
"id": None,
239+
"llm.conversation_id": "my-awesome-id",
240+
"llm.foo": "bar",
241+
"span_id": None,
242+
"trace_id": "trace-id",
243+
"request_id": "f4908827-3db9-4742-9103-2bbc34578b03",
244+
"content": "Model does not exist.",
245+
"role": "user",
246+
"completion_id": None,
247+
"response.model": "does-not-exist",
248+
"sequence": 0,
249+
"vendor": "bedrock",
250+
"ingest_source": "Python",
251+
},
252+
),
253+
]

0 commit comments

Comments
 (0)