Skip to content

Commit fbaf291

Browse files
xrmxxrmx
authored
elastic-opentelemetry-instrumentation-openai: test calls with model not found (#22)
And remove the assumptions that expected it to be always present.
1 parent b7ba48a commit fbaf291

11 files changed

+710
-11
lines changed

‎instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,15 @@
3535
_record_operation_duration_metric,
3636
_set_span_attributes_from_response,
3737
_set_embeddings_span_attributes_from_response,
38+
_span_name_from_span_attributes,
3839
)
3940
from opentelemetry.instrumentation.openai.package import _instruments
4041
from opentelemetry.instrumentation.openai.version import __version__
4142
from opentelemetry.instrumentation.openai.wrappers import StreamWrapper
4243
from opentelemetry.metrics import get_meter
4344
from opentelemetry.semconv._incubating.attributes.gen_ai_attributes import (
4445
GEN_AI_COMPLETION,
45-
GEN_AI_OPERATION_NAME,
4646
GEN_AI_PROMPT,
47-
GEN_AI_REQUEST_MODEL,
4847
)
4948
from opentelemetry.semconv._incubating.metrics.gen_ai_metrics import (
5049
create_gen_ai_client_token_usage,
@@ -123,7 +122,7 @@ def _chat_completion_wrapper(self, wrapped, instance, args, kwargs):
123122

124123
span_attributes = _get_span_attributes_from_wrapper(instance, kwargs)
125124

126-
span_name = f"{span_attributes[GEN_AI_OPERATION_NAME]} {span_attributes[GEN_AI_REQUEST_MODEL]}"
125+
span_name = _span_name_from_span_attributes(span_attributes)
127126
with self.tracer.start_as_current_span(
128127
name=span_name,
129128
kind=SpanKind.CLIENT,
@@ -185,7 +184,7 @@ async def _async_chat_completion_wrapper(self, wrapped, instance, args, kwargs):
185184

186185
span_attributes = _get_span_attributes_from_wrapper(instance, kwargs)
187186

188-
span_name = f"{span_attributes[GEN_AI_OPERATION_NAME]} {span_attributes[GEN_AI_REQUEST_MODEL]}"
187+
span_name = _span_name_from_span_attributes(span_attributes)
189188
with self.tracer.start_as_current_span(
190189
name=span_name,
191190
kind=SpanKind.CLIENT,
@@ -244,7 +243,7 @@ async def _async_chat_completion_wrapper(self, wrapped, instance, args, kwargs):
244243
def _embeddings_wrapper(self, wrapped, instance, args, kwargs):
245244
span_attributes = _get_embeddings_span_attributes_from_wrapper(instance, kwargs)
246245

247-
span_name = f"{span_attributes[GEN_AI_OPERATION_NAME]} {span_attributes[GEN_AI_REQUEST_MODEL]}"
246+
span_name = _span_name_from_span_attributes(span_attributes)
248247
with self.tracer.start_as_current_span(
249248
name=span_name,
250249
kind=SpanKind.CLIENT,
@@ -274,7 +273,7 @@ def _embeddings_wrapper(self, wrapped, instance, args, kwargs):
274273
async def _async_embeddings_wrapper(self, wrapped, instance, args, kwargs):
275274
span_attributes = _get_embeddings_span_attributes_from_wrapper(instance, kwargs)
276275

277-
span_name = f"{span_attributes[GEN_AI_OPERATION_NAME]} {span_attributes[GEN_AI_REQUEST_MODEL]}"
276+
span_name = _span_name_from_span_attributes(span_attributes)
278277
with self.tracer.start_as_current_span(
279278
name=span_name,
280279
kind=SpanKind.CLIENT,

‎instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
)
4141
from opentelemetry.metrics import Histogram
4242
from opentelemetry.trace import Span
43+
from opentelemetry.util.types import Attributes
4344

4445
if TYPE_CHECKING:
4546
from openai.types import CompletionUsage
@@ -125,10 +126,12 @@ def _attributes_from_client(client):
125126
def _get_span_attributes_from_wrapper(instance, kwargs):
126127
span_attributes = {
127128
GEN_AI_OPERATION_NAME: "chat",
128-
GEN_AI_REQUEST_MODEL: kwargs["model"],
129129
GEN_AI_SYSTEM: "openai",
130130
}
131131

132+
if (request_model := kwargs.get("model")) is not None:
133+
span_attributes[GEN_AI_REQUEST_MODEL] = request_model
134+
132135
if client := getattr(instance, "_client", None):
133136
span_attributes.update(_attributes_from_client(client))
134137

@@ -150,13 +153,24 @@ def _get_span_attributes_from_wrapper(instance, kwargs):
150153
return span_attributes
151154

152155

156+
def _span_name_from_span_attributes(attributes: Attributes) -> str:
157+
request_model = attributes.get(GEN_AI_REQUEST_MODEL)
158+
return (
159+
f"{attributes[GEN_AI_OPERATION_NAME]} {request_model}"
160+
if request_model
161+
else f"{attributes[GEN_AI_OPERATION_NAME]}"
162+
)
163+
164+
153165
def _get_embeddings_span_attributes_from_wrapper(instance, kwargs):
154166
span_attributes = {
155167
GEN_AI_OPERATION_NAME: "embeddings",
156-
GEN_AI_REQUEST_MODEL: kwargs["model"],
157168
GEN_AI_SYSTEM: "openai",
158169
}
159170

171+
if (request_model := kwargs.get("model")) is not None:
172+
span_attributes[GEN_AI_REQUEST_MODEL] = request_model
173+
160174
if client := getattr(instance, "_client", None):
161175
span_attributes.update(_attributes_from_client(client))
162176

‎instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[azure_provider_chat_completions].yaml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
interactions:
2+
- request:
3+
body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which
4+
ocean contains the falkland islands?"}], "model": "not-found-model"}'
5+
headers:
6+
accept:
7+
- application/json
8+
accept-encoding:
9+
- gzip, deflate
10+
api-key:
11+
- test_azure_api_key
12+
authorization:
13+
- Bearer test_openai_api_key
14+
connection:
15+
- keep-alive
16+
content-length:
17+
- '142'
18+
content-type:
19+
- application/json
20+
host:
21+
- test.openai.azure.com
22+
user-agent:
23+
- AzureOpenAI/Python 1.34.0
24+
x-stainless-arch:
25+
- x64
26+
x-stainless-async:
27+
- 'false'
28+
x-stainless-lang:
29+
- python
30+
x-stainless-os:
31+
- Linux
32+
x-stainless-package-version:
33+
- 1.34.0
34+
x-stainless-runtime:
35+
- CPython
36+
x-stainless-runtime-version:
37+
- 3.10.12
38+
method: POST
39+
uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/chat/completions?api-version=2023-03-15-preview
40+
response:
41+
body:
42+
string: '{"error":{"code":"DeploymentNotFound", "message":"The API deployment
43+
for this resource does not exist. If you created the deployment within the
44+
last 5 minutes, please wait a moment and try again."}}'
45+
headers:
46+
Content-Length:
47+
- '198'
48+
Content-Type:
49+
- application/json
50+
Date:
51+
- Thu, 10 Oct 2024 13:28:08 GMT
52+
Set-Cookie: test_set_cookie
53+
Strict-Transport-Security:
54+
- max-age=31536000; includeSubDomains; preload
55+
apim-request-id:
56+
- a9022d64-309b-4b6b-a425-a797f00b6cff
57+
openai-organization: test_openai_org_key
58+
x-content-type-options:
59+
- nosniff
60+
x-ms-region:
61+
- East US
62+
status:
63+
code: 404
64+
message: Not Found
65+
version: 1

‎instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[ollama_provider_chat_completions].yaml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
interactions:
2+
- request:
3+
body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which
4+
ocean contains the falkland islands?"}], "model": "not-found-model"}'
5+
headers:
6+
accept:
7+
- application/json
8+
accept-encoding:
9+
- gzip, deflate
10+
authorization:
11+
- Bearer test_openai_api_key
12+
connection:
13+
- keep-alive
14+
content-length:
15+
- '142'
16+
content-type:
17+
- application/json
18+
host:
19+
- localhost:11434
20+
user-agent:
21+
- OpenAI/Python 1.34.0
22+
x-stainless-arch:
23+
- x64
24+
x-stainless-async:
25+
- 'false'
26+
x-stainless-lang:
27+
- python
28+
x-stainless-os:
29+
- Linux
30+
x-stainless-package-version:
31+
- 1.34.0
32+
x-stainless-runtime:
33+
- CPython
34+
x-stainless-runtime-version:
35+
- 3.10.12
36+
method: POST
37+
uri: http://localhost:11434/v1/chat/completions
38+
response:
39+
body:
40+
string: '{"error":{"message":"model \"not-found-model\" not found, try pulling
41+
it first","type":"api_error","param":null,"code":null}}
42+
43+
'
44+
headers:
45+
Content-Length:
46+
- '126'
47+
Content-Type:
48+
- application/json
49+
Date:
50+
- Thu, 10 Oct 2024 13:37:48 GMT
51+
Set-Cookie: test_set_cookie
52+
openai-organization: test_openai_org_key
53+
status:
54+
code: 404
55+
message: Not Found
56+
version: 1

‎instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[openai_provider_chat_completions].yaml‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
interactions:
2+
- request:
3+
body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which
4+
ocean contains the falkland islands?"}], "model": "not-found-model"}'
5+
headers:
6+
accept:
7+
- application/json
8+
accept-encoding:
9+
- gzip, deflate
10+
authorization:
11+
- Bearer test_openai_api_key
12+
connection:
13+
- keep-alive
14+
content-length:
15+
- '142'
16+
content-type:
17+
- application/json
18+
host:
19+
- api.openai.com
20+
user-agent:
21+
- OpenAI/Python 1.34.0
22+
x-stainless-arch:
23+
- x64
24+
x-stainless-async:
25+
- 'false'
26+
x-stainless-lang:
27+
- python
28+
x-stainless-os:
29+
- Linux
30+
x-stainless-package-version:
31+
- 1.34.0
32+
x-stainless-runtime:
33+
- CPython
34+
x-stainless-runtime-version:
35+
- 3.10.12
36+
method: POST
37+
uri: https://api.openai.com/v1/chat/completions
38+
response:
39+
body:
40+
string: "{\n \"error\": {\n \"message\": \"The model `not-found-model`
41+
does not exist or you do not have access to it.\",\n \"type\": \"invalid_request_error\",\n
42+
\ \"param\": null,\n \"code\": \"model_not_found\"\n }\n}\n"
43+
headers:
44+
CF-Cache-Status:
45+
- DYNAMIC
46+
CF-RAY:
47+
- 8d06f3a529a63865-LHR
48+
Connection:
49+
- keep-alive
50+
Content-Type:
51+
- application/json; charset=utf-8
52+
Date:
53+
- Thu, 10 Oct 2024 13:28:07 GMT
54+
Server:
55+
- cloudflare
56+
Set-Cookie: test_set_cookie
57+
Transfer-Encoding:
58+
- chunked
59+
X-Content-Type-Options:
60+
- nosniff
61+
alt-svc:
62+
- h3=":443"; ma=86400
63+
content-length:
64+
- '221'
65+
openai-organization: test_openai_org_key
66+
strict-transport-security:
67+
- max-age=31536000; includeSubDomains; preload
68+
vary:
69+
- Origin
70+
x-request-id:
71+
- req_a61de9eb10d55c0c52033fd9e6715e6e
72+
status:
73+
code: 404
74+
message: Not Found
75+
version: 1

‎instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[azure_provider_embeddings].yaml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
interactions:
2+
- request:
3+
body: '{"input": ["South Atlantic Ocean."], "model": "not-found-model", "encoding_format":
4+
"base64"}'
5+
headers:
6+
accept:
7+
- application/json
8+
accept-encoding:
9+
- gzip, deflate
10+
api-key:
11+
- test_azure_api_key
12+
authorization:
13+
- Bearer test_openai_api_key
14+
connection:
15+
- keep-alive
16+
content-length:
17+
- '93'
18+
content-type:
19+
- application/json
20+
host:
21+
- test.openai.azure.com
22+
user-agent:
23+
- AzureOpenAI/Python 1.50.0
24+
x-stainless-arch:
25+
- x64
26+
x-stainless-async:
27+
- 'false'
28+
x-stainless-lang:
29+
- python
30+
x-stainless-os:
31+
- Linux
32+
x-stainless-package-version:
33+
- 1.50.0
34+
x-stainless-retry-count:
35+
- '0'
36+
x-stainless-runtime:
37+
- CPython
38+
x-stainless-runtime-version:
39+
- 3.10.12
40+
method: POST
41+
uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/embeddings?api-version=2023-05-15
42+
response:
43+
body:
44+
string: '{"error":{"code":"DeploymentNotFound", "message":"The API deployment
45+
for this resource does not exist. If you created the deployment within the
46+
last 5 minutes, please wait a moment and try again."}}'
47+
headers:
48+
Content-Length:
49+
- '198'
50+
Content-Type:
51+
- application/json
52+
Date:
53+
- Tue, 29 Oct 2024 08:43:31 GMT
54+
Set-Cookie: test_set_cookie
55+
Strict-Transport-Security:
56+
- max-age=31536000; includeSubDomains; preload
57+
apim-request-id:
58+
- 975b7870-9f64-4e72-9d3d-ecfd22f4a68d
59+
openai-organization: test_openai_org_key
60+
x-content-type-options:
61+
- nosniff
62+
x-ms-region:
63+
- East US
64+
status:
65+
code: 404
66+
message: Not Found
67+
version: 1

0 commit comments

Comments
 (0)