Skip to content

elastic-opentelemetry-instrumentation-openai: test calls with model not found #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xrmx merged 1 commit into from
Nov 14, 2024
Merged

elastic-opentelemetry-instrumentation-openai: test calls with model not found #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions ...opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,15 @@
_record_operation_duration_metric,
_set_span_attributes_from_response,
_set_embeddings_span_attributes_from_response,
_span_name_from_span_attributes,
)
from opentelemetry.instrumentation.openai.package import _instruments
from opentelemetry.instrumentation.openai.version import __version__
from opentelemetry.instrumentation.openai.wrappers import StreamWrapper
from opentelemetry.metrics import get_meter
from opentelemetry.semconv._incubating.attributes.gen_ai_attributes import (
GEN_AI_COMPLETION,
GEN_AI_OPERATION_NAME,
GEN_AI_PROMPT,
GEN_AI_REQUEST_MODEL,
)
from opentelemetry.semconv._incubating.metrics.gen_ai_metrics import (
create_gen_ai_client_token_usage,
Expand Down Expand Up @@ -123,7 +122,7 @@ def _chat_completion_wrapper(self, wrapped, instance, args, kwargs):

span_attributes = _get_span_attributes_from_wrapper(instance, kwargs)

span_name = f"{span_attributes[GEN_AI_OPERATION_NAME]} {span_attributes[GEN_AI_REQUEST_MODEL]}"
span_name = _span_name_from_span_attributes(span_attributes)
with self.tracer.start_as_current_span(
name=span_name,
kind=SpanKind.CLIENT,
Expand Down Expand Up @@ -185,7 +184,7 @@ async def _async_chat_completion_wrapper(self, wrapped, instance, args, kwargs):

span_attributes = _get_span_attributes_from_wrapper(instance, kwargs)

span_name = f"{span_attributes[GEN_AI_OPERATION_NAME]} {span_attributes[GEN_AI_REQUEST_MODEL]}"
span_name = _span_name_from_span_attributes(span_attributes)
with self.tracer.start_as_current_span(
name=span_name,
kind=SpanKind.CLIENT,
Expand Down Expand Up @@ -244,7 +243,7 @@ async def _async_chat_completion_wrapper(self, wrapped, instance, args, kwargs):
def _embeddings_wrapper(self, wrapped, instance, args, kwargs):
span_attributes = _get_embeddings_span_attributes_from_wrapper(instance, kwargs)

span_name = f"{span_attributes[GEN_AI_OPERATION_NAME]} {span_attributes[GEN_AI_REQUEST_MODEL]}"
span_name = _span_name_from_span_attributes(span_attributes)
with self.tracer.start_as_current_span(
name=span_name,
kind=SpanKind.CLIENT,
Expand Down Expand Up @@ -274,7 +273,7 @@ def _embeddings_wrapper(self, wrapped, instance, args, kwargs):
async def _async_embeddings_wrapper(self, wrapped, instance, args, kwargs):
span_attributes = _get_embeddings_span_attributes_from_wrapper(instance, kwargs)

span_name = f"{span_attributes[GEN_AI_OPERATION_NAME]} {span_attributes[GEN_AI_REQUEST_MODEL]}"
span_name = _span_name_from_span_attributes(span_attributes)
with self.tracer.start_as_current_span(
name=span_name,
kind=SpanKind.CLIENT,
Expand Down
18 changes: 16 additions & 2 deletions ...-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
)
from opentelemetry.metrics import Histogram
from opentelemetry.trace import Span
from opentelemetry.util.types import Attributes

if TYPE_CHECKING:
from openai.types import CompletionUsage
Expand Down Expand Up @@ -125,10 +126,12 @@ def _attributes_from_client(client):
def _get_span_attributes_from_wrapper(instance, kwargs):
span_attributes = {
GEN_AI_OPERATION_NAME: "chat",
GEN_AI_REQUEST_MODEL: kwargs["model"],
GEN_AI_SYSTEM: "openai",
}

if (request_model := kwargs.get("model")) is not None:
span_attributes[GEN_AI_REQUEST_MODEL] = request_model

if client := getattr(instance, "_client", None):
span_attributes.update(_attributes_from_client(client))

Expand All @@ -150,13 +153,24 @@ def _get_span_attributes_from_wrapper(instance, kwargs):
return span_attributes


def _span_name_from_span_attributes(attributes: Attributes) -> str:
request_model = attributes.get(GEN_AI_REQUEST_MODEL)
return (
f"{attributes[GEN_AI_OPERATION_NAME]} {request_model}"
if request_model
else f"{attributes[GEN_AI_OPERATION_NAME]}"
)


def _get_embeddings_span_attributes_from_wrapper(instance, kwargs):
span_attributes = {
GEN_AI_OPERATION_NAME: "embeddings",
GEN_AI_REQUEST_MODEL: kwargs["model"],
GEN_AI_SYSTEM: "openai",
}

if (request_model := kwargs.get("model")) is not None:
span_attributes[GEN_AI_REQUEST_MODEL] = request_model

if client := getattr(instance, "_client", None):
span_attributes.update(_attributes_from_client(client))

Expand Down
65 changes: 65 additions & 0 deletions ...es/.test_chat_completions/test_with_model_not_found[azure_provider_chat_completions].yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
interactions:
- request:
body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which
ocean contains the falkland islands?"}], "model": "not-found-model"}'
headers:
accept:
- application/json
accept-encoding:
- gzip, deflate
api-key:
- test_azure_api_key
authorization:
- Bearer test_openai_api_key
connection:
- keep-alive
content-length:
- '142'
content-type:
- application/json
host:
- test.openai.azure.com
user-agent:
- AzureOpenAI/Python 1.34.0
x-stainless-arch:
- x64
x-stainless-async:
- 'false'
x-stainless-lang:
- python
x-stainless-os:
- Linux
x-stainless-package-version:
- 1.34.0
x-stainless-runtime:
- CPython
x-stainless-runtime-version:
- 3.10.12
method: POST
uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/chat/completions?api-version=2023-03-15-preview
response:
body:
string: '{"error":{"code":"DeploymentNotFound", "message":"The API deployment
for this resource does not exist. If you created the deployment within the
last 5 minutes, please wait a moment and try again."}}'
headers:
Content-Length:
- '198'
Content-Type:
- application/json
Date:
- Thu, 10 Oct 2024 13:28:08 GMT
Set-Cookie: test_set_cookie
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
apim-request-id:
- a9022d64-309b-4b6b-a425-a797f00b6cff
openai-organization: test_openai_org_key
x-content-type-options:
- nosniff
x-ms-region:
- East US
status:
code: 404
message: Not Found
version: 1
56 changes: 56 additions & 0 deletions ...s/.test_chat_completions/test_with_model_not_found[ollama_provider_chat_completions].yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
interactions:
- request:
body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which
ocean contains the falkland islands?"}], "model": "not-found-model"}'
headers:
accept:
- application/json
accept-encoding:
- gzip, deflate
authorization:
- Bearer test_openai_api_key
connection:
- keep-alive
content-length:
- '142'
content-type:
- application/json
host:
- localhost:11434
user-agent:
- OpenAI/Python 1.34.0
x-stainless-arch:
- x64
x-stainless-async:
- 'false'
x-stainless-lang:
- python
x-stainless-os:
- Linux
x-stainless-package-version:
- 1.34.0
x-stainless-runtime:
- CPython
x-stainless-runtime-version:
- 3.10.12
method: POST
uri: http://localhost:11434/v1/chat/completions
response:
body:
string: '{"error":{"message":"model \"not-found-model\" not found, try pulling
it first","type":"api_error","param":null,"code":null}}

'
headers:
Content-Length:
- '126'
Content-Type:
- application/json
Date:
- Thu, 10 Oct 2024 13:37:48 GMT
Set-Cookie: test_set_cookie
openai-organization: test_openai_org_key
status:
code: 404
message: Not Found
version: 1
75 changes: 75 additions & 0 deletions ...s/.test_chat_completions/test_with_model_not_found[openai_provider_chat_completions].yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
interactions:
- request:
body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which
ocean contains the falkland islands?"}], "model": "not-found-model"}'
headers:
accept:
- application/json
accept-encoding:
- gzip, deflate
authorization:
- Bearer test_openai_api_key
connection:
- keep-alive
content-length:
- '142'
content-type:
- application/json
host:
- api.openai.com
user-agent:
- OpenAI/Python 1.34.0
x-stainless-arch:
- x64
x-stainless-async:
- 'false'
x-stainless-lang:
- python
x-stainless-os:
- Linux
x-stainless-package-version:
- 1.34.0
x-stainless-runtime:
- CPython
x-stainless-runtime-version:
- 3.10.12
method: POST
uri: https://api.openai.com/v1/chat/completions
response:
body:
string: "{\n \"error\": {\n \"message\": \"The model `not-found-model`
does not exist or you do not have access to it.\",\n \"type\": \"invalid_request_error\",\n
\ \"param\": null,\n \"code\": \"model_not_found\"\n }\n}\n"
headers:
CF-Cache-Status:
- DYNAMIC
CF-RAY:
- 8d06f3a529a63865-LHR
Connection:
- keep-alive
Content-Type:
- application/json; charset=utf-8
Date:
- Thu, 10 Oct 2024 13:28:07 GMT
Server:
- cloudflare
Set-Cookie: test_set_cookie
Transfer-Encoding:
- chunked
X-Content-Type-Options:
- nosniff
alt-svc:
- h3=":443"; ma=86400
content-length:
- '221'
openai-organization: test_openai_org_key
strict-transport-security:
- max-age=31536000; includeSubDomains; preload
vary:
- Origin
x-request-id:
- req_a61de9eb10d55c0c52033fd9e6715e6e
status:
code: 404
message: Not Found
version: 1
67 changes: 67 additions & 0 deletions ...nai/tests/cassettes/.test_embeddings/test_model_not_found[azure_provider_embeddings].yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
interactions:
- request:
body: '{"input": ["South Atlantic Ocean."], "model": "not-found-model", "encoding_format":
"base64"}'
headers:
accept:
- application/json
accept-encoding:
- gzip, deflate
api-key:
- test_azure_api_key
authorization:
- Bearer test_openai_api_key
connection:
- keep-alive
content-length:
- '93'
content-type:
- application/json
host:
- test.openai.azure.com
user-agent:
- AzureOpenAI/Python 1.50.0
x-stainless-arch:
- x64
x-stainless-async:
- 'false'
x-stainless-lang:
- python
x-stainless-os:
- Linux
x-stainless-package-version:
- 1.50.0
x-stainless-retry-count:
- '0'
x-stainless-runtime:
- CPython
x-stainless-runtime-version:
- 3.10.12
method: POST
uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/embeddings?api-version=2023-05-15
response:
body:
string: '{"error":{"code":"DeploymentNotFound", "message":"The API deployment
for this resource does not exist. If you created the deployment within the
last 5 minutes, please wait a moment and try again."}}'
headers:
Content-Length:
- '198'
Content-Type:
- application/json
Date:
- Tue, 29 Oct 2024 08:43:31 GMT
Set-Cookie: test_set_cookie
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
apim-request-id:
- 975b7870-9f64-4e72-9d3d-ecfd22f4a68d
openai-organization: test_openai_org_key
x-content-type-options:
- nosniff
x-ms-region:
- East US
status:
code: 404
message: Not Found
version: 1
Loading