From 9edd62a2a760059170ade2d5d6cdd35e973d346d Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 13 Nov 2024 12:08:12 +0100 Subject: [PATCH] elastic-opentelemetry-instrumentation-openai: test calls with model not found And remove the assumptions that expected it to be always present. --- .../instrumentation/openai/__init__.py | 11 +- .../instrumentation/openai/helpers.py | 18 +- ...ound[azure_provider_chat_completions].yaml | 65 +++++++ ...und[ollama_provider_chat_completions].yaml | 56 ++++++ ...und[openai_provider_chat_completions].yaml | 75 +++++++++ ..._not_found[azure_provider_embeddings].yaml | 67 ++++++++ ...not_found[ollama_provider_embeddings].yaml | 56 ++++++ ...not_found[openai_provider_embeddings].yaml | 75 +++++++++ .../tests/conftest.py | 6 +- .../tests/test_chat_completions.py | 159 ++++++++++++++++++ .../tests/test_embeddings.py | 133 ++++++++++++++- 11 files changed, 710 insertions(+), 11 deletions(-) create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[azure_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[ollama_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[openai_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[azure_provider_embeddings].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[ollama_provider_embeddings].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[openai_provider_embeddings].yaml diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py index 819f956..de7f406 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py @@ -35,6 +35,7 @@ _record_operation_duration_metric, _set_span_attributes_from_response, _set_embeddings_span_attributes_from_response, + _span_name_from_span_attributes, ) from opentelemetry.instrumentation.openai.package import _instruments from opentelemetry.instrumentation.openai.version import __version__ @@ -42,9 +43,7 @@ from opentelemetry.metrics import get_meter from opentelemetry.semconv._incubating.attributes.gen_ai_attributes import ( GEN_AI_COMPLETION, - GEN_AI_OPERATION_NAME, GEN_AI_PROMPT, - GEN_AI_REQUEST_MODEL, ) from opentelemetry.semconv._incubating.metrics.gen_ai_metrics import ( create_gen_ai_client_token_usage, @@ -123,7 +122,7 @@ def _chat_completion_wrapper(self, wrapped, instance, args, kwargs): span_attributes = _get_span_attributes_from_wrapper(instance, kwargs) - span_name = f"{span_attributes[GEN_AI_OPERATION_NAME]} {span_attributes[GEN_AI_REQUEST_MODEL]}" + span_name = _span_name_from_span_attributes(span_attributes) with self.tracer.start_as_current_span( name=span_name, kind=SpanKind.CLIENT, @@ -185,7 +184,7 @@ async def _async_chat_completion_wrapper(self, wrapped, instance, args, kwargs): span_attributes = _get_span_attributes_from_wrapper(instance, kwargs) - span_name = f"{span_attributes[GEN_AI_OPERATION_NAME]} {span_attributes[GEN_AI_REQUEST_MODEL]}" + span_name = _span_name_from_span_attributes(span_attributes) with self.tracer.start_as_current_span( name=span_name, kind=SpanKind.CLIENT, @@ -244,7 +243,7 @@ async def _async_chat_completion_wrapper(self, wrapped, instance, args, kwargs): def _embeddings_wrapper(self, wrapped, instance, args, kwargs): span_attributes = _get_embeddings_span_attributes_from_wrapper(instance, kwargs) - span_name = f"{span_attributes[GEN_AI_OPERATION_NAME]} {span_attributes[GEN_AI_REQUEST_MODEL]}" + span_name = _span_name_from_span_attributes(span_attributes) with self.tracer.start_as_current_span( name=span_name, kind=SpanKind.CLIENT, @@ -274,7 +273,7 @@ def _embeddings_wrapper(self, wrapped, instance, args, kwargs): async def _async_embeddings_wrapper(self, wrapped, instance, args, kwargs): span_attributes = _get_embeddings_span_attributes_from_wrapper(instance, kwargs) - span_name = f"{span_attributes[GEN_AI_OPERATION_NAME]} {span_attributes[GEN_AI_REQUEST_MODEL]}" + span_name = _span_name_from_span_attributes(span_attributes) with self.tracer.start_as_current_span( name=span_name, kind=SpanKind.CLIENT, diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py index 918495e..e36a4d8 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py @@ -40,6 +40,7 @@ ) from opentelemetry.metrics import Histogram from opentelemetry.trace import Span +from opentelemetry.util.types import Attributes if TYPE_CHECKING: from openai.types import CompletionUsage @@ -125,10 +126,12 @@ def _attributes_from_client(client): def _get_span_attributes_from_wrapper(instance, kwargs): span_attributes = { GEN_AI_OPERATION_NAME: "chat", - GEN_AI_REQUEST_MODEL: kwargs["model"], GEN_AI_SYSTEM: "openai", } + if (request_model := kwargs.get("model")) is not None: + span_attributes[GEN_AI_REQUEST_MODEL] = request_model + if client := getattr(instance, "_client", None): span_attributes.update(_attributes_from_client(client)) @@ -150,13 +153,24 @@ def _get_span_attributes_from_wrapper(instance, kwargs): return span_attributes +def _span_name_from_span_attributes(attributes: Attributes) -> str: + request_model = attributes.get(GEN_AI_REQUEST_MODEL) + return ( + f"{attributes[GEN_AI_OPERATION_NAME]} {request_model}" + if request_model + else f"{attributes[GEN_AI_OPERATION_NAME]}" + ) + + def _get_embeddings_span_attributes_from_wrapper(instance, kwargs): span_attributes = { GEN_AI_OPERATION_NAME: "embeddings", - GEN_AI_REQUEST_MODEL: kwargs["model"], GEN_AI_SYSTEM: "openai", } + if (request_model := kwargs.get("model")) is not None: + span_attributes[GEN_AI_REQUEST_MODEL] = request_model + if client := getattr(instance, "_client", None): span_attributes.update(_attributes_from_client(client)) diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[azure_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[azure_provider_chat_completions].yaml new file mode 100644 index 0000000..15acf23 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[azure_provider_chat_completions].yaml @@ -0,0 +1,65 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "not-found-model"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + api-key: + - test_azure_api_key + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '142' + content-type: + - application/json + host: + - test.openai.azure.com + user-agent: + - AzureOpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/chat/completions?api-version=2023-03-15-preview + response: + body: + string: '{"error":{"code":"DeploymentNotFound", "message":"The API deployment + for this resource does not exist. If you created the deployment within the + last 5 minutes, please wait a moment and try again."}}' + headers: + Content-Length: + - '198' + Content-Type: + - application/json + Date: + - Thu, 10 Oct 2024 13:28:08 GMT + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + apim-request-id: + - a9022d64-309b-4b6b-a425-a797f00b6cff + openai-organization: test_openai_org_key + x-content-type-options: + - nosniff + x-ms-region: + - East US + status: + code: 404 + message: Not Found +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[ollama_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[ollama_provider_chat_completions].yaml new file mode 100644 index 0000000..df5db8b --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[ollama_provider_chat_completions].yaml @@ -0,0 +1,56 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "not-found-model"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '142' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: '{"error":{"message":"model \"not-found-model\" not found, try pulling + it first","type":"api_error","param":null,"code":null}} + + ' + headers: + Content-Length: + - '126' + Content-Type: + - application/json + Date: + - Thu, 10 Oct 2024 13:37:48 GMT + Set-Cookie: test_set_cookie + openai-organization: test_openai_org_key + status: + code: 404 + message: Not Found +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[openai_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[openai_provider_chat_completions].yaml new file mode 100644 index 0000000..c86cae0 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_with_model_not_found[openai_provider_chat_completions].yaml @@ -0,0 +1,75 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "not-found-model"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '142' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"error\": {\n \"message\": \"The model `not-found-model` + does not exist or you do not have access to it.\",\n \"type\": \"invalid_request_error\",\n + \ \"param\": null,\n \"code\": \"model_not_found\"\n }\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d06f3a529a63865-LHR + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 10 Oct 2024 13:28:07 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '221' + openai-organization: test_openai_org_key + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Origin + x-request-id: + - req_a61de9eb10d55c0c52033fd9e6715e6e + status: + code: 404 + message: Not Found +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[azure_provider_embeddings].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[azure_provider_embeddings].yaml new file mode 100644 index 0000000..eceb999 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[azure_provider_embeddings].yaml @@ -0,0 +1,67 @@ +interactions: +- request: + body: '{"input": ["South Atlantic Ocean."], "model": "not-found-model", "encoding_format": + "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + api-key: + - test_azure_api_key + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '93' + content-type: + - application/json + host: + - test.openai.azure.com + user-agent: + - AzureOpenAI/Python 1.50.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.50.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/embeddings?api-version=2023-05-15 + response: + body: + string: '{"error":{"code":"DeploymentNotFound", "message":"The API deployment + for this resource does not exist. If you created the deployment within the + last 5 minutes, please wait a moment and try again."}}' + headers: + Content-Length: + - '198' + Content-Type: + - application/json + Date: + - Tue, 29 Oct 2024 08:43:31 GMT + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + apim-request-id: + - 975b7870-9f64-4e72-9d3d-ecfd22f4a68d + openai-organization: test_openai_org_key + x-content-type-options: + - nosniff + x-ms-region: + - East US + status: + code: 404 + message: Not Found +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[ollama_provider_embeddings].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[ollama_provider_embeddings].yaml new file mode 100644 index 0000000..6f68173 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[ollama_provider_embeddings].yaml @@ -0,0 +1,56 @@ +interactions: +- request: + body: '{"input": ["South Atlantic Ocean."], "model": "not-found-model", "encoding_format": + "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '93' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: http://localhost:11434/v1/embeddings + response: + body: + string: '{"error":{"message":"model \"not-found-model\" not found, try pulling + it first","type":"api_error","param":null,"code":null}} + + ' + headers: + Content-Length: + - '126' + Content-Type: + - application/json + Date: + - Thu, 10 Oct 2024 14:14:28 GMT + Set-Cookie: test_set_cookie + openai-organization: test_openai_org_key + status: + code: 404 + message: Not Found +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[openai_provider_embeddings].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[openai_provider_embeddings].yaml new file mode 100644 index 0000000..04c44d5 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_embeddings/test_model_not_found[openai_provider_embeddings].yaml @@ -0,0 +1,75 @@ +interactions: +- request: + body: '{"input": ["South Atlantic Ocean."], "model": "not-found-model", "encoding_format": + "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '93' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: "{\n \"error\": {\n \"message\": \"The model `not-found-model` + does not exist or you do not have access to it.\",\n \"type\": \"invalid_request_error\",\n + \ \"param\": null,\n \"code\": \"model_not_found\"\n }\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d07372d1c22d1fa-LHR + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 10 Oct 2024 14:14:13 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '221' + openai-organization: test_openai_org_key + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Origin + x-request-id: + - req_2f53ac48cd8ac55089fc27329069bfa2 + status: + code: 404 + message: Not Found +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/conftest.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/conftest.py index ac78e4e..c8e123f 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/conftest.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/conftest.py @@ -352,7 +352,9 @@ def assert_operation_duration_metric(provider, metric: Histogram, attributes: di ) -def assert_error_operation_duration_metric(provider, metric: Histogram, attributes: dict, data_point: float): +def assert_error_operation_duration_metric( + provider, metric: Histogram, attributes: dict, data_point: float, value_delta: float = 0.5 +): assert metric.name == "gen_ai.client.operation.duration" default_attributes = { "gen_ai.operation.name": provider.operation_name, @@ -372,7 +374,7 @@ def assert_error_operation_duration_metric(provider, metric: Histogram, attribut attributes={**default_attributes, **attributes}, ), ], - est_value_delta=0.5, + est_value_delta=value_delta, ) diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_chat_completions.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_chat_completions.py index fea26f1..869a274 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_chat_completions.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_chat_completions.py @@ -15,6 +15,7 @@ # limitations under the License. import json +import re from unittest import mock import openai @@ -611,6 +612,7 @@ def test_connection_error(provider_str, model, duration, trace_exporter, metrics operation_duration_metric, attributes=attributes, data_point=duration, + value_delta=1.0, ) @@ -1570,3 +1572,160 @@ async def test_async_tools_with_capture_content( input_data_point=input_tokens, output_data_point=output_tokens, ) + + +test_without_model_parameter_test_data = [ + ( + "openai_provider_chat_completions", + "api.openai.com", + 443, + 5, + ), + ( + "azure_provider_chat_completions", + "test.openai.azure.com", + 443, + 5, + ), + ( + "ollama_provider_chat_completions", + "localhost", + 11434, + 5, + ), +] + + +@pytest.mark.vcr() +@pytest.mark.parametrize("provider_str,server_address,server_port,duration", test_without_model_parameter_test_data) +def test_without_model_parameter( + provider_str, + server_address, + server_port, + duration, + trace_exporter, + metrics_reader, + request, +): + provider = request.getfixturevalue(provider_str) + + client = provider.get_client() + + messages = [ + { + "role": "user", + "content": "Answer in up to 3 words: Which ocean contains the falkland islands?", + } + ] + + with pytest.raises( + TypeError, + match=re.escape( + "Missing required arguments; Expected either ('messages' and 'model') or ('messages', 'model' and 'stream') arguments to be given" + ), + ): + client.chat.completions.create(messages=messages) + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == "chat" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.ERROR + + assert dict(span.attributes) == { + ERROR_TYPE: "TypeError", + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_SYSTEM: "openai", + SERVER_ADDRESS: server_address, + SERVER_PORT: server_port, + } + + (operation_duration_metric,) = get_sorted_metrics(metrics_reader) + attributes = {"error.type": "TypeError", "server.address": server_address, "server.port": server_port} + assert_error_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, data_point=duration, value_delta=5 + ) + + +test_with_model_not_found_test_data = [ + ( + "openai_provider_chat_completions", + "api.openai.com", + 443, + "The model `not-found-model` does not exist or you do not have access to it.", + 0.00230291485786438, + ), + ( + "azure_provider_chat_completions", + "test.openai.azure.com", + 443, + "The API deployment for this resource does not exist. If you created the deployment within the last 5 minutes, please wait a moment and try again.", + 0.00230291485786438, + ), + ( + "ollama_provider_chat_completions", + "localhost", + 11434, + 'model "not-found-model" not found, try pulling it first', + 0.00230291485786438, + ), +] + + +@pytest.mark.vcr() +@pytest.mark.parametrize( + "provider_str,server_address,server_port,exception,duration", test_with_model_not_found_test_data +) +def test_with_model_not_found( + provider_str, + server_address, + server_port, + exception, + duration, + trace_exporter, + metrics_reader, + request, +): + provider = request.getfixturevalue(provider_str) + + client = provider.get_client() + + messages = [ + { + "role": "user", + "content": "Answer in up to 3 words: Which ocean contains the falkland islands?", + } + ] + + with pytest.raises(openai.NotFoundError, match="Error code: 404.*" + re.escape(exception)): + client.chat.completions.create(model="not-found-model", messages=messages) + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == "chat not-found-model" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.ERROR + + assert dict(span.attributes) == { + ERROR_TYPE: "NotFoundError", + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: "not-found-model", + GEN_AI_SYSTEM: "openai", + SERVER_ADDRESS: server_address, + SERVER_PORT: server_port, + } + + (operation_duration_metric,) = get_sorted_metrics(metrics_reader) + attributes = { + "gen_ai.request.model": "not-found-model", + "error.type": "NotFoundError", + "server.address": server_address, + "server.port": server_port, + } + assert_error_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, data_point=duration + ) diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_embeddings.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_embeddings.py index 79e1de6..fec7745 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_embeddings.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_embeddings.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re + import openai import pytest from opentelemetry.instrumentation.openai.helpers import GEN_AI_REQUEST_ENCODING_FORMAT @@ -177,6 +179,7 @@ def test_connection_error(provider_str, model, duration, trace_exporter, metrics operation_duration_metric, attributes=attributes, data_point=duration, + value_delta=1.0, ) @@ -279,7 +282,7 @@ async def test_async_all_the_client_options( test_async_connection_error_test_data = [ ("openai_provider_embeddings", "text-embedding-3-small", 0.2263190783560276), - ("azure_provider_embeddings", "text-embedding-3-small", 0.0017870571464300156), + ("azure_provider_embeddings", "text-embedding-3-small", 0.0036478489999751673), ("ollama_provider_embeddings", "all-minilm:33m", 0.0030461717396974564), ] @@ -325,4 +328,132 @@ async def test_async_connection_error(provider_str, model, duration, trace_expor operation_duration_metric, attributes=attributes, data_point=duration, + value_delta=1.0, + ) + + +test_without_model_parameter_test_data = [ + ("openai_provider_embeddings", "api.openai.com", 443, 4.2263190783560276), + ("azure_provider_embeddings", "test.openai.azure.com", 443, 4.0017870571464300156), + ("ollama_provider_embeddings", "localhost", 11434, 4.10461717396974564), +] + + +@pytest.mark.vcr() +@pytest.mark.parametrize("provider_str,server_address,server_port,duration", test_without_model_parameter_test_data) +def test_without_model_parameter( + provider_str, server_address, server_port, duration, trace_exporter, metrics_reader, request +): + provider = request.getfixturevalue(provider_str) + client = provider.get_client() + + text = "South Atlantic Ocean." + with pytest.raises(TypeError, match=re.escape("create() missing 1 required keyword-only argument: 'model'")): + client.embeddings.create(input=[text]) + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == "embeddings" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.ERROR + + assert dict(span.attributes) == { + ERROR_TYPE: "TypeError", + GEN_AI_OPERATION_NAME: "embeddings", + GEN_AI_SYSTEM: "openai", + SERVER_ADDRESS: server_address, + SERVER_PORT: server_port, + } + + attributes = { + "error.type": "TypeError", + "server.address": server_address, + "server.port": server_port, + "gen_ai.operation.name": "embeddings", + } + (operation_duration_metric,) = get_sorted_metrics(metrics_reader) + assert_error_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, data_point=duration, value_delta=5 + ) + + +test_model_not_found_test_data = [ + ( + "openai_provider_embeddings", + "api.openai.com", + 443, + openai.NotFoundError, + "The model `not-found-model` does not exist or you do not have access to it.", + 0.05915193818509579, + ), + ( + "azure_provider_embeddings", + "test.openai.azure.com", + 443, + openai.NotFoundError, + "The API deployment for this resource does not exist", + 0.0015850383788347244, + ), + ( + "ollama_provider_embeddings", + "localhost", + 11434, + openai.NotFoundError, + 'model "not-found-model" not found, try pulling it first', + 0.087132233195006854, + ), +] + + +@pytest.mark.vcr() +@pytest.mark.parametrize( + "provider_str,server_address,server_port,exception,exception_message,duration", test_model_not_found_test_data +) +def test_model_not_found( + provider_str, + server_address, + server_port, + exception, + exception_message, + duration, + trace_exporter, + metrics_reader, + request, +): + provider = request.getfixturevalue(provider_str) + client = provider.get_client() + + text = "South Atlantic Ocean." + with pytest.raises(exception, match=re.escape(exception_message)): + client.embeddings.create(model="not-found-model", input=[text]) + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == "embeddings not-found-model" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.ERROR + + assert dict(span.attributes) == { + ERROR_TYPE: exception.__qualname__, + GEN_AI_OPERATION_NAME: "embeddings", + GEN_AI_REQUEST_MODEL: "not-found-model", + GEN_AI_SYSTEM: "openai", + SERVER_ADDRESS: server_address, + SERVER_PORT: server_port, + } + + attributes = { + "error.type": exception.__qualname__, + "server.address": server_address, + "server.port": server_port, + "gen_ai.operation.name": "embeddings", + "gen_ai.request.model": "not-found-model", + } + (operation_duration_metric,) = get_sorted_metrics(metrics_reader) + assert_error_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, data_point=duration )