From 11c0a50f861817463faf9ca03a5ad11d7ecc16a0 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 13 Nov 2024 15:36:05 +0100 Subject: [PATCH 1/2] elastic-opentelemetry-instrumentation-openai: Bump to latest sdk for sdk events --- .../dev-requirements.txt | 44 +++++++++++-------- .../pyproject.toml | 8 ++-- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements.txt b/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements.txt index 964ab18..cc4fd02 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements.txt +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/dev-requirements.txt @@ -6,13 +6,13 @@ # annotated-types==0.7.0 # via pydantic -anyio==4.5.0 +anyio==4.6.2.post1 # via # httpx # openai asgiref==3.8.1 # via opentelemetry-test-utils -build==1.2.2 +build==1.2.2.post1 # via pip-tools certifi==2024.8.30 # via @@ -32,7 +32,7 @@ exceptiongroup==1.2.2 # pytest h11==0.14.0 # via httpcore -httpcore==1.0.5 +httpcore==1.0.6 # via httpx httpx==0.27.2 # via openai @@ -41,48 +41,52 @@ idna==3.10 # anyio # httpx # yarl -importlib-metadata==8.4.0 +importlib-metadata==8.5.0 # via opentelemetry-api iniconfig==2.0.0 # via pytest -jiter==0.5.0 +jiter==0.7.1 # via openai multidict==6.1.0 # via yarl numpy==2.1.3 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) -openai==1.50.2 +openai==1.54.4 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) -opentelemetry-api==1.27.0 +opentelemetry-api==1.28.1 # via # elastic-opentelemetry-instrumentation-openai (pyproject.toml) # opentelemetry-instrumentation # opentelemetry-sdk # opentelemetry-semantic-conventions # opentelemetry-test-utils -opentelemetry-instrumentation==0.48b0 +opentelemetry-instrumentation==0.49b1 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) -opentelemetry-sdk==1.27.0 +opentelemetry-sdk==1.28.1 # via opentelemetry-test-utils -opentelemetry-semantic-conventions==0.48b0 +opentelemetry-semantic-conventions==0.49b1 # via # elastic-opentelemetry-instrumentation-openai (pyproject.toml) + # opentelemetry-instrumentation # opentelemetry-sdk -opentelemetry-test-utils==0.48b0 +opentelemetry-test-utils==0.49b1 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) -packaging==24.1 +packaging==24.2 # via # build + # opentelemetry-instrumentation # pytest pip-tools==7.4.1 # via elastic-opentelemetry-instrumentation-openai (pyproject.toml) pluggy==1.5.0 # via pytest +propcache==0.2.0 + # via yarl pydantic==2.9.2 # via openai pydantic-core==2.23.4 # via pydantic -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools @@ -102,12 +106,12 @@ sniffio==1.3.1 # anyio # httpx # openai -tomli==2.0.1 +tomli==2.1.0 # via # build # pip-tools # pytest -tqdm==4.66.5 +tqdm==4.67.0 # via openai typing-extensions==4.12.2 # via @@ -118,20 +122,22 @@ typing-extensions==4.12.2 # opentelemetry-sdk # pydantic # pydantic-core -vcrpy==6.0.1 +urllib3==2.2.3 + # via vcrpy +vcrpy==6.0.2 # via # elastic-opentelemetry-instrumentation-openai (pyproject.toml) # pytest-vcr -wheel==0.44.0 +wheel==0.45.0 # via pip-tools wrapt==1.16.0 # via # deprecated # opentelemetry-instrumentation # vcrpy -yarl==1.11.1 +yarl==1.17.1 # via vcrpy -zipp==3.20.2 +zipp==3.21.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/pyproject.toml b/instrumentation/elastic-opentelemetry-instrumentation-openai/pyproject.toml index ae47566..367ad86 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/pyproject.toml +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/pyproject.toml @@ -25,10 +25,10 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - # 1.27.0 is required for Events API - "opentelemetry-api >= 1.27.0", - "opentelemetry-instrumentation >= 0.48b0", - "opentelemetry-semantic-conventions >= 0.48b0", + # 1.28.1 is required for Events API/SDK + "opentelemetry-api ~= 1.28.1", + "opentelemetry-instrumentation ~= 0.49b1", + "opentelemetry-semantic-conventions ~= 0.49b1", ] [project.readme] From 208bca8421eeb0f2df60e35eb6d900df914ab127 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 13 Nov 2024 15:36:50 +0100 Subject: [PATCH 2/2] elastic-opentelemetry-instrumentation-openai: implement log events support --- .../instrumentation/openai/__init__.py | 103 +- .../openai/environment_variables.py | 2 + .../instrumentation/openai/helpers.py | 192 +- .../instrumentation/openai/wrappers.py | 29 +- ...ents[azure_provider_chat_completions].yaml | 82 + ...nts[ollama_provider_chat_completions].yaml | 57 + ...nts[openai_provider_chat_completions].yaml | 97 + ...ents[azure_provider_chat_completions].yaml | 100 + ...nts[ollama_provider_chat_completions].yaml | 97 + ...nts[openai_provider_chat_completions].yaml | 110 ++ ...ents[azure_provider_chat_completions].yaml | 90 + ...nts[openai_provider_chat_completions].yaml | 110 ++ ...ents[azure_provider_chat_completions].yaml | 82 + ...nts[ollama_provider_chat_completions].yaml | 56 + ...nts[openai_provider_chat_completions].yaml | 97 + ...ents[azure_provider_chat_completions].yaml | 85 + ...nts[openai_provider_chat_completions].yaml | 102 + ...ent[ollama_provider_chat_completions].yaml | 264 +++ ...ent[openai_provider_chat_completions].yaml | 139 ++ ...nts[ollama_provider_chat_completions].yaml | 266 +++ ...nts[openai_provider_chat_completions].yaml | 143 ++ ...ents[azure_provider_chat_completions].yaml | 124 ++ ...nts[ollama_provider_chat_completions].yaml | 153 ++ ...nts[openai_provider_chat_completions].yaml | 132 ++ ...ents[azure_provider_chat_completions].yaml | 90 + ...nts[ollama_provider_chat_completions].yaml | 64 + ...nts[openai_provider_chat_completions].yaml | 110 ++ ...ents[azure_provider_chat_completions].yaml | 179 ++ ...nts[openai_provider_chat_completions].yaml | 215 ++ .../tests/conftest.py | 100 +- .../tests/test_chat_completions.py | 1729 ++++++++++++++++- .../tests/test_embeddings.py | 20 +- .../tests/utils.py | 8 +- 33 files changed, 5072 insertions(+), 155 deletions(-) create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_basic_with_capture_content_log_events[azure_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_basic_with_capture_content_log_events[ollama_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_basic_with_capture_content_log_events[openai_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_stream_with_capture_content_log_events[azure_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_stream_with_capture_content_log_events[ollama_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_stream_with_capture_content_log_events[openai_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_tools_with_capture_content_log_events[azure_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_tools_with_capture_content_log_events[openai_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_basic_with_capture_content_log_events[azure_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_basic_with_capture_content_log_events[ollama_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_basic_with_capture_content_log_events[openai_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_multiple_choices_with_capture_content_log_events[azure_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_multiple_choices_with_capture_content_log_events[openai_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content[ollama_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content[openai_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content_log_events[ollama_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content_log_events[openai_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_tools_and_capture_content_log_events[azure_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_tools_and_capture_content_log_events[ollama_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_tools_and_capture_content_log_events[openai_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_capture_content_log_events[azure_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_capture_content_log_events[ollama_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_capture_content_log_events[openai_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_followup_and_capture_content_log_events[azure_provider_chat_completions].yaml create mode 100644 instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_followup_and_capture_content_log_events[openai_provider_chat_completions].yaml diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py index de7f406..1bd33f2 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py @@ -22,17 +22,22 @@ from wrapt import register_post_import_hook, wrap_function_wrapper +from opentelemetry._events import get_event_logger from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from opentelemetry.instrumentation.utils import unwrap from opentelemetry.instrumentation.openai.environment_variables import ( ELASTIC_OTEL_GENAI_CAPTURE_CONTENT, + ELASTIC_OTEL_GENAI_EVENTS, ) from opentelemetry.instrumentation.openai.helpers import ( _get_embeddings_span_attributes_from_wrapper, + _get_event_attributes, _get_span_attributes_from_wrapper, _message_from_choice, _record_token_usage_metrics, _record_operation_duration_metric, + _send_log_events_from_messages, + _send_log_events_from_choices, _set_span_attributes_from_response, _set_embeddings_span_attributes_from_response, _span_name_from_span_attributes, @@ -72,14 +77,36 @@ def _instrument(self, **kwargs): **kwargs: Optional arguments ``tracer_provider``: a TracerProvider, defaults to global ``meter_provider``: a MeterProvider, defaults to global + ``event_logger_provider``: a EventLoggerProvider, defaults to global ``capture_content``: to enable content capturing, defaults to False """ capture_content = "true" if kwargs.get("capture_content") else "false" self.capture_content = os.environ.get(ELASTIC_OTEL_GENAI_CAPTURE_CONTENT, capture_content).lower() == "true" + + # we support 3 values for deciding how to send events: + # - "latest" to match latest semconv, as 1.27.0 it's span + # - "log" to send log events + # - "span" to send span events (default) + genai_events = os.environ.get(ELASTIC_OTEL_GENAI_EVENTS, "latest").lower() + self.event_kind = "log" if genai_events == "log" else "span" + tracer_provider = kwargs.get("tracer_provider") - self.tracer = get_tracer(__name__, __version__, tracer_provider, schema_url=Schemas.V1_27_0.value) + self.tracer = get_tracer( + __name__, + __version__, + tracer_provider, + schema_url=Schemas.V1_27_0.value, + ) meter_provider = kwargs.get("meter_provider") - self.meter = get_meter(__name__, __version__, meter_provider, schema_url=Schemas.V1_27_0.value) + self.meter = get_meter( + __name__, + __version__, + meter_provider, + schema_url=Schemas.V1_27_0.value, + ) + event_logger_provider = kwargs.get("event_logger_provider") + self.event_logger = get_event_logger(__name__, event_logger_provider) + self.token_usage_metric = create_gen_ai_client_token_usage(self.meter) self.operation_duration_metric = create_gen_ai_client_operation_duration(self.meter) @@ -121,6 +148,7 @@ def _chat_completion_wrapper(self, wrapped, instance, args, kwargs): logger.debug(f"openai.resources.chat.completions.Completions.create kwargs: {kwargs}") span_attributes = _get_span_attributes_from_wrapper(instance, kwargs) + event_attributes = _get_event_attributes() span_name = _span_name_from_span_attributes(span_attributes) with self.tracer.start_as_current_span( @@ -130,13 +158,17 @@ def _chat_completion_wrapper(self, wrapped, instance, args, kwargs): # this is important to avoid having the span closed before ending the stream end_on_exit=False, ) as span: + # TODO: more fine grained depending on the message.role? if self.capture_content: messages = kwargs.get("messages", []) - prompt = [message for message in messages] - try: - span.add_event(EVENT_GEN_AI_CONTENT_PROMPT, attributes={GEN_AI_PROMPT: json.dumps(prompt)}) - except TypeError: - logger.error(f"Failed to serialize {EVENT_GEN_AI_CONTENT_PROMPT}") + + if self.event_kind == "log": + _send_log_events_from_messages(self.event_logger, messages=messages, attributes=event_attributes) + else: + try: + span.add_event(EVENT_GEN_AI_CONTENT_PROMPT, attributes={GEN_AI_PROMPT: json.dumps(messages)}) + except TypeError: + logger.error(f"Failed to serialize {EVENT_GEN_AI_CONTENT_PROMPT}") start_time = default_timer() try: @@ -153,6 +185,9 @@ def _chat_completion_wrapper(self, wrapped, instance, args, kwargs): stream=result, span=span, capture_content=self.capture_content, + event_kind=self.event_kind, + event_attributes=event_attributes, + event_logger=self.event_logger, start_time=start_time, token_usage_metric=self.token_usage_metric, operation_duration_metric=self.operation_duration_metric, @@ -166,14 +201,19 @@ def _chat_completion_wrapper(self, wrapped, instance, args, kwargs): _record_operation_duration_metric(self.operation_duration_metric, span, start_time) if self.capture_content: - # same format as the prompt - completion = [_message_from_choice(choice) for choice in result.choices] - try: - span.add_event( - EVENT_GEN_AI_CONTENT_COMPLETION, attributes={GEN_AI_COMPLETION: json.dumps(completion)} + if self.event_kind == "log": + _send_log_events_from_choices( + self.event_logger, choices=result.choices, attributes=event_attributes ) - except TypeError: - logger.error(f"Failed to serialize {EVENT_GEN_AI_CONTENT_COMPLETION}") + else: + # same format as the prompt + completion = [_message_from_choice(choice) for choice in result.choices] + try: + span.add_event( + EVENT_GEN_AI_CONTENT_COMPLETION, attributes={GEN_AI_COMPLETION: json.dumps(completion)} + ) + except TypeError: + logger.error(f"Failed to serialize {EVENT_GEN_AI_CONTENT_COMPLETION}") span.end() @@ -183,6 +223,7 @@ async def _async_chat_completion_wrapper(self, wrapped, instance, args, kwargs): logger.debug(f"openai.resources.chat.completions.AsyncCompletions.create kwargs: {kwargs}") span_attributes = _get_span_attributes_from_wrapper(instance, kwargs) + event_attributes = _get_event_attributes() span_name = _span_name_from_span_attributes(span_attributes) with self.tracer.start_as_current_span( @@ -194,10 +235,14 @@ async def _async_chat_completion_wrapper(self, wrapped, instance, args, kwargs): ) as span: if self.capture_content: messages = kwargs.get("messages", []) - try: - span.add_event(EVENT_GEN_AI_CONTENT_PROMPT, attributes={GEN_AI_PROMPT: json.dumps(messages)}) - except TypeError: - logger.error(f"Failed to serialize {EVENT_GEN_AI_CONTENT_PROMPT}") + + if self.event_kind == "log": + _send_log_events_from_messages(self.event_logger, messages=messages, attributes=event_attributes) + else: + try: + span.add_event(EVENT_GEN_AI_CONTENT_PROMPT, attributes={GEN_AI_PROMPT: json.dumps(messages)}) + except TypeError: + logger.error(f"Failed to serialize {EVENT_GEN_AI_CONTENT_PROMPT}") start_time = default_timer() try: @@ -214,6 +259,9 @@ async def _async_chat_completion_wrapper(self, wrapped, instance, args, kwargs): stream=result, span=span, capture_content=self.capture_content, + event_kind=self.event_kind, + event_attributes=event_attributes, + event_logger=self.event_logger, start_time=start_time, token_usage_metric=self.token_usage_metric, operation_duration_metric=self.operation_duration_metric, @@ -227,14 +275,19 @@ async def _async_chat_completion_wrapper(self, wrapped, instance, args, kwargs): _record_operation_duration_metric(self.operation_duration_metric, span, start_time) if self.capture_content: - # same format as the prompt - completion = [_message_from_choice(choice) for choice in result.choices] - try: - span.add_event( - EVENT_GEN_AI_CONTENT_COMPLETION, attributes={GEN_AI_COMPLETION: json.dumps(completion)} + if self.event_kind == "log": + _send_log_events_from_choices( + self.event_logger, choices=result.choices, attributes=event_attributes ) - except TypeError: - logger.error(f"Failed to serialize {EVENT_GEN_AI_CONTENT_COMPLETION}") + else: + # same format as the prompt + completion = [_message_from_choice(choice) for choice in result.choices] + try: + span.add_event( + EVENT_GEN_AI_CONTENT_COMPLETION, attributes={GEN_AI_COMPLETION: json.dumps(completion)} + ) + except TypeError: + logger.error(f"Failed to serialize {EVENT_GEN_AI_CONTENT_COMPLETION}") span.end() diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/environment_variables.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/environment_variables.py index b5ba5ee..20f6280 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/environment_variables.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/environment_variables.py @@ -15,3 +15,5 @@ # limitations under the License. ELASTIC_OTEL_GENAI_CAPTURE_CONTENT = "ELASTIC_OTEL_GENAI_CAPTURE_CONTENT" + +ELASTIC_OTEL_GENAI_EVENTS = "ELASTIC_OTEL_GENAI_EVENTS" diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py index e36a4d8..24c32dc 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/helpers.py @@ -14,11 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from timeit import default_timer from typing import TYPE_CHECKING +from opentelemetry._events import Event, EventLogger from opentelemetry.semconv.attributes.error_attributes import ERROR_TYPE from opentelemetry.semconv.attributes.server_attributes import SERVER_ADDRESS, SERVER_PORT from opentelemetry.semconv._incubating.attributes.gen_ai_attributes import ( @@ -42,13 +42,23 @@ from opentelemetry.trace import Span from opentelemetry.util.types import Attributes +EVENT_GEN_AI_ASSISTANT_MESSAGE = "gen_ai.assistant.message" +EVENT_GEN_AI_CHOICE = "gen_ai.choice" +EVENT_GEN_AI_USER_MESSAGE = "gen_ai.user.message" +EVENT_GEN_AI_SYSTEM_MESSAGE = "gen_ai.system.message" +EVENT_GEN_AI_TOOL_MESSAGE = "gen_ai.tool.message" + +# elastic specific attributes +GEN_AI_REQUEST_ENCODING_FORMAT = "gen_ai.request.encoding_format" + +# As this is only used for a type annotation, only import from openai module +# when running type checker like pyright since we otherwise don't want to import +# it before the app. if TYPE_CHECKING: from openai.types import CompletionUsage else: CompletionUsage = None -GEN_AI_REQUEST_ENCODING_FORMAT = "gen_ai.request.encoding_format" - def _set_span_attributes_from_response( span: Span, response_id: str, model: str, choices, usage: CompletionUsage @@ -69,44 +79,63 @@ def _set_embeddings_span_attributes_from_response(span: Span, model: str, usage: span.set_attribute(GEN_AI_USAGE_INPUT_TOKENS, usage.prompt_tokens) -def _decode_function_arguments(arguments: str): - try: - return json.loads(arguments) - except (TypeError, json.JSONDecodeError): - return None - - def _message_from_choice(choice): """Format a choice into a message of the same shape of the prompt""" if tool_calls := getattr(choice.message, "tool_calls", None): - tool_call = tool_calls[0] - return {"role": choice.message.role, "content": _decode_function_arguments(tool_call.function.arguments)} + return { + "role": choice.message.role, + "content": "", + "tool_calls": [ + { + "id": tool_call.id, + "type": tool_call.type, + "function": { + "name": tool_call.function.name, + "arguments": tool_call.function.arguments, + }, + } + for tool_call in tool_calls + ], + } else: return {"role": choice.message.role, "content": choice.message.content} def _message_from_stream_choices(choices): """Format an iterable of choices into a message of the same shape of the prompt""" - message = {"role": None, "content": ""} - tools_arguments = "" + messages = {} + tool_calls = {} for choice in choices: + messages.setdefault(choice.index, {"role": None, "content": ""}) + message = messages[choice.index] if choice.delta.role: message["role"] = choice.delta.role if choice.delta.content: message["content"] += choice.delta.content + if choice.delta.tool_calls: for call in choice.delta.tool_calls: + tool_calls.setdefault(choice.index, {}) + tool_calls[choice.index].setdefault(call.index, {"function": {"arguments": ""}}) + tool_call = tool_calls[choice.index][call.index] if call.function.arguments: - tools_arguments += call.function.arguments + tool_call["function"]["arguments"] += call.function.arguments + if call.function.name: + tool_call["function"]["name"] = call.function.name + if call.id: + tool_call["id"] = call.id + if call.type: + tool_call["type"] = call.type - if tools_arguments: - if decoded_arguments := _decode_function_arguments(tools_arguments): - message["content"] = decoded_arguments + for message_index in tool_calls: + message = messages[message_index] + message["tool_calls"] = [arguments for _, arguments in sorted(tool_calls[message_index].items())] - return message + # assumes there's only one message + return [message for _, message in sorted(messages.items())][0] -def _attributes_from_client(client): +def _attributes_from_client(client) -> Attributes: span_attributes = {} if base_url := getattr(client, "_base_url", None): @@ -123,12 +152,13 @@ def _attributes_from_client(client): return span_attributes -def _get_span_attributes_from_wrapper(instance, kwargs): +def _get_span_attributes_from_wrapper(instance, kwargs) -> Attributes: span_attributes = { GEN_AI_OPERATION_NAME: "chat", GEN_AI_SYSTEM: "openai", } + # on some azure clients the model was not mandatory if (request_model := kwargs.get("model")) is not None: span_attributes[GEN_AI_REQUEST_MODEL] = request_model @@ -162,7 +192,7 @@ def _span_name_from_span_attributes(attributes: Attributes) -> str: ) -def _get_embeddings_span_attributes_from_wrapper(instance, kwargs): +def _get_embeddings_span_attributes_from_wrapper(instance, kwargs) -> Attributes: span_attributes = { GEN_AI_OPERATION_NAME: "embeddings", GEN_AI_SYSTEM: "openai", @@ -180,7 +210,11 @@ def _get_embeddings_span_attributes_from_wrapper(instance, kwargs): return span_attributes -def _get_attributes_if_set(span: Span, names: Iterable) -> dict: +def _get_event_attributes() -> Attributes: + return {GEN_AI_SYSTEM: "openai"} + + +def _get_attributes_if_set(span: Span, names: Iterable) -> Attributes: """Returns a dict with any attribute found in the span attributes""" attributes = span.attributes return {name: attributes[name] for name in names if name in attributes} @@ -219,3 +253,113 @@ def _record_operation_duration_metric(metric: Histogram, span: Span, start: floa ) duration_s = default_timer() - start metric.record(duration_s, operation_duration_metric_attrs) + + +def _key_or_property(obj, name): + if isinstance(obj, Mapping): + return obj[name] + return getattr(obj, name) + + +def _serialize_tool_calls_for_event(tool_calls): + return [ + { + "id": _key_or_property(tool_call, "id"), + "type": _key_or_property(tool_call, "type"), + "function": { + "name": _key_or_property(_key_or_property(tool_call, "function"), "name"), + "arguments": _key_or_property(_key_or_property(tool_call, "function"), "arguments"), + }, + } + for tool_call in tool_calls + ] + + +def _send_log_events_from_messages(event_logger: EventLogger, messages, attributes: Attributes): + for message in messages: + if message["role"] == "system": + event = Event(name=EVENT_GEN_AI_SYSTEM_MESSAGE, body={"content": message["content"]}, attributes=attributes) + event_logger.emit(event) + elif message["role"] == "user": + event = Event(name=EVENT_GEN_AI_USER_MESSAGE, body={"content": message["content"]}, attributes=attributes) + event_logger.emit(event) + elif message["role"] == "assistant": + body = {} + if content := message.get("content"): + body["content"] = content + tool_calls = _serialize_tool_calls_for_event(message.get("tool_calls", [])) + if tool_calls: + body["tool_calls"] = tool_calls + event = Event( + name=EVENT_GEN_AI_ASSISTANT_MESSAGE, + body=body, + attributes=attributes, + ) + event_logger.emit(event) + elif message["role"] == "tool": + event = Event( + name=EVENT_GEN_AI_TOOL_MESSAGE, + body={"content": message["content"], "id": message["tool_call_id"]}, + attributes=attributes, + ) + event_logger.emit(event) + + +def _send_log_events_from_choices(event_logger: EventLogger, choices, attributes: Attributes): + for choice in choices: + tool_calls = _serialize_tool_calls_for_event(choice.message.tool_calls or []) + body = {"finish_reason": choice.finish_reason, "index": choice.index, "message": {}} + if tool_calls: + body["message"]["tool_calls"] = tool_calls + if choice.message.content: + body["message"]["content"] = choice.message.content + + event = Event(name=EVENT_GEN_AI_CHOICE, body=body, attributes=attributes) + event_logger.emit(event) + + +def _send_log_events_from_stream_choices(event_logger: EventLogger, choices, span: Span, attributes: Attributes): + body = {} + message = {} + message_content = "" + tool_calls = {} + for choice in choices: + if choice.delta.content: + message_content += choice.delta.content + if choice.delta.tool_calls: + for call in choice.delta.tool_calls: + tool_calls.setdefault(call.index, {"function": {"arguments": ""}}) + tool_call = tool_calls[call.index] + if call.function.arguments: + tool_call["function"]["arguments"] += call.function.arguments + if call.function.name: + tool_call["function"]["name"] = call.function.name + if call.id: + tool_call["id"] = call.id + if call.type: + tool_call["type"] = call.type + if choice.finish_reason: + body["finish_reason"] = choice.finish_reason + body["index"] = choice.index + + if message_content: + message["content"] = message_content + if tool_calls: + message["tool_calls"] = [call for _, call in sorted(tool_calls.items())] + + body = { + "finish_reason": choice.finish_reason, + "index": choice.index, + "message": message, + } + # StreamWrapper is consumed after start_as_current_span exits, so capture the current span + ctx = span.get_span_context() + event = Event( + name=EVENT_GEN_AI_CHOICE, + body=body, + attributes=attributes, + trace_id=ctx.trace_id, + span_id=ctx.span_id, + trace_flags=ctx.trace_flags, + ) + event_logger.emit(event) diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/wrappers.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/wrappers.py index 4a5403c..fd19db2 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/wrappers.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/wrappers.py @@ -16,12 +16,15 @@ import json import logging +from typing import Literal +from opentelemetry._events import EventLogger from opentelemetry.instrumentation.openai.helpers import ( _message_from_stream_choices, _record_token_usage_metrics, _record_operation_duration_metric, _set_span_attributes_from_response, + _send_log_events_from_stream_choices, ) from opentelemetry.metrics import Histogram from opentelemetry.semconv.attributes.error_attributes import ERROR_TYPE @@ -30,6 +33,7 @@ ) from opentelemetry.trace import Span from opentelemetry.trace.status import StatusCode +from opentelemetry.util.types import Attributes EVENT_GEN_AI_CONTENT_COMPLETION = "gen_ai.content.completion" @@ -42,6 +46,9 @@ def __init__( stream, span: Span, capture_content: bool, + event_kind: Literal["log", "span"], + event_attributes: Attributes, + event_logger: EventLogger, start_time: float, token_usage_metric: Histogram, operation_duration_metric: Histogram, @@ -49,6 +56,9 @@ def __init__( self.stream = stream self.span = span self.capture_content = capture_content + self.event_kind = event_kind + self.event_attributes = event_attributes + self.event_logger = event_logger self.token_usage_metric = token_usage_metric self.operation_duration_metric = operation_duration_metric self.start_time = start_time @@ -74,14 +84,19 @@ def end(self, exc=None): _record_token_usage_metrics(self.token_usage_metric, self.span, self.usage) if self.capture_content: - # same format as the prompt - completion = [_message_from_stream_choices(self.choices)] - try: - self.span.add_event( - EVENT_GEN_AI_CONTENT_COMPLETION, attributes={GEN_AI_COMPLETION: json.dumps(completion)} + if self.event_kind == "log": + _send_log_events_from_stream_choices( + self.event_logger, choices=self.choices, span=self.span, attributes=self.event_attributes ) - except TypeError: - logger.error(f"Failed to serialize {EVENT_GEN_AI_CONTENT_COMPLETION}") + else: + # same format as the prompt + completion = [_message_from_stream_choices(self.choices)] + try: + self.span.add_event( + EVENT_GEN_AI_CONTENT_COMPLETION, attributes={GEN_AI_COMPLETION: json.dumps(completion)} + ) + except TypeError: + logger.error(f"Failed to serialize {EVENT_GEN_AI_CONTENT_COMPLETION}") self.span.end() diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_basic_with_capture_content_log_events[azure_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_basic_with_capture_content_log_events[azure_provider_chat_completions].yaml new file mode 100644 index 0000000..5567e0b --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_basic_with_capture_content_log_events[azure_provider_chat_completions].yaml @@ -0,0 +1,82 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + api-key: + - test_azure_api_key + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '138' + content-type: + - application/json + host: + - test.openai.azure.com + user-agent: + - AsyncAzureOpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/chat/completions?api-version=2023-03-15-preview + response: + body: + string: '{"choices":[{"finish_reason":"stop","index":0,"message":{"content":"South + Atlantic Ocean.","role":"assistant"}}],"created":1728909022,"id":"chatcmpl-AIEVKUfan2sD0ScQLHPSMYn7Fet3Y","model":"gpt-4o-mini","object":"chat.completion","system_fingerprint":"fp_878413d04d","usage":{"completion_tokens":4,"prompt_tokens":24,"total_tokens":28}} + + ' + headers: + Content-Length: + - '336' + Content-Type: + - application/json + Date: + - Mon, 14 Oct 2024 12:30:21 GMT + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + apim-request-id: + - aa7e6892-0666-4025-9c04-90d2a1f32987 + azureml-model-session: + - d009-20240926131256 + openai-organization: test_openai_org_key + x-accel-buffering: + - 'no' + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '142' + x-ms-client-request-id: + - aa7e6892-0666-4025-9c04-90d2a1f32987 + x-ms-rai-invoked: + - 'true' + x-ms-region: + - East US + x-ratelimit-remaining-requests: + - '906' + x-ratelimit-remaining-tokens: + - '88284' + x-request-id: + - ce524e90-581f-4d11-b86f-0e5f2b47f5fc + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_basic_with_capture_content_log_events[ollama_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_basic_with_capture_content_log_events[ollama_provider_chat_completions].yaml new file mode 100644 index 0000000..02a3eb7 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_basic_with_capture_content_log_events[ollama_provider_chat_completions].yaml @@ -0,0 +1,57 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "qwen2.5:0.5b"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '139' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - AsyncOpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: '{"id":"chatcmpl-472","object":"chat.completion","created":1728909023,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"message":{"role":"assistant","content":"The + Falkland Islands are located within the southern waters of the South Atlantic + Ocean."},"finish_reason":"stop"}],"usage":{"prompt_tokens":46,"completion_tokens":17,"total_tokens":63}} + + ' + headers: + Content-Length: + - '375' + Content-Type: + - application/json + Date: + - Mon, 14 Oct 2024 12:30:23 GMT + Set-Cookie: test_set_cookie + openai-organization: test_openai_org_key + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_basic_with_capture_content_log_events[openai_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_basic_with_capture_content_log_events[openai_provider_chat_completions].yaml new file mode 100644 index 0000000..058eba5 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_basic_with_capture_content_log_events[openai_provider_chat_completions].yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '138' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-AIEVJxmbMtOSyVurjk73oqE0uQhAX\",\n \"object\": + \"chat.completion\",\n \"created\": 1728909021,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"South Atlantic Ocean.\",\n \"refusal\": + null\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n + \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 24,\n \"completion_tokens\": + 4,\n \"total_tokens\": 28,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0\n }\n },\n \"system_fingerprint\": \"fp_e2bde53e6e\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d279486aaf1bae7-MXP + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 14 Oct 2024 12:30:21 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '643' + openai-organization: test_openai_org_key + openai-processing-ms: + - '575' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9997' + x-ratelimit-remaining-tokens: + - '199966' + x-ratelimit-reset-requests: + - 24.489s + x-ratelimit-reset-tokens: + - 10ms + x-request-id: + - req_ffe4e6ff8b66c569921f76182eab65cd + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_stream_with_capture_content_log_events[azure_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_stream_with_capture_content_log_events[azure_provider_chat_completions].yaml new file mode 100644 index 0000000..1355593 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_stream_with_capture_content_log_events[azure_provider_chat_completions].yaml @@ -0,0 +1,100 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "gpt-4o-mini", "stream": true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + api-key: + - test_azure_api_key + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '154' + content-type: + - application/json + host: + - test.openai.azure.com + user-agent: + - AsyncAzureOpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/chat/completions?api-version=2023-03-15-preview + response: + body: + string: 'data: {"choices":[{"delta":{"role":"assistant"},"finish_reason":null,"index":0}],"created":1728909025,"id":"chatcmpl-AIEVNaNWKhuoMnRJtcPUJNjbAz9Ib","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"content":"South"},"finish_reason":null,"index":0}],"created":1728909025,"id":"chatcmpl-AIEVNaNWKhuoMnRJtcPUJNjbAz9Ib","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"content":" Atlantic"},"finish_reason":null,"index":0}],"created":1728909025,"id":"chatcmpl-AIEVNaNWKhuoMnRJtcPUJNjbAz9Ib","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"content":" Ocean"},"finish_reason":null,"index":0}],"created":1728909025,"id":"chatcmpl-AIEVNaNWKhuoMnRJtcPUJNjbAz9Ib","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"content":"."},"finish_reason":null,"index":0}],"created":1728909025,"id":"chatcmpl-AIEVNaNWKhuoMnRJtcPUJNjbAz9Ib","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{},"finish_reason":"stop","index":0}],"created":1728909025,"id":"chatcmpl-AIEVNaNWKhuoMnRJtcPUJNjbAz9Ib","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: [DONE] + + + ' + headers: + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Mon, 14 Oct 2024 12:30:25 GMT + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + apim-request-id: + - 7db487c8-658f-408a-8cb9-33c834be990c + azureml-model-session: + - d017-20240921115920 + openai-organization: test_openai_org_key + x-accel-buffering: + - 'no' + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '60' + x-ms-client-request-id: + - 7db487c8-658f-408a-8cb9-33c834be990c + x-ms-rai-invoked: + - 'true' + x-ms-region: + - East US + x-ratelimit-remaining-requests: + - '905' + x-ratelimit-remaining-tokens: + - '87627' + x-request-id: + - 6a36c78f-3505-4ae8-8450-124faf425179 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_stream_with_capture_content_log_events[ollama_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_stream_with_capture_content_log_events[ollama_provider_chat_completions].yaml new file mode 100644 index 0000000..1be0613 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_stream_with_capture_content_log_events[ollama_provider_chat_completions].yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "qwen2.5:0.5b", "stream": + true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '155' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - AsyncOpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-466","object":"chat.completion.chunk","created":1728909025,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"The"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-466","object":"chat.completion.chunk","created":1728909025,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + Falk"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-466","object":"chat.completion.chunk","created":1728909025,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"land"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-466","object":"chat.completion.chunk","created":1728909025,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + Islands"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-466","object":"chat.completion.chunk","created":1728909025,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + contain"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-466","object":"chat.completion.chunk","created":1728909025,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + the"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-466","object":"chat.completion.chunk","created":1728909025,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + South"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-466","object":"chat.completion.chunk","created":1728909025,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + Atlantic"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-466","object":"chat.completion.chunk","created":1728909025,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + Ocean"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-466","object":"chat.completion.chunk","created":1728909025,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null}]} + + + data: {"id":"chatcmpl-466","object":"chat.completion.chunk","created":1728909025,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + Content-Type: + - text/event-stream + Date: + - Mon, 14 Oct 2024 12:30:25 GMT + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + openai-organization: test_openai_org_key + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_stream_with_capture_content_log_events[openai_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_stream_with_capture_content_log_events[openai_provider_chat_completions].yaml new file mode 100644 index 0000000..0294556 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_stream_with_capture_content_log_events[openai_provider_chat_completions].yaml @@ -0,0 +1,110 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "gpt-4o-mini", "stream": true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '154' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-AIEVLVduixLA39qqzjDIfJ2u2dPcJ","object":"chat.completion.chunk","created":1728909023,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVLVduixLA39qqzjDIfJ2u2dPcJ","object":"chat.completion.chunk","created":1728909023,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"South"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVLVduixLA39qqzjDIfJ2u2dPcJ","object":"chat.completion.chunk","created":1728909023,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + Atlantic"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVLVduixLA39qqzjDIfJ2u2dPcJ","object":"chat.completion.chunk","created":1728909023,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + Ocean"},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVLVduixLA39qqzjDIfJ2u2dPcJ","object":"chat.completion.chunk","created":1728909023,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVLVduixLA39qqzjDIfJ2u2dPcJ","object":"chat.completion.chunk","created":1728909023,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d279493ad9ebb14-MXP + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Mon, 14 Oct 2024 12:30:24 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: test_openai_org_key + openai-processing-ms: + - '729' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9996' + x-ratelimit-remaining-tokens: + - '199966' + x-ratelimit-reset-requests: + - 31.045s + x-ratelimit-reset-tokens: + - 10ms + x-request-id: + - req_d4104d88846db07a1d0156b1eaee178a + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_tools_with_capture_content_log_events[azure_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_tools_with_capture_content_log_events[azure_provider_chat_completions].yaml new file mode 100644 index 0000000..09c193f --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_tools_with_capture_content_log_events[azure_provider_chat_completions].yaml @@ -0,0 +1,90 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful customer + support assistant. Use the supplied tools to assist the user."}, {"role": "user", + "content": "Hi, can you tell me the delivery date for my order?"}, {"role": + "assistant", "content": "Hi there! I can help with that. Can you please provide + your order ID?"}, {"role": "user", "content": "i think it is order_12345"}], + "model": "gpt-4o-mini", "tools": [{"type": "function", "function": {"name": + "get_delivery_date", "description": "Get the delivery date for a customer''s + order. Call this whenever you need to know the delivery date, for example when + a customer asks ''Where is my package''", "parameters": {"type": "object", "properties": + {"order_id": {"type": "string", "description": "The customer''s order ID."}}, + "required": ["order_id"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + api-key: + - test_azure_api_key + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '842' + content-type: + - application/json + host: + - test.openai.azure.com + user-agent: + - AsyncAzureOpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/chat/completions?api-version=2023-03-15-preview + response: + body: + string: '{"choices":[{"finish_reason":"tool_calls","index":0,"message":{"content":null,"role":"assistant","tool_calls":[{"function":{"arguments":"{\"order_id\":\"order_12345\"}","name":"get_delivery_date"},"id":"call_uO2RgeshT5Xmbwg778qN14d5","type":"function"}]}}],"created":1728909027,"id":"chatcmpl-AIEVPAAgrPcEpkQHXamdD9JpNNPep","model":"gpt-4o-mini","object":"chat.completion","system_fingerprint":"fp_878413d04d","usage":{"completion_tokens":19,"prompt_tokens":140,"total_tokens":159}} + + ' + headers: + Content-Length: + - '483' + Content-Type: + - application/json + Date: + - Mon, 14 Oct 2024 12:30:27 GMT + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + apim-request-id: + - 5a2ef9fb-86d6-4f22-830a-509cf6ffa56a + azureml-model-session: + - d010-20240925083720 + openai-organization: test_openai_org_key + x-accel-buffering: + - 'no' + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '270' + x-ms-client-request-id: + - 5a2ef9fb-86d6-4f22-830a-509cf6ffa56a + x-ms-rai-invoked: + - 'true' + x-ms-region: + - East US + x-ratelimit-remaining-requests: + - '904' + x-ratelimit-remaining-tokens: + - '86926' + x-request-id: + - dc8bd70f-cf73-4c46-95ce-3ea309c22f45 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_tools_with_capture_content_log_events[openai_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_tools_with_capture_content_log_events[openai_provider_chat_completions].yaml new file mode 100644 index 0000000..1f4a595 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_async_tools_with_capture_content_log_events[openai_provider_chat_completions].yaml @@ -0,0 +1,110 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful customer + support assistant. Use the supplied tools to assist the user."}, {"role": "user", + "content": "Hi, can you tell me the delivery date for my order?"}, {"role": + "assistant", "content": "Hi there! I can help with that. Can you please provide + your order ID?"}, {"role": "user", "content": "i think it is order_12345"}], + "model": "gpt-4o-mini", "tools": [{"type": "function", "function": {"name": + "get_delivery_date", "description": "Get the delivery date for a customer''s + order. Call this whenever you need to know the delivery date, for example when + a customer asks ''Where is my package''", "parameters": {"type": "object", "properties": + {"order_id": {"type": "string", "description": "The customer''s order ID."}}, + "required": ["order_id"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '842' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-AIEVOegIWRWjNVbjxs4iASh4SNKAj\",\n \"object\": + \"chat.completion\",\n \"created\": 1728909026,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_n4WJq7bu6UsgjdqeNYxRmGno\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"get_delivery_date\",\n + \ \"arguments\": \"{\\\"order_id\\\":\\\"order_12345\\\"}\"\n + \ }\n }\n ],\n \"refusal\": null\n },\n + \ \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n + \ ],\n \"usage\": {\n \"prompt_tokens\": 140,\n \"completion_tokens\": + 19,\n \"total_tokens\": 159,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0\n }\n },\n \"system_fingerprint\": \"fp_8552ec53e1\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d2794a2f9a15252-MXP + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 14 Oct 2024 12:30:26 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '918' + openai-organization: test_openai_org_key + openai-processing-ms: + - '757' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9995' + x-ratelimit-remaining-tokens: + - '199921' + x-ratelimit-reset-requests: + - 37.186s + x-ratelimit-reset-tokens: + - 23ms + x-request-id: + - req_562b73aa01066c19e08aa19d847e33f7 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_basic_with_capture_content_log_events[azure_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_basic_with_capture_content_log_events[azure_provider_chat_completions].yaml new file mode 100644 index 0000000..65f4c8d --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_basic_with_capture_content_log_events[azure_provider_chat_completions].yaml @@ -0,0 +1,82 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + api-key: + - test_azure_api_key + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '138' + content-type: + - application/json + host: + - test.openai.azure.com + user-agent: + - AzureOpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/chat/completions?api-version=2023-03-15-preview + response: + body: + string: '{"choices":[{"finish_reason":"stop","index":0,"message":{"content":"South + Atlantic Ocean.","role":"assistant"}}],"created":1728909016,"id":"chatcmpl-AIEVEmwAmuw8qGX1PViCfm0kTe9O8","model":"gpt-4o-mini","object":"chat.completion","system_fingerprint":"fp_878413d04d","usage":{"completion_tokens":4,"prompt_tokens":24,"total_tokens":28}} + + ' + headers: + Content-Length: + - '336' + Content-Type: + - application/json + Date: + - Mon, 14 Oct 2024 12:30:16 GMT + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + apim-request-id: + - b67935a7-4d51-4ab6-9c26-c099dd0e08b4 + azureml-model-session: + - d021-20240921171145 + openai-organization: test_openai_org_key + x-accel-buffering: + - 'no' + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '132' + x-ms-client-request-id: + - b67935a7-4d51-4ab6-9c26-c099dd0e08b4 + x-ms-rai-invoked: + - 'true' + x-ms-region: + - East US + x-ratelimit-remaining-requests: + - '908' + x-ratelimit-remaining-tokens: + - '89642' + x-request-id: + - 3fef4a36-547f-4d50-ad2e-a87c5876562c + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_basic_with_capture_content_log_events[ollama_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_basic_with_capture_content_log_events[ollama_provider_chat_completions].yaml new file mode 100644 index 0000000..6188934 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_basic_with_capture_content_log_events[ollama_provider_chat_completions].yaml @@ -0,0 +1,56 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "qwen2.5:0.5b"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '139' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: '{"id":"chatcmpl-694","object":"chat.completion","created":1728909017,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"message":{"role":"assistant","content":"Atlantic + Ocean"},"finish_reason":"stop"}],"usage":{"prompt_tokens":46,"completion_tokens":3,"total_tokens":49}} + + ' + headers: + Content-Length: + - '300' + Content-Type: + - application/json + Date: + - Mon, 14 Oct 2024 12:30:17 GMT + Set-Cookie: test_set_cookie + openai-organization: test_openai_org_key + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_basic_with_capture_content_log_events[openai_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_basic_with_capture_content_log_events[openai_provider_chat_completions].yaml new file mode 100644 index 0000000..295e41d --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_basic_with_capture_content_log_events[openai_provider_chat_completions].yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '138' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-AIEVEddriZ8trWDORY6MdqNgqRkDX\",\n \"object\": + \"chat.completion\",\n \"created\": 1728909016,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Atlantic Ocean.\",\n \"refusal\": + null\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n + \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 24,\n \"completion_tokens\": + 3,\n \"total_tokens\": 27,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0\n }\n },\n \"system_fingerprint\": \"fp_e2bde53e6e\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d2794643dccba92-MXP + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 14 Oct 2024 12:30:16 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '637' + openai-organization: test_openai_org_key + openai-processing-ms: + - '596' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9998' + x-ratelimit-remaining-tokens: + - '199966' + x-ratelimit-reset-requests: + - 12.705s + x-ratelimit-reset-tokens: + - 10ms + x-request-id: + - req_f69fac0b2dfa1cd26f6b487cb2413713 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_multiple_choices_with_capture_content_log_events[azure_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_multiple_choices_with_capture_content_log_events[azure_provider_chat_completions].yaml new file mode 100644 index 0000000..48a08b1 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_multiple_choices_with_capture_content_log_events[azure_provider_chat_completions].yaml @@ -0,0 +1,85 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "gpt-4o-mini", "n": 2}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + api-key: + - test_azure_api_key + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '146' + content-type: + - application/json + host: + - test.openai.azure.com + user-agent: + - AzureOpenAI/Python 1.50.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.50.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/chat/completions?api-version=2023-03-15-preview + response: + body: + string: '{"choices":[{"finish_reason":"stop","index":0,"message":{"content":"South + Atlantic Ocean.","role":"assistant"}},{"finish_reason":"stop","index":1,"message":{"content":"South + Atlantic Ocean.","role":"assistant"}}],"created":1730200441,"id":"chatcmpl-ANeSfhGk22jizSPywSdR4MGCOdc76","model":"gpt-4o-mini","object":"chat.completion","system_fingerprint":"fp_d54531d9eb","usage":{"completion_tokens":8,"prompt_tokens":24,"total_tokens":32}} + + ' + headers: + Content-Length: + - '436' + Content-Type: + - application/json + Date: + - Tue, 29 Oct 2024 11:14:01 GMT + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + apim-request-id: + - f75b87aa-0c69-4819-bb46-4b18a3acc8b1 + azureml-model-session: + - d034-20241024145218 + openai-organization: test_openai_org_key + x-accel-buffering: + - 'no' + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '121' + x-ms-client-request-id: + - f75b87aa-0c69-4819-bb46-4b18a3acc8b1 + x-ms-rai-invoked: + - 'true' + x-ms-region: + - East US + x-ratelimit-remaining-requests: + - '909' + x-ratelimit-remaining-tokens: + - '89686' + x-request-id: + - c68236c0-ed69-4b12-944f-1bc002a4a85d + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_multiple_choices_with_capture_content_log_events[openai_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_multiple_choices_with_capture_content_log_events[openai_provider_chat_completions].yaml new file mode 100644 index 0000000..cdb0e9a --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_multiple_choices_with_capture_content_log_events[openai_provider_chat_completions].yaml @@ -0,0 +1,102 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Answer in up to 3 words: Which + ocean contains the falkland islands?"}], "model": "gpt-4o-mini", "n": 2}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '146' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.50.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.50.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-ANeSeH4fwAhE7sU211OIx0aI6K16o\",\n \"object\": + \"chat.completion\",\n \"created\": 1730200440,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"South Atlantic Ocean.\",\n \"refusal\": + null\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n + \ },\n {\n \"index\": 1,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"South Atlantic Ocean.\",\n \"refusal\": + null\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n + \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 24,\n \"completion_tokens\": + 8,\n \"total_tokens\": 32,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0\n }\n },\n \"system_fingerprint\": \"fp_8bfc6a7dc2\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8da2bd4efc765232-MXP + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 29 Oct 2024 11:14:01 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '853' + openai-organization: test_openai_org_key + openai-processing-ms: + - '555' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199950' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 15ms + x-request-id: + - req_208e8d7f784c381eb55ac6769c9a9597 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content[ollama_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content[ollama_provider_chat_completions].yaml new file mode 100644 index 0000000..7e3dcae --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content[ollama_provider_chat_completions].yaml @@ -0,0 +1,264 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful assistant + providing weather updates."}, {"role": "user", "content": "What is the weather + in New York and London?"}], "model": "qwen2.5:0.5b", "stream": true, "tools": + [{"type": "function", "function": {"name": "get_weather", "strict": true, "parameters": + {"type": "object", "properties": {"location": {"type": "string"}}, "required": + ["location"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '445' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"To"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + provide"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + you"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + with"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + the"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + most"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + current"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + information"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + about"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + the"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + weather"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":","},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + I"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + need"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + to"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + know"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + which"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + cities"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + you"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + are"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + interested"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + in"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + Could"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + we"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + please"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + specify"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + the"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + name"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + of"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + each"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + city"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"?"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + For"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + instance"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":","},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + both"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"New"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + York"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\","},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"London"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + or"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + individual"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + names"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + like"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571924,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571925,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"New"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571925,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + York"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571925,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + City"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571925,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\"."},"finish_reason":null}]} + + + data: {"id":"chatcmpl-142","object":"chat.completion.chunk","created":1728571925,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + Content-Type: + - text/event-stream + Date: + - Thu, 10 Oct 2024 14:52:04 GMT + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + openai-organization: test_openai_org_key + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content[openai_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content[openai_provider_chat_completions].yaml new file mode 100644 index 0000000..5ec32ae --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content[openai_provider_chat_completions].yaml @@ -0,0 +1,139 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful assistant + providing weather updates."}, {"role": "user", "content": "What is the weather + in New York and London?"}], "model": "gpt-4o-mini", "stream": true, "tools": + [{"type": "function", "function": {"name": "get_weather", "strict": true, "parameters": + {"type": "object", "properties": {"location": {"type": "string"}}, "required": + ["location"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '444' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"role":"assistant","content":null},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_uDsEOSTauJkgNI8ciF0AvU0X","type":"function","function":{"name":"get_weather","arguments":""}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"lo"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"catio"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"n\": + \"N"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ew + Y"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ork\"}"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_If8zqvcIX9JYzEYJ02dlpoBX","type":"function","function":{"name":"get_weather","arguments":""}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"lo"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"catio"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"n\": + \"L"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"ondo"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"n\"}"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP","object":"chat.completion.chunk","created":1728571920,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}]} + + + data: [DONE] + + + ' + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d076e83de049ec5-CDG + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Thu, 10 Oct 2024 14:52:00 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: test_openai_org_key + openai-processing-ms: + - '625' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199957' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 12ms + x-request-id: + - req_08a37326d1686b41a1f1ef7dcdc2d2f9 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content_log_events[ollama_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content_log_events[ollama_provider_chat_completions].yaml new file mode 100644 index 0000000..d3dfaef --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content_log_events[ollama_provider_chat_completions].yaml @@ -0,0 +1,266 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful assistant + providing weather updates."}, {"role": "user", "content": "What is the weather + in New York City and London?"}], "model": "qwen2.5:0.5b", "stream": true, "tools": + [{"type": "function", "function": {"name": "get_weather", "strict": true, "parameters": + {"type": "object", "properties": {"location": {"type": "string"}}, "required": + ["location"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '450' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\u003ctool_call\u003e"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"{\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + "},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"name"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\":"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"get"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"_weather"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\",\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + "},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"arguments"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\":"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + {\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" "},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"location"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\":"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"New"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + York"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":","},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + NY"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\"\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + "},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + }\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"}\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\u003c/tool_call\u003e"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\u003ctool_call\u003e"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"{\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + "},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"name"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\":"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"get"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"_weather"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\",\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + "},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"arguments"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\":"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + {\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" "},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"location"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\":"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"London"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":","},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + UK"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\"\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + "},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + }\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"}\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\u003c/tool_call\u003e"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-986","object":"chat.completion.chunk","created":1728902553,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + Content-Type: + - text/event-stream + Date: + - Mon, 14 Oct 2024 10:42:33 GMT + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + openai-organization: test_openai_org_key + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content_log_events[openai_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content_log_events[openai_provider_chat_completions].yaml new file mode 100644 index 0000000..5b254dd --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_parallel_tools_and_capture_content_log_events[openai_provider_chat_completions].yaml @@ -0,0 +1,143 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful assistant + providing weather updates."}, {"role": "user", "content": "What is the weather + in New York City and London?"}], "model": "gpt-4o-mini", "stream": true, "tools": + [{"type": "function", "function": {"name": "get_weather", "strict": true, "parameters": + {"type": "object", "properties": {"location": {"type": "string"}}, "required": + ["location"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '449' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"role":"assistant","content":null},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_m8FzMvtVd3wjksWMeRCWkPDK","type":"function","function":{"name":"get_weather","arguments":""}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"lo"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"catio"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"n\": + \"N"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ew + Y"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ork + C"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ity\"}"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_4WcXUPtB1wlKUy1lOrguqAtC","type":"function","function":{"name":"get_weather","arguments":""}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"lo"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"catio"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"n\": + \"L"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"ondo"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"n\"}"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs","object":"chat.completion.chunk","created":1728902549,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}]} + + + data: [DONE] + + + ' + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d26f681a8120e56-MXP + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Mon, 14 Oct 2024 10:42:31 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: test_openai_org_key + openai-processing-ms: + - '2211' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199956' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 13ms + x-request-id: + - req_edb2118c6f6bfb46ab38b815de243d4d + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_tools_and_capture_content_log_events[azure_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_tools_and_capture_content_log_events[azure_provider_chat_completions].yaml new file mode 100644 index 0000000..a73b813 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_tools_and_capture_content_log_events[azure_provider_chat_completions].yaml @@ -0,0 +1,124 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful customer + support assistant. Use the supplied tools to assist the user."}, {"role": "user", + "content": "Hi, can you tell me the delivery date for my order?"}, {"role": + "assistant", "content": "Hi there! I can help with that. Can you please provide + your order ID?"}, {"role": "user", "content": "i think it is order_12345"}], + "model": "gpt-4o-mini", "stream": true, "tools": [{"type": "function", "function": + {"name": "get_delivery_date", "description": "Get the delivery date for a customer''s + order. Call this whenever you need to know the delivery date, for example when + a customer asks ''Where is my package''", "parameters": {"type": "object", "properties": + {"order_id": {"type": "string", "description": "The customer''s order ID."}}, + "required": ["order_id"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + api-key: + - test_azure_api_key + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '858' + content-type: + - application/json + host: + - test.openai.azure.com + user-agent: + - AzureOpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/chat/completions?api-version=2023-03-15-preview + response: + body: + string: 'data: {"choices":[{"delta":{"role":"assistant","tool_calls":[{"function":{"arguments":"","name":"get_delivery_date"},"id":"call_XNHRbrreMnt9ReHJfNH30mom","index":0,"type":"function"}]},"finish_reason":null,"index":0}],"created":1728909019,"id":"chatcmpl-AIEVHRdUM6ip3Uolr8CcrlGhchugq","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"{\""},"index":0}]},"finish_reason":null,"index":0}],"created":1728909019,"id":"chatcmpl-AIEVHRdUM6ip3Uolr8CcrlGhchugq","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"order"},"index":0}]},"finish_reason":null,"index":0}],"created":1728909019,"id":"chatcmpl-AIEVHRdUM6ip3Uolr8CcrlGhchugq","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"_id"},"index":0}]},"finish_reason":null,"index":0}],"created":1728909019,"id":"chatcmpl-AIEVHRdUM6ip3Uolr8CcrlGhchugq","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"\":\""},"index":0}]},"finish_reason":null,"index":0}],"created":1728909019,"id":"chatcmpl-AIEVHRdUM6ip3Uolr8CcrlGhchugq","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"order"},"index":0}]},"finish_reason":null,"index":0}],"created":1728909019,"id":"chatcmpl-AIEVHRdUM6ip3Uolr8CcrlGhchugq","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"_"},"index":0}]},"finish_reason":null,"index":0}],"created":1728909019,"id":"chatcmpl-AIEVHRdUM6ip3Uolr8CcrlGhchugq","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"123"},"index":0}]},"finish_reason":null,"index":0}],"created":1728909019,"id":"chatcmpl-AIEVHRdUM6ip3Uolr8CcrlGhchugq","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"45"},"index":0}]},"finish_reason":null,"index":0}],"created":1728909019,"id":"chatcmpl-AIEVHRdUM6ip3Uolr8CcrlGhchugq","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"\"}"},"index":0}]},"finish_reason":null,"index":0}],"created":1728909019,"id":"chatcmpl-AIEVHRdUM6ip3Uolr8CcrlGhchugq","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: {"choices":[{"delta":{},"finish_reason":"tool_calls","index":0}],"created":1728909019,"id":"chatcmpl-AIEVHRdUM6ip3Uolr8CcrlGhchugq","model":"gpt-4o-mini","object":"chat.completion.chunk","system_fingerprint":"fp_878413d04d"} + + + data: [DONE] + + + ' + headers: + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Mon, 14 Oct 2024 12:30:19 GMT + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + apim-request-id: + - 1bb18f17-07a0-4c26-a747-724303e11ebf + azureml-model-session: + - d017-20240921115920 + openai-organization: test_openai_org_key + x-accel-buffering: + - 'no' + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '123' + x-ms-client-request-id: + - 1bb18f17-07a0-4c26-a747-724303e11ebf + x-ms-rai-invoked: + - 'true' + x-ms-region: + - East US + x-ratelimit-remaining-requests: + - '907' + x-ratelimit-remaining-tokens: + - '88941' + x-request-id: + - 5a07aea0-8492-4144-8c21-5c9b81d82ea1 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_tools_and_capture_content_log_events[ollama_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_tools_and_capture_content_log_events[ollama_provider_chat_completions].yaml new file mode 100644 index 0000000..71afce8 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_tools_and_capture_content_log_events[ollama_provider_chat_completions].yaml @@ -0,0 +1,153 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful customer + support assistant. Use the supplied tools to assist the user."}, {"role": "user", + "content": "Hi, can you tell me the delivery date for my order?"}, {"role": + "assistant", "content": "Hi there! I can help with that. Can you please provide + your order ID?"}, {"role": "user", "content": "i think it is order_12345"}], + "model": "qwen2.5:0.5b", "stream": true, "tools": [{"type": "function", "function": + {"name": "get_delivery_date", "description": "Get the delivery date for a customer''s + order. Call this whenever you need to know the delivery date, for example when + a customer asks ''Where is my package''", "parameters": {"type": "object", "properties": + {"order_id": {"type": "string", "description": "The customer''s order ID."}}, + "required": ["order_id"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '859' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\u003ctool_call\u003e"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"{\""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"name"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\":"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"get"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"_delivery"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"_date"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\","},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"arguments"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\":"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + {\""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"order"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"_id"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\":"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" + \""},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"order"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"_"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"1"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"2"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"3"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"4"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"5"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\"}}\n"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"\u003c/tool_call\u003e"},"finish_reason":null}]} + + + data: {"id":"chatcmpl-436","object":"chat.completion.chunk","created":1728909020,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}]} + + + data: [DONE] + + + ' + headers: + Content-Type: + - text/event-stream + Date: + - Mon, 14 Oct 2024 12:30:20 GMT + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + openai-organization: test_openai_org_key + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_tools_and_capture_content_log_events[openai_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_tools_and_capture_content_log_events[openai_provider_chat_completions].yaml new file mode 100644 index 0000000..cbc21b0 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_stream_with_tools_and_capture_content_log_events[openai_provider_chat_completions].yaml @@ -0,0 +1,132 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful customer + support assistant. Use the supplied tools to assist the user."}, {"role": "user", + "content": "Hi, can you tell me the delivery date for my order?"}, {"role": + "assistant", "content": "Hi there! I can help with that. Can you please provide + your order ID?"}, {"role": "user", "content": "i think it is order_12345"}], + "model": "gpt-4o-mini", "stream": true, "tools": [{"type": "function", "function": + {"name": "get_delivery_date", "description": "Get the delivery date for a customer''s + order. Call this whenever you need to know the delivery date, for example when + a customer asks ''Where is my package''", "parameters": {"type": "object", "properties": + {"order_id": {"type": "string", "description": "The customer''s order ID."}}, + "required": ["order_id"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '858' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-AIEVFr8IGqjRC2wxrGU3tcRjNbGKf","object":"chat.completion.chunk","created":1728909017,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_BQ6tpzuq28epoO6jzUNSdG6r","type":"function","function":{"name":"get_delivery_date","arguments":""}}],"refusal":null},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVFr8IGqjRC2wxrGU3tcRjNbGKf","object":"chat.completion.chunk","created":1728909017,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVFr8IGqjRC2wxrGU3tcRjNbGKf","object":"chat.completion.chunk","created":1728909017,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"order"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVFr8IGqjRC2wxrGU3tcRjNbGKf","object":"chat.completion.chunk","created":1728909017,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"_id"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVFr8IGqjRC2wxrGU3tcRjNbGKf","object":"chat.completion.chunk","created":1728909017,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\""}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVFr8IGqjRC2wxrGU3tcRjNbGKf","object":"chat.completion.chunk","created":1728909017,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"order"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVFr8IGqjRC2wxrGU3tcRjNbGKf","object":"chat.completion.chunk","created":1728909017,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"_"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVFr8IGqjRC2wxrGU3tcRjNbGKf","object":"chat.completion.chunk","created":1728909017,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"123"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVFr8IGqjRC2wxrGU3tcRjNbGKf","object":"chat.completion.chunk","created":1728909017,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"45"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVFr8IGqjRC2wxrGU3tcRjNbGKf","object":"chat.completion.chunk","created":1728909017,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]},"logprobs":null,"finish_reason":null}]} + + + data: {"id":"chatcmpl-AIEVFr8IGqjRC2wxrGU3tcRjNbGKf","object":"chat.completion.chunk","created":1728909017,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}]} + + + data: [DONE] + + + ' + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d27946fec104c45-MXP + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Mon, 14 Oct 2024 12:30:18 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: test_openai_org_key + openai-processing-ms: + - '621' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9997' + x-ratelimit-remaining-tokens: + - '199921' + x-ratelimit-reset-requests: + - 19.444s + x-ratelimit-reset-tokens: + - 23ms + x-request-id: + - req_a90f027e4c3cd9eb3c0a6732a9b805f1 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_capture_content_log_events[azure_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_capture_content_log_events[azure_provider_chat_completions].yaml new file mode 100644 index 0000000..53b25b4 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_capture_content_log_events[azure_provider_chat_completions].yaml @@ -0,0 +1,90 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful customer + support assistant. Use the supplied tools to assist the user."}, {"role": "user", + "content": "Hi, can you tell me the delivery date for my order?"}, {"role": + "assistant", "content": "Hi there! I can help with that. Can you please provide + your order ID?"}, {"role": "user", "content": "i think it is order_12345"}], + "model": "gpt-4o-mini", "tools": [{"type": "function", "function": {"name": + "get_delivery_date", "description": "Get the delivery date for a customer''s + order. Call this whenever you need to know the delivery date, for example when + a customer asks ''Where is my package''", "parameters": {"type": "object", "properties": + {"order_id": {"type": "string", "description": "The customer''s order ID."}}, + "required": ["order_id"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + api-key: + - test_azure_api_key + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '842' + content-type: + - application/json + host: + - test.openai.azure.com + user-agent: + - AzureOpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/chat/completions?api-version=2023-03-15-preview + response: + body: + string: '{"choices":[{"finish_reason":"tool_calls","index":0,"message":{"content":null,"role":"assistant","tool_calls":[{"function":{"arguments":"{\"order_id\":\"order_12345\"}","name":"get_delivery_date"},"id":"call_Xyr5OWcqhvuW62vUx3sPPruH","type":"function"}]}}],"created":1728909012,"id":"chatcmpl-AIEVA7wwx1Cxy8O31zWDnxgUETAeN","model":"gpt-4o-mini","object":"chat.completion","system_fingerprint":"fp_878413d04d","usage":{"completion_tokens":19,"prompt_tokens":140,"total_tokens":159}} + + ' + headers: + Content-Length: + - '483' + Content-Type: + - application/json + Date: + - Mon, 14 Oct 2024 12:30:12 GMT + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + apim-request-id: + - a141176e-94e3-486c-9085-230741a42457 + azureml-model-session: + - d005-20241009165114 + openai-organization: test_openai_org_key + x-accel-buffering: + - 'no' + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '417' + x-ms-client-request-id: + - a141176e-94e3-486c-9085-230741a42457 + x-ms-rai-invoked: + - 'true' + x-ms-region: + - East US + x-ratelimit-remaining-requests: + - '909' + x-ratelimit-remaining-tokens: + - '90299' + x-request-id: + - 320ccb63-87b7-40df-9d5d-538d1dc7fcad + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_capture_content_log_events[ollama_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_capture_content_log_events[ollama_provider_chat_completions].yaml new file mode 100644 index 0000000..663b6e1 --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_capture_content_log_events[ollama_provider_chat_completions].yaml @@ -0,0 +1,64 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful customer + support assistant. Use the supplied tools to assist the user."}, {"role": "user", + "content": "Hi, can you tell me the delivery date for my order?"}, {"role": + "assistant", "content": "Hi there! I can help with that. Can you please provide + your order ID?"}, {"role": "user", "content": "i think it is order_12345"}], + "model": "qwen2.5:0.5b", "tools": [{"type": "function", "function": {"name": + "get_delivery_date", "description": "Get the delivery date for a customer''s + order. Call this whenever you need to know the delivery date, for example when + a customer asks ''Where is my package''", "parameters": {"type": "object", "properties": + {"order_id": {"type": "string", "description": "The customer''s order ID."}}, + "required": ["order_id"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '843' + content-type: + - application/json + host: + - localhost:11434 + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: http://localhost:11434/v1/chat/completions + response: + body: + string: '{"id":"chatcmpl-339","object":"chat.completion","created":1728909015,"model":"qwen2.5:0.5b","system_fingerprint":"fp_ollama","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_3h40tlh2","type":"function","function":{"name":"get_delivery_date","arguments":"{\"order_id\":\"order_12345\"}"}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":241,"completion_tokens":28,"total_tokens":269}} + + ' + headers: + Content-Length: + - '436' + Content-Type: + - application/json + Date: + - Mon, 14 Oct 2024 12:30:15 GMT + Set-Cookie: test_set_cookie + openai-organization: test_openai_org_key + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_capture_content_log_events[openai_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_capture_content_log_events[openai_provider_chat_completions].yaml new file mode 100644 index 0000000..5991e9b --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_capture_content_log_events[openai_provider_chat_completions].yaml @@ -0,0 +1,110 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful customer + support assistant. Use the supplied tools to assist the user."}, {"role": "user", + "content": "Hi, can you tell me the delivery date for my order?"}, {"role": + "assistant", "content": "Hi there! I can help with that. Can you please provide + your order ID?"}, {"role": "user", "content": "i think it is order_12345"}], + "model": "gpt-4o-mini", "tools": [{"type": "function", "function": {"name": + "get_delivery_date", "description": "Get the delivery date for a customer''s + order. Call this whenever you need to know the delivery date, for example when + a customer asks ''Where is my package''", "parameters": {"type": "object", "properties": + {"order_id": {"type": "string", "description": "The customer''s order ID."}}, + "required": ["order_id"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '842' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.34.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.34.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-AIEV9MTVUJ4HtPJm6pro1FgWlWQ2g\",\n \"object\": + \"chat.completion\",\n \"created\": 1728909011,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_jQEwfwLUeVLVsxWaz8N0c8Yp\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"get_delivery_date\",\n + \ \"arguments\": \"{\\\"order_id\\\":\\\"order_12345\\\"}\"\n + \ }\n }\n ],\n \"refusal\": null\n },\n + \ \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n + \ ],\n \"usage\": {\n \"prompt_tokens\": 140,\n \"completion_tokens\": + 19,\n \"total_tokens\": 159,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0\n }\n },\n \"system_fingerprint\": \"fp_8552ec53e1\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d2794478f680de8-MXP + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 14 Oct 2024 12:30:12 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '918' + openai-organization: test_openai_org_key + openai-processing-ms: + - '954' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199921' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 23ms + x-request-id: + - req_899caaee8931b70c71856c7315a7312f + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_followup_and_capture_content_log_events[azure_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_followup_and_capture_content_log_events[azure_provider_chat_completions].yaml new file mode 100644 index 0000000..cbd79ad --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_followup_and_capture_content_log_events[azure_provider_chat_completions].yaml @@ -0,0 +1,179 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful assistant + providing weather updates."}, {"role": "user", "content": "What is the weather + in New York City and London?"}], "model": "gpt-4o-mini", "tools": [{"type": + "function", "function": {"name": "get_weather", "strict": true, "parameters": + {"type": "object", "properties": {"location": {"type": "string"}}, "required": + ["location"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + api-key: + - test_azure_api_key + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '433' + content-type: + - application/json + host: + - test.openai.azure.com + user-agent: + - AzureOpenAI/Python 1.50.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.50.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/chat/completions?api-version=2023-03-15-preview + response: + body: + string: '{"choices":[{"finish_reason":"tool_calls","index":0,"message":{"content":null,"role":"assistant","tool_calls":[{"function":{"arguments":"{\"location\": + \"New York City\"}","name":"get_weather"},"id":"call_TYKLGjzizbih3K9ouYXAEwMI","type":"function"},{"function":{"arguments":"{\"location\": + \"London\"}","name":"get_weather"},"id":"call_MKJdMkxrfGOtWqKdw7Ey0iFU","type":"function"}]}}],"created":1730217198,"id":"chatcmpl-ANiownjaqRi57SCL9HzkcnO6WeVhp","model":"gpt-4o-mini","object":"chat.completion","system_fingerprint":"fp_d54531d9eb","usage":{"completion_tokens":46,"prompt_tokens":57,"total_tokens":103}} + + ' + headers: + Content-Length: + - '611' + Content-Type: + - application/json + Date: + - Tue, 29 Oct 2024 15:53:18 GMT + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + apim-request-id: + - 0d20754c-4d9d-44e5-8561-c2c6f69d4380 + azureml-model-session: + - d036-20241024163555 + openai-organization: test_openai_org_key + x-accel-buffering: + - 'no' + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '5250' + x-ms-client-request-id: + - 0d20754c-4d9d-44e5-8561-c2c6f69d4380 + x-ms-rai-invoked: + - 'true' + x-ms-region: + - East US + x-ratelimit-remaining-requests: + - '909' + x-ratelimit-remaining-tokens: + - '90333' + x-request-id: + - 866c111e-4e36-4f2d-b320-235e0721f623 + status: + code: 200 + message: OK +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful assistant + providing weather updates."}, {"role": "user", "content": "What is the weather + in New York City and London?"}, {"role": "assistant", "tool_calls": [{"id": + "call_TYKLGjzizbih3K9ouYXAEwMI", "function": {"arguments": "{\"location\": \"New + York City\"}", "name": "get_weather"}, "type": "function"}, {"id": "call_MKJdMkxrfGOtWqKdw7Ey0iFU", + "function": {"arguments": "{\"location\": \"London\"}", "name": "get_weather"}, + "type": "function"}]}, {"role": "tool", "content": "25 degrees and sunny", "tool_call_id": + "call_TYKLGjzizbih3K9ouYXAEwMI"}, {"role": "tool", "content": "15 degrees and + raining", "tool_call_id": "call_MKJdMkxrfGOtWqKdw7Ey0iFU"}], "model": "gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + api-key: + - test_azure_api_key + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '742' + content-type: + - application/json + host: + - test.openai.azure.com + user-agent: + - AzureOpenAI/Python 1.50.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.50.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://test.openai.azure.com/openai/deployments/test-azure-deployment/chat/completions?api-version=2023-03-15-preview + response: + body: + string: '{"choices":[{"finish_reason":"stop","index":0,"message":{"content":"The + current weather is as follows:\n\n- **New York City**: 25 degrees and sunny\n- + **London**: 15 degrees and raining\n\nLet me know if you need more information!","role":"assistant"}}],"created":1730217199,"id":"chatcmpl-ANioxBVkM3W9XRWl7rLK0tMEFxmC0","model":"gpt-4o-mini","object":"chat.completion","system_fingerprint":"fp_d54531d9eb","usage":{"completion_tokens":40,"prompt_tokens":99,"total_tokens":139}} + + ' + headers: + Content-Length: + - '479' + Content-Type: + - application/json + Date: + - Tue, 29 Oct 2024 15:53:18 GMT + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + apim-request-id: + - ec236cd8-fa08-4ac0-9ea2-cec96990ffe5 + azureml-model-session: + - d006-20241024084752 + openai-organization: test_openai_org_key + x-accel-buffering: + - 'no' + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '492' + x-ms-client-request-id: + - ec236cd8-fa08-4ac0-9ea2-cec96990ffe5 + x-ms-rai-invoked: + - 'true' + x-ms-region: + - East US + x-ratelimit-remaining-requests: + - '908' + x-ratelimit-remaining-tokens: + - '89654' + x-request-id: + - 450b1944-43cf-4edf-ad42-ff7acbfe7045 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_followup_and_capture_content_log_events[openai_provider_chat_completions].yaml b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_followup_and_capture_content_log_events[openai_provider_chat_completions].yaml new file mode 100644 index 0000000..6246dbf --- /dev/null +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/cassettes/.test_chat_completions/test_tools_with_followup_and_capture_content_log_events[openai_provider_chat_completions].yaml @@ -0,0 +1,215 @@ +interactions: +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful assistant + providing weather updates."}, {"role": "user", "content": "What is the weather + in New York City and London?"}], "model": "gpt-4o-mini", "tools": [{"type": + "function", "function": {"name": "get_weather", "strict": true, "parameters": + {"type": "object", "properties": {"location": {"type": "string"}}, "required": + ["location"], "additionalProperties": false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '433' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.50.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.50.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-ANiop2XcXbTSq8swBkuxNauL2ZCpp\",\n \"object\": + \"chat.completion\",\n \"created\": 1730217191,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_WUdxyJgUagWwal3Oq9ZevTO0\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n + \ \"arguments\": \"{\\\"location\\\": \\\"New York City\\\"}\"\n + \ }\n },\n {\n \"id\": \"call_BAZEZMlsHRPFvG5yYPNIPFA8\",\n + \ \"type\": \"function\",\n \"function\": {\n \"name\": + \"get_weather\",\n \"arguments\": \"{\\\"location\\\": \\\"London\\\"}\"\n + \ }\n }\n ],\n \"refusal\": null\n },\n + \ \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n + \ ],\n \"usage\": {\n \"prompt_tokens\": 57,\n \"completion_tokens\": + 46,\n \"total_tokens\": 103,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0\n }\n },\n \"system_fingerprint\": \"fp_f59a81427f\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8da45642195e5261-MXP + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 29 Oct 2024 15:53:11 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '1155' + openai-organization: test_openai_org_key + openai-processing-ms: + - '1027' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199956' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 13ms + x-request-id: + - req_9020588902482d8196158fcf8203a9ee + status: + code: 200 + message: OK +- request: + body: '{"messages": [{"role": "system", "content": "You are a helpful assistant + providing weather updates."}, {"role": "user", "content": "What is the weather + in New York City and London?"}, {"role": "assistant", "tool_calls": [{"id": + "call_WUdxyJgUagWwal3Oq9ZevTO0", "function": {"arguments": "{\"location\": \"New + York City\"}", "name": "get_weather"}, "type": "function"}, {"id": "call_BAZEZMlsHRPFvG5yYPNIPFA8", + "function": {"arguments": "{\"location\": \"London\"}", "name": "get_weather"}, + "type": "function"}]}, {"role": "tool", "content": "25 degrees and sunny", "tool_call_id": + "call_WUdxyJgUagWwal3Oq9ZevTO0"}, {"role": "tool", "content": "15 degrees and + raining", "tool_call_id": "call_BAZEZMlsHRPFvG5yYPNIPFA8"}], "model": "gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '742' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.50.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.50.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-ANioqFAl6knUGNXUHM97w9fnolOUA\",\n \"object\": + \"chat.completion\",\n \"created\": 1730217192,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"The current weather is as follows:\\n\\n- + **New York City**: 25 degrees and sunny\\n- **London**: 15 degrees and raining\",\n + \ \"refusal\": null\n },\n \"logprobs\": null,\n \"finish_reason\": + \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 99,\n \"completion_tokens\": + 30,\n \"total_tokens\": 129,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0\n }\n },\n \"system_fingerprint\": \"fp_f59a81427f\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8da45649ed485261-MXP + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 29 Oct 2024 15:53:13 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '741' + openai-organization: test_openai_org_key + openai-processing-ms: + - '1119' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9998' + x-ratelimit-remaining-tokens: + - '199942' + x-ratelimit-reset-requests: + - 16.076s + x-ratelimit-reset-tokens: + - 17ms + x-request-id: + - req_05dcf7d7979cddd2ca4d74b2ac762add + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/conftest.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/conftest.py index c8e123f..ace6ee6 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/conftest.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/conftest.py @@ -16,13 +16,22 @@ import re import os +from typing import Sequence, Union from urllib.parse import parse_qs, urlparse import openai import pytest from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry._events import set_event_logger_provider from opentelemetry.instrumentation.openai import OpenAIInstrumentor from opentelemetry.metrics import Histogram +from opentelemetry.sdk._events import EventLoggerProvider +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import ( + InMemoryLogExporter, + SimpleLogRecordProcessor, +) from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import ( InMemoryMetricReader, @@ -61,9 +70,24 @@ def metrics_reader(): return memory_reader +@pytest.fixture(scope="session") +def logs_exporter(): + exporter = InMemoryLogExporter() + logger_provider = LoggerProvider() + set_logger_provider(logger_provider) + + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(exporter)) + + event_logger_provider = EventLoggerProvider(logger_provider=logger_provider) + set_event_logger_provider(event_logger_provider) + + return exporter + + @pytest.fixture(autouse=True) -def clear_exporter(trace_exporter, metrics_reader): +def clear_exporter(trace_exporter, metrics_reader, logs_exporter): trace_exporter.clear() + logs_exporter.clear() # TODO: should drop autouse and use it explicitly? @@ -328,8 +352,15 @@ def ollama_provider_embeddings(): return OllamaProvider.from_env(operation_name="embeddings") -# data_point = 0.006543334107846022 -def assert_operation_duration_metric(provider, metric: Histogram, attributes: dict, data_point: float): +def assert_operation_duration_metric( + provider, + metric: Histogram, + attributes: dict, + min_data_point: float, + max_data_point: float = None, + sum_data_point: float = None, + count: int = 1, +): assert metric.name == "gen_ai.client.operation.duration" default_attributes = { "gen_ai.operation.name": provider.operation_name, @@ -337,14 +368,24 @@ def assert_operation_duration_metric(provider, metric: Histogram, attributes: di "server.address": provider.server_address, "server.port": provider.server_port, } + # handle the simple cases of 1 or 2 data points in the histogram with just min_data_point + if max_data_point is None: + max_data_point = min_data_point + if sum_data_point is None: + if count == 1: + sum_data_point = min_data_point + elif count == 2: + sum_data_point = min_data_point + max_data_point + else: + raise ValueError("Missing sum_data_point with more than 2 values") assert_metric_expected( metric, [ create_histogram_data_point( - count=1, - sum_data_point=data_point, - max_data_point=data_point, - min_data_point=data_point, + count=count, + sum_data_point=sum_data_point, + max_data_point=max_data_point, + min_data_point=min_data_point, attributes={**default_attributes, **attributes}, ), ], @@ -378,7 +419,9 @@ def assert_error_operation_duration_metric( ) -def assert_token_usage_input_metric(provider, metric: Histogram, attributes: dict, input_data_point: int): +def assert_token_usage_input_metric( + provider, metric: Histogram, attributes: dict, input_data_point: int, count: int = 1 +): assert metric.name == "gen_ai.client.token.usage" default_attributes = { "gen_ai.operation.name": provider.operation_name, @@ -391,7 +434,7 @@ def assert_token_usage_input_metric(provider, metric: Histogram, attributes: dic metric, [ create_histogram_data_point( - count=1, + count=count, sum_data_point=input_data_point, max_data_point=input_data_point, min_data_point=input_data_point, @@ -402,7 +445,12 @@ def assert_token_usage_input_metric(provider, metric: Histogram, attributes: dic def assert_token_usage_metric( - provider, metric: Histogram, attributes: dict, input_data_point: int, output_data_point: int + provider, + metric: Histogram, + attributes: dict, + input_data_point: Union[int, Sequence[int]], + output_data_point: Union[int, Sequence[int]], + count: int = 1, ): assert metric.name == "gen_ai.client.token.usage" default_attributes = { @@ -412,21 +460,37 @@ def assert_token_usage_metric( "server.port": provider.server_port, "gen_ai.token.type": "input", } + + if count == 1: + assert isinstance(input_data_point, int) + assert isinstance(output_data_point, int) + sum_input = min_input = max_input = input_data_point + sum_output = min_output = max_output = output_data_point + else: + assert isinstance(input_data_point, Sequence) + assert isinstance(output_data_point, Sequence) + sum_input = sum(input_data_point) + min_input = min(input_data_point) + max_input = max(input_data_point) + sum_output = sum(output_data_point) + min_output = min(output_data_point) + max_output = max(output_data_point) + assert_metric_expected( metric, [ create_histogram_data_point( - count=1, - sum_data_point=input_data_point, - max_data_point=input_data_point, - min_data_point=input_data_point, + count=count, + sum_data_point=sum_input, + max_data_point=max_input, + min_data_point=min_input, attributes={**default_attributes, **attributes, "gen_ai.token.type": "input"}, ), create_histogram_data_point( - count=1, - sum_data_point=output_data_point, - max_data_point=output_data_point, - min_data_point=output_data_point, + count=count, + sum_data_point=sum_output, + max_data_point=max_output, + min_data_point=min_output, attributes={**default_attributes, **attributes, "gen_ai.token.type": "output"}, ), ], diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_chat_completions.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_chat_completions.py index 869a274..70d7f14 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_chat_completions.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_chat_completions.py @@ -46,7 +46,7 @@ assert_operation_duration_metric, assert_token_usage_metric, ) -from .utils import get_sorted_metrics +from .utils import get_sorted_metrics, logrecords_from_logs providers = ["openai_provider_chat_completions", "ollama_provider_chat_completions", "azure_provider_chat_completions"] @@ -145,7 +145,9 @@ def test_basic( GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) assert_token_usage_metric( provider, token_usage_metric, @@ -263,7 +265,9 @@ def test_all_the_client_options( GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) assert_token_usage_metric( provider, token_usage_metric, @@ -307,6 +311,138 @@ def test_all_the_client_options( ] +test_multiple_choices_capture_content_log_events_test_data = [ + ( + "openai_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + "South Atlantic Ocean.", + "chatcmpl-ANeSeH4fwAhE7sU211OIx0aI6K16o", + 24, + 8, + 0.006761051714420319, + ), + ( + "azure_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini", + "South Atlantic Ocean.", + "chatcmpl-ANeSfhGk22jizSPywSdR4MGCOdc76", + 24, + 8, + 0.002889830619096756, + ), + # ollama does not support n>1 +] + + +@pytest.mark.vcr() +@pytest.mark.parametrize( + "provider_str,model,response_model,content,response_id,input_tokens,output_tokens,duration", + test_multiple_choices_capture_content_log_events_test_data, +) +def test_multiple_choices_with_capture_content_log_events( + provider_str, + model, + response_model, + content, + response_id, + input_tokens, + output_tokens, + duration, + trace_exporter, + metrics_reader, + logs_exporter, + request, +): + provider = request.getfixturevalue(provider_str) + + client = provider.get_client() + + # Redo the instrumentation dance to be affected by the environment variable + OpenAIInstrumentor().uninstrument() + with mock.patch.dict( + "os.environ", {"ELASTIC_OTEL_GENAI_CAPTURE_CONTENT": "true", "ELASTIC_OTEL_GENAI_EVENTS": "log"} + ): + OpenAIInstrumentor().instrument() + + messages = [ + { + "role": "user", + "content": "Answer in up to 3 words: Which ocean contains the falkland islands?", + } + ] + + chat_completion = client.chat.completions.create(model=model, messages=messages, n=2) + + assert chat_completion.choices[0].message.content == content + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == f"chat {model}" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.UNSET + + assert dict(span.attributes) == { + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: model, + GEN_AI_SYSTEM: "openai", + GEN_AI_RESPONSE_ID: response_id, + GEN_AI_RESPONSE_MODEL: response_model, + GEN_AI_RESPONSE_FINISH_REASONS: ("stop", "stop"), + GEN_AI_USAGE_INPUT_TOKENS: input_tokens, + GEN_AI_USAGE_OUTPUT_TOKENS: output_tokens, + SERVER_ADDRESS: provider.server_address, + SERVER_PORT: provider.server_port, + } + + logs = logs_exporter.get_finished_logs() + assert len(logs) == 3 + log_records = logrecords_from_logs(logs) + user_message, choice, second_choice = log_records + assert user_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert user_message.body == {"content": "Answer in up to 3 words: Which ocean contains the falkland islands?"} + assert choice.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.choice"} + + expected_body = { + "finish_reason": "stop", + "index": 0, + "message": { + "content": content, + }, + } + assert dict(choice.body) == expected_body + + assert second_choice.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.choice"} + + second_expected_body = { + "finish_reason": "stop", + "index": 1, + "message": { + "content": content, + }, + } + assert dict(second_choice.body) == second_expected_body + + operation_duration_metric, token_usage_metric = get_sorted_metrics(metrics_reader) + attributes = { + GEN_AI_REQUEST_MODEL: model, + GEN_AI_RESPONSE_MODEL: response_model, + } + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) + assert_token_usage_metric( + provider, + token_usage_metric, + attributes=attributes, + input_data_point=input_tokens, + output_data_point=output_tokens, + ) + + @pytest.mark.vcr() @pytest.mark.parametrize( "provider_str,model,response_model,content,response_id,input_tokens,output_tokens,duration", @@ -395,7 +531,9 @@ def test_function_calling_with_tools( GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) assert_token_usage_metric( provider, token_usage_metric, @@ -412,6 +550,7 @@ def test_function_calling_with_tools( "gpt-4o-mini-2024-07-18", "South Atlantic Ocean.", "chatcmpl-AEGM5OhimYEMDsRq20IQCBx4vzf2Z", + "call_yU31CceO4OliQuZgPdsSPRXT", 140, 19, 0.006761051714420319, @@ -422,6 +561,7 @@ def test_function_calling_with_tools( "gpt-4o-mini", "South Atlantic Ocean", "chatcmpl-AEGM6niNxWuulOpOMXNelXUZqF443", + "call_EOPkr9g0pp71Wt6LEteCHWkZ", 140, 19, 0.002889830619096756, @@ -432,6 +572,7 @@ def test_function_calling_with_tools( "qwen2.5:0.5b", "The Falklands Islands are located in the oceans south of South America.", "chatcmpl-556", + "call_sr2j6oa1", 241, 28, 0.002600736916065216, @@ -441,7 +582,7 @@ def test_function_calling_with_tools( @pytest.mark.vcr() @pytest.mark.parametrize( - "provider_str,model,response_model,content,response_id,input_tokens,output_tokens,duration", + "provider_str,model,response_model,content,response_id,function_call_id,input_tokens,output_tokens,duration", test_tools_with_capture_content_test_data, ) def test_tools_with_capture_content( @@ -450,6 +591,7 @@ def test_tools_with_capture_content( response_model, content, response_id, + function_call_id, input_tokens, output_tokens, duration, @@ -531,7 +673,9 @@ def test_tools_with_capture_content( assert dict(prompt_event.attributes) == {"gen_ai.prompt": json.dumps(messages)} assert completion_event.name == "gen_ai.content.completion" assert dict(completion_event.attributes) == { - "gen_ai.completion": '[{"role": "assistant", "content": {"order_id": "order_12345"}}]' + "gen_ai.completion": '[{"role": "assistant", "content": "", "tool_calls": [{"id": "' + + function_call_id + + '", "type": "function", "function": {"name": "get_delivery_date", "arguments": "{\\"order_id\\":\\"order_12345\\"}"}}]}]' } operation_duration_metric, token_usage_metric = get_sorted_metrics(metrics_reader) @@ -539,7 +683,9 @@ def test_tools_with_capture_content( GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) assert_token_usage_metric( provider, token_usage_metric, @@ -549,17 +695,194 @@ def test_tools_with_capture_content( ) -test_connection_error_test_data = [ +test_tools_with_capture_content_log_events_test_data = [ ( "openai_provider_chat_completions", "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + "South Atlantic Ocean.", + "chatcmpl-AIEV9MTVUJ4HtPJm6pro1FgWlWQ2g", + "call_jQEwfwLUeVLVsxWaz8N0c8Yp", + 140, + 19, 0.006761051714420319, ), ( "azure_provider_chat_completions", "gpt-4o-mini", + "gpt-4o-mini", + "South Atlantic Ocean", + "chatcmpl-AIEVA7wwx1Cxy8O31zWDnxgUETAeN", + "call_Xyr5OWcqhvuW62vUx3sPPruH", + 140, + 19, 0.002889830619096756, ), + ( + "ollama_provider_chat_completions", + "qwen2.5:0.5b", + "qwen2.5:0.5b", + "The Falklands Islands are located in the oceans south of South America.", + "chatcmpl-339", + "call_3h40tlh2", + 241, + 28, + 0.002600736916065216, + ), +] + + +@pytest.mark.vcr() +@pytest.mark.parametrize( + "provider_str,model,response_model,content,response_id,function_call_id,input_tokens,output_tokens,duration", + test_tools_with_capture_content_log_events_test_data, +) +def test_tools_with_capture_content_log_events( + provider_str, + model, + response_model, + content, + response_id, + function_call_id, + input_tokens, + output_tokens, + duration, + trace_exporter, + logs_exporter, + metrics_reader, + request, +): + provider = request.getfixturevalue(provider_str) + client = provider.get_client() + + # Redo the instrumentation dance to be affected by the environment variable + OpenAIInstrumentor().uninstrument() + with mock.patch.dict( + "os.environ", {"ELASTIC_OTEL_GENAI_CAPTURE_CONTENT": "true", "ELASTIC_OTEL_GENAI_EVENTS": "log"} + ): + OpenAIInstrumentor().instrument() + + tools = [ + { + "type": "function", + "function": { + "name": "get_delivery_date", + "description": "Get the delivery date for a customer's order. Call this whenever you need to know the delivery date, for example when a customer asks 'Where is my package'", + "parameters": { + "type": "object", + "properties": { + "order_id": { + "type": "string", + "description": "The customer's order ID.", + }, + }, + "required": ["order_id"], + "additionalProperties": False, + }, + }, + } + ] + + messages = [ + { + "role": "system", + "content": "You are a helpful customer support assistant. Use the supplied tools to assist the user.", + }, + {"role": "user", "content": "Hi, can you tell me the delivery date for my order?"}, + { + "role": "assistant", + "content": "Hi there! I can help with that. Can you please provide your order ID?", + }, + {"role": "user", "content": "i think it is order_12345"}, + ] + + response = client.chat.completions.create(model=model, messages=messages, tools=tools) + tool_call = response.choices[0].message.tool_calls[0] + assert tool_call.function.name == "get_delivery_date" + assert json.loads(tool_call.function.arguments) == {"order_id": "order_12345"} + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == f"chat {model}" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.UNSET + + assert dict(span.attributes) == { + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: model, + GEN_AI_SYSTEM: "openai", + GEN_AI_RESPONSE_ID: response_id, + GEN_AI_RESPONSE_MODEL: response_model, + GEN_AI_RESPONSE_FINISH_REASONS: ("tool_calls",), + GEN_AI_USAGE_INPUT_TOKENS: input_tokens, + GEN_AI_USAGE_OUTPUT_TOKENS: output_tokens, + SERVER_ADDRESS: provider.server_address, + SERVER_PORT: provider.server_port, + } + + logs = logs_exporter.get_finished_logs() + assert len(logs) == 5 + log_records = logrecords_from_logs(logs) + system_message, user_message, assistant_message, second_user_message, choice = log_records + assert system_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.system.message"} + assert system_message.body == { + "content": "You are a helpful customer support assistant. Use the supplied tools to assist the user." + } + assert user_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert user_message.body == {"content": "Hi, can you tell me the delivery date for my order?"} + assert assistant_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.assistant.message"} + assert assistant_message.body == { + "content": "Hi there! I can help with that. Can you please provide your order ID?" + } + assert second_user_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert second_user_message.body == {"content": "i think it is order_12345"} + assert choice.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.choice"} + + expected_body = { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "tool_calls": [ + { + "function": {"arguments": '{"order_id":"order_12345"}', "name": "get_delivery_date"}, + "id": function_call_id, + "type": "function", + }, + ], + }, + } + assert dict(choice.body) == expected_body + + operation_duration_metric, token_usage_metric = get_sorted_metrics(metrics_reader) + attributes = { + GEN_AI_REQUEST_MODEL: model, + GEN_AI_RESPONSE_MODEL: response_model, + } + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) + assert_token_usage_metric( + provider, + token_usage_metric, + attributes=attributes, + input_data_point=input_tokens, + output_data_point=output_tokens, + ) + + +test_connection_error_test_data = [ + ( + "openai_provider_chat_completions", + "gpt-4o-mini", + 0.006761051714420319, + ), + ( + "azure_provider_chat_completions", + "gpt-4o-mini", + 0.971050308085978, + ), ( "ollama_provider_chat_completions", "qwen2.5:0.5b", @@ -722,7 +1045,9 @@ def test_basic_with_capture_content( GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) assert_token_usage_metric( provider, token_usage_metric, @@ -732,13 +1057,15 @@ def test_basic_with_capture_content( ) -test_stream_test_data = [ +test_basic_with_capture_content_log_events_test_data = [ ( "openai_provider_chat_completions", "gpt-4o-mini", "gpt-4o-mini-2024-07-18", - "South Atlantic Ocean.", - "chatcmpl-AEGTAvX2YSIO9EQwMleHTB91Cgn4G", + "Atlantic Ocean.", + "chatcmpl-AIEVEddriZ8trWDORY6MdqNgqRkDX", + 24, + 3, 0.006761051714420319, ), ( @@ -746,24 +1073,151 @@ def test_basic_with_capture_content( "gpt-4o-mini", "gpt-4o-mini", "South Atlantic Ocean.", - "chatcmpl-AEGTBkVVthwTc3DgRp6RGKJxyY1pE", + "chatcmpl-AIEVEmwAmuw8qGX1PViCfm0kTe9O8", + 24, + 4, 0.002889830619096756, ), ( "ollama_provider_chat_completions", "qwen2.5:0.5b", "qwen2.5:0.5b", - "The Falkland Islands, also known as the Argentinian Islands or British South America Land (BSAL), is an archipelago of several small islands and peninsulas located off the coast of Argentina. It contains nine uninhabited Falkland Islands, plus a mix of uninhabitable territory and other small features that are not considered part of the Falkland Islands' current administrative divisions.", - "chatcmpl-415", + "Atlantic Ocean", + "chatcmpl-694", + 46, + 3, 0.002600736916065216, ), ] @pytest.mark.vcr() -@pytest.mark.parametrize("provider_str,model,response_model,content,response_id,duration", test_stream_test_data) -def test_stream( - provider_str, model, response_model, content, response_id, duration, trace_exporter, metrics_reader, request +@pytest.mark.parametrize( + "provider_str,model,response_model,content,response_id,input_tokens,output_tokens,duration", + test_basic_with_capture_content_log_events_test_data, +) +def test_basic_with_capture_content_log_events( + provider_str, + model, + response_model, + content, + response_id, + input_tokens, + output_tokens, + duration, + trace_exporter, + logs_exporter, + metrics_reader, + request, +): + provider = request.getfixturevalue(provider_str) + client = provider.get_client() + + # Redo the instrumentation dance to be affected by the environment variable + OpenAIInstrumentor().uninstrument() + with mock.patch.dict( + "os.environ", {"ELASTIC_OTEL_GENAI_CAPTURE_CONTENT": "true", "ELASTIC_OTEL_GENAI_EVENTS": "log"} + ): + OpenAIInstrumentor().instrument() + + messages = [ + { + "role": "user", + "content": "Answer in up to 3 words: Which ocean contains the falkland islands?", + } + ] + + chat_completion = client.chat.completions.create(model=model, messages=messages) + + assert chat_completion.choices[0].message.content == content + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == f"chat {model}" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.UNSET + + assert dict(span.attributes) == { + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: model, + GEN_AI_SYSTEM: "openai", + GEN_AI_RESPONSE_ID: response_id, + GEN_AI_RESPONSE_MODEL: response_model, + GEN_AI_RESPONSE_FINISH_REASONS: ("stop",), + GEN_AI_USAGE_INPUT_TOKENS: input_tokens, + GEN_AI_USAGE_OUTPUT_TOKENS: output_tokens, + SERVER_ADDRESS: provider.server_address, + SERVER_PORT: provider.server_port, + } + + logs = logs_exporter.get_finished_logs() + assert len(logs) == 2 + log_records = logrecords_from_logs(logs) + user_message, choice = log_records + assert dict(user_message.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert dict(user_message.body) == {"content": "Answer in up to 3 words: Which ocean contains the falkland islands?"} + assert dict(choice.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.choice"} + + expected_body = { + "finish_reason": "stop", + "index": 0, + "message": { + "content": content, + }, + } + assert dict(choice.body) == expected_body + + operation_duration_metric, token_usage_metric = get_sorted_metrics(metrics_reader) + attributes = { + GEN_AI_REQUEST_MODEL: model, + GEN_AI_RESPONSE_MODEL: response_model, + } + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) + assert_token_usage_metric( + provider, + token_usage_metric, + attributes=attributes, + input_data_point=input_tokens, + output_data_point=output_tokens, + ) + + +test_stream_test_data = [ + ( + "openai_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + "South Atlantic Ocean.", + "chatcmpl-AEGTAvX2YSIO9EQwMleHTB91Cgn4G", + 0.006761051714420319, + ), + ( + "azure_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini", + "South Atlantic Ocean.", + "chatcmpl-AEGTBkVVthwTc3DgRp6RGKJxyY1pE", + 0.002889830619096756, + ), + ( + "ollama_provider_chat_completions", + "qwen2.5:0.5b", + "qwen2.5:0.5b", + "The Falkland Islands, also known as the Argentinian Islands or British South America Land (BSAL), is an archipelago of several small islands and peninsulas located off the coast of Argentina. It contains nine uninhabited Falkland Islands, plus a mix of uninhabitable territory and other small features that are not considered part of the Falkland Islands' current administrative divisions.", + "chatcmpl-415", + 0.002600736916065216, + ), +] + + +@pytest.mark.vcr() +@pytest.mark.parametrize("provider_str,model,response_model,content,response_id,duration", test_stream_test_data) +def test_stream( + provider_str, model, response_model, content, response_id, duration, trace_exporter, metrics_reader, request ): provider = request.getfixturevalue(provider_str) client = provider.get_client() @@ -805,7 +1259,9 @@ def test_stream( GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) # FIXME: add custom ollama @@ -895,7 +1351,9 @@ def test_stream_with_include_usage_option( GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) assert_token_usage_metric( provider, token_usage_metric, @@ -911,7 +1369,9 @@ def test_stream_with_include_usage_option( "gpt-4o-mini", "gpt-4o-mini-2024-07-18", "", - '{"order_id": "order_12345"}', + { + "gen_ai.completion": '[{"role": "assistant", "content": "", "tool_calls": [{"function": {"arguments": "{\\"order_id\\":\\"order_12345\\"}", "name": "get_delivery_date"}, "id": "call_Hb0tFTds0zHfckULpnZ4t8XL", "type": "function"}]}]' + }, "chatcmpl-AEGTFZ2zBPeLJlZ1EA10ZEDA12VfO", "tool_calls", 0.006761051714420319, @@ -921,7 +1381,9 @@ def test_stream_with_include_usage_option( "gpt-4o-mini", "gpt-4o-mini", "", - '{"order_id": "order_12345"}', + { + "gen_ai.completion": '[{"role": "assistant", "content": "", "tool_calls": [{"function": {"arguments": "{\\"order_id\\":\\"order_12345\\"}", "name": "get_delivery_date"}, "id": "call_96LHcDPBXtgIxgxbFvuJjTYU", "type": "function"}]}]' + }, "chatcmpl-AEGTHbbvqBK2BE52I8GByDCM2dypS", "tool_calls", 0.002889830619096756, @@ -931,9 +1393,13 @@ def test_stream_with_include_usage_option( "qwen2.5:0.5b", "qwen2.5:0.5b", '\n{"name": "get_delivery_date", "arguments": {"order_id": "order_12345"}}\n', - json.dumps( - '\n{"name": "get_delivery_date", "arguments": {"order_id": "order_12345"}}\n' - ), + { + "gen_ai.completion": '[{"role": "assistant", "content": ' + + json.dumps( + '\n{"name": "get_delivery_date", "arguments": {"order_id": "order_12345"}}\n' + ) + + "}]" + }, "chatcmpl-598", "stop", 0.002600736916065216, @@ -1030,59 +1496,854 @@ def test_stream_with_tools_and_capture_content( assert prompt_event.name == "gen_ai.content.prompt" assert dict(prompt_event.attributes) == {"gen_ai.prompt": json.dumps(messages)} assert completion_event.name == "gen_ai.content.completion" - assert dict(completion_event.attributes) == { - "gen_ai.completion": '[{"role": "assistant", "content": ' + completion_content + "}]" - } + assert dict(completion_event.attributes) == completion_content (operation_duration_metric,) = get_sorted_metrics(metrics_reader) attributes = { GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) -test_async_basic_test_data = [ +test_stream_with_tools_and_capture_content_log_events_test_data = [ ( "openai_provider_chat_completions", "gpt-4o-mini", "gpt-4o-mini-2024-07-18", - "South Atlantic Ocean.", - "chatcmpl-AEGqstM7lJ74sLzKJxMyhZZJixowh", - 24, - 4, + "", + '{"order_id": "order_12345"}', + "chatcmpl-AIEVFr8IGqjRC2wxrGU3tcRjNbGKf", + "tool_calls", + "call_BQ6tpzuq28epoO6jzUNSdG6r", 0.006761051714420319, ), ( "azure_provider_chat_completions", "gpt-4o-mini", "gpt-4o-mini", - "South Atlantic Ocean.", - "chatcmpl-AEGqs0IFKsicvFYSaiyWFpZVr6OHe", - 24, - 4, + "", + '{"order_id": "order_12345"}', + "chatcmpl-AIEVHRdUM6ip3Uolr8CcrlGhchugq", + "tool_calls", + "call_XNHRbrreMnt9ReHJfNH30mom", 0.002889830619096756, ), ( "ollama_provider_chat_completions", "qwen2.5:0.5b", "qwen2.5:0.5b", - "Atlantic Ocean", - "chatcmpl-425", - 46, - 3, + '\n{"name": "get_delivery_date", "arguments": {"order_id": "order_12345"}}\n', + json.dumps( + '\n{"name": "get_delivery_date", "arguments": {"order_id": "order_12345"}}\n' + ), + "chatcmpl-436", + "stop", + "ciao", 0.002600736916065216, ), ] -@pytest.mark.asyncio @pytest.mark.vcr() @pytest.mark.parametrize( - "provider_str,model,response_model,content,response_id,input_tokens,output_tokens,duration", - test_async_basic_test_data, + "provider_str,model,response_model,content,completion_content,response_id,finish_reason,function_call_id,duration", + test_stream_with_tools_and_capture_content_log_events_test_data, ) -async def test_async_basic( +def test_stream_with_tools_and_capture_content_log_events( + provider_str, + model, + response_model, + content, + completion_content, + response_id, + finish_reason, + function_call_id, + duration, + trace_exporter, + logs_exporter, + metrics_reader, + request, +): + provider = request.getfixturevalue(provider_str) + client = provider.get_client() + + # Redo the instrumentation dance to be affected by the environment variable + OpenAIInstrumentor().uninstrument() + with mock.patch.dict( + "os.environ", {"ELASTIC_OTEL_GENAI_CAPTURE_CONTENT": "true", "ELASTIC_OTEL_GENAI_EVENTS": "log"} + ): + OpenAIInstrumentor().instrument() + + tools = [ + { + "type": "function", + "function": { + "name": "get_delivery_date", + "description": "Get the delivery date for a customer's order. Call this whenever you need to know the delivery date, for example when a customer asks 'Where is my package'", + "parameters": { + "type": "object", + "properties": { + "order_id": { + "type": "string", + "description": "The customer's order ID.", + }, + }, + "required": ["order_id"], + "additionalProperties": False, + }, + }, + } + ] + + messages = [ + { + "role": "system", + "content": "You are a helpful customer support assistant. Use the supplied tools to assist the user.", + }, + {"role": "user", "content": "Hi, can you tell me the delivery date for my order?"}, + { + "role": "assistant", + "content": "Hi there! I can help with that. Can you please provide your order ID?", + }, + {"role": "user", "content": "i think it is order_12345"}, + ] + + chat_completion = client.chat.completions.create(model=model, messages=messages, tools=tools, stream=True) + + chunks = [chunk.choices[0].delta.content or "" for chunk in chat_completion if chunk.choices] + assert "".join(chunks) == content + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == f"chat {model}" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.UNSET + + assert dict(span.attributes) == { + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: model, + GEN_AI_SYSTEM: "openai", + GEN_AI_RESPONSE_ID: response_id, + GEN_AI_RESPONSE_MODEL: response_model, + GEN_AI_RESPONSE_FINISH_REASONS: (finish_reason,), + SERVER_ADDRESS: provider.server_address, + SERVER_PORT: provider.server_port, + } + + logs = logs_exporter.get_finished_logs() + assert len(logs) == 5 + log_records = logrecords_from_logs(logs) + system_message, user_message, assistant_message, second_user_message, choice = log_records + assert system_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.system.message"} + assert system_message.body == { + "content": "You are a helpful customer support assistant. Use the supplied tools to assist the user." + } + assert user_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert user_message.body == {"content": "Hi, can you tell me the delivery date for my order?"} + assert assistant_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.assistant.message"} + assert assistant_message.body == { + "content": "Hi there! I can help with that. Can you please provide your order ID?" + } + assert second_user_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert second_user_message.body == {"content": "i think it is order_12345"} + assert choice.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.choice"} + + if finish_reason == "tool_calls": + expected_body = { + "finish_reason": finish_reason, + "index": 0, + "message": { + "tool_calls": [ + { + "function": {"arguments": '{"order_id":"order_12345"}', "name": "get_delivery_date"}, + "id": function_call_id, + "type": "function", + }, + ] + }, + } + else: + expected_body = { + "finish_reason": finish_reason, + "index": 0, + "message": { + "content": content, + }, + } + assert dict(choice.body) == expected_body + + span_ctx = span.get_span_context() + assert choice.trace_id == span_ctx.trace_id + assert choice.span_id == span_ctx.span_id + assert choice.trace_flags == span_ctx.trace_flags + + (operation_duration_metric,) = get_sorted_metrics(metrics_reader) + attributes = { + GEN_AI_REQUEST_MODEL: model, + GEN_AI_RESPONSE_MODEL: response_model, + } + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) + + +# Azure is not tested because only gpt-4o version 2024-08-06 supports structured output: +# openai.BadRequestError: Error code: 400 - {'error': {'message': 'Structured output is not allowed.', 'type': 'invalid_request_error', 'param': None, 'code': None}} +test_stream_with_parallel_tools_and_capture_content_test_data = [ + ( + "openai_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + "", + { + "gen_ai.completion": '[{"role": "assistant", "content": "", "tool_calls": [{"function": {"arguments": "{\\"location\\": \\"New York\\"}", "name": "get_weather"}, "id": "call_uDsEOSTauJkgNI8ciF0AvU0X", "type": "function"}, {"function": {"arguments": "{\\"location\\": \\"London\\"}", "name": "get_weather"}, "id": "call_If8zqvcIX9JYzEYJ02dlpoBX", "type": "function"}]}]' + }, + "chatcmpl-AGooCqGLiGVX21z77Wlic8pSD93XP", + "tool_calls", + 0.006761051714420319, + ), + ( + "ollama_provider_chat_completions", + "qwen2.5:0.5b", + "qwen2.5:0.5b", + 'To provide you with the most current information about the weather, I need to know which cities you are interested in. Could we please specify the name of each city? For instance, both "New York", "London" or individual names like "New York City".', + { + "gen_ai.completion": '[{"role": "assistant", "content": "To provide you with the most current information about the weather, I need to know which cities you are interested in. Could we please specify the name of each city? For instance, both \\"New York\\", \\"London\\" or individual names like \\"New York City\\"."}]' + }, + "chatcmpl-142", + "stop", + 0.002600736916065216, + ), +] + + +@pytest.mark.vcr() +@pytest.mark.parametrize( + "provider_str,model,response_model,content,completion_content,response_id,finish_reason,duration", + test_stream_with_parallel_tools_and_capture_content_test_data, +) +def test_stream_with_parallel_tools_and_capture_content( + provider_str, + model, + response_model, + content, + completion_content, + response_id, + finish_reason, + duration, + trace_exporter, + metrics_reader, + request, +): + provider = request.getfixturevalue(provider_str) + client = provider.get_client() + + # Redo the instrumentation dance to be affected by the environment variable + OpenAIInstrumentor().uninstrument() + with mock.patch.dict("os.environ", {"ELASTIC_OTEL_GENAI_CAPTURE_CONTENT": "true"}): + OpenAIInstrumentor().instrument() + + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + }, + "required": ["location"], + "additionalProperties": False, + }, + }, + } + ] + + messages = [ + { + "role": "system", + "content": "You are a helpful assistant providing weather updates.", + }, + {"role": "user", "content": "What is the weather in New York City and London?"}, + ] + + chat_completion = client.chat.completions.create(model=model, messages=messages, tools=tools, stream=True) + + chunks = [chunk.choices[0].delta.content or "" for chunk in chat_completion if chunk.choices] + assert "".join(chunks) == content + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == f"chat {model}" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.UNSET + + assert dict(span.attributes) == { + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: model, + GEN_AI_SYSTEM: "openai", + GEN_AI_RESPONSE_ID: response_id, + GEN_AI_RESPONSE_MODEL: response_model, + GEN_AI_RESPONSE_FINISH_REASONS: (finish_reason,), + SERVER_ADDRESS: provider.server_address, + SERVER_PORT: provider.server_port, + } + + assert len(span.events) == 2 + prompt_event, completion_event = span.events + assert prompt_event.name == "gen_ai.content.prompt" + assert dict(prompt_event.attributes) == {"gen_ai.prompt": json.dumps(messages)} + assert completion_event.name == "gen_ai.content.completion" + assert dict(completion_event.attributes) == completion_content + + (operation_duration_metric,) = get_sorted_metrics(metrics_reader) + attributes = { + GEN_AI_REQUEST_MODEL: model, + GEN_AI_RESPONSE_MODEL: response_model, + } + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) + + +# Azure is not tested because only gpt-4o version 2024-08-06 supports structured output: +# openai.BadRequestError: Error code: 400 - {'error': {'message': 'Structured output is not allowed.', 'type': 'invalid_request_error', 'param': None, 'code': None}} +test_stream_with_parallel_tools_and_capture_content_log_events_test_data = [ + ( + "openai_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + "", + json.dumps(""), + "chatcmpl-AICov4avW9uwU1rfxlUzPKGG5BiCs", + "tool_calls", + 0.006761051714420319, + ), + ( + "ollama_provider_chat_completions", + "qwen2.5:0.5b", + "qwen2.5:0.5b", + "\n" + + json.dumps({"name": "get_weather", "arguments": {"location": "New York, NY"}}, indent=2) + + "\n\n\n" + + json.dumps({"name": "get_weather", "arguments": {"location": "London, UK"}}, indent=2) + + "\n", + '{"message": {"content":"\n' + + json.dumps({"name": "get_weather", "arguments": {"location": "New York, NY"}}, indent=2) + + "\n\n\n" + + json.dumps({"name": "get_weather", "arguments": {"location": "London, UK"}}, indent=2) + + "\n", + "chatcmpl-986", + "stop", + 0.002600736916065216, + ), +] + + +@pytest.mark.vcr() +@pytest.mark.parametrize( + "provider_str,model,response_model,content,completion_content,response_id,finish_reason,duration", + test_stream_with_parallel_tools_and_capture_content_log_events_test_data, +) +def test_stream_with_parallel_tools_and_capture_content_log_events( + provider_str, + model, + response_model, + content, + completion_content, + response_id, + finish_reason, + duration, + trace_exporter, + metrics_reader, + logs_exporter, + request, +): + provider = request.getfixturevalue(provider_str) + client = provider.get_client() + + # Redo the instrumentation dance to be affected by the environment variable + OpenAIInstrumentor().uninstrument() + with mock.patch.dict( + "os.environ", {"ELASTIC_OTEL_GENAI_CAPTURE_CONTENT": "true", "ELASTIC_OTEL_GENAI_EVENTS": "log"} + ): + OpenAIInstrumentor().instrument() + + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + }, + "required": ["location"], + "additionalProperties": False, + }, + }, + } + ] + + messages = [ + { + "role": "system", + "content": "You are a helpful assistant providing weather updates.", + }, + {"role": "user", "content": "What is the weather in New York City and London?"}, + ] + + chat_completion = client.chat.completions.create(model=model, messages=messages, tools=tools, stream=True) + + chunks = [chunk.choices[0].delta.content or "" for chunk in chat_completion if chunk.choices] + assert "".join(chunks) == content + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == f"chat {model}" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.UNSET + + assert dict(span.attributes) == { + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: model, + GEN_AI_SYSTEM: "openai", + GEN_AI_RESPONSE_ID: response_id, + GEN_AI_RESPONSE_MODEL: response_model, + GEN_AI_RESPONSE_FINISH_REASONS: (finish_reason,), + SERVER_ADDRESS: provider.server_address, + SERVER_PORT: provider.server_port, + } + + logs = logs_exporter.get_finished_logs() + assert len(logs) == 3 + log_records = logrecords_from_logs(logs) + system_message, user_message, choice = log_records + assert system_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.system.message"} + assert system_message.body == {"content": "You are a helpful assistant providing weather updates."} + assert user_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert user_message.body == {"content": "What is the weather in New York City and London?"} + assert choice.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.choice"} + + if finish_reason == "tool_calls": + expected_body = { + "finish_reason": finish_reason, + "index": 0, + "message": { + "tool_calls": [ + { + "function": {"arguments": '{"location": "New York City"}', "name": "get_weather"}, + "id": "call_m8FzMvtVd3wjksWMeRCWkPDK", + "type": "function", + }, + { + "function": {"arguments": '{"location": "London"}', "name": "get_weather"}, + "id": "call_4WcXUPtB1wlKUy1lOrguqAtC", + "type": "function", + }, + ] + }, + } + else: + expected_body = { + "finish_reason": finish_reason, + "index": 0, + "message": { + "content": content, + }, + } + assert dict(choice.body) == expected_body + + span_ctx = span.get_span_context() + assert choice.trace_id == span_ctx.trace_id + assert choice.span_id == span_ctx.span_id + assert choice.trace_flags == span_ctx.trace_flags + + (operation_duration_metric,) = get_sorted_metrics(metrics_reader) + attributes = { + GEN_AI_REQUEST_MODEL: model, + GEN_AI_RESPONSE_MODEL: response_model, + } + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) + + +test_tools_with_followup_and_capture_content_log_events_test_data = [ + ( + "openai_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + None, + json.dumps(""), + "tool_calls", + 0.007433261722326279, + ), + ( + "azure_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini", + None, + json.dumps(""), + "tool_calls", + 0.003254897892475128, + ), + # ollama does not return tool calls +] + + +@pytest.mark.vcr() +@pytest.mark.parametrize( + "provider_str,model,response_model,content,completion_content,finish_reason,duration", + test_tools_with_followup_and_capture_content_log_events_test_data, +) +def test_tools_with_followup_and_capture_content_log_events( + provider_str, + model, + response_model, + content, + completion_content, + finish_reason, + duration, + trace_exporter, + metrics_reader, + logs_exporter, + request, +): + provider = request.getfixturevalue(provider_str) + client = provider.get_client() + + # Redo the instrumentation dance to be affected by the environment variable + OpenAIInstrumentor().uninstrument() + with mock.patch.dict( + "os.environ", {"ELASTIC_OTEL_GENAI_CAPTURE_CONTENT": "true", "ELASTIC_OTEL_GENAI_EVENTS": "log"} + ): + OpenAIInstrumentor().instrument() + + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + }, + "required": ["location"], + "additionalProperties": False, + }, + }, + } + ] + + messages = [ + { + "role": "system", + "content": "You are a helpful assistant providing weather updates.", + }, + {"role": "user", "content": "What is the weather in New York City and London?"}, + ] + + first_response = client.chat.completions.create(model=model, messages=messages, tools=tools) + + assert first_response.choices[0].message.content == content + + first_reponse_message = first_response.choices[0].message + if hasattr(first_reponse_message, "to_dict"): + previous_message = first_response.choices[0].message.to_dict() + else: + # old pydantic from old openai client + previous_message = first_response.choices[0].message.model_dump() + followup_messages = [ + { + "role": "assistant", + "tool_calls": previous_message["tool_calls"], + }, + { + "role": "tool", + "content": "25 degrees and sunny", + "tool_call_id": previous_message["tool_calls"][0]["id"], + }, + { + "role": "tool", + "content": "15 degrees and raining", + "tool_call_id": previous_message["tool_calls"][1]["id"], + }, + ] + + second_response = client.chat.completions.create(model=model, messages=messages + followup_messages) + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 2 + + first_span, second_span = spans + assert first_span.name == f"chat {model}" + assert first_span.kind == SpanKind.CLIENT + assert first_span.status.status_code == StatusCode.UNSET + + assert dict(first_span.attributes) == { + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: model, + GEN_AI_SYSTEM: "openai", + GEN_AI_RESPONSE_ID: first_response.id, + GEN_AI_RESPONSE_MODEL: response_model, + GEN_AI_RESPONSE_FINISH_REASONS: (finish_reason,), + GEN_AI_USAGE_INPUT_TOKENS: first_response.usage.prompt_tokens, + GEN_AI_USAGE_OUTPUT_TOKENS: first_response.usage.completion_tokens, + SERVER_ADDRESS: provider.server_address, + SERVER_PORT: provider.server_port, + } + + assert second_span.name == f"chat {model}" + assert second_span.kind == SpanKind.CLIENT + assert second_span.status.status_code == StatusCode.UNSET + + assert dict(second_span.attributes) == { + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: model, + GEN_AI_SYSTEM: "openai", + GEN_AI_RESPONSE_ID: second_response.id, + GEN_AI_RESPONSE_MODEL: response_model, + GEN_AI_RESPONSE_FINISH_REASONS: ("stop",), + GEN_AI_USAGE_INPUT_TOKENS: second_response.usage.prompt_tokens, + GEN_AI_USAGE_OUTPUT_TOKENS: second_response.usage.completion_tokens, + SERVER_ADDRESS: provider.server_address, + SERVER_PORT: provider.server_port, + } + + logs = logs_exporter.get_finished_logs() + assert len(logs) == 9 + log_records = logrecords_from_logs(logs) + + # first call events + system_message, user_message, choice = log_records[:3] + assert system_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.system.message"} + assert system_message.body == {"content": "You are a helpful assistant providing weather updates."} + assert user_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert user_message.body == {"content": "What is the weather in New York City and London?"} + assert choice.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.choice"} + + expected_body = { + "finish_reason": finish_reason, + "index": 0, + "message": { + "tool_calls": [ + { + "function": {"arguments": '{"location": "New York City"}', "name": "get_weather"}, + "id": previous_message["tool_calls"][0]["id"], + "type": "function", + }, + { + "function": {"arguments": '{"location": "London"}', "name": "get_weather"}, + "id": previous_message["tool_calls"][1]["id"], + "type": "function", + }, + ] + }, + } + assert dict(choice.body) == expected_body + + # second call events + system_message, user_message, assistant_message, first_tool, second_tool, choice = log_records[3:] + assert system_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.system.message"} + assert system_message.body == {"content": "You are a helpful assistant providing weather updates."} + assert user_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert user_message.body == {"content": "What is the weather in New York City and London?"} + assert assistant_message.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.assistant.message"} + assert assistant_message.body == {"tool_calls": previous_message["tool_calls"]} + assert first_tool.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.tool.message"} + first_tool_response = previous_message["tool_calls"][0] + assert first_tool.body == {"content": "25 degrees and sunny", "id": first_tool_response["id"]} + assert second_tool.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.tool.message"} + second_tool_response = previous_message["tool_calls"][1] + assert second_tool.body == {"content": "15 degrees and raining", "id": second_tool_response["id"]} + assert choice.attributes == {"gen_ai.system": "openai", "event.name": "gen_ai.choice"} + assert choice.body == { + "index": 0, + "finish_reason": "stop", + "message": {"content": second_response.choices[0].message.content}, + } + + operation_duration_metric, token_usage_metric = get_sorted_metrics(metrics_reader) + attributes = { + GEN_AI_REQUEST_MODEL: model, + GEN_AI_RESPONSE_MODEL: response_model, + } + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration, count=2 + ) + assert_token_usage_metric( + provider, + token_usage_metric, + attributes=attributes, + input_data_point=[first_response.usage.prompt_tokens, second_response.usage.prompt_tokens], + output_data_point=[first_response.usage.completion_tokens, second_response.usage.completion_tokens], + count=2, + ) + + +test_async_basic_test_data = [ + ( + "openai_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + "South Atlantic Ocean.", + "chatcmpl-AEGqstM7lJ74sLzKJxMyhZZJixowh", + 24, + 4, + 0.006761051714420319, + ), + ( + "azure_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini", + "South Atlantic Ocean.", + "chatcmpl-AEGqs0IFKsicvFYSaiyWFpZVr6OHe", + 24, + 4, + 0.002889830619096756, + ), + ( + "ollama_provider_chat_completions", + "qwen2.5:0.5b", + "qwen2.5:0.5b", + "Atlantic Ocean", + "chatcmpl-425", + 46, + 3, + 0.002600736916065216, + ), +] + + +@pytest.mark.asyncio +@pytest.mark.vcr() +@pytest.mark.parametrize( + "provider_str,model,response_model,content,response_id,input_tokens,output_tokens,duration", + test_async_basic_test_data, +) +async def test_async_basic( + provider_str, + model, + response_model, + content, + response_id, + input_tokens, + output_tokens, + duration, + trace_exporter, + metrics_reader, + request, +): + provider = request.getfixturevalue(provider_str) + client = provider.get_async_client() + + messages = [ + { + "role": "user", + "content": "Answer in up to 3 words: Which ocean contains the falkland islands?", + } + ] + + chat_completion = await client.chat.completions.create(model=model, messages=messages) + + assert chat_completion.choices[0].message.content == content + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == f"chat {model}" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.UNSET + + assert dict(span.attributes) == { + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: model, + GEN_AI_SYSTEM: "openai", + GEN_AI_RESPONSE_ID: response_id, + GEN_AI_RESPONSE_MODEL: response_model, + GEN_AI_RESPONSE_FINISH_REASONS: ("stop",), + GEN_AI_USAGE_INPUT_TOKENS: input_tokens, + GEN_AI_USAGE_OUTPUT_TOKENS: output_tokens, + SERVER_ADDRESS: provider.server_address, + SERVER_PORT: provider.server_port, + } + assert span.events == () + + operation_duration_metric, token_usage_metric = get_sorted_metrics(metrics_reader) + attributes = { + GEN_AI_REQUEST_MODEL: model, + GEN_AI_RESPONSE_MODEL: response_model, + } + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) + assert_token_usage_metric( + provider, + token_usage_metric, + attributes=attributes, + input_data_point=input_tokens, + output_data_point=output_tokens, + ) + + +test_async_basic_with_capture_content_test_data = [ + ( + "openai_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + "South Atlantic Ocean.", + "chatcmpl-AEGquKdflabT1q3yPFXFggu0hn6Cg", + 24, + 4, + 0.006761051714420319, + ), + ( + "azure_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini", + "South Atlantic Ocean.", + "chatcmpl-AEGqv0J1RJgG477zKEayzIFWtBfoN", + 24, + 4, + 0.002889830619096756, + ), + ( + "ollama_provider_chat_completions", + "qwen2.5:0.5b", + "qwen2.5:0.5b", + "Antarctica", + "chatcmpl-280", + 46, + 4, + 0.002600736916065216, + ), +] + + +@pytest.mark.asyncio +@pytest.mark.vcr() +@pytest.mark.parametrize( + "provider_str,model,response_model,content,response_id,input_tokens,output_tokens,duration", + test_async_basic_with_capture_content_test_data, +) +async def test_async_basic_with_capture_content( provider_str, model, response_model, @@ -1098,6 +2359,11 @@ async def test_async_basic( provider = request.getfixturevalue(provider_str) client = provider.get_async_client() + # Redo the instrumentation dance to be affected by the environment variable + OpenAIInstrumentor().uninstrument() + with mock.patch.dict("os.environ", {"ELASTIC_OTEL_GENAI_CAPTURE_CONTENT": "true"}): + OpenAIInstrumentor().instrument() + messages = [ { "role": "user", @@ -1129,14 +2395,23 @@ async def test_async_basic( SERVER_ADDRESS: provider.server_address, SERVER_PORT: provider.server_port, } - assert span.events == () + assert len(span.events) == 2 + prompt_event, completion_event = span.events + assert prompt_event.name == "gen_ai.content.prompt" + assert dict(prompt_event.attributes) == {"gen_ai.prompt": json.dumps(messages)} + assert completion_event.name == "gen_ai.content.completion" + assert dict(completion_event.attributes) == { + "gen_ai.completion": ('[{"role": "assistant", "content": "' + content + '"}]') + } operation_duration_metric, token_usage_metric = get_sorted_metrics(metrics_reader) attributes = { GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) assert_token_usage_metric( provider, token_usage_metric, @@ -1146,13 +2421,13 @@ async def test_async_basic( ) -test_async_basic_with_capture_content_test_data = [ +test_async_basic_with_capture_content_log_events_test_data = [ ( "openai_provider_chat_completions", "gpt-4o-mini", "gpt-4o-mini-2024-07-18", "South Atlantic Ocean.", - "chatcmpl-AEGquKdflabT1q3yPFXFggu0hn6Cg", + "chatcmpl-AIEVJxmbMtOSyVurjk73oqE0uQhAX", 24, 4, 0.006761051714420319, @@ -1162,7 +2437,7 @@ async def test_async_basic( "gpt-4o-mini", "gpt-4o-mini", "South Atlantic Ocean.", - "chatcmpl-AEGqv0J1RJgG477zKEayzIFWtBfoN", + "chatcmpl-AIEVKUfan2sD0ScQLHPSMYn7Fet3Y", 24, 4, 0.002889830619096756, @@ -1171,10 +2446,10 @@ async def test_async_basic( "ollama_provider_chat_completions", "qwen2.5:0.5b", "qwen2.5:0.5b", - "Antarctica", - "chatcmpl-280", + "The Falkland Islands are located within the southern waters of the South Atlantic Ocean.", + "chatcmpl-472", 46, - 4, + 17, 0.002600736916065216, ), ] @@ -1184,9 +2459,9 @@ async def test_async_basic( @pytest.mark.vcr() @pytest.mark.parametrize( "provider_str,model,response_model,content,response_id,input_tokens,output_tokens,duration", - test_async_basic_with_capture_content_test_data, + test_async_basic_with_capture_content_log_events_test_data, ) -async def test_async_basic_with_capture_content( +async def test_async_basic_with_capture_content_log_events( provider_str, model, response_model, @@ -1196,6 +2471,7 @@ async def test_async_basic_with_capture_content( output_tokens, duration, trace_exporter, + logs_exporter, metrics_reader, request, ): @@ -1204,7 +2480,9 @@ async def test_async_basic_with_capture_content( # Redo the instrumentation dance to be affected by the environment variable OpenAIInstrumentor().uninstrument() - with mock.patch.dict("os.environ", {"ELASTIC_OTEL_GENAI_CAPTURE_CONTENT": "true"}): + with mock.patch.dict( + "os.environ", {"ELASTIC_OTEL_GENAI_CAPTURE_CONTENT": "true", "ELASTIC_OTEL_GENAI_EVENTS": "log"} + ): OpenAIInstrumentor().instrument() messages = [ @@ -1238,21 +2516,32 @@ async def test_async_basic_with_capture_content( SERVER_ADDRESS: provider.server_address, SERVER_PORT: provider.server_port, } - assert len(span.events) == 2 - prompt_event, completion_event = span.events - assert prompt_event.name == "gen_ai.content.prompt" - assert dict(prompt_event.attributes) == {"gen_ai.prompt": json.dumps(messages)} - assert completion_event.name == "gen_ai.content.completion" - assert dict(completion_event.attributes) == { - "gen_ai.completion": ('[{"role": "assistant", "content": "' + content + '"}]') + + logs = logs_exporter.get_finished_logs() + assert len(logs) == 2 + log_records = logrecords_from_logs(logs) + user_message, choice = log_records + assert dict(user_message.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert dict(user_message.body) == {"content": "Answer in up to 3 words: Which ocean contains the falkland islands?"} + assert dict(choice.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.choice"} + + expected_body = { + "finish_reason": "stop", + "index": 0, + "message": { + "content": content, + }, } + assert dict(choice.body) == expected_body operation_duration_metric, token_usage_metric = get_sorted_metrics(metrics_reader) attributes = { GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) assert_token_usage_metric( provider, token_usage_metric, @@ -1336,7 +2625,9 @@ async def test_async_stream( GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) test_async_stream_with_capture_content_test_data = [ @@ -1435,7 +2726,127 @@ async def test_async_stream_with_capture_content( GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) + + +test_async_stream_with_capture_content_log_events_test_data = [ + ( + "openai_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + "South Atlantic Ocean.", + "chatcmpl-AIEVLVduixLA39qqzjDIfJ2u2dPcJ", + 0.006761051714420319, + ), + ( + "azure_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini", + "South Atlantic Ocean.", + "chatcmpl-AIEVNaNWKhuoMnRJtcPUJNjbAz9Ib", + 0.002889830619096756, + ), + ( + "ollama_provider_chat_completions", + "qwen2.5:0.5b", + "qwen2.5:0.5b", + "The Falkland Islands contain the South Atlantic Ocean.", + "chatcmpl-466", + 0.002600736916065216, + ), +] + + +@pytest.mark.vcr() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "provider_str,model,response_model,content,response_id,duration", + test_async_stream_with_capture_content_log_events_test_data, +) +async def test_async_stream_with_capture_content_log_events( + provider_str, + model, + response_model, + content, + response_id, + duration, + trace_exporter, + logs_exporter, + metrics_reader, + request, +): + provider = request.getfixturevalue(provider_str) + client = provider.get_async_client() + + # Redo the instrumentation dance to be affected by the environment variable + OpenAIInstrumentor().uninstrument() + with mock.patch.dict( + "os.environ", {"ELASTIC_OTEL_GENAI_CAPTURE_CONTENT": "true", "ELASTIC_OTEL_GENAI_EVENTS": "log"} + ): + OpenAIInstrumentor().instrument() + messages = [ + { + "role": "user", + "content": "Answer in up to 3 words: Which ocean contains the falkland islands?", + } + ] + + chat_completion = await client.chat.completions.create(model=model, messages=messages, stream=True) + + chunks = [chunk.choices[0].delta.content or "" async for chunk in chat_completion if chunk.choices] + assert "".join(chunks) == content + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == f"chat {model}" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.UNSET + + assert dict(span.attributes) == { + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: model, + GEN_AI_SYSTEM: "openai", + GEN_AI_RESPONSE_ID: response_id, + GEN_AI_RESPONSE_MODEL: response_model, + GEN_AI_RESPONSE_FINISH_REASONS: ("stop",), + SERVER_ADDRESS: provider.server_address, + SERVER_PORT: provider.server_port, + } + + logs = logs_exporter.get_finished_logs() + assert len(logs) == 2 + log_records = logrecords_from_logs(logs) + user_message, choice = log_records + assert dict(user_message.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert dict(user_message.body) == {"content": "Answer in up to 3 words: Which ocean contains the falkland islands?"} + assert dict(choice.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.choice"} + + expected_body = { + "finish_reason": "stop", + "index": 0, + "message": { + "content": content, + }, + } + assert dict(choice.body) == expected_body + + span_ctx = span.get_span_context() + assert choice.trace_id == span_ctx.trace_id + assert choice.span_id == span_ctx.span_id + assert choice.trace_flags == span_ctx.trace_flags + + (operation_duration_metric,) = get_sorted_metrics(metrics_reader) + attributes = { + GEN_AI_REQUEST_MODEL: model, + GEN_AI_RESPONSE_MODEL: response_model, + } + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) # FIXME: ollama has empty tool_calls @@ -1444,8 +2855,8 @@ async def test_async_stream_with_capture_content( "openai_provider_chat_completions", "gpt-4o-mini", "gpt-4o-mini-2024-07-18", - "South Atlantic Ocean.", "chatcmpl-AEGr0SsAhppNLpPXpTtnmBiGGViQb", + "call_vZtzXVh5oO3k1IpFfuRejWHv", 140, 19, 0.006761051714420319, @@ -1454,8 +2865,8 @@ async def test_async_stream_with_capture_content( "azure_provider_chat_completions", "gpt-4o-mini", "gpt-4o-mini", - "South Atlantic Ocean", "chatcmpl-AEGr1X6ieLFOD5hlXZBx2BL2BrSLe", + "call_46cgdzJzy50oQJwWeUVWIwC3", 140, 19, 0.002889830619096756, @@ -1466,15 +2877,15 @@ async def test_async_stream_with_capture_content( @pytest.mark.vcr() @pytest.mark.asyncio @pytest.mark.parametrize( - "provider_str,model,response_model,content,response_id,input_tokens,output_tokens,duration", + "provider_str,model,response_model,response_id,function_call_id,input_tokens,output_tokens,duration", test_async_tools_with_capture_content_test_data, ) async def test_async_tools_with_capture_content( provider_str, model, response_model, - content, response_id, + function_call_id, input_tokens, output_tokens, duration, @@ -1556,15 +2967,187 @@ async def test_async_tools_with_capture_content( assert dict(prompt_event.attributes) == {"gen_ai.prompt": json.dumps(messages)} assert completion_event.name == "gen_ai.content.completion" assert dict(completion_event.attributes) == { - "gen_ai.completion": '[{"role": "assistant", "content": {"order_id": "order_12345"}}]' + "gen_ai.completion": '[{"role": "assistant", "content": "", "tool_calls": [{"id": "' + + function_call_id + + '", "type": "function", "function": {"name": "get_delivery_date", "arguments": "{\\"order_id\\":\\"order_12345\\"}"}}]}]' + } + + operation_duration_metric, token_usage_metric = get_sorted_metrics(metrics_reader) + attributes = { + GEN_AI_REQUEST_MODEL: model, + GEN_AI_RESPONSE_MODEL: response_model, + } + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) + assert_token_usage_metric( + provider, + token_usage_metric, + attributes=attributes, + input_data_point=input_tokens, + output_data_point=output_tokens, + ) + + +# FIXME: ollama has empty tool_calls +test_async_tools_with_capture_content_log_events_test_data = [ + ( + "openai_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + "chatcmpl-AIEVOegIWRWjNVbjxs4iASh4SNKAj", + "call_n4WJq7bu6UsgjdqeNYxRmGno", + "", + 140, + 19, + 0.006761051714420319, + ), + ( + "azure_provider_chat_completions", + "gpt-4o-mini", + "gpt-4o-mini", + "chatcmpl-AIEVPAAgrPcEpkQHXamdD9JpNNPep", + "call_uO2RgeshT5Xmbwg778qN14d5", + "", + 140, + 19, + 0.002889830619096756, + ), +] + + +@pytest.mark.vcr() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "provider_str,model,response_model,response_id,function_call_id,choice_content,input_tokens,output_tokens,duration", + test_async_tools_with_capture_content_log_events_test_data, +) +async def test_async_tools_with_capture_content_log_events( + provider_str, + model, + response_model, + response_id, + function_call_id, + choice_content, + input_tokens, + output_tokens, + duration, + trace_exporter, + logs_exporter, + metrics_reader, + request, +): + provider = request.getfixturevalue(provider_str) + client = provider.get_async_client() + + # Redo the instrumentation dance to be affected by the environment variable + OpenAIInstrumentor().uninstrument() + with mock.patch.dict( + "os.environ", {"ELASTIC_OTEL_GENAI_CAPTURE_CONTENT": "true", "ELASTIC_OTEL_GENAI_EVENTS": "log"} + ): + OpenAIInstrumentor().instrument() + + tools = [ + { + "type": "function", + "function": { + "name": "get_delivery_date", + "description": "Get the delivery date for a customer's order. Call this whenever you need to know the delivery date, for example when a customer asks 'Where is my package'", + "parameters": { + "type": "object", + "properties": { + "order_id": { + "type": "string", + "description": "The customer's order ID.", + }, + }, + "required": ["order_id"], + "additionalProperties": False, + }, + }, + } + ] + + messages = [ + { + "role": "system", + "content": "You are a helpful customer support assistant. Use the supplied tools to assist the user.", + }, + {"role": "user", "content": "Hi, can you tell me the delivery date for my order?"}, + { + "role": "assistant", + "content": "Hi there! I can help with that. Can you please provide your order ID?", + }, + {"role": "user", "content": "i think it is order_12345"}, + ] + + response = await client.chat.completions.create(model=model, messages=messages, tools=tools) + tool_call = response.choices[0].message.tool_calls[0] + assert tool_call.function.name == "get_delivery_date" + assert json.loads(tool_call.function.arguments) == {"order_id": "order_12345"} + + spans = trace_exporter.get_finished_spans() + assert len(spans) == 1 + + span = spans[0] + assert span.name == f"chat {model}" + assert span.kind == SpanKind.CLIENT + assert span.status.status_code == StatusCode.UNSET + + assert dict(span.attributes) == { + GEN_AI_OPERATION_NAME: "chat", + GEN_AI_REQUEST_MODEL: model, + GEN_AI_SYSTEM: "openai", + GEN_AI_RESPONSE_ID: response_id, + GEN_AI_RESPONSE_MODEL: response_model, + GEN_AI_RESPONSE_FINISH_REASONS: ("tool_calls",), + GEN_AI_USAGE_INPUT_TOKENS: input_tokens, + GEN_AI_USAGE_OUTPUT_TOKENS: output_tokens, + SERVER_ADDRESS: provider.server_address, + SERVER_PORT: provider.server_port, + } + + logs = logs_exporter.get_finished_logs() + assert len(logs) == 5 + log_records = logrecords_from_logs(logs) + system_message, user_message, assistant_message, second_user_message, choice = log_records + assert dict(system_message.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.system.message"} + assert dict(system_message.body) == { + "content": "You are a helpful customer support assistant. Use the supplied tools to assist the user." + } + assert dict(user_message.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert dict(user_message.body) == {"content": "Hi, can you tell me the delivery date for my order?"} + assert dict(assistant_message.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.assistant.message"} + assert dict(assistant_message.body) == { + "content": "Hi there! I can help with that. Can you please provide your order ID?" } + assert dict(second_user_message.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.user.message"} + assert dict(second_user_message.body) == {"content": "i think it is order_12345"} + assert dict(choice.attributes) == {"gen_ai.system": "openai", "event.name": "gen_ai.choice"} + + expected_body = { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "tool_calls": [ + { + "function": {"arguments": '{"order_id":"order_12345"}', "name": "get_delivery_date"}, + "id": function_call_id, + "type": "function", + }, + ], + }, + } + assert dict(choice.body) == expected_body operation_duration_metric, token_usage_metric = get_sorted_metrics(metrics_reader) attributes = { GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: response_model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) assert_token_usage_metric( provider, token_usage_metric, diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_embeddings.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_embeddings.py index fec7745..b86183a 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_embeddings.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/test_embeddings.py @@ -80,7 +80,9 @@ def test_basic(provider_str, model, input_tokens, duration, trace_exporter, metr GEN_AI_RESPONSE_MODEL: model, } operation_duration_metric, token_usage_metric = get_sorted_metrics(metrics_reader) - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) assert_token_usage_input_metric(provider, token_usage_metric, attributes=attributes, input_data_point=input_tokens) @@ -128,15 +130,15 @@ def test_all_the_client_options(provider_str, model, input_tokens, duration, tra GEN_AI_RESPONSE_MODEL: model, } assert_operation_duration_metric( - provider, operation_duration_metric, attributes=attributes, data_point=0.2263190783560276 + provider, operation_duration_metric, attributes=attributes, min_data_point=0.2263190783560276 ) assert_token_usage_input_metric(provider, token_usage_metric, attributes=attributes, input_data_point=4) test_connection_error_data = [ - ("openai_provider_embeddings", "text-embedding-3-small", 0.2263190783560276), - ("azure_provider_embeddings", "text-embedding-3-small", 0.0017870571464300156), - ("ollama_provider_embeddings", "all-minilm:33m", 0.0030461717396974564), + ("openai_provider_embeddings", "text-embedding-3-small", 0.460242404602468), + ("azure_provider_embeddings", "text-embedding-3-small", 0.4328950522467494), + ("ollama_provider_embeddings", "all-minilm:33m", 0.4006666960194707), ] @@ -226,7 +228,9 @@ async def test_async_basic(provider_str, model, input_tokens, duration, trace_ex GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) assert_token_usage_input_metric(provider, token_usage_metric, attributes=attributes, input_data_point=input_tokens) @@ -276,7 +280,9 @@ async def test_async_all_the_client_options( GEN_AI_REQUEST_MODEL: model, GEN_AI_RESPONSE_MODEL: model, } - assert_operation_duration_metric(provider, operation_duration_metric, attributes=attributes, data_point=duration) + assert_operation_duration_metric( + provider, operation_duration_metric, attributes=attributes, min_data_point=duration + ) assert_token_usage_input_metric(provider, token_usage_metric, attributes=attributes, input_data_point=input_tokens) diff --git a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/utils.py b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/utils.py index 8a98dd8..61bf75a 100644 --- a/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/utils.py +++ b/instrumentation/elastic-opentelemetry-instrumentation-openai/tests/utils.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Sequence +from typing import Mapping, Optional, Sequence +from opentelemetry.sdk._logs._internal import LogData from opentelemetry.sdk.metrics._internal.point import Metric from opentelemetry.sdk.metrics.export import ( InMemoryMetricReader, @@ -21,6 +22,7 @@ HistogramDataPoint, NumberDataPoint, ) +from opentelemetry.util.types import AttributeValue def get_sorted_metrics(memory_metrics_reader: InMemoryMetricReader): @@ -117,3 +119,7 @@ def create_histogram_data_point(sum_data_point, count, max_data_point, min_data_ bucket_counts=[], explicit_bounds=[], ) + + +def logrecords_from_logs(logs: Sequence[LogData]) -> Sequence[Mapping[str, AttributeValue]]: + return [log.log_record for log in logs]