From 953305e8e18acbf503b1ed3b4d9cce683f0a99ea Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 25 Mar 2025 13:08:18 -0700 Subject: [PATCH 01/26] Update attribute extraction to support dict as well as object. --- .../openai_agents/attributes/__init__.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/agentops/instrumentation/openai_agents/attributes/__init__.py b/agentops/instrumentation/openai_agents/attributes/__init__.py index df987951c..be5e38621 100644 --- a/agentops/instrumentation/openai_agents/attributes/__init__.py +++ b/agentops/instrumentation/openai_agents/attributes/__init__.py @@ -34,7 +34,7 @@ def _extract_attributes_from_mapping(span_data: Any, attribute_mapping: Attribut """Helper function to extract attributes based on a mapping. Args: - span_data: The span data object to extract attributes from + span_data: The span data object or dict to extract attributes from attribute_mapping: Dictionary mapping target attributes to source attributes Returns: @@ -43,22 +43,22 @@ def _extract_attributes_from_mapping(span_data: Any, attribute_mapping: Attribut attributes = {} for target_attr, source_attr in attribute_mapping.items(): if hasattr(span_data, source_attr): + # Use getattr to handle properties value = getattr(span_data, source_attr) - - # Skip if value is None or empty - if value is None or (isinstance(value, (list, dict, str)) and not value): - continue - - # Join lists to comma-separated strings - if source_attr == "tools" or source_attr == "handoffs": - if isinstance(value, list): - value = ",".join(value) - else: - value = str(value) - # Serialize complex objects - elif isinstance(value, (dict, list, object)) and not isinstance(value, (str, int, float, bool)): - value = safe_serialize(value) - - attributes[target_attr] = value + elif isinstance(span_data, dict) and source_attr in span_data: + # Use direct key access for dicts + value = span_data[source_attr] + else: + continue + + # Skip if value is None or empty + if value is None or (isinstance(value, (list, dict, str)) and not value): + continue + + # Serialize complex objects + elif isinstance(value, (dict, list, object)) and not isinstance(value, (str, int, float, bool)): + value = safe_serialize(value) + + attributes[target_attr] = value return attributes \ No newline at end of file From ab8dad144661ead321033b0e2cd282a7229a0448 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 25 Mar 2025 13:31:30 -0700 Subject: [PATCH 02/26] Adjust tests to match serialization format of `list[str]`. Patch JSON encode in tests to handle MagicMock objects. --- .../openai_agents/test_openai_agents.py | 2 +- .../test_openai_agents_attributes.py | 37 +++++++++++++++++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/tests/unit/instrumentation/openai_agents/test_openai_agents.py b/tests/unit/instrumentation/openai_agents/test_openai_agents.py index 26fe9d79f..34db816ff 100644 --- a/tests/unit/instrumentation/openai_agents/test_openai_agents.py +++ b/tests/unit/instrumentation/openai_agents/test_openai_agents.py @@ -288,7 +288,7 @@ def test_span_hierarchy_and_attributes(self, instrumentation): assert parent_captured_attributes[AgentAttributes.AGENT_NAME] == "parent_agent" assert parent_captured_attributes[WorkflowAttributes.WORKFLOW_INPUT] == "parent input" assert parent_captured_attributes[WorkflowAttributes.FINAL_OUTPUT] == "parent output" - assert parent_captured_attributes[AgentAttributes.AGENT_TOOLS] == "tool1,tool2" + assert parent_captured_attributes[AgentAttributes.AGENT_TOOLS] == '["tool1", "tool2"]' # JSON encoded is fine. # Verify child span attributes assert child_captured_attributes[AgentAttributes.AGENT_NAME] == "child_agent" diff --git a/tests/unit/instrumentation/openai_agents/test_openai_agents_attributes.py b/tests/unit/instrumentation/openai_agents/test_openai_agents_attributes.py index 3c0908f13..5c39c1830 100644 --- a/tests/unit/instrumentation/openai_agents/test_openai_agents_attributes.py +++ b/tests/unit/instrumentation/openai_agents/test_openai_agents_attributes.py @@ -116,8 +116,36 @@ def load_fixture(fixture_name): @pytest.fixture(autouse=True) def mock_external_dependencies(): """Mock any external dependencies to avoid actual API calls or slow operations""" - with patch('importlib.metadata.version', return_value='1.0.0'): - with patch('agentops.helpers.serialization.safe_serialize', side_effect=lambda x: str(x)[:100]): + # Create a more comprehensive mock for JSON serialization + # This will directly patch the json.dumps function which is used inside safe_serialize + + # Store the original json.dumps function + original_dumps = json.dumps + + # Create a wrapper for json.dumps that handles MagicMock objects + def json_dumps_wrapper(*args, **kwargs): + """ + Our JSON encode method doesn't play well with MagicMock objects and gets stuck iun a recursive loop. + Patch the functionality to return a simple string instead of trying to serialize the object. + """ + # If the first argument is a MagicMock, return a simple string + if args and hasattr(args[0], '__module__') and 'mock' in args[0].__module__.lower(): + return '"mock_object"' + # Otherwise, use the original function with a custom encoder that handles MagicMock objects + cls = kwargs.get('cls', None) + if not cls: + # Use our own encoder that handles MagicMock objects + class MagicMockJSONEncoder(json.JSONEncoder): + def default(self, obj): + if hasattr(obj, '__module__') and 'mock' in obj.__module__.lower(): + return 'mock_object' + return super().default(obj) + kwargs['cls'] = MagicMockJSONEncoder + # Call the original dumps with our encoder + return original_dumps(*args, **kwargs) + + with patch('json.dumps', side_effect=json_dumps_wrapper): + with patch('importlib.metadata.version', return_value='1.0.0'): with patch('agentops.instrumentation.openai_agents.LIBRARY_NAME', 'openai'): with patch('agentops.instrumentation.openai_agents.LIBRARY_VERSION', '1.0.0'): yield @@ -138,7 +166,8 @@ def test_common_instrumentation_attributes(self): # Verify values assert attrs[InstrumentationAttributes.NAME] == "agentops" - assert attrs[InstrumentationAttributes.VERSION] == get_agentops_version() # Use actual version + # Don't call get_agentops_version() again, just verify it's in the dictionary + assert InstrumentationAttributes.VERSION in attrs assert attrs[InstrumentationAttributes.LIBRARY_NAME] == LIBRARY_NAME def test_agent_span_attributes(self): @@ -158,7 +187,7 @@ def test_agent_span_attributes(self): assert attrs[AgentAttributes.AGENT_NAME] == "test_agent" assert attrs[WorkflowAttributes.WORKFLOW_INPUT] == "test input" assert attrs[WorkflowAttributes.FINAL_OUTPUT] == "test output" - assert attrs[AgentAttributes.AGENT_TOOLS] == "tool1,tool2" + assert attrs[AgentAttributes.AGENT_TOOLS] == '["tool1", "tool2"]' # JSON-serialized string is fine. # LLM_PROMPTS is handled in common.py now so we don't test for it directly def test_function_span_attributes(self): From fd4b4f8cbb692af37e7f81771f37eb1d129f4ea0 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 25 Mar 2025 15:16:24 -0700 Subject: [PATCH 03/26] Instrumentor and wrappers for OpenAI responses. --- agentops/instrumentation/openai/__init__.py | 9 ++ .../openai/attributes/__init__.py | 7 + .../instrumentation/openai/instrumentor.py | 111 ++++++++++++++++ agentops/instrumentation/openai/wrappers.py | 122 ++++++++++++++++++ 4 files changed, 249 insertions(+) create mode 100644 agentops/instrumentation/openai/__init__.py create mode 100644 agentops/instrumentation/openai/attributes/__init__.py create mode 100644 agentops/instrumentation/openai/instrumentor.py create mode 100644 agentops/instrumentation/openai/wrappers.py diff --git a/agentops/instrumentation/openai/__init__.py b/agentops/instrumentation/openai/__init__.py new file mode 100644 index 000000000..de7ec77dc --- /dev/null +++ b/agentops/instrumentation/openai/__init__.py @@ -0,0 +1,9 @@ +"""OpenAI API instrumentation for AgentOps. + +This package provides OpenTelemetry-based instrumentation for OpenAI API calls, +extending the third-party instrumentation to add support for OpenAI responses. +""" + +from agentops.instrumentation.openai.instrumentor import OpenAIInstrumentor + +__all__ = ["OpenAIInstrumentor"] \ No newline at end of file diff --git a/agentops/instrumentation/openai/attributes/__init__.py b/agentops/instrumentation/openai/attributes/__init__.py new file mode 100644 index 000000000..719a92a32 --- /dev/null +++ b/agentops/instrumentation/openai/attributes/__init__.py @@ -0,0 +1,7 @@ +"""Attribute helpers for OpenAI API instrumentation. + +This package contains helpers for extracting attributes from OpenAI API responses +for use in OpenTelemetry spans. +""" + +# Will contain attribute extraction helpers in the future \ No newline at end of file diff --git a/agentops/instrumentation/openai/instrumentor.py b/agentops/instrumentation/openai/instrumentor.py new file mode 100644 index 000000000..cd6c48496 --- /dev/null +++ b/agentops/instrumentation/openai/instrumentor.py @@ -0,0 +1,111 @@ +"""OpenAI API Instrumentation for AgentOps + +This module provides instrumentation for the OpenAI API, extending the third-party +OpenTelemetry instrumentation to add support for OpenAI responses. + +We subclass the OpenAIV1Instrumentor from the third-party package and add our own +wrapper for the new `openai.responses` call pattern used in the Agents SDK. + +Notes on OpenAI Responses API structure: +- The module is located at openai.resources.responses +- The main class is Responses which inherits from SyncAPIResource +- The create() method generates model responses and returns a Response object +- Key parameters for create(): + - input: Union[str, ResponseInputParam] - Text or other input to the model + - model: Union[str, ChatModel] - The model to use + - tools: Iterable[ToolParam] - Tools for the model to use + - stream: Optional[Literal[False]] - Streaming is handled by a separate method +- The Response object contains response data including usage information + +When instrumenting, we need to: +1. Wrap the Responses.create method +2. Extract data from both the request parameters and response object +3. Create spans with appropriate attributes for observability +""" +from typing import Collection + +from wrapt import wrap_function_wrapper +from opentelemetry.instrumentation.utils import unwrap + +# Import third-party OpenAIV1Instrumentor +from opentelemetry.instrumentation.openai.v1 import OpenAIV1Instrumentor as ThirdPartyOpenAIV1Instrumentor +from opentelemetry.trace import get_tracer +from opentelemetry.instrumentation.openai.version import __version__ + +from agentops.logging import logger +from agentops.instrumentation.openai.wrappers import sync_responses_wrapper, async_responses_wrapper + +# Methods to wrap beyond what the third-party instrumentation handles +WRAPPED_METHODS = [ + { + "package": "openai.resources.responses", + "object": "Responses", + "method": "create", + "wrapper": sync_responses_wrapper, + }, + { + "package": "openai.resources.responses", + "object": "AsyncResponses", + "method": "create", + "wrapper": async_responses_wrapper, + }, +] + + +class OpenAIInstrumentor(ThirdPartyOpenAIV1Instrumentor): + """An instrumentor for OpenAI API that extends the third-party implementation.""" + + def instrumentation_dependencies(self) -> Collection[str]: + """Return packages required for instrumentation.""" + return ["openai >= 1.0.0"] + + def _instrument(self, **kwargs): + """Instrument the OpenAI API, extending the third-party instrumentation. + + This implementation calls the parent _instrument method to handle + standard OpenAI API endpoints, then adds our own instrumentation for + the responses module. + """ + # Call the parent _instrument method first to handle all standard cases + super()._instrument(**kwargs) + + tracer_provider = kwargs.get("tracer_provider") + tracer = get_tracer(__name__, __version__, tracer_provider) + + # Add our own wrappers for additional modules + for wrapped_method in WRAPPED_METHODS: + package = wrapped_method["package"] + object_name = wrapped_method["object"] + method_name = wrapped_method["method"] + wrapper_func = wrapped_method["wrapper"] + + try: + wrap_function_wrapper( + package, + f"{object_name}.{method_name}", + wrapper_func(tracer), + ) + logger.debug(f"Successfully wrapped {package}.{object_name}.{method_name}") + except (AttributeError, ModuleNotFoundError) as e: + logger.debug(f"Failed to wrap {package}.{object_name}.{method_name}: {e}") + + logger.debug("Successfully instrumented OpenAI API with AgentOps extensions") + + def _uninstrument(self, **kwargs): + """Remove instrumentation from OpenAI API.""" + # Call the parent _uninstrument method to handle the standard instrumentations + super()._uninstrument(**kwargs) + + # Unwrap our additional methods + for wrapped_method in WRAPPED_METHODS: + package = wrapped_method["package"] + object_name = wrapped_method["object"] + method_name = wrapped_method["method"] + + try: + unwrap(f"{package}.{object_name}", method_name) + logger.debug(f"Successfully unwrapped {package}.{object_name}.{method_name}") + except Exception as e: + logger.debug(f"Failed to unwrap {package}.{object_name}.{method_name}: {e}") + + logger.debug("Successfully removed OpenAI API instrumentation with AgentOps extensions") \ No newline at end of file diff --git a/agentops/instrumentation/openai/wrappers.py b/agentops/instrumentation/openai/wrappers.py new file mode 100644 index 000000000..2d90e102e --- /dev/null +++ b/agentops/instrumentation/openai/wrappers.py @@ -0,0 +1,122 @@ +"""Wrapper functions for OpenAI API instrumentation. + +This module contains wrapper functions for the OpenAI API calls, +including both synchronous and asynchronous variants. +""" +from opentelemetry import context as context_api +from opentelemetry.instrumentation.utils import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.trace import SpanKind, Status, StatusCode +import time + + +def _create_response_span(tracer, span_name="openai.responses.create"): + """Create a span for an OpenAI Responses API call. + + Args: + tracer: The OpenTelemetry tracer to use + span_name: The name to use for the span + + Returns: + A span for the API call + """ + return tracer.start_span( + span_name, + kind=SpanKind.CLIENT, + ) + + +def _handle_response_span(span, start_time, success=True, exception=None): + """Handle common tasks for a response span. + + Args: + span: The span to handle + start_time: The start time of the operation + success: Whether the operation was successful + exception: Any exception that occurred + """ + # Set status based on success + if success: + span.set_status(Status(StatusCode.OK)) + else: + span.record_exception(exception) + span.set_status(Status(StatusCode.ERROR, str(exception))) + + # Record the duration + duration = time.time() - start_time + span.set_attribute("duration_s", duration) + + +def sync_responses_wrapper(tracer): + """Wrapper for synchronous openai.resources.responses.Responses.create API calls. + + This wrapper creates a span for tracking OpenAI Responses API calls. + The detailed attribute extraction will be handled separately. + + Args: + tracer: The OpenTelemetry tracer to use for creating spans + + Returns: + A wrapper function that instruments the synchronous OpenAI Responses API + """ + def wrapper(wrapped, instance, args, kwargs): + # Skip instrumentation if it's suppressed in the current context + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return wrapped(*args, **kwargs) + + # Start a span for the responses API call + with tracer.start_as_current_span( + "openai.responses.create", + kind=SpanKind.CLIENT, + ) as span: + # Record the start time for duration calculation + start_time = time.time() + + # Execute the wrapped function and get the response + try: + response = wrapped(*args, **kwargs) + _handle_response_span(span, start_time, success=True) + except Exception as e: + _handle_response_span(span, start_time, success=False, exception=e) + raise + + return response + + return wrapper + + +def async_responses_wrapper(tracer): + """Wrapper for asynchronous openai.resources.responses.AsyncResponses.create API calls. + + This wrapper creates a span for tracking asynchronous OpenAI Responses API calls. + The detailed attribute extraction will be handled separately. + + Args: + tracer: The OpenTelemetry tracer to use for creating spans + + Returns: + A wrapper function that instruments the asynchronous OpenAI Responses API + """ + async def wrapper(wrapped, instance, args, kwargs): + # Skip instrumentation if it's suppressed in the current context + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return await wrapped(*args, **kwargs) + + # Start a span for the responses API call + with tracer.start_as_current_span( + "openai.responses.create", + kind=SpanKind.CLIENT, + ) as span: + # Record the start time for duration calculation + start_time = time.time() + + # Execute the wrapped function and get the response + try: + response = await wrapped(*args, **kwargs) + _handle_response_span(span, start_time, success=True) + except Exception as e: + _handle_response_span(span, start_time, success=False, exception=e) + raise + + return response + + return wrapper \ No newline at end of file From 4c6b12fc3a6be68ff80a7be8a3d9819cdaa7cc6e Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 25 Mar 2025 15:21:23 -0700 Subject: [PATCH 04/26] Collect base usage attributes, too. --- agentops/instrumentation/openai_agents/attributes/response.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/agentops/instrumentation/openai_agents/attributes/response.py b/agentops/instrumentation/openai_agents/attributes/response.py index 62a62b909..def98b56c 100644 --- a/agentops/instrumentation/openai_agents/attributes/response.py +++ b/agentops/instrumentation/openai_agents/attributes/response.py @@ -294,6 +294,10 @@ def get_response_usage_attributes(usage: 'ResponseUsage') -> AttributeMap: # ) attributes = {} + attributes.update(_extract_attributes_from_mapping( + usage.__dict__, + RESPONSE_USAGE_ATTRIBUTES)) + # input_tokens_details is a dict if it exists if hasattr(usage, 'input_tokens_details'): input_details = usage.input_tokens_details From 5589ec64523dd049d1fbe146d269152e1254feb6 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 25 Mar 2025 15:51:36 -0700 Subject: [PATCH 05/26] Move Response attribute parsing to openai module. Move common attribute parsing to common module. --- agentops/instrumentation/common/__init__.py | 0 agentops/instrumentation/common/attributes.py | 138 ++++++++++++++++++ .../attributes/response.py | 0 .../openai_agents/attributes/__init__.py | 64 -------- .../openai_agents/attributes/common.py | 103 ++++++------- .../instrumentation/openai_agents/exporter.py | 8 +- 6 files changed, 192 insertions(+), 121 deletions(-) create mode 100644 agentops/instrumentation/common/__init__.py create mode 100644 agentops/instrumentation/common/attributes.py rename agentops/instrumentation/{openai_agents => openai}/attributes/response.py (100%) diff --git a/agentops/instrumentation/common/__init__.py b/agentops/instrumentation/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/agentops/instrumentation/common/attributes.py b/agentops/instrumentation/common/attributes.py new file mode 100644 index 000000000..5f8908a57 --- /dev/null +++ b/agentops/instrumentation/common/attributes.py @@ -0,0 +1,138 @@ +"""Attribute processing modules for OpenAI Agents instrumentation. + +This package provides specialized getter functions that extract and format +OpenTelemetry-compatible attributes from span data. Each function follows a +consistent pattern: + +1. Takes span data (or specific parts of span data) as input +2. Processes the data according to semantic conventions +3. Returns a dictionary of formatted attributes + +The modules are organized by functional domain: + +- common: Core attribute extraction functions for all span types +- tokens: Token usage extraction and processing +- model: Model information and parameter extraction +- completion: Completion content and tool call processing + +Each getter function is focused on a single responsibility and does not +modify any global state. Functions are designed to be composable, allowing +different attribute types to be combined as needed in the exporter. + +The separation of attribute extraction (getters in this module) from +attribute application (managed by exporter) follows the principle of +separation of concerns. +""" +from typing import Dict, Any +from agentops.logging import logger +from agentops.helpers import safe_serialize, get_agentops_version +from agentops.semconv import ( + CoreAttributes, + InstrumentationAttributes, + WorkflowAttributes, +) + +# target_attribute_key: source_attribute +AttributeMap = Dict[str, Any] + + +# Common attribute mapping for all span types +COMMON_ATTRIBUTES: AttributeMap = { + CoreAttributes.TRACE_ID: "trace_id", + CoreAttributes.SPAN_ID: "span_id", + CoreAttributes.PARENT_ID: "parent_id", +} + + +def _extract_attributes_from_mapping(span_data: Any, attribute_mapping: AttributeMap) -> AttributeMap: + """Helper function to extract attributes based on a mapping. + + Args: + span_data: The span data object or dict to extract attributes from + attribute_mapping: Dictionary mapping target attributes to source attributes + + Returns: + Dictionary of extracted attributes + """ + attributes = {} + for target_attr, source_attr in attribute_mapping.items(): + if hasattr(span_data, source_attr): + # Use getattr to handle properties + value = getattr(span_data, source_attr) + elif isinstance(span_data, dict) and source_attr in span_data: + # Use direct key access for dicts + value = span_data[source_attr] + else: + continue + + # Skip if value is None or empty + if value is None or (isinstance(value, (list, dict, str)) and not value): + continue + + # Serialize complex objects + elif isinstance(value, (dict, list, object)) and not isinstance(value, (str, int, float, bool)): + value = safe_serialize(value) + + attributes[target_attr] = value + + return attributes + + +def get_common_attributes() -> AttributeMap: + """Get common instrumentation attributes used across traces and spans. + + Returns: + Dictionary of common instrumentation attributes + """ + return { + InstrumentationAttributes.NAME: "agentops", + InstrumentationAttributes.VERSION: get_agentops_version(), + } + + +def get_base_trace_attributes(trace: Any) -> AttributeMap: + """Create the base attributes dictionary for an OpenTelemetry trace. + + Args: + trace: The trace object to extract attributes from + + Returns: + Dictionary containing base trace attributes + """ + if not hasattr(trace, 'trace_id'): + logger.warning("Cannot create trace attributes: missing trace_id") + return {} + + attributes = { + WorkflowAttributes.WORKFLOW_NAME: trace.name, + CoreAttributes.TRACE_ID: trace.trace_id, + WorkflowAttributes.WORKFLOW_STEP_TYPE: "trace", + **get_common_attributes(), + } + + return attributes + + +def get_base_span_attributes(span: Any) -> AttributeMap: + """Create the base attributes dictionary for an OpenTelemetry span. + + Args: + span: The span object to extract attributes from + + Returns: + Dictionary containing base span attributes + """ + span_id = getattr(span, 'span_id', 'unknown') + trace_id = getattr(span, 'trace_id', 'unknown') + parent_id = getattr(span, 'parent_id', None) + + attributes = { + CoreAttributes.TRACE_ID: trace_id, + CoreAttributes.SPAN_ID: span_id, + **get_common_attributes(), + } + + if parent_id: + attributes[CoreAttributes.PARENT_ID] = parent_id + + return attributes \ No newline at end of file diff --git a/agentops/instrumentation/openai_agents/attributes/response.py b/agentops/instrumentation/openai/attributes/response.py similarity index 100% rename from agentops/instrumentation/openai_agents/attributes/response.py rename to agentops/instrumentation/openai/attributes/response.py diff --git a/agentops/instrumentation/openai_agents/attributes/__init__.py b/agentops/instrumentation/openai_agents/attributes/__init__.py index be5e38621..e69de29bb 100644 --- a/agentops/instrumentation/openai_agents/attributes/__init__.py +++ b/agentops/instrumentation/openai_agents/attributes/__init__.py @@ -1,64 +0,0 @@ -"""Attribute processing modules for OpenAI Agents instrumentation. - -This package provides specialized getter functions that extract and format -OpenTelemetry-compatible attributes from span data. Each function follows a -consistent pattern: - -1. Takes span data (or specific parts of span data) as input -2. Processes the data according to semantic conventions -3. Returns a dictionary of formatted attributes - -The modules are organized by functional domain: - -- common: Core attribute extraction functions for all span types -- tokens: Token usage extraction and processing -- model: Model information and parameter extraction -- completion: Completion content and tool call processing - -Each getter function is focused on a single responsibility and does not -modify any global state. Functions are designed to be composable, allowing -different attribute types to be combined as needed in the exporter. - -The separation of attribute extraction (getters in this module) from -attribute application (managed by exporter) follows the principle of -separation of concerns. -""" -from typing import Dict, Any -from agentops.helpers import safe_serialize - - -# target_attribute_key: source_attribute -AttributeMap = Dict[str, Any] - -def _extract_attributes_from_mapping(span_data: Any, attribute_mapping: AttributeMap) -> AttributeMap: - """Helper function to extract attributes based on a mapping. - - Args: - span_data: The span data object or dict to extract attributes from - attribute_mapping: Dictionary mapping target attributes to source attributes - - Returns: - Dictionary of extracted attributes - """ - attributes = {} - for target_attr, source_attr in attribute_mapping.items(): - if hasattr(span_data, source_attr): - # Use getattr to handle properties - value = getattr(span_data, source_attr) - elif isinstance(span_data, dict) and source_attr in span_data: - # Use direct key access for dicts - value = span_data[source_attr] - else: - continue - - # Skip if value is None or empty - if value is None or (isinstance(value, (list, dict, str)) and not value): - continue - - # Serialize complex objects - elif isinstance(value, (dict, list, object)) and not isinstance(value, (str, int, float, bool)): - value = safe_serialize(value) - - attributes[target_attr] = value - - return attributes \ No newline at end of file diff --git a/agentops/instrumentation/openai_agents/attributes/common.py b/agentops/instrumentation/openai_agents/attributes/common.py index d3f532e1f..e715bf94d 100644 --- a/agentops/instrumentation/openai_agents/attributes/common.py +++ b/agentops/instrumentation/openai_agents/attributes/common.py @@ -6,7 +6,6 @@ """ from typing import Any from agentops.logging import logger -from agentops.helpers import get_agentops_version from agentops.semconv import ( CoreAttributes, AgentAttributes, @@ -14,21 +13,16 @@ SpanAttributes, InstrumentationAttributes ) + +from agentops.instrumentation.common.attributes import AttributeMap, _extract_attributes_from_mapping +from agentops.instrumentation.common.attributes import get_common_attributes +from agentops.instrumentation.openai.attributes.response import get_response_response_attributes + from agentops.instrumentation.openai_agents import LIBRARY_NAME, LIBRARY_VERSION -from agentops.instrumentation.openai_agents.attributes import AttributeMap, _extract_attributes_from_mapping from agentops.instrumentation.openai_agents.attributes.model import extract_model_config -from agentops.instrumentation.openai_agents.attributes.response import get_response_response_attributes from agentops.instrumentation.openai_agents.attributes.completion import get_generation_output_attributes -# Common attribute mapping for all span types -COMMON_ATTRIBUTES: AttributeMap = { - CoreAttributes.TRACE_ID: "trace_id", - CoreAttributes.SPAN_ID: "span_id", - CoreAttributes.PARENT_ID: "parent_id", -} - - # Attribute mapping for AgentSpanData AGENT_SPAN_ATTRIBUTES: AttributeMap = { AgentAttributes.AGENT_NAME: "name", @@ -69,76 +63,71 @@ } -def get_common_instrumentation_attributes() -> AttributeMap: - """Get common instrumentation attributes used across traces and spans. +def get_library_attributes() -> AttributeMap: + """Get common attributes for the OpenAI Agents library. Returns: - Dictionary of common instrumentation attributes + Dictionary of common library attributes """ return { - InstrumentationAttributes.NAME: "agentops", - InstrumentationAttributes.VERSION: get_agentops_version(), - InstrumentationAttributes.LIBRARY_NAME: LIBRARY_NAME, - InstrumentationAttributes.LIBRARY_VERSION: LIBRARY_VERSION, + InstrumentationAttributes.NAME: LIBRARY_NAME, + InstrumentationAttributes.VERSION: LIBRARY_VERSION, } -def get_base_trace_attributes(trace: Any) -> AttributeMap: - """Create the base attributes dictionary for an OpenTelemetry trace. +def get_agent_span_attributes(span_data: Any) -> AttributeMap: + """Extract attributes from an AgentSpanData object. + + Agents are requests made to the `openai.agents` endpoint. Args: - trace: The trace object to extract attributes from + span_data: The AgentSpanData object Returns: - Dictionary containing base trace attributes + Dictionary of attributes for agent span """ - if not hasattr(trace, 'trace_id'): - logger.warning("Cannot create trace attributes: missing trace_id") - return {} - - attributes = { - WorkflowAttributes.WORKFLOW_NAME: trace.name, - CoreAttributes.TRACE_ID: trace.trace_id, - WorkflowAttributes.WORKFLOW_STEP_TYPE: "trace", - **get_common_instrumentation_attributes() - } + attributes = _extract_attributes_from_mapping(span_data, AGENT_SPAN_ATTRIBUTES) + attributes.update(get_common_attributes()) + attributes.update(get_library_attributes()) return attributes -def get_base_span_attributes(span: Any) -> AttributeMap: - """Create the base attributes dictionary for an OpenTelemetry span. +def get_function_span_attributes(span_data: Any) -> AttributeMap: + """Extract attributes from a FunctionSpanData object. + + Functions are requests made to the `openai.functions` endpoint. Args: - span: The span object to extract attributes from + span_data: The FunctionSpanData object Returns: - Dictionary containing base span attributes + Dictionary of attributes for function span """ - span_id = getattr(span, 'span_id', 'unknown') - trace_id = getattr(span, 'trace_id', 'unknown') - parent_id = getattr(span, 'parent_id', None) - - attributes = { - CoreAttributes.TRACE_ID: trace_id, - CoreAttributes.SPAN_ID: span_id, - **get_common_instrumentation_attributes(), - } + attributes = _extract_attributes_from_mapping(span_data, FUNCTION_SPAN_ATTRIBUTES) + attributes.update(get_common_attributes()) + attributes.update(get_library_attributes()) - if parent_id: - attributes[CoreAttributes.PARENT_ID] = parent_id - return attributes -get_agent_span_attributes = lambda span_data: \ - _extract_attributes_from_mapping(span_data, AGENT_SPAN_ATTRIBUTES) - -get_function_span_attributes = lambda span_data: \ - _extract_attributes_from_mapping(span_data, FUNCTION_SPAN_ATTRIBUTES) +def get_handoff_span_attributes(span_data: Any) -> AttributeMap: + """Extract attributes from a HandoffSpanData object. + + Handoffs are requests made to the `openai.handoffs` endpoint. + + Args: + span_data: The HandoffSpanData object + + Returns: + Dictionary of attributes for handoff span + """ + attributes = _extract_attributes_from_mapping(span_data, HANDOFF_SPAN_ATTRIBUTES) + attributes.update(get_common_attributes()) + attributes.update(get_library_attributes()) + + return attributes -get_handoff_span_attributes = lambda span_data: \ - _extract_attributes_from_mapping(span_data, HANDOFF_SPAN_ATTRIBUTES) def get_response_span_attributes(span_data: Any) -> AttributeMap: @@ -160,6 +149,8 @@ def get_response_span_attributes(span_data: Any) -> AttributeMap: """ # Get basic attributes from mapping attributes = _extract_attributes_from_mapping(span_data, RESPONSE_SPAN_ATTRIBUTES) + attributes.update(get_common_attributes()) + attributes.update(get_library_attributes()) if span_data.response: attributes.update(get_response_response_attributes(span_data.response)) @@ -185,6 +176,8 @@ def get_generation_span_attributes(span_data: Any) -> AttributeMap: Dictionary of attributes for generation span """ attributes = _extract_attributes_from_mapping(span_data, GENERATION_SPAN_ATTRIBUTES) + attributes.update(get_common_attributes()) + attributes.update(get_library_attributes()) # Process output for GenerationSpanData if available if span_data.output: diff --git a/agentops/instrumentation/openai_agents/exporter.py b/agentops/instrumentation/openai_agents/exporter.py index d40f6e3d0..a7708410d 100644 --- a/agentops/instrumentation/openai_agents/exporter.py +++ b/agentops/instrumentation/openai_agents/exporter.py @@ -26,10 +26,14 @@ from agentops.semconv import ( CoreAttributes, ) -from agentops.instrumentation.openai_agents import LIBRARY_NAME, LIBRARY_VERSION -from agentops.instrumentation.openai_agents.attributes.common import ( + +from agentops.instrumentation.common.attributes import ( get_base_trace_attributes, get_base_span_attributes, +) + +from agentops.instrumentation.openai_agents import LIBRARY_NAME, LIBRARY_VERSION +from agentops.instrumentation.openai_agents.attributes.common import ( get_span_attributes, ) From f2dca863a193eaef7c2be7cd7fbcf98ac9cb97da Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 25 Mar 2025 16:33:43 -0700 Subject: [PATCH 06/26] Include tags in parent span. Helpers for accessing global config and tags. Tests for helpers and common insrumentation attributes. --- agentops/helpers/__init__.py | 2 + agentops/helpers/config.py | 31 ++ agentops/instrumentation/common/attributes.py | 50 ++- .../openai/attributes/response.py | 2 +- .../openai_agents/attributes/common.py | 15 + .../openai_agents/attributes/completion.py | 2 +- agentops/semconv/core.py | 2 + tests/unit/helpers/test_helpers.py | 73 +++++ .../test_openai_agents_attributes.py | 37 +-- .../instrumentation/test_common_attributes.py | 285 ++++++++++++++++++ 10 files changed, 432 insertions(+), 67 deletions(-) create mode 100644 agentops/helpers/config.py create mode 100644 tests/unit/helpers/test_helpers.py create mode 100644 tests/unit/instrumentation/test_common_attributes.py diff --git a/agentops/helpers/__init__.py b/agentops/helpers/__init__.py index ba8c1aad7..1f148bca9 100644 --- a/agentops/helpers/__init__.py +++ b/agentops/helpers/__init__.py @@ -20,6 +20,7 @@ from .version import get_agentops_version, check_agentops_update from .debug import debug_print_function_params from .env import get_env_bool, get_env_int, get_env_list +from .config import get_tags_from_config __all__ = [ "get_ISO_time", @@ -45,4 +46,5 @@ "get_env_bool", "get_env_int", "get_env_list", + "get_tags_from_config", ] diff --git a/agentops/helpers/config.py b/agentops/helpers/config.py new file mode 100644 index 000000000..ce7a3e945 --- /dev/null +++ b/agentops/helpers/config.py @@ -0,0 +1,31 @@ +"""Helper functions for accessing configuration values. + +This module provides utility functions for accessing configuration values from +the global Config object in a safe way. +""" + +from typing import List, Any + +from agentops.config import Config + + +def get_config() -> Config: + """Get the global configuration object from the Client singleton. + + Returns: + The Config instance from the global Client + """ + from agentops import get_client + return get_client().config + + +def get_tags_from_config() -> List[str]: + """Get tags from the global configuration. + + Returns: + List of tags if they exist in the configuration, or empty list + """ + config = get_config() + if config.default_tags: + return list(config.default_tags) + return [] # Return empty list for empty tags set \ No newline at end of file diff --git a/agentops/instrumentation/common/attributes.py b/agentops/instrumentation/common/attributes.py index 5f8908a57..95558c6b9 100644 --- a/agentops/instrumentation/common/attributes.py +++ b/agentops/instrumentation/common/attributes.py @@ -1,31 +1,27 @@ -"""Attribute processing modules for OpenAI Agents instrumentation. +"""Common attribute processing utilities shared across all instrumentors. -This package provides specialized getter functions that extract and format -OpenTelemetry-compatible attributes from span data. Each function follows a -consistent pattern: +This module provides core utilities for extracting and formatting +OpenTelemetry-compatible attributes from span data. These functions +are provider-agnostic and used by all instrumentors in the AgentOps +package. -1. Takes span data (or specific parts of span data) as input -2. Processes the data according to semantic conventions -3. Returns a dictionary of formatted attributes +The module includes: -The modules are organized by functional domain: +1. Helper functions for attribute extraction and mapping +2. Common attribute getters used across all providers +3. Base trace and span attribute functions -- common: Core attribute extraction functions for all span types -- tokens: Token usage extraction and processing -- model: Model information and parameter extraction -- completion: Completion content and tool call processing +All functions follow a consistent pattern: +- Accept span/trace data as input +- Process according to semantic conventions +- Return a dictionary of formatted attributes -Each getter function is focused on a single responsibility and does not -modify any global state. Functions are designed to be composable, allowing -different attribute types to be combined as needed in the exporter. - -The separation of attribute extraction (getters in this module) from -attribute application (managed by exporter) follows the principle of -separation of concerns. +These utilities ensure consistent attribute handling across different +LLM service instrumentors while maintaining separation of concerns. """ -from typing import Dict, Any +from typing import Dict, Any, Optional, List from agentops.logging import logger -from agentops.helpers import safe_serialize, get_agentops_version +from agentops.helpers import safe_serialize, get_agentops_version, get_tags_from_config from agentops.semconv import ( CoreAttributes, InstrumentationAttributes, @@ -36,14 +32,6 @@ AttributeMap = Dict[str, Any] -# Common attribute mapping for all span types -COMMON_ATTRIBUTES: AttributeMap = { - CoreAttributes.TRACE_ID: "trace_id", - CoreAttributes.SPAN_ID: "span_id", - CoreAttributes.PARENT_ID: "parent_id", -} - - def _extract_attributes_from_mapping(span_data: Any, attribute_mapping: AttributeMap) -> AttributeMap: """Helper function to extract attributes based on a mapping. @@ -110,6 +98,10 @@ def get_base_trace_attributes(trace: Any) -> AttributeMap: **get_common_attributes(), } + # Add tags from the config to the trace attributes (these should only be added to the trace) + if tags := get_tags_from_config(): + attributes[CoreAttributes.TAGS] = tags + return attributes diff --git a/agentops/instrumentation/openai/attributes/response.py b/agentops/instrumentation/openai/attributes/response.py index 62a62b909..0567de00a 100644 --- a/agentops/instrumentation/openai/attributes/response.py +++ b/agentops/instrumentation/openai/attributes/response.py @@ -6,7 +6,7 @@ MessageAttributes, ToolAttributes, ) -from agentops.instrumentation.openai_agents.attributes import ( +from agentops.instrumentation.common.attributes import ( AttributeMap, _extract_attributes_from_mapping, ) diff --git a/agentops/instrumentation/openai_agents/attributes/common.py b/agentops/instrumentation/openai_agents/attributes/common.py index e715bf94d..e78647233 100644 --- a/agentops/instrumentation/openai_agents/attributes/common.py +++ b/agentops/instrumentation/openai_agents/attributes/common.py @@ -63,6 +63,21 @@ } +def get_common_instrumentation_attributes() -> AttributeMap: + """Get common instrumentation attributes for the OpenAI Agents instrumentation. + + This combines the generic AgentOps attributes with OpenAI Agents specific library attributes. + + Returns: + Dictionary of common instrumentation attributes + """ + attributes = get_common_attributes() + attributes.update({ + InstrumentationAttributes.LIBRARY_NAME: LIBRARY_NAME, + InstrumentationAttributes.LIBRARY_VERSION: LIBRARY_VERSION, + }) + return attributes + def get_library_attributes() -> AttributeMap: """Get common attributes for the OpenAI Agents library. diff --git a/agentops/instrumentation/openai_agents/attributes/completion.py b/agentops/instrumentation/openai_agents/attributes/completion.py index 18dbd98f5..01ace15a1 100644 --- a/agentops/instrumentation/openai_agents/attributes/completion.py +++ b/agentops/instrumentation/openai_agents/attributes/completion.py @@ -5,7 +5,7 @@ """ from typing import Any, Dict -from agentops.instrumentation.openai_agents.attributes import AttributeMap +from agentops.instrumentation.common.attributes import AttributeMap from agentops.logging import logger from agentops.helpers.serialization import model_to_dict diff --git a/agentops/semconv/core.py b/agentops/semconv/core.py index 56edfcd8f..d6b6d9022 100644 --- a/agentops/semconv/core.py +++ b/agentops/semconv/core.py @@ -11,6 +11,8 @@ class CoreAttributes: IN_FLIGHT = "agentops.in-flight" # Whether the span is in-flight EXPORT_IMMEDIATELY = "agentops.export.immediate" # Whether the span should be exported immediately + TAGS = "agentops.tags" # Tags passed to agentops.init + # Trace context attributes TRACE_ID = "trace.id" # Trace ID SPAN_ID = "span.id" # Span ID diff --git a/tests/unit/helpers/test_helpers.py b/tests/unit/helpers/test_helpers.py new file mode 100644 index 000000000..0f3a65f71 --- /dev/null +++ b/tests/unit/helpers/test_helpers.py @@ -0,0 +1,73 @@ +"""Tests for the config helper functions.""" + +import pytest + +from agentops.helpers.config import get_config, get_tags_from_config +from agentops.config import Config + + +class TestConfigHelpers: + """Test suite for configuration helper functions.""" + + def test_get_config_returns_valid_instance(self): + """Test that get_config returns a valid Config instance.""" + # Call the helper function + config = get_config() + + # Verify it returns a Config instance + assert isinstance(config, Config) + + # Verify it has the expected attributes + assert hasattr(config, 'api_key') + assert hasattr(config, 'endpoint') + assert hasattr(config, 'default_tags') + + def test_get_config_returns_singleton(self): + """Test that get_config returns the same Config instance each time.""" + # Call the helper function twice + config1 = get_config() + config2 = get_config() + + # Verify they are the same object (singleton pattern) + assert config1 is config2 + assert isinstance(config1, Config) + + def test_get_tags_from_config_with_actual_config(self): + """Test that get_tags_from_config returns tags from the actual application config.""" + # Get the actual application config + config = get_config() + original_tags = config.default_tags + + try: + # Temporarily set some test tags + test_tags = {"test_tag1", "test_tag2"} + config.default_tags = test_tags + + # Call the helper function + tags = get_tags_from_config() + + # Verify it returns the expected tags + assert isinstance(tags, list) + assert set(tags) == test_tags + finally: + # Restore the original tags + config.default_tags = original_tags + + def test_get_tags_from_config_with_empty_tags(self): + """Test that get_tags_from_config returns an empty list when no tags are set.""" + # Get the actual application config + config = get_config() + original_tags = config.default_tags + + try: + # Temporarily set empty tags + config.default_tags = set() + + # Call the helper function + tags = get_tags_from_config() + + # Verify it returns an empty list + assert tags == [] + finally: + # Restore the original tags + config.default_tags = original_tags \ No newline at end of file diff --git a/tests/unit/instrumentation/openai_agents/test_openai_agents_attributes.py b/tests/unit/instrumentation/openai_agents/test_openai_agents_attributes.py index 5c39c1830..8bfcae9c2 100644 --- a/tests/unit/instrumentation/openai_agents/test_openai_agents_attributes.py +++ b/tests/unit/instrumentation/openai_agents/test_openai_agents_attributes.py @@ -25,8 +25,6 @@ get_response_span_attributes, get_span_attributes, get_common_instrumentation_attributes, - get_base_trace_attributes, - get_base_span_attributes, ) # Import model-related functions @@ -642,37 +640,4 @@ def test_get_model_attributes(self): assert attrs[SpanAttributes.LLM_RESPONSE_MODEL] == "gpt-4" assert attrs[SpanAttributes.LLM_SYSTEM] == "openai" - def test_get_base_trace_attributes(self): - """Test base trace attributes generation""" - # Create a simple trace object - class TraceObj: - def __init__(self): - self.name = "test_workflow" - self.trace_id = "trace123" - - trace = TraceObj() - attrs = get_base_trace_attributes(trace) - - # Verify core trace attributes - assert attrs[WorkflowAttributes.WORKFLOW_NAME] == "test_workflow" - assert attrs[CoreAttributes.TRACE_ID] == "trace123" - assert attrs[WorkflowAttributes.WORKFLOW_STEP_TYPE] == "trace" - assert attrs[InstrumentationAttributes.NAME] == "agentops" - - def test_get_base_span_attributes(self): - """Test base span attributes generation""" - # Create a simple span object - class SpanObj: - def __init__(self): - self.span_id = "span456" - self.trace_id = "trace123" - self.parent_id = "parent789" - - span = SpanObj() - attrs = get_base_span_attributes(span) - - # Verify core span attributes - assert attrs[CoreAttributes.SPAN_ID] == "span456" - assert attrs[CoreAttributes.TRACE_ID] == "trace123" - assert attrs[CoreAttributes.PARENT_ID] == "parent789" - assert attrs[InstrumentationAttributes.NAME] == "agentops" \ No newline at end of file + # Common attribute tests have been moved to test_common_attributes.py \ No newline at end of file diff --git a/tests/unit/instrumentation/test_common_attributes.py b/tests/unit/instrumentation/test_common_attributes.py new file mode 100644 index 000000000..34477e6e4 --- /dev/null +++ b/tests/unit/instrumentation/test_common_attributes.py @@ -0,0 +1,285 @@ +""" +Tests for Common Attributes Module + +This module contains tests for the common attribute processing utilities that are shared +across all instrumentors in the AgentOps package. +""" + +import pytest +from unittest.mock import patch, MagicMock +from typing import Dict, Any, List, Set + +from agentops.instrumentation.common.attributes import ( + AttributeMap, + _extract_attributes_from_mapping, + get_common_attributes, + get_base_trace_attributes, + get_base_span_attributes +) +from agentops.helpers import get_tags_from_config + +from agentops.semconv import ( + CoreAttributes, + InstrumentationAttributes, + WorkflowAttributes, +) + + +class TestCommonAttributes: + """Test suite for common attribute processing utilities""" + + def test_extract_attributes_from_mapping(self): + """Test extraction of attributes based on mapping""" + # Create a simple span data object with attributes + class SpanData: + def __init__(self): + self.trace_id = "trace123" + self.span_id = "span456" + self.parent_id = "parent789" + self.name = "test_span" + + span_data = SpanData() + + # Define a mapping + mapping = { + "target.trace_id": "trace_id", + "target.span_id": "span_id", + "target.parent_id": "parent_id", + "target.name": "name", + "target.missing": "missing_attr" # This attribute doesn't exist + } + + # Extract attributes + attributes = _extract_attributes_from_mapping(span_data, mapping) + + # Verify extracted attributes + assert attributes["target.trace_id"] == "trace123" + assert attributes["target.span_id"] == "span456" + assert attributes["target.parent_id"] == "parent789" + assert attributes["target.name"] == "test_span" + assert "target.missing" not in attributes # Missing attribute should be skipped + + def test_extract_attributes_from_dict(self): + """Test extraction of attributes from a dictionary""" + # Create a dictionary with attributes + span_data = { + "trace_id": "trace123", + "span_id": "span456", + "parent_id": "parent789", + "name": "test_span" + } + + # Define a mapping + mapping = { + "target.trace_id": "trace_id", + "target.span_id": "span_id", + "target.parent_id": "parent_id", + "target.name": "name", + "target.missing": "missing_key" # This key doesn't exist + } + + # Extract attributes + attributes = _extract_attributes_from_mapping(span_data, mapping) + + # Verify extracted attributes + assert attributes["target.trace_id"] == "trace123" + assert attributes["target.span_id"] == "span456" + assert attributes["target.parent_id"] == "parent789" + assert attributes["target.name"] == "test_span" + assert "target.missing" not in attributes # Missing key should be skipped + + def test_extract_attributes_handles_none_empty_values(self): + """Test that the extraction function properly handles None and empty values""" + # Create a span data object with None and empty values + class SpanData: + def __init__(self): + self.none_attr = None + self.empty_str = "" + self.empty_list = [] + self.empty_dict = {} + self.valid_attr = "valid_value" + + span_data = SpanData() + + # Define a mapping + mapping = { + "target.none": "none_attr", + "target.empty_str": "empty_str", + "target.empty_list": "empty_list", + "target.empty_dict": "empty_dict", + "target.valid": "valid_attr" + } + + # Extract attributes + attributes = _extract_attributes_from_mapping(span_data, mapping) + + # Verify that None and empty values are skipped + assert "target.none" not in attributes + assert "target.empty_str" not in attributes + assert "target.empty_list" not in attributes + assert "target.empty_dict" not in attributes + assert attributes["target.valid"] == "valid_value" + + def test_extract_attributes_serializes_complex_objects(self): + """Test that complex objects are properly serialized during extraction""" + # Create a dictionary with complex values + span_data = { + "complex_obj": {"attr1": "value1", "attr2": "value2"}, + "dict_obj": {"key1": "value1", "key2": "value2"}, + "list_obj": ["item1", "item2"] + } + + # Define a mapping + mapping = { + "target.complex": "complex_obj", + "target.dict": "dict_obj", + "target.list": "list_obj" + } + + # Extract attributes with serialization + attributes = _extract_attributes_from_mapping(span_data, mapping) + + # Verify that complex objects are serialized to strings + assert isinstance(attributes["target.complex"], str) + assert isinstance(attributes["target.dict"], str) + assert isinstance(attributes["target.list"], str) + + # Check that serialized values contain expected content + assert "value1" in attributes["target.complex"] + assert "value2" in attributes["target.complex"] + assert "key1" in attributes["target.dict"] + assert "key2" in attributes["target.dict"] + assert "item1" in attributes["target.list"] + assert "item2" in attributes["target.list"] + + def test_get_common_attributes(self): + """Test that common instrumentation attributes are correctly generated""" + # Get common attributes + attributes = get_common_attributes() + + # Verify required keys and values + assert InstrumentationAttributes.NAME in attributes + assert InstrumentationAttributes.VERSION in attributes + assert attributes[InstrumentationAttributes.NAME] == "agentops" + + def test_get_base_trace_attributes(self): + """Test generation of base trace attributes""" + # Create a simple trace object + class TraceObj: + def __init__(self): + self.name = "test_workflow" + self.trace_id = "trace123" + + trace = TraceObj() + + # Get base trace attributes + attributes = get_base_trace_attributes(trace) + + # Verify core trace attributes + assert attributes[WorkflowAttributes.WORKFLOW_NAME] == "test_workflow" + assert attributes[CoreAttributes.TRACE_ID] == "trace123" + assert attributes[WorkflowAttributes.WORKFLOW_STEP_TYPE] == "trace" + assert attributes[InstrumentationAttributes.NAME] == "agentops" + + # Test error case when trace_id is missing + class InvalidTrace: + def __init__(self): + self.name = "invalid_workflow" + # No trace_id + + invalid_trace = InvalidTrace() + invalid_attributes = get_base_trace_attributes(invalid_trace) + assert invalid_attributes == {} + + def test_get_base_span_attributes(self): + """Test generation of base span attributes""" + # Create a simple span object + class SpanObj: + def __init__(self): + self.span_id = "span456" + self.trace_id = "trace123" + self.parent_id = "parent789" + + span = SpanObj() + + # Get base span attributes + attributes = get_base_span_attributes(span) + + # Verify core span attributes + assert attributes[CoreAttributes.SPAN_ID] == "span456" + assert attributes[CoreAttributes.TRACE_ID] == "trace123" + assert attributes[CoreAttributes.PARENT_ID] == "parent789" + assert attributes[InstrumentationAttributes.NAME] == "agentops" + + # Test without parent_id + class SpanWithoutParent: + def __init__(self): + self.span_id = "span456" + self.trace_id = "trace123" + # No parent_id + + span_without_parent = SpanWithoutParent() + attributes_without_parent = get_base_span_attributes(span_without_parent) + + # Verify parent_id is not included + assert CoreAttributes.PARENT_ID not in attributes_without_parent + + def test_get_tags_from_config(self): + """Test retrieval of tags from the configuration""" + # Mock the get_config function + mock_config = MagicMock() + mock_config.default_tags = {"tag1", "tag2", "tag3"} + + with patch("agentops.helpers.config.get_config", return_value=mock_config): + # Get tags from config + tags = get_tags_from_config() + + # Verify tags are returned as a list + assert isinstance(tags, list) + assert set(tags) == {"tag1", "tag2", "tag3"} + + def test_get_tags_from_config_handles_empty_tags(self): + """Test that empty tags are handled correctly""" + # Mock the get_config function with empty tags + mock_config = MagicMock() + mock_config.default_tags = set() + + with patch("agentops.helpers.config.get_config", return_value=mock_config): + # Get tags from config + tags = get_tags_from_config() + + # Verify empty tags returns an empty list + assert tags == [] # This should pass if get_tags_from_config returns [] for empty sets + + # Removed test_get_tags_from_config_handles_error since we're not handling errors anymore + + def test_tags_added_to_trace_attributes(self): + """Test that tags are added to trace attributes but not span attributes""" + # Create test objects + class TraceObj: + def __init__(self): + self.name = "test_workflow" + self.trace_id = "trace123" + + class SpanObj: + def __init__(self): + self.span_id = "span456" + self.trace_id = "trace123" + + trace = TraceObj() + span = SpanObj() + + # Mock the get_tags_from_config function to return test tags + with patch("agentops.instrumentation.common.attributes.get_tags_from_config", + return_value=["test_tag1", "test_tag2"]): + # Get attributes for both trace and span + trace_attributes = get_base_trace_attributes(trace) + span_attributes = get_base_span_attributes(span) + + + # Verify tags are added to trace attributes + assert CoreAttributes.TAGS in trace_attributes + assert trace_attributes[CoreAttributes.TAGS] == ["test_tag1", "test_tag2"] + + # Verify tags are NOT added to span attributes + assert CoreAttributes.TAGS not in span_attributes \ No newline at end of file From e1e350643176d920841fe5f6f947cac3e8b1ac0c Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 25 Mar 2025 16:40:03 -0700 Subject: [PATCH 07/26] Add tags to an example. --- examples/agents-example/hello_world.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/agents-example/hello_world.py b/examples/agents-example/hello_world.py index bcc0a15ba..7cdcdb0ec 100644 --- a/examples/agents-example/hello_world.py +++ b/examples/agents-example/hello_world.py @@ -9,7 +9,7 @@ import agentops async def main(): - agentops.init() + agentops.init(tags=["test", "openai-agents"]) agent = Agent( name="Hello World Agent", From d7a9f6f96ca2a7bff3aa79700b8cf26eacd8bd8e Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 25 Mar 2025 16:46:06 -0700 Subject: [PATCH 08/26] Remove duplicate library attributes. --- .../openai_agents/attributes/common.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/agentops/instrumentation/openai_agents/attributes/common.py b/agentops/instrumentation/openai_agents/attributes/common.py index e78647233..55c2475b3 100644 --- a/agentops/instrumentation/openai_agents/attributes/common.py +++ b/agentops/instrumentation/openai_agents/attributes/common.py @@ -7,7 +7,6 @@ from typing import Any from agentops.logging import logger from agentops.semconv import ( - CoreAttributes, AgentAttributes, WorkflowAttributes, SpanAttributes, @@ -78,17 +77,6 @@ def get_common_instrumentation_attributes() -> AttributeMap: }) return attributes -def get_library_attributes() -> AttributeMap: - """Get common attributes for the OpenAI Agents library. - - Returns: - Dictionary of common library attributes - """ - return { - InstrumentationAttributes.NAME: LIBRARY_NAME, - InstrumentationAttributes.VERSION: LIBRARY_VERSION, - } - def get_agent_span_attributes(span_data: Any) -> AttributeMap: """Extract attributes from an AgentSpanData object. @@ -103,7 +91,6 @@ def get_agent_span_attributes(span_data: Any) -> AttributeMap: """ attributes = _extract_attributes_from_mapping(span_data, AGENT_SPAN_ATTRIBUTES) attributes.update(get_common_attributes()) - attributes.update(get_library_attributes()) return attributes @@ -121,7 +108,6 @@ def get_function_span_attributes(span_data: Any) -> AttributeMap: """ attributes = _extract_attributes_from_mapping(span_data, FUNCTION_SPAN_ATTRIBUTES) attributes.update(get_common_attributes()) - attributes.update(get_library_attributes()) return attributes @@ -139,7 +125,6 @@ def get_handoff_span_attributes(span_data: Any) -> AttributeMap: """ attributes = _extract_attributes_from_mapping(span_data, HANDOFF_SPAN_ATTRIBUTES) attributes.update(get_common_attributes()) - attributes.update(get_library_attributes()) return attributes @@ -165,7 +150,6 @@ def get_response_span_attributes(span_data: Any) -> AttributeMap: # Get basic attributes from mapping attributes = _extract_attributes_from_mapping(span_data, RESPONSE_SPAN_ATTRIBUTES) attributes.update(get_common_attributes()) - attributes.update(get_library_attributes()) if span_data.response: attributes.update(get_response_response_attributes(span_data.response)) @@ -192,7 +176,6 @@ def get_generation_span_attributes(span_data: Any) -> AttributeMap: """ attributes = _extract_attributes_from_mapping(span_data, GENERATION_SPAN_ATTRIBUTES) attributes.update(get_common_attributes()) - attributes.update(get_library_attributes()) # Process output for GenerationSpanData if available if span_data.output: From f30f43a9de2f334df222c114c049c2a9803c29df Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 25 Mar 2025 16:59:13 -0700 Subject: [PATCH 09/26] Pass OpenAI responses objects through our new instrumentor. --- agentops/instrumentation/__init__.py | 4 +- agentops/instrumentation/openai/__init__.py | 25 +++++++++- .../openai/attributes/common.py | 47 +++++++++++++++++++ .../openai/attributes/response.py | 2 + .../instrumentation/openai/instrumentor.py | 6 ++- agentops/instrumentation/openai/wrappers.py | 24 ++++++++-- .../instrumentation/openai_agents/__init__.py | 5 +- 7 files changed, 101 insertions(+), 12 deletions(-) create mode 100644 agentops/instrumentation/openai/attributes/common.py diff --git a/agentops/instrumentation/__init__.py b/agentops/instrumentation/__init__.py index 367334d21..732af7666 100644 --- a/agentops/instrumentation/__init__.py +++ b/agentops/instrumentation/__init__.py @@ -23,7 +23,7 @@ class InstrumentorLoader: We use the `provider_import_name` to determine if the library is installed i n the environment. - `modue_name` is the name of the module to import from. + `module_name` is the name of the module to import from. `class_name` is the name of the class to instantiate from the module. `provider_import_name` is the name of the package to check for availability. """ @@ -53,7 +53,7 @@ def get_instance(self) -> BaseInstrumentor: available_instrumentors: list[InstrumentorLoader] = [ InstrumentorLoader( - module_name="opentelemetry.instrumentation.openai", + module_name="agentops.instrumentation.openai", class_name="OpenAIInstrumentor", provider_import_name="openai", ), diff --git a/agentops/instrumentation/openai/__init__.py b/agentops/instrumentation/openai/__init__.py index de7ec77dc..c36be0faa 100644 --- a/agentops/instrumentation/openai/__init__.py +++ b/agentops/instrumentation/openai/__init__.py @@ -3,7 +3,28 @@ This package provides OpenTelemetry-based instrumentation for OpenAI API calls, extending the third-party instrumentation to add support for OpenAI responses. """ +from agentops.logging import logger -from agentops.instrumentation.openai.instrumentor import OpenAIInstrumentor -__all__ = ["OpenAIInstrumentor"] \ No newline at end of file +def get_version() -> str: + """Get the version of the agents SDK, or 'unknown' if not found""" + try: + from importlib.metadata import version + + return version("openai") + except ImportError: + logger.debug("Could not find OpenAI Agents SDK version") + return "unknown" + + +LIBRARY_NAME = "openai" +LIBRARY_VERSION: str = get_version() + +# Import after defining constants to avoid circular imports +from agentops.instrumentation.openai.instrumentor import OpenAIInstrumentor # noqa: E402 + +__all__ = [ + "LIBRARY_NAME", + "LIBRARY_VERSION", + "OpenAIInstrumentor", +] diff --git a/agentops/instrumentation/openai/attributes/common.py b/agentops/instrumentation/openai/attributes/common.py new file mode 100644 index 000000000..eefb456fc --- /dev/null +++ b/agentops/instrumentation/openai/attributes/common.py @@ -0,0 +1,47 @@ +from typing import Any +from agentops.semconv import ( + InstrumentationAttributes +) +from agentops.instrumentation.openai import LIBRARY_NAME, LIBRARY_VERSION +from agentops.instrumentation.common.attributes import AttributeMap +from agentops.instrumentation.common.attributes import get_common_attributes +from agentops.instrumentation.openai.attributes.response import get_response_response_attributes + + +# Attribute mapping for ResponseSpanData +RESPONSE_SPAN_ATTRIBUTES: AttributeMap = { +} + + +def get_common_instrumentation_attributes() -> AttributeMap: + """Get common instrumentation attributes for the OpenAI Agents instrumentation. + + This combines the generic AgentOps attributes with OpenAI Agents specific library attributes. + + Returns: + Dictionary of common instrumentation attributes + """ + attributes = get_common_attributes() + attributes.update({ + InstrumentationAttributes.LIBRARY_NAME: LIBRARY_NAME, + InstrumentationAttributes.LIBRARY_VERSION: LIBRARY_VERSION, + }) + return attributes + + +def get_response_span_attributes(span_data: Any) -> AttributeMap: + """Extract attributes from a ResponseSpanData object. + + Responses are requests made to the `openai.responses` endpoint. + + Args: + span_data: The ResponseSpanData object + + Returns: + Dictionary of attributes for response span + """ + attributes = get_response_response_attributes(span_data) + attributes.update(get_common_attributes()) + + return attributes + diff --git a/agentops/instrumentation/openai/attributes/response.py b/agentops/instrumentation/openai/attributes/response.py index 0567de00a..07d890367 100644 --- a/agentops/instrumentation/openai/attributes/response.py +++ b/agentops/instrumentation/openai/attributes/response.py @@ -105,6 +105,8 @@ } +# TODO we call this `response_response` because in OpenAI Agents the `Response` is nested +# in a `ResponseSpan` object def get_response_response_attributes(response: 'Response') -> AttributeMap: """Handles interpretation of an openai Response object.""" # Response( diff --git a/agentops/instrumentation/openai/instrumentor.py b/agentops/instrumentation/openai/instrumentor.py index cd6c48496..cf1c05d8f 100644 --- a/agentops/instrumentation/openai/instrumentor.py +++ b/agentops/instrumentation/openai/instrumentor.py @@ -34,6 +34,7 @@ from agentops.logging import logger from agentops.instrumentation.openai.wrappers import sync_responses_wrapper, async_responses_wrapper +from agentops.instrumentation.openai.attributes.response import get_response_response_attributes # Methods to wrap beyond what the third-party instrumentation handles WRAPPED_METHODS = [ @@ -42,12 +43,14 @@ "object": "Responses", "method": "create", "wrapper": sync_responses_wrapper, + "formatter": get_response_response_attributes, }, { "package": "openai.resources.responses", "object": "AsyncResponses", "method": "create", "wrapper": async_responses_wrapper, + "formatter": get_response_response_attributes, }, ] @@ -78,12 +81,13 @@ def _instrument(self, **kwargs): object_name = wrapped_method["object"] method_name = wrapped_method["method"] wrapper_func = wrapped_method["wrapper"] + formatter = wrapped_method["formatter"] try: wrap_function_wrapper( package, f"{object_name}.{method_name}", - wrapper_func(tracer), + wrapper_func(tracer, formatter), ) logger.debug(f"Successfully wrapped {package}.{object_name}.{method_name}") except (AttributeError, ModuleNotFoundError) as e: diff --git a/agentops/instrumentation/openai/wrappers.py b/agentops/instrumentation/openai/wrappers.py index 2d90e102e..616551915 100644 --- a/agentops/instrumentation/openai/wrappers.py +++ b/agentops/instrumentation/openai/wrappers.py @@ -46,14 +46,15 @@ def _handle_response_span(span, start_time, success=True, exception=None): span.set_attribute("duration_s", duration) -def sync_responses_wrapper(tracer): +def sync_responses_wrapper(tracer, formatter): """Wrapper for synchronous openai.resources.responses.Responses.create API calls. This wrapper creates a span for tracking OpenAI Responses API calls. - The detailed attribute extraction will be handled separately. + It uses the provided formatter to extract attributes from the response. Args: tracer: The OpenTelemetry tracer to use for creating spans + formatter: Function to extract attributes from the response object Returns: A wrapper function that instruments the synchronous OpenAI Responses API @@ -74,6 +75,13 @@ def wrapper(wrapped, instance, args, kwargs): # Execute the wrapped function and get the response try: response = wrapped(*args, **kwargs) + + # Use the formatter to extract and set attributes from response + if response: + attributes = formatter(response) + for key, value in attributes.items(): + span.set_attribute(key, value) + _handle_response_span(span, start_time, success=True) except Exception as e: _handle_response_span(span, start_time, success=False, exception=e) @@ -84,14 +92,15 @@ def wrapper(wrapped, instance, args, kwargs): return wrapper -def async_responses_wrapper(tracer): +def async_responses_wrapper(tracer, formatter): """Wrapper for asynchronous openai.resources.responses.AsyncResponses.create API calls. This wrapper creates a span for tracking asynchronous OpenAI Responses API calls. - The detailed attribute extraction will be handled separately. + It uses the provided formatter to extract attributes from the response. Args: tracer: The OpenTelemetry tracer to use for creating spans + formatter: Function to extract attributes from the response object Returns: A wrapper function that instruments the asynchronous OpenAI Responses API @@ -112,6 +121,13 @@ async def wrapper(wrapped, instance, args, kwargs): # Execute the wrapped function and get the response try: response = await wrapped(*args, **kwargs) + + # Use the formatter to extract and set attributes from response + if response: + attributes = formatter(response) + for key, value in attributes.items(): + span.set_attribute(key, value) + _handle_response_span(span, start_time, success=True) except Exception as e: _handle_response_span(span, start_time, success=False, exception=e) diff --git a/agentops/instrumentation/openai_agents/__init__.py b/agentops/instrumentation/openai_agents/__init__.py index f3e9ce66e..1b3fb4b1c 100644 --- a/agentops/instrumentation/openai_agents/__init__.py +++ b/agentops/instrumentation/openai_agents/__init__.py @@ -21,15 +21,14 @@ def get_version() -> str: try: from importlib.metadata import version - library_version = version("openai-agents") - return library_version + return version("openai-agents") except ImportError: logger.debug("Could not find OpenAI Agents SDK version") return "unknown" LIBRARY_NAME = "openai-agents" -LIBRARY_VERSION: str = get_version() # Actual OpenAI Agents SDK version +LIBRARY_VERSION: str = get_version() # Import after defining constants to avoid circular imports from .instrumentor import OpenAIAgentsInstrumentor # noqa: E402 From e942f94c1cc62c87cd6fc34e9b429b443465fb91 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 25 Mar 2025 17:46:11 -0700 Subject: [PATCH 10/26] Incorporate common attributes, too. --- .../openai/attributes/common.py | 18 ++++++++------- .../instrumentation/openai/instrumentor.py | 23 ++++++++++--------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/agentops/instrumentation/openai/attributes/common.py b/agentops/instrumentation/openai/attributes/common.py index eefb456fc..9bfedf5ad 100644 --- a/agentops/instrumentation/openai/attributes/common.py +++ b/agentops/instrumentation/openai/attributes/common.py @@ -1,4 +1,4 @@ -from typing import Any +from agentops.logging import logger from agentops.semconv import ( InstrumentationAttributes ) @@ -7,10 +7,11 @@ from agentops.instrumentation.common.attributes import get_common_attributes from agentops.instrumentation.openai.attributes.response import get_response_response_attributes +try: + from openai.types.responses import Response +except ImportError as e: + logger.debug(f"[agentops.instrumentation.openai_agents] Could not import OpenAI Agents SDK types: {e}") -# Attribute mapping for ResponseSpanData -RESPONSE_SPAN_ATTRIBUTES: AttributeMap = { -} def get_common_instrumentation_attributes() -> AttributeMap: @@ -29,18 +30,19 @@ def get_common_instrumentation_attributes() -> AttributeMap: return attributes -def get_response_span_attributes(span_data: Any) -> AttributeMap: +def get_response_attributes(response: Response) -> AttributeMap: """Extract attributes from a ResponseSpanData object. Responses are requests made to the `openai.responses` endpoint. Args: - span_data: The ResponseSpanData object + response: The `openai` `Response` object Returns: - Dictionary of attributes for response span + Dictionary of attributes for the span """ - attributes = get_response_response_attributes(span_data) + # TODO include prompt(s) + attributes = get_response_response_attributes(response) attributes.update(get_common_attributes()) return attributes diff --git a/agentops/instrumentation/openai/instrumentor.py b/agentops/instrumentation/openai/instrumentor.py index cf1c05d8f..b13e678ea 100644 --- a/agentops/instrumentation/openai/instrumentor.py +++ b/agentops/instrumentation/openai/instrumentor.py @@ -23,18 +23,17 @@ 3. Create spans with appropriate attributes for observability """ from typing import Collection - from wrapt import wrap_function_wrapper +from opentelemetry.trace import get_tracer from opentelemetry.instrumentation.utils import unwrap -# Import third-party OpenAIV1Instrumentor from opentelemetry.instrumentation.openai.v1 import OpenAIV1Instrumentor as ThirdPartyOpenAIV1Instrumentor -from opentelemetry.trace import get_tracer -from opentelemetry.instrumentation.openai.version import __version__ from agentops.logging import logger +from agentops.instrumentation.openai import LIBRARY_NAME, LIBRARY_VERSION from agentops.instrumentation.openai.wrappers import sync_responses_wrapper, async_responses_wrapper -from agentops.instrumentation.openai.attributes.response import get_response_response_attributes +from agentops.instrumentation.openai.attributes.common import get_response_attributes + # Methods to wrap beyond what the third-party instrumentation handles WRAPPED_METHODS = [ @@ -43,14 +42,14 @@ "object": "Responses", "method": "create", "wrapper": sync_responses_wrapper, - "formatter": get_response_response_attributes, + "formatter": get_response_attributes, }, { "package": "openai.resources.responses", "object": "AsyncResponses", "method": "create", "wrapper": async_responses_wrapper, - "formatter": get_response_response_attributes, + "formatter": get_response_attributes, }, ] @@ -58,9 +57,11 @@ class OpenAIInstrumentor(ThirdPartyOpenAIV1Instrumentor): """An instrumentor for OpenAI API that extends the third-party implementation.""" - def instrumentation_dependencies(self) -> Collection[str]: - """Return packages required for instrumentation.""" - return ["openai >= 1.0.0"] + # TODO we should only activate the `responses` feature if we are above a certain version, + # otherwise fallback to the third-party implementation + # def instrumentation_dependencies(self) -> Collection[str]: + # """Return packages required for instrumentation.""" + # return ["openai >= 1.0.0"] def _instrument(self, **kwargs): """Instrument the OpenAI API, extending the third-party instrumentation. @@ -73,7 +74,7 @@ def _instrument(self, **kwargs): super()._instrument(**kwargs) tracer_provider = kwargs.get("tracer_provider") - tracer = get_tracer(__name__, __version__, tracer_provider) + tracer = get_tracer(LIBRARY_NAME, LIBRARY_VERSION, tracer_provider) # Add our own wrappers for additional modules for wrapped_method in WRAPPED_METHODS: From 9b2c5f76551498ae2ad062183ad90d524bcd2a6d Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 26 Mar 2025 13:55:04 -0700 Subject: [PATCH 11/26] Add indexed PROMPT semconv to MessageAttributes. Provide reusable wrapping functionality from instrumentation.common. Include prompts in OpenAI Responses attributes. --- agentops/instrumentation/common/README.md | 65 +++++++ agentops/instrumentation/common/wrappers.py | 170 ++++++++++++++++++ .../openai/attributes/common.py | 40 +++-- .../openai/attributes/response.py | 62 ++++++- .../instrumentation/openai/instrumentor.py | 76 +++----- agentops/instrumentation/openai/wrappers.py | 138 -------------- agentops/semconv/message.py | 4 + 7 files changed, 350 insertions(+), 205 deletions(-) create mode 100644 agentops/instrumentation/common/README.md create mode 100644 agentops/instrumentation/common/wrappers.py delete mode 100644 agentops/instrumentation/openai/wrappers.py diff --git a/agentops/instrumentation/common/README.md b/agentops/instrumentation/common/README.md new file mode 100644 index 000000000..341c4d35d --- /dev/null +++ b/agentops/instrumentation/common/README.md @@ -0,0 +1,65 @@ +# AgentOps Instrumentation Common Module + +The `agentops.instrumentation.common` module provides shared utilities for OpenTelemetry instrumentation across different LLM service providers. + +## Core Components + +### Attribute Handler Example + +Attribute handlers extract data from method inputs and outputs: + +```python +from typing import Optional, Any, Tuple, Dict +from agentops.instrumentation.common.attributes import AttributeMap +from agentops.semconv import SpanAttributes + +def my_attribute_handler(args: Optional[Tuple] = None, kwargs: Optional[Dict] = None, return_value: Optional[Any] = None) -> AttributeMap: + attributes = {} + + # Extract attributes from kwargs (method inputs) + if kwargs: + if "model" in kwargs: + attributes[SpanAttributes.MODEL_NAME] = kwargs["model"] + # ... + + # Extract attributes from return value (method outputs) + if return_value: + if hasattr(return_value, "model"): + attributes[SpanAttributes.LLM_RESPONSE_MODEL] = return_value.model + # ... + + return attributes +``` + +### `WrapConfig` Class + +Config object defining how a method should be wrapped: + +```python +from agentops.instrumentation.common.wrappers import WrapConfig +from opentelemetry.trace import SpanKind + +config = WrapConfig( + trace_name="llm.completion", # Name that will appear in trace spans + package="openai.resources", # Path to the module containing the class + class_name="Completions", # Name of the class containing the method + method_name="create", # Name of the method to wrap + handler=my_attribute_handler, # Function that extracts attributes + span_kind=SpanKind.CLIENT # Type of span to create +) +``` + +### Wrapping/Unwrapping Methods + +```python +from opentelemetry.trace import get_tracer +from agentops.instrumentation.common.wrappers import wrap, unwrap + +# Create a tracer and wrap a method +tracer = get_tracer("openai", "0.0.0") +wrap(config, tracer) + +# Later, unwrap the method +unwrap(config) +``` + diff --git a/agentops/instrumentation/common/wrappers.py b/agentops/instrumentation/common/wrappers.py new file mode 100644 index 000000000..36537db2c --- /dev/null +++ b/agentops/instrumentation/common/wrappers.py @@ -0,0 +1,170 @@ +"""Common wrapper utilities for OpenTelemetry instrumentation. + +This module provides common utilities for creating and managing wrappers +around functions and methods for OpenTelemetry instrumentation. It includes +a configuration class for wrapping methods, helper functions for updating +spans with attributes, and functions for creating and applying wrappers. +""" + +from typing import Any, Optional, Tuple, Dict, Callable +from dataclasses import dataclass +from wrapt import wrap_function_wrapper +from opentelemetry.instrumentation.utils import unwrap as _unwrap +from opentelemetry.trace import Tracer +from opentelemetry.trace import Span, SpanKind, Status, StatusCode +from opentelemetry import context as context_api +from opentelemetry.instrumentation.utils import _SUPPRESS_INSTRUMENTATION_KEY + +from agentops.instrumentation.common.attributes import AttributeMap + + +AttributeHandler = Callable[[Optional[Tuple], Optional[Dict], Optional[Any]], AttributeMap] + +@dataclass +class WrapConfig: + """Configuration for wrapping a method with OpenTelemetry instrumentation. + + This class defines how a method should be wrapped for instrumentation, + including what package, class, and method to wrap, what span attributes + to set, and how to name the resulting trace spans. + + Attributes: + trace_name: The name to use for the trace span + package: The package containing the target class + class_name: The name of the class containing the method + method_name: The name of the method to wrap + handler: A function that extracts attributes from args, kwargs, or return value + span_kind: The kind of span to create (default: CLIENT) + """ + trace_name: str + package: str + class_name: str + method_name: str + handler: AttributeHandler + span_kind: SpanKind = SpanKind.CLIENT + + def __repr__(self): + return f"{self.package}.{self.class_name}.{self.method_name}" + + +def _update_span(span: Span, attributes: AttributeMap) -> None: + """Update a span with the provided attributes. + + Args: + span: The OpenTelemetry span to update + attributes: A dictionary of attributes to set on the span + """ + for key, value in attributes.items(): + span.set_attribute(key, value) + + +def _finish_span_success(span: Span) -> None: + """Mark a span as successful by setting its status to OK. + + Args: + span: The OpenTelemetry span to update + """ + span.set_status(Status(StatusCode.OK)) + + +def _finish_span_error(span: Span, exception: Exception) -> None: + """Mark a span as failed by recording the exception and setting error status. + + Args: + span: The OpenTelemetry span to update + exception: The exception that caused the error + """ + span.record_exception(exception) + span.set_status(Status(StatusCode.ERROR, str(exception))) + + +def _create_wrapper(wrap_config: WrapConfig, tracer: Tracer): + """Create a wrapper function for the specified configuration. + + This function creates a wrapper that: + 1. Creates a new span for the wrapped method + 2. Sets attributes on the span based on input arguments + 3. Calls the wrapped method + 4. Sets attributes on the span based on the return value + 5. Handles exceptions by recording them on the span + + Args: + wrap_config: Configuration for the wrapper + tracer: The OpenTelemetry tracer to use for creating spans + + Returns: + A wrapper function compatible with wrapt.wrap_function_wrapper + """ + handler = wrap_config.handler + + def wrapper(wrapped, instance, args, kwargs): + # Skip instrumentation if it's suppressed in the current context + # TODO I don't understand what this actually does + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return wrapped(*args, **kwargs) + + return_value = None + + with tracer.start_as_current_span( + wrap_config.trace_name, + kind=wrap_config.span_kind, + ) as span: + try: + # Add the input attributes to the span before execution + attributes = handler(args=args, kwargs=kwargs) + _update_span(span, attributes) + + return_value = wrapped(*args, **kwargs) + + # Add the output attributes to the span after execution + attributes = handler(return_value=return_value) + _update_span(span, attributes) + _finish_span_success(span) + except Exception as e: + # Add everything we have in the case of an error + attributes = handler(args=args, kwargs=kwargs, return_value=return_value) + _update_span(span, attributes) + _finish_span_error(span, e) + raise + + return return_value + + return wrapper + + +def wrap(wrap_config: WrapConfig, tracer: Tracer): + """Wrap a method with OpenTelemetry instrumentation. + + This function applies the wrapper created by _create_wrapper + to the method specified in the wrap_config. + + Args: + wrap_config: Configuration specifying what to wrap and how + tracer: The OpenTelemetry tracer to use for creating spans + + Returns: + The result of wrap_function_wrapper (typically None) + """ + return wrap_function_wrapper( + wrap_config.package, + f"{wrap_config.class_name}.{wrap_config.method_name}", + _create_wrapper(wrap_config, tracer), + ) + + +def unwrap(wrap_config: WrapConfig): + """Remove instrumentation wrapper from a method. + + This function removes the wrapper applied by wrap(). + + Args: + wrap_config: Configuration specifying what to unwrap + + Returns: + The result of the unwrap operation (typically None) + """ + return _unwrap( + f"{wrap_config.package}.{wrap_config.class_name}", + wrap_config.method_name, + ) + diff --git a/agentops/instrumentation/openai/attributes/common.py b/agentops/instrumentation/openai/attributes/common.py index 9bfedf5ad..b9611709b 100644 --- a/agentops/instrumentation/openai/attributes/common.py +++ b/agentops/instrumentation/openai/attributes/common.py @@ -1,16 +1,19 @@ +from typing import Optional, Tuple, Dict from agentops.logging import logger from agentops.semconv import ( InstrumentationAttributes ) from agentops.instrumentation.openai import LIBRARY_NAME, LIBRARY_VERSION -from agentops.instrumentation.common.attributes import AttributeMap -from agentops.instrumentation.common.attributes import get_common_attributes -from agentops.instrumentation.openai.attributes.response import get_response_response_attributes +from agentops.instrumentation.common.attributes import AttributeMap, get_common_attributes +from agentops.instrumentation.openai.attributes.response import ( + get_response_kwarg_attributes, + get_response_response_attributes, +) try: from openai.types.responses import Response except ImportError as e: - logger.debug(f"[agentops.instrumentation.openai_agents] Could not import OpenAI Agents SDK types: {e}") + logger.debug(f"[agentops.instrumentation.openai] Could not import OpenAI types: {e}") @@ -30,20 +33,25 @@ def get_common_instrumentation_attributes() -> AttributeMap: return attributes -def get_response_attributes(response: Response) -> AttributeMap: - """Extract attributes from a ResponseSpanData object. - - Responses are requests made to the `openai.responses` endpoint. +def get_response_attributes(args: Optional[Tuple] = None, kwargs: Optional[Dict] = None, return_value: Optional[Response] = None) -> AttributeMap: + """ - Args: - response: The `openai` `Response` object - - Returns: - Dictionary of attributes for the span """ - # TODO include prompt(s) - attributes = get_response_response_attributes(response) - attributes.update(get_common_attributes()) + # We can get an context object before, and after the request is made, so + # conditionally handle the data we have available. + attributes = get_common_instrumentation_attributes() + + # Parse the keyword arguments to extract relevant attributes + # We do not ever get `args` from this method call since it is a keyword-only method + if kwargs: + attributes.update(get_response_kwarg_attributes(kwargs)) + + # Parse the return value to extract relevant attributes + if return_value: + if isinstance(return_value, Response): + attributes.update(get_response_response_attributes(return_value)) + else: + logger.debug(F"[agentops.instrumentation.openai] Got an unexpected return type: {type(return_value)}") return attributes diff --git a/agentops/instrumentation/openai/attributes/response.py b/agentops/instrumentation/openai/attributes/response.py index dee6bf4f4..da3996406 100644 --- a/agentops/instrumentation/openai/attributes/response.py +++ b/agentops/instrumentation/openai/attributes/response.py @@ -1,4 +1,4 @@ -from typing import Any, List +from typing import Any, List, Union from agentops.logging import logger from agentops.helpers import safe_serialize from agentops.semconv import ( @@ -21,6 +21,7 @@ ResponseOutputText, ResponseReasoningItem, ResponseFunctionToolCall, + ResponseInputParam, # ResponseComputerToolCall, # ResponseFileSearchToolCall, # ResponseFunctionWebSearch, @@ -105,8 +106,63 @@ } -# TODO we call this `response_response` because in OpenAI Agents the `Response` is nested -# in a `ResponseSpan` object +def get_response_kwarg_attributes(kwargs: dict) -> AttributeMap: + """Handles interpretation of openai Responses.create method keyword arguments.""" + + # Just gather the attributes that are not present in the Response object + # TODO We could gather more here and have more context available in the + # event of an error during the request execution. + + # Method signature for `Responses.create`: + # input: Union[str, ResponseInputParam], + # model: Union[str, ChatModel], + # include: Optional[List[ResponseIncludable]] | NotGiven = NOT_GIVEN, + # instructions: Optional[str] | NotGiven = NOT_GIVEN, + # max_output_tokens: Optional[int] | NotGiven = NOT_GIVEN, + # metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + # parallel_tool_calls: Optional[bool] | NotGiven = NOT_GIVEN, + # previous_response_id: Optional[str] | NotGiven = NOT_GIVEN, + # reasoning: Optional[Reasoning] | NotGiven = NOT_GIVEN, + # store: Optional[bool] | NotGiven = NOT_GIVEN, + # stream: Optional[Literal[False]] | NotGiven = NOT_GIVEN, + # temperature: Optional[float] | NotGiven = NOT_GIVEN, + # text: ResponseTextConfigParam | NotGiven = NOT_GIVEN, + # tool_choice: response_create_params.ToolChoice | NotGiven = NOT_GIVEN, + # tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN, + # top_p: Optional[float] | NotGiven = NOT_GIVEN, + # truncation: Optional[Literal["auto", "disabled"]] | NotGiven = NOT_GIVEN, + # user: str | NotGiven = NOT_GIVEN, + # extra_headers: Headers | None = None, + # extra_query: Query | None = None, + # extra_body: Body | None = None, + # timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + attributes = {} + + # `input` can either be a `str` of a complex object type + _input: Union[str, ResponseInputParam] = kwargs.get("input") + if isinstance(_input, str): + attributes[MessageAttributes.PROMPT_ROLE.format(i=0)] = "user" + attributes[MessageAttributes.PROMPT_CONTENT.format(i=0)] = _input + elif isinstance(_input, ResponseInputParam): + for i, prompt in enumerate(_input.prompts): + # Object type is pretty diverse, so we handle common attributes, but do so + # conditionally because not all attributes are guaranteed to exist + if hasattr(prompt, "type"): + attributes[MessageAttributes.PROMPT_TYPE.format(i=i)] = prompt.type + if hasattr(prompt, "role"): + attributes[MessageAttributes.PROMPT_ROLE.format(i=i)] = prompt.role + if hasattr(prompt, "content"): + attributes[MessageAttributes.PROMPT_CONTENT.format(i=i)] = prompt.content + + # `model` is always `str` (`ChatModel` type is just a string literal) + _model: str = kwargs.get("model") + attributes[SpanAttributes.LLM_REQUEST_MODEL] = _model + + return attributes + + +# We call this `response_response` because in OpenAI Agents the `Response` is +# a return type from the `responses` module def get_response_response_attributes(response: 'Response') -> AttributeMap: """Handles interpretation of an openai Response object.""" # Response( diff --git a/agentops/instrumentation/openai/instrumentor.py b/agentops/instrumentation/openai/instrumentor.py index b13e678ea..9d6ea798b 100644 --- a/agentops/instrumentation/openai/instrumentor.py +++ b/agentops/instrumentation/openai/instrumentor.py @@ -22,35 +22,32 @@ 2. Extract data from both the request parameters and response object 3. Create spans with appropriate attributes for observability """ -from typing import Collection -from wrapt import wrap_function_wrapper +from typing import List, Collection from opentelemetry.trace import get_tracer -from opentelemetry.instrumentation.utils import unwrap - from opentelemetry.instrumentation.openai.v1 import OpenAIV1Instrumentor as ThirdPartyOpenAIV1Instrumentor from agentops.logging import logger +from agentops.instrumentation.common.wrappers import WrapConfig, wrap, unwrap from agentops.instrumentation.openai import LIBRARY_NAME, LIBRARY_VERSION -from agentops.instrumentation.openai.wrappers import sync_responses_wrapper, async_responses_wrapper from agentops.instrumentation.openai.attributes.common import get_response_attributes # Methods to wrap beyond what the third-party instrumentation handles -WRAPPED_METHODS = [ - { - "package": "openai.resources.responses", - "object": "Responses", - "method": "create", - "wrapper": sync_responses_wrapper, - "formatter": get_response_attributes, - }, - { - "package": "openai.resources.responses", - "object": "AsyncResponses", - "method": "create", - "wrapper": async_responses_wrapper, - "formatter": get_response_attributes, - }, +WRAPPED_METHODS: List[WrapConfig] = [ + WrapConfig( + trace_name="openai.responses.create", + package="openai.resources.responses", + class_name="Responses", + method_name="create", + handler=get_response_attributes, + ), + WrapConfig( + trace_name="openai.responses.create", + package="openai.resources.responses", + class_name="AsyncResponses", + method_name="create", + handler=get_response_attributes, + ), ] @@ -70,47 +67,30 @@ def _instrument(self, **kwargs): standard OpenAI API endpoints, then adds our own instrumentation for the responses module. """ - # Call the parent _instrument method first to handle all standard cases super()._instrument(**kwargs) tracer_provider = kwargs.get("tracer_provider") tracer = get_tracer(LIBRARY_NAME, LIBRARY_VERSION, tracer_provider) - # Add our own wrappers for additional modules - for wrapped_method in WRAPPED_METHODS: - package = wrapped_method["package"] - object_name = wrapped_method["object"] - method_name = wrapped_method["method"] - wrapper_func = wrapped_method["wrapper"] - formatter = wrapped_method["formatter"] - + for wrap_config in WRAPPED_METHODS: try: - wrap_function_wrapper( - package, - f"{object_name}.{method_name}", - wrapper_func(tracer, formatter), - ) - logger.debug(f"Successfully wrapped {package}.{object_name}.{method_name}") + wrap(wrap_config, tracer) + logger.debug(f"Successfully wrapped {wrap_config}") except (AttributeError, ModuleNotFoundError) as e: - logger.debug(f"Failed to wrap {package}.{object_name}.{method_name}: {e}") + logger.debug(f"Failed to wrap {wrap_config}: {e}") - logger.debug("Successfully instrumented OpenAI API with AgentOps extensions") + logger.debug("Successfully instrumented OpenAI API with Response extensions") def _uninstrument(self, **kwargs): """Remove instrumentation from OpenAI API.""" - # Call the parent _uninstrument method to handle the standard instrumentations super()._uninstrument(**kwargs) - # Unwrap our additional methods - for wrapped_method in WRAPPED_METHODS: - package = wrapped_method["package"] - object_name = wrapped_method["object"] - method_name = wrapped_method["method"] - + for wrap_config in WRAPPED_METHODS: try: - unwrap(f"{package}.{object_name}", method_name) - logger.debug(f"Successfully unwrapped {package}.{object_name}.{method_name}") + unwrap(wrap_config) + logger.debug(f"Successfully unwrapped {wrap_config}") except Exception as e: - logger.debug(f"Failed to unwrap {package}.{object_name}.{method_name}: {e}") + logger.debug(f"Failed to unwrap {wrap_config}: {e}") - logger.debug("Successfully removed OpenAI API instrumentation with AgentOps extensions") \ No newline at end of file + logger.debug("Successfully removed OpenAI API instrumentation with Response extensions") + diff --git a/agentops/instrumentation/openai/wrappers.py b/agentops/instrumentation/openai/wrappers.py deleted file mode 100644 index 616551915..000000000 --- a/agentops/instrumentation/openai/wrappers.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Wrapper functions for OpenAI API instrumentation. - -This module contains wrapper functions for the OpenAI API calls, -including both synchronous and asynchronous variants. -""" -from opentelemetry import context as context_api -from opentelemetry.instrumentation.utils import _SUPPRESS_INSTRUMENTATION_KEY -from opentelemetry.trace import SpanKind, Status, StatusCode -import time - - -def _create_response_span(tracer, span_name="openai.responses.create"): - """Create a span for an OpenAI Responses API call. - - Args: - tracer: The OpenTelemetry tracer to use - span_name: The name to use for the span - - Returns: - A span for the API call - """ - return tracer.start_span( - span_name, - kind=SpanKind.CLIENT, - ) - - -def _handle_response_span(span, start_time, success=True, exception=None): - """Handle common tasks for a response span. - - Args: - span: The span to handle - start_time: The start time of the operation - success: Whether the operation was successful - exception: Any exception that occurred - """ - # Set status based on success - if success: - span.set_status(Status(StatusCode.OK)) - else: - span.record_exception(exception) - span.set_status(Status(StatusCode.ERROR, str(exception))) - - # Record the duration - duration = time.time() - start_time - span.set_attribute("duration_s", duration) - - -def sync_responses_wrapper(tracer, formatter): - """Wrapper for synchronous openai.resources.responses.Responses.create API calls. - - This wrapper creates a span for tracking OpenAI Responses API calls. - It uses the provided formatter to extract attributes from the response. - - Args: - tracer: The OpenTelemetry tracer to use for creating spans - formatter: Function to extract attributes from the response object - - Returns: - A wrapper function that instruments the synchronous OpenAI Responses API - """ - def wrapper(wrapped, instance, args, kwargs): - # Skip instrumentation if it's suppressed in the current context - if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return wrapped(*args, **kwargs) - - # Start a span for the responses API call - with tracer.start_as_current_span( - "openai.responses.create", - kind=SpanKind.CLIENT, - ) as span: - # Record the start time for duration calculation - start_time = time.time() - - # Execute the wrapped function and get the response - try: - response = wrapped(*args, **kwargs) - - # Use the formatter to extract and set attributes from response - if response: - attributes = formatter(response) - for key, value in attributes.items(): - span.set_attribute(key, value) - - _handle_response_span(span, start_time, success=True) - except Exception as e: - _handle_response_span(span, start_time, success=False, exception=e) - raise - - return response - - return wrapper - - -def async_responses_wrapper(tracer, formatter): - """Wrapper for asynchronous openai.resources.responses.AsyncResponses.create API calls. - - This wrapper creates a span for tracking asynchronous OpenAI Responses API calls. - It uses the provided formatter to extract attributes from the response. - - Args: - tracer: The OpenTelemetry tracer to use for creating spans - formatter: Function to extract attributes from the response object - - Returns: - A wrapper function that instruments the asynchronous OpenAI Responses API - """ - async def wrapper(wrapped, instance, args, kwargs): - # Skip instrumentation if it's suppressed in the current context - if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return await wrapped(*args, **kwargs) - - # Start a span for the responses API call - with tracer.start_as_current_span( - "openai.responses.create", - kind=SpanKind.CLIENT, - ) as span: - # Record the start time for duration calculation - start_time = time.time() - - # Execute the wrapped function and get the response - try: - response = await wrapped(*args, **kwargs) - - # Use the formatter to extract and set attributes from response - if response: - attributes = formatter(response) - for key, value in attributes.items(): - span.set_attribute(key, value) - - _handle_response_span(span, start_time, success=True) - except Exception as e: - _handle_response_span(span, start_time, success=False, exception=e) - raise - - return response - - return wrapper \ No newline at end of file diff --git a/agentops/semconv/message.py b/agentops/semconv/message.py index 365e8e671..d31b17f3d 100644 --- a/agentops/semconv/message.py +++ b/agentops/semconv/message.py @@ -4,6 +4,10 @@ class MessageAttributes: """Semantic conventions for message-related attributes in AI systems.""" + PROMPT_ROLE = "gen_ai.prompt.{i}.role" # Role of the prompt message + PROMPT_CONTENT = "gen_ai.prompt.{i}.content" # Content of the prompt message + PROMPT_TYPE = "gen_ai.prompt.{i}.type" # Type of the prompt message + # Indexed completions (with {i} for interpolation) COMPLETION_ID = "gen_ai.completion.{i}.id" # Unique identifier for the completion From d58af5847490f5aa1fa497c1b16f21feed0b675e Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 26 Mar 2025 14:16:06 -0700 Subject: [PATCH 12/26] Type checking. --- agentops/instrumentation/common/wrappers.py | 2 +- .../instrumentation/openai/attributes/response.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/agentops/instrumentation/common/wrappers.py b/agentops/instrumentation/common/wrappers.py index 36537db2c..9c33962f4 100644 --- a/agentops/instrumentation/common/wrappers.py +++ b/agentops/instrumentation/common/wrappers.py @@ -8,7 +8,7 @@ from typing import Any, Optional, Tuple, Dict, Callable from dataclasses import dataclass -from wrapt import wrap_function_wrapper +from wrapt import wrap_function_wrapper # type: ignore from opentelemetry.instrumentation.utils import unwrap as _unwrap from opentelemetry.trace import Tracer from opentelemetry.trace import Span, SpanKind, Status, StatusCode diff --git a/agentops/instrumentation/openai/attributes/response.py b/agentops/instrumentation/openai/attributes/response.py index da3996406..98cc9b34d 100644 --- a/agentops/instrumentation/openai/attributes/response.py +++ b/agentops/instrumentation/openai/attributes/response.py @@ -138,13 +138,14 @@ def get_response_kwarg_attributes(kwargs: dict) -> AttributeMap: # timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, attributes = {} - # `input` can either be a `str` of a complex object type - _input: Union[str, ResponseInputParam] = kwargs.get("input") + # `input` can either be a `str` or a list of many internal types, so we duck + # type our way into some usable common attributes + _input: Union[str, list, None] = kwargs.get("input") if isinstance(_input, str): attributes[MessageAttributes.PROMPT_ROLE.format(i=0)] = "user" attributes[MessageAttributes.PROMPT_CONTENT.format(i=0)] = _input - elif isinstance(_input, ResponseInputParam): - for i, prompt in enumerate(_input.prompts): + elif isinstance(_input, list): + for i, prompt in enumerate(_input): # Object type is pretty diverse, so we handle common attributes, but do so # conditionally because not all attributes are guaranteed to exist if hasattr(prompt, "type"): @@ -153,9 +154,11 @@ def get_response_kwarg_attributes(kwargs: dict) -> AttributeMap: attributes[MessageAttributes.PROMPT_ROLE.format(i=i)] = prompt.role if hasattr(prompt, "content"): attributes[MessageAttributes.PROMPT_CONTENT.format(i=i)] = prompt.content + else: + logger.debug(f"[agentops.instrumentation.openai_agents] '{type(_input)}' is not a recognized input type.") # `model` is always `str` (`ChatModel` type is just a string literal) - _model: str = kwargs.get("model") + _model: str = str(kwargs.get("model")) attributes[SpanAttributes.LLM_REQUEST_MODEL] = _model return attributes From 9d2d493b9092ee3fc2930b04f06739f8b263f223 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 26 Mar 2025 15:09:07 -0700 Subject: [PATCH 13/26] Test coverage for instrumentation.common --- tests/unit/instrumentation/common/__init__.py | 7 + .../instrumentation/common/test_attributes.py | 264 ++++++++++++ .../instrumentation/common/test_wrappers.py | 394 ++++++++++++++++++ 3 files changed, 665 insertions(+) create mode 100644 tests/unit/instrumentation/common/__init__.py create mode 100644 tests/unit/instrumentation/common/test_attributes.py create mode 100644 tests/unit/instrumentation/common/test_wrappers.py diff --git a/tests/unit/instrumentation/common/__init__.py b/tests/unit/instrumentation/common/__init__.py new file mode 100644 index 000000000..80051e471 --- /dev/null +++ b/tests/unit/instrumentation/common/__init__.py @@ -0,0 +1,7 @@ +""" +Unit tests for the common instrumentation utilities. + +This package contains tests for the shared utilities used across +OpenTelemetry instrumentation modules, including wrappers, attributes, +and other common functionality. +""" \ No newline at end of file diff --git a/tests/unit/instrumentation/common/test_attributes.py b/tests/unit/instrumentation/common/test_attributes.py new file mode 100644 index 000000000..cea2448c1 --- /dev/null +++ b/tests/unit/instrumentation/common/test_attributes.py @@ -0,0 +1,264 @@ +""" +Unit tests for the common attributes module. + +This module tests the functionality of the common attribute processing utilities +shared across all instrumentors, including attribute extraction, common attribute +getters, and base trace/span attribute functions. +""" + +import pytest +from unittest.mock import MagicMock, patch + +from agentops.instrumentation.common.attributes import ( + _extract_attributes_from_mapping, + get_common_attributes, + get_base_trace_attributes, + get_base_span_attributes, + AttributeMap +) +from agentops.semconv import ( + CoreAttributes, + InstrumentationAttributes, + WorkflowAttributes, +) + + +class TestAttributeExtraction: + """Tests for attribute extraction utilities.""" + + def test_extract_attributes_from_object(self): + """Test extracting attributes from an object.""" + # Create a test object with attributes + class TestObject: + def __init__(self): + self.attr1 = "value1" + self.attr2 = 42 + self.attr3 = None + self.attr4 = [] + self.attr5 = {} + self.attr6 = ["list", "of", "values"] + self.attr7 = {"key": "value"} + + test_obj = TestObject() + + # Define a mapping of target attributes to source attributes + mapping = { + "target_attr1": "attr1", + "target_attr2": "attr2", + "target_attr3": "attr3", # None value, should be skipped + "target_attr4": "attr4", # Empty list, should be skipped + "target_attr5": "attr5", # Empty dict, should be skipped + "target_attr6": "attr6", # List value, should be handled + "target_attr7": "attr7", # Dict value, should be handled + "target_attr8": "missing_attr", # Missing attribute, should be skipped + } + + # Extract attributes + attributes = _extract_attributes_from_mapping(test_obj, mapping) + + # Verify extracted attributes + assert "target_attr1" in attributes + assert attributes["target_attr1"] == "value1" + assert "target_attr2" in attributes + assert attributes["target_attr2"] == 42 + assert "target_attr3" not in attributes # None value should be skipped + assert "target_attr4" not in attributes # Empty list should be skipped + assert "target_attr5" not in attributes # Empty dict should be skipped + assert "target_attr6" in attributes # List value should be handled + assert attributes["target_attr6"] == '["list", "of", "values"]' # JSON encoded + assert "target_attr7" in attributes # Dict value should be handled + assert attributes["target_attr7"] == '{"key": "value"}' # JSON encoded + assert "target_attr8" not in attributes # Missing attribute should be skipped + + def test_extract_attributes_from_dict(self): + """Test extracting attributes from a dictionary.""" + # Create a test dictionary + test_dict = { + "attr1": "value1", + "attr2": 42, + "attr3": None, + "attr4": [], + "attr5": {}, + "attr6": ["list", "of", "values"], + "attr7": {"key": "value"}, + } + + # Define a mapping of target attributes to source attributes + mapping = { + "target_attr1": "attr1", + "target_attr2": "attr2", + "target_attr3": "attr3", # None value, should be skipped + "target_attr4": "attr4", # Empty list, should be skipped + "target_attr5": "attr5", # Empty dict, should be skipped + "target_attr6": "attr6", # List value, should be handled + "target_attr7": "attr7", # Dict value, should be handled + "target_attr8": "missing_attr", # Missing key, should be skipped + } + + # Extract attributes + attributes = _extract_attributes_from_mapping(test_dict, mapping) + + # Verify extracted attributes + assert "target_attr1" in attributes + assert attributes["target_attr1"] == "value1" + assert "target_attr2" in attributes + assert attributes["target_attr2"] == 42 + assert "target_attr3" not in attributes # None value should be skipped + assert "target_attr4" not in attributes # Empty list should be skipped + assert "target_attr5" not in attributes # Empty dict should be skipped + assert "target_attr6" in attributes # List value should be handled + assert attributes["target_attr6"] == '["list", "of", "values"]' # JSON encoded + assert "target_attr7" in attributes # Dict value should be handled + assert attributes["target_attr7"] == '{"key": "value"}' # JSON encoded + assert "target_attr8" not in attributes # Missing key should be skipped + + +class TestCommonAttributes: + """Tests for common attribute getters.""" + + def test_get_common_attributes(self): + """Test getting common instrumentation attributes.""" + # Mock the version function to return a fixed value + with patch("agentops.instrumentation.common.attributes.get_agentops_version", return_value="0.1.2"): + # Get common attributes + attributes = get_common_attributes() + + # Verify attributes + assert InstrumentationAttributes.NAME in attributes + assert attributes[InstrumentationAttributes.NAME] == "agentops" + assert InstrumentationAttributes.VERSION in attributes + assert attributes[InstrumentationAttributes.VERSION] == "0.1.2" + + def test_get_base_trace_attributes_with_valid_trace(self): + """Test getting base trace attributes with a valid trace.""" + # Create a mock trace + class MockTrace: + def __init__(self): + self.trace_id = "test_trace_id" + self.name = "test_trace_name" + + mock_trace = MockTrace() + + # Mock the common attributes and tags functions + with patch("agentops.instrumentation.common.attributes.get_common_attributes", + return_value={InstrumentationAttributes.NAME: "agentops", InstrumentationAttributes.VERSION: "0.1.2"}): + with patch("agentops.instrumentation.common.attributes.get_tags_from_config", + return_value={"tag1": "value1", "tag2": "value2"}): + # Get base trace attributes + attributes = get_base_trace_attributes(mock_trace) + + # Verify attributes + assert CoreAttributes.TRACE_ID in attributes + assert attributes[CoreAttributes.TRACE_ID] == "test_trace_id" + assert WorkflowAttributes.WORKFLOW_NAME in attributes + assert attributes[WorkflowAttributes.WORKFLOW_NAME] == "test_trace_name" + assert WorkflowAttributes.WORKFLOW_STEP_TYPE in attributes + assert attributes[WorkflowAttributes.WORKFLOW_STEP_TYPE] == "trace" + assert InstrumentationAttributes.NAME in attributes + assert attributes[InstrumentationAttributes.NAME] == "agentops" + assert InstrumentationAttributes.VERSION in attributes + assert attributes[InstrumentationAttributes.VERSION] == "0.1.2" + assert CoreAttributes.TAGS in attributes + assert attributes[CoreAttributes.TAGS] == {"tag1": "value1", "tag2": "value2"} + + def test_get_base_trace_attributes_with_invalid_trace(self): + """Test getting base trace attributes with an invalid trace (missing trace_id).""" + # Create a mock trace without trace_id + class MockTrace: + def __init__(self): + self.name = "test_trace_name" + + mock_trace = MockTrace() + + # Mock the logger + with patch("agentops.instrumentation.common.attributes.logger.warning") as mock_warning: + # Get base trace attributes + attributes = get_base_trace_attributes(mock_trace) + + # Verify logger was called + mock_warning.assert_called_once_with("Cannot create trace attributes: missing trace_id") + + # Verify attributes is empty + assert attributes == {} + + def test_get_base_span_attributes_with_basic_span(self): + """Test getting base span attributes with a basic span.""" + # Create a mock span + class MockSpan: + def __init__(self): + self.span_id = "test_span_id" + self.trace_id = "test_trace_id" + + mock_span = MockSpan() + + # Mock the common attributes function + with patch("agentops.instrumentation.common.attributes.get_common_attributes", + return_value={InstrumentationAttributes.NAME: "agentops", InstrumentationAttributes.VERSION: "0.1.2"}): + # Get base span attributes + attributes = get_base_span_attributes(mock_span) + + # Verify attributes + assert CoreAttributes.SPAN_ID in attributes + assert attributes[CoreAttributes.SPAN_ID] == "test_span_id" + assert CoreAttributes.TRACE_ID in attributes + assert attributes[CoreAttributes.TRACE_ID] == "test_trace_id" + assert InstrumentationAttributes.NAME in attributes + assert attributes[InstrumentationAttributes.NAME] == "agentops" + assert InstrumentationAttributes.VERSION in attributes + assert attributes[InstrumentationAttributes.VERSION] == "0.1.2" + assert CoreAttributes.PARENT_ID not in attributes # No parent_id in span + + def test_get_base_span_attributes_with_parent(self): + """Test getting base span attributes with a span that has a parent.""" + # Create a mock span with parent_id + class MockSpan: + def __init__(self): + self.span_id = "test_span_id" + self.trace_id = "test_trace_id" + self.parent_id = "test_parent_id" + + mock_span = MockSpan() + + # Mock the common attributes function + with patch("agentops.instrumentation.common.attributes.get_common_attributes", + return_value={InstrumentationAttributes.NAME: "agentops", InstrumentationAttributes.VERSION: "0.1.2"}): + # Get base span attributes + attributes = get_base_span_attributes(mock_span) + + # Verify attributes + assert CoreAttributes.SPAN_ID in attributes + assert attributes[CoreAttributes.SPAN_ID] == "test_span_id" + assert CoreAttributes.TRACE_ID in attributes + assert attributes[CoreAttributes.TRACE_ID] == "test_trace_id" + assert CoreAttributes.PARENT_ID in attributes + assert attributes[CoreAttributes.PARENT_ID] == "test_parent_id" + assert InstrumentationAttributes.NAME in attributes + assert attributes[InstrumentationAttributes.NAME] == "agentops" + assert InstrumentationAttributes.VERSION in attributes + assert attributes[InstrumentationAttributes.VERSION] == "0.1.2" + + def test_get_base_span_attributes_with_unknown_values(self): + """Test getting base span attributes with a span that has unknown values.""" + # Create a mock object that doesn't have the expected attributes + mock_object = object() + + # Mock the common attributes function + with patch("agentops.instrumentation.common.attributes.get_common_attributes", + return_value={InstrumentationAttributes.NAME: "agentops", InstrumentationAttributes.VERSION: "0.1.2"}): + # Get base span attributes + attributes = get_base_span_attributes(mock_object) + + # Verify attributes + assert CoreAttributes.SPAN_ID in attributes + assert attributes[CoreAttributes.SPAN_ID] == "unknown" + assert CoreAttributes.TRACE_ID in attributes + assert attributes[CoreAttributes.TRACE_ID] == "unknown" + assert CoreAttributes.PARENT_ID not in attributes # No parent_id + assert InstrumentationAttributes.NAME in attributes + assert attributes[InstrumentationAttributes.NAME] == "agentops" + assert InstrumentationAttributes.VERSION in attributes + assert attributes[InstrumentationAttributes.VERSION] == "0.1.2" + + +if __name__ == "__main__": + pytest.main() \ No newline at end of file diff --git a/tests/unit/instrumentation/common/test_wrappers.py b/tests/unit/instrumentation/common/test_wrappers.py new file mode 100644 index 000000000..73fcac4e3 --- /dev/null +++ b/tests/unit/instrumentation/common/test_wrappers.py @@ -0,0 +1,394 @@ +""" +Unit tests for the common wrappers module. + +This module tests the functionality of the common wrapper utilities +for OpenTelemetry instrumentation, including WrapConfig, _update_span, +_finish_span_success, _finish_span_error, _create_wrapper, wrap, and unwrap. +""" + +import pytest +from unittest.mock import MagicMock, patch +from typing import Dict, Any, Optional, Tuple + +from opentelemetry.trace import SpanKind + +from agentops.instrumentation.common.wrappers import ( + WrapConfig, _update_span, _finish_span_success, _finish_span_error, + _create_wrapper, wrap, unwrap, AttributeHandler +) +from agentops.instrumentation.common.attributes import AttributeMap +from tests.unit.instrumentation.mock_span import MockTracingSpan + + +class TestWrapConfig: + """Tests for the WrapConfig class.""" + + def test_wrap_config_initialization(self): + """Test that WrapConfig is initialized properly with default values.""" + # Create a simple attribute handler + def dummy_handler( + args: Optional[Tuple] = None, + kwargs: Optional[Dict] = None, + return_value: Optional[Any] = None + ) -> AttributeMap: + return {"key": "value"} + + # Initialize a WrapConfig + config = WrapConfig( + trace_name="test_trace", + package="test_package", + class_name="TestClass", + method_name="test_method", + handler=dummy_handler + ) + + # Verify config values + assert config.trace_name == "test_trace" + assert config.package == "test_package" + assert config.class_name == "TestClass" + assert config.method_name == "test_method" + assert config.handler == dummy_handler + assert config.span_kind == SpanKind.CLIENT # Default value + + def test_wrap_config_repr(self): + """Test the string representation of WrapConfig.""" + # Create a simple attribute handler + def dummy_handler( + args: Optional[Tuple] = None, + kwargs: Optional[Dict] = None, + return_value: Optional[Any] = None + ) -> AttributeMap: + return {"key": "value"} + + # Initialize a WrapConfig + config = WrapConfig( + trace_name="test_trace", + package="test_package", + class_name="TestClass", + method_name="test_method", + handler=dummy_handler + ) + + # Verify the string representation + assert repr(config) == "test_package.TestClass.test_method" + + def test_wrap_config_with_custom_span_kind(self): + """Test that WrapConfig accepts a custom span kind.""" + # Create a simple attribute handler + def dummy_handler( + args: Optional[Tuple] = None, + kwargs: Optional[Dict] = None, + return_value: Optional[Any] = None + ) -> AttributeMap: + return {"key": "value"} + + # Initialize a WrapConfig with custom span kind + config = WrapConfig( + trace_name="test_trace", + package="test_package", + class_name="TestClass", + method_name="test_method", + handler=dummy_handler, + span_kind=SpanKind.SERVER + ) + + # Verify the span kind + assert config.span_kind == SpanKind.SERVER + + +class TestSpanHelpers: + """Tests for span helper functions.""" + + def test_update_span(self): + """Test _update_span sets attributes on a span.""" + # Create a mock span + mock_span = MockTracingSpan() + + # Define attributes to set + attributes = {"key1": "value1", "key2": 42, "key3": True} + + # Call _update_span + _update_span(mock_span, attributes) + + # Verify attributes were set + for key, value in attributes.items(): + assert mock_span.attributes[key] == value + + def test_finish_span_success(self): + """Test _finish_span_success sets status to OK.""" + # Create a mock span + mock_span = MockTracingSpan() + + # Call _finish_span_success + _finish_span_success(mock_span) + + # Verify status was set + assert mock_span.status is not None + # The actual object is a real Status with StatusCode.OK + # We're not checking the exact type, just that it was called with OK status code + + def test_finish_span_error(self): + """Test _finish_span_error sets error status and records exception.""" + # Create a mock span + mock_span = MockTracingSpan() + + # Create a test exception + test_exception = ValueError("Test error") + + # Call _finish_span_error + _finish_span_error(mock_span, test_exception) + + # Verify status was set to ERROR + assert mock_span.status is not None + # The actual object is a real Status with StatusCode.ERROR + # We're not checking the exact type, just that it was called with ERROR status code + + # Verify exception was recorded + assert len(mock_span.events) == 1 + assert mock_span.events[0]["name"] == "exception" + assert mock_span.events[0]["exception"] == test_exception + + +class TestCreateWrapper: + """Tests for _create_wrapper and wrapper functionality.""" + + def test_create_wrapper_success_path(self): + """Test wrapper created by _create_wrapper handles success path.""" + # Create a mock tracer + mock_tracer = MagicMock() + mock_span = MockTracingSpan() + + # Mock start_as_current_span to return our mock span + mock_tracer.start_as_current_span.return_value.__enter__.return_value = mock_span + + # Create a simple attribute handler + def dummy_handler( + args: Optional[Tuple] = None, + kwargs: Optional[Dict] = None, + return_value: Optional[Any] = None + ) -> AttributeMap: + result = {} + if args: + result["args"] = str(args) + if kwargs: + result["kwargs"] = str(kwargs) + if return_value is not None: + result["return_value"] = str(return_value) + return result + + # Create a WrapConfig + config = WrapConfig( + trace_name="test_trace", + package="test_package", + class_name="TestClass", + method_name="test_method", + handler=dummy_handler + ) + + # Create the wrapper + wrapper = _create_wrapper(config, mock_tracer) + + # Create a mock wrapped function + def mock_wrapped(*args, **kwargs): + return "success" + + # Call the wrapper with the wrapped function + result = wrapper(mock_wrapped, None, ("arg1", "arg2"), {"kwarg1": "value1"}) + + # Verify the result + assert result == "success" + + # Verify tracer was called correctly + mock_tracer.start_as_current_span.assert_called_once_with( + "test_trace", + kind=SpanKind.CLIENT + ) + + # Verify attributes were set on the span + assert "args" in mock_span.attributes + assert "('arg1', 'arg2')" == mock_span.attributes["args"] + assert "kwargs" in mock_span.attributes + assert "{'kwarg1': 'value1'}" == mock_span.attributes["kwargs"] + assert "return_value" in mock_span.attributes + assert "success" == mock_span.attributes["return_value"] + + def test_create_wrapper_error_path(self): + """Test wrapper created by _create_wrapper handles error path.""" + # Create a mock tracer + mock_tracer = MagicMock() + mock_span = MockTracingSpan() + + # Mock start_as_current_span to return our mock span + mock_tracer.start_as_current_span.return_value.__enter__.return_value = mock_span + + # Create a simple attribute handler + def dummy_handler( + args: Optional[Tuple] = None, + kwargs: Optional[Dict] = None, + return_value: Optional[Any] = None + ) -> AttributeMap: + result = {} + if args: + result["args"] = str(args) + if kwargs: + result["kwargs"] = str(kwargs) + if return_value is not None: + result["return_value"] = str(return_value) + return result + + # Create a WrapConfig + config = WrapConfig( + trace_name="test_trace", + package="test_package", + class_name="TestClass", + method_name="test_method", + handler=dummy_handler + ) + + # Create the wrapper + wrapper = _create_wrapper(config, mock_tracer) + + # Create a mock wrapped function that raises an exception + def mock_wrapped(*args, **kwargs): + raise ValueError("Test error") + + # Call the wrapper with the wrapped function, expecting an exception + with pytest.raises(ValueError, match="Test error"): + wrapper(mock_wrapped, None, ("arg1", "arg2"), {"kwarg1": "value1"}) + + # Verify tracer was called correctly + mock_tracer.start_as_current_span.assert_called_once_with( + "test_trace", + kind=SpanKind.CLIENT + ) + + # Verify attributes were set on the span + assert "args" in mock_span.attributes + assert "('arg1', 'arg2')" == mock_span.attributes["args"] + assert "kwargs" in mock_span.attributes + assert "{'kwarg1': 'value1'}" == mock_span.attributes["kwargs"] + + # Verify exception was recorded + assert len(mock_span.events) == 1 + assert mock_span.events[0]["name"] == "exception" + assert isinstance(mock_span.events[0]["exception"], ValueError) + + def test_create_wrapper_suppressed_instrumentation(self): + """Test wrapper respects suppressed instrumentation context.""" + # Create a mock tracer + mock_tracer = MagicMock() + + # Create a simple attribute handler + def dummy_handler( + args: Optional[Tuple] = None, + kwargs: Optional[Dict] = None, + return_value: Optional[Any] = None + ) -> AttributeMap: + return {} + + # Create a WrapConfig + config = WrapConfig( + trace_name="test_trace", + package="test_package", + class_name="TestClass", + method_name="test_method", + handler=dummy_handler + ) + + # Create the wrapper + wrapper = _create_wrapper(config, mock_tracer) + + # Create a mock wrapped function + mock_wrapped = MagicMock(return_value="success") + + # Mock the context_api to return True for suppressed instrumentation + with patch('agentops.instrumentation.common.wrappers.context_api.get_value', return_value=True): + result = wrapper(mock_wrapped, None, ("arg1", "arg2"), {"kwarg1": "value1"}) + + # Verify the result + assert result == "success" + + # Verify tracer was NOT called + mock_tracer.start_as_current_span.assert_not_called() + + # Verify wrapped function was called directly + mock_wrapped.assert_called_once_with("arg1", "arg2", kwarg1="value1") + + +class TestWrapUnwrap: + """Tests for wrap and unwrap functions.""" + + def test_wrap_function(self): + """Test that wrap calls wrap_function_wrapper with correct arguments.""" + # Create a simple attribute handler + def dummy_handler( + args: Optional[Tuple] = None, + kwargs: Optional[Dict] = None, + return_value: Optional[Any] = None + ) -> AttributeMap: + return {} + + # Create a WrapConfig + config = WrapConfig( + trace_name="test_trace", + package="test_package", + class_name="TestClass", + method_name="test_method", + handler=dummy_handler + ) + + # Create a mock tracer + mock_tracer = MagicMock() + + # Mock wrap_function_wrapper + with patch('agentops.instrumentation.common.wrappers.wrap_function_wrapper') as mock_wrap: + # Mock _create_wrapper to return a simple function + with patch('agentops.instrumentation.common.wrappers._create_wrapper') as mock_create_wrapper: + mock_create_wrapper.return_value = lambda *args: None + + # Call wrap + wrap(config, mock_tracer) + + # Verify wrap_function_wrapper was called correctly + mock_wrap.assert_called_once_with( + "test_package", + "TestClass.test_method", + mock_create_wrapper.return_value + ) + + # Verify _create_wrapper was called correctly + mock_create_wrapper.assert_called_once_with(config, mock_tracer) + + def test_unwrap_function(self): + """Test that unwrap calls _unwrap with correct arguments.""" + # Create a simple attribute handler + def dummy_handler( + args: Optional[Tuple] = None, + kwargs: Optional[Dict] = None, + return_value: Optional[Any] = None + ) -> AttributeMap: + return {} + + # Create a WrapConfig + config = WrapConfig( + trace_name="test_trace", + package="test_package", + class_name="TestClass", + method_name="test_method", + handler=dummy_handler + ) + + # Mock _unwrap + with patch('agentops.instrumentation.common.wrappers._unwrap') as mock_unwrap: + # Call unwrap + unwrap(config) + + # Verify _unwrap was called correctly + mock_unwrap.assert_called_once_with( + "test_package.TestClass", + "test_method" + ) + + +if __name__ == "__main__": + pytest.main() \ No newline at end of file From 12b559b04f37e8341478b992d1ece6d728542a68 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 26 Mar 2025 16:07:01 -0700 Subject: [PATCH 14/26] Type in method def should be string in case of missing import. --- agentops/instrumentation/openai/attributes/common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/agentops/instrumentation/openai/attributes/common.py b/agentops/instrumentation/openai/attributes/common.py index b9611709b..37e75e5d2 100644 --- a/agentops/instrumentation/openai/attributes/common.py +++ b/agentops/instrumentation/openai/attributes/common.py @@ -16,7 +16,6 @@ logger.debug(f"[agentops.instrumentation.openai] Could not import OpenAI types: {e}") - def get_common_instrumentation_attributes() -> AttributeMap: """Get common instrumentation attributes for the OpenAI Agents instrumentation. @@ -33,7 +32,7 @@ def get_common_instrumentation_attributes() -> AttributeMap: return attributes -def get_response_attributes(args: Optional[Tuple] = None, kwargs: Optional[Dict] = None, return_value: Optional[Response] = None) -> AttributeMap: +def get_response_attributes(args: Optional[Tuple] = None, kwargs: Optional[Dict] = None, return_value: Optional['Response'] = None) -> AttributeMap: """ """ From 9d778456d557430c0ed4d3bc23f842355eb80d03 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 26 Mar 2025 16:07:26 -0700 Subject: [PATCH 15/26] Wrap third party module imports from openai in try except block --- .../openai/v1/assistant_wrappers.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/third_party/opentelemetry/instrumentation/openai/v1/assistant_wrappers.py b/third_party/opentelemetry/instrumentation/openai/v1/assistant_wrappers.py index 9dd604cde..84c07fdba 100644 --- a/third_party/opentelemetry/instrumentation/openai/v1/assistant_wrappers.py +++ b/third_party/opentelemetry/instrumentation/openai/v1/assistant_wrappers.py @@ -13,10 +13,21 @@ from opentelemetry.instrumentation.openai.utils import _with_tracer_wrapper, dont_throw from opentelemetry.instrumentation.openai.shared.config import Config -from openai._legacy_response import LegacyAPIResponse -from openai.types.beta.threads.run import Run +logger = logging.getLogger(__name__) # noqa + +try: + from openai._legacy_response import LegacyAPIResponse +except (ImportError, ModuleNotFoundError): + # This was removed from the `openai` package at some point + logger.debug("LegacyAPIResponse not found in openai package") + LegacyAPIResponse = None + +try: + from openai.types.beta.threads.run import Run +except (ImportError, ModuleNotFoundError): + logger.debug("Run not found in openai package") + Run = None -logger = logging.getLogger(__name__) assistants = {} runs = {} From d5cdb57bf99bb0ed28e081ab4320382422f3199b Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 26 Mar 2025 16:23:53 -0700 Subject: [PATCH 16/26] OpenAI instrumentation tests. (Relocated to openai_core to avoid import hijack) --- .../instrumentation/openai_core/__init__.py | 0 .../openai_core/test_common_attributes.py | 170 ++++++++ .../openai_core/test_instrumentor.py | 209 +++++++++ .../openai_core/test_response_attributes.py | 404 ++++++++++++++++++ .../instrumentation/test_common_attributes.py | 285 ------------ 5 files changed, 783 insertions(+), 285 deletions(-) create mode 100644 tests/unit/instrumentation/openai_core/__init__.py create mode 100644 tests/unit/instrumentation/openai_core/test_common_attributes.py create mode 100644 tests/unit/instrumentation/openai_core/test_instrumentor.py create mode 100644 tests/unit/instrumentation/openai_core/test_response_attributes.py delete mode 100644 tests/unit/instrumentation/test_common_attributes.py diff --git a/tests/unit/instrumentation/openai_core/__init__.py b/tests/unit/instrumentation/openai_core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/instrumentation/openai_core/test_common_attributes.py b/tests/unit/instrumentation/openai_core/test_common_attributes.py new file mode 100644 index 000000000..8e02b8a17 --- /dev/null +++ b/tests/unit/instrumentation/openai_core/test_common_attributes.py @@ -0,0 +1,170 @@ +""" +Tests for OpenAI common attribute handling + +This module contains tests for common attribute handling functions in the OpenAI instrumentation, +specifically focusing on the get_response_attributes function which combines various attribute +extraction functions. +""" + +import pytest +from unittest.mock import MagicMock, patch + +from agentops.instrumentation.openai.attributes.common import ( + get_common_instrumentation_attributes, + get_response_attributes +) +from agentops.instrumentation.openai import LIBRARY_NAME, LIBRARY_VERSION +from agentops.semconv import ( + SpanAttributes, + MessageAttributes, + InstrumentationAttributes +) + + +class MockResponse: + """Mock Response object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + + +class TestCommonAttributes: + """Tests for OpenAI common attribute handling functions""" + + def test_get_common_instrumentation_attributes(self): + """Test that common instrumentation attributes are correctly generated""" + # Call the function + attributes = get_common_instrumentation_attributes() + + # Verify library attributes are set + assert InstrumentationAttributes.LIBRARY_NAME in attributes + assert attributes[InstrumentationAttributes.LIBRARY_NAME] == LIBRARY_NAME + assert InstrumentationAttributes.LIBRARY_VERSION in attributes + assert attributes[InstrumentationAttributes.LIBRARY_VERSION] == LIBRARY_VERSION + + # Verify common attributes from parent function are included + # (these would be added by get_common_attributes) + assert InstrumentationAttributes.NAME in attributes + + def test_get_response_attributes_with_kwargs(self): + """Test that response attributes are correctly extracted from kwargs""" + # Create kwargs + kwargs = { + "input": "What is the capital of France?", + "model": "gpt-4o", + "temperature": 0.7, + "top_p": 1.0, + } + + # Mock the kwarg extraction function + with patch('agentops.instrumentation.openai.attributes.common.get_response_kwarg_attributes') as mock_kwarg_attributes: + mock_kwarg_attributes.return_value = { + MessageAttributes.PROMPT_ROLE.format(i=0): "user", + MessageAttributes.PROMPT_CONTENT.format(i=0): "What is the capital of France?", + SpanAttributes.LLM_REQUEST_MODEL: "gpt-4o" + } + + # Call the function + attributes = get_response_attributes(kwargs=kwargs) + + # Verify kwarg extraction was called + mock_kwarg_attributes.assert_called_once_with(kwargs) + + # Verify attributes from kwarg extraction are included + assert MessageAttributes.PROMPT_ROLE.format(i=0) in attributes + assert attributes[MessageAttributes.PROMPT_ROLE.format(i=0)] == "user" + assert MessageAttributes.PROMPT_CONTENT.format(i=0) in attributes + assert attributes[MessageAttributes.PROMPT_CONTENT.format(i=0)] == "What is the capital of France?" + assert SpanAttributes.LLM_REQUEST_MODEL in attributes + assert attributes[SpanAttributes.LLM_REQUEST_MODEL] == "gpt-4o" + + def test_get_response_attributes_with_return_value(self): + """Test that response attributes are correctly extracted from return value""" + # Create a mock Response object with all required attributes + response = MockResponse({ + "id": "resp_12345", + "model": "gpt-4o", + "instructions": "You are a helpful assistant.", + "output": [], + "tools": [], + "reasoning": None, + "usage": None, + "__dict__": { + "id": "resp_12345", + "model": "gpt-4o", + "instructions": "You are a helpful assistant.", + "output": [], + "tools": [], + "reasoning": None, + "usage": None + } + }) + + # Use direct patching of Response class check instead + with patch('agentops.instrumentation.openai.attributes.common.Response', MockResponse): + # Call the function + attributes = get_response_attributes(return_value=response) + + # Verify attributes are included without mocking the specific function + # Just verify some basic attributes are set + assert InstrumentationAttributes.LIBRARY_NAME in attributes + assert attributes[InstrumentationAttributes.LIBRARY_NAME] == LIBRARY_NAME + assert InstrumentationAttributes.LIBRARY_VERSION in attributes + assert attributes[InstrumentationAttributes.LIBRARY_VERSION] == LIBRARY_VERSION + + def test_get_response_attributes_with_both(self): + """Test that response attributes are correctly extracted from both kwargs and return value""" + # Create kwargs + kwargs = { + "input": "What is the capital of France?", + "model": "gpt-4o", + "temperature": 0.7, + "top_p": 1.0, + } + + # Create a mock Response object with all required attributes + response = MockResponse({ + "id": "resp_12345", + "model": "gpt-4o", + "instructions": "You are a helpful assistant.", + "output": [], + "tools": [], + "reasoning": None, + "usage": None, + "__dict__": { + "id": "resp_12345", + "model": "gpt-4o", + "instructions": "You are a helpful assistant.", + "output": [], + "tools": [], + "reasoning": None, + "usage": None + } + }) + + # Instead of mocking the internal functions, test the integration directly + with patch('agentops.instrumentation.openai.attributes.common.Response', MockResponse): + # Call the function + attributes = get_response_attributes(kwargs=kwargs, return_value=response) + + # Verify the key response attributes are in the final attributes dict + assert InstrumentationAttributes.LIBRARY_NAME in attributes + assert attributes[InstrumentationAttributes.LIBRARY_NAME] == LIBRARY_NAME + + def test_get_response_attributes_with_unexpected_return_type(self): + """Test handling of unexpected return value type""" + # Create an object that's not a Response + not_a_response = "not a response" + + # Should log a debug message but not raise an exception + with patch('agentops.instrumentation.openai.attributes.common.logger.debug') as mock_logger: + # Call the function + attributes = get_response_attributes(return_value=not_a_response) + + # Verify debug message was logged + mock_logger.assert_called_once() + assert "unexpected return type" in mock_logger.call_args[0][0] + + # Verify common attributes are still present + assert InstrumentationAttributes.NAME in attributes + assert InstrumentationAttributes.LIBRARY_NAME in attributes \ No newline at end of file diff --git a/tests/unit/instrumentation/openai_core/test_instrumentor.py b/tests/unit/instrumentation/openai_core/test_instrumentor.py new file mode 100644 index 000000000..6509de406 --- /dev/null +++ b/tests/unit/instrumentation/openai_core/test_instrumentor.py @@ -0,0 +1,209 @@ +""" +Tests for OpenAI API Instrumentation + +This module contains tests for properly handling and serializing data from the OpenAI API. +It verifies that our instrumentation correctly captures and instruments API calls, +specifically focusing on the newer Response API format used in the Agents SDK. + +The OpenAI Instrumentor extends the third-party OpenTelemetry instrumentor and adds +our own wrapper for the newer Response API format. +""" + +import json +import os +import pytest +from unittest.mock import MagicMock, patch + +from opentelemetry.trace import get_tracer, StatusCode + +from agentops.instrumentation.openai.instrumentor import OpenAIInstrumentor +from agentops.instrumentation.common.wrappers import WrapConfig +from agentops.instrumentation.openai import LIBRARY_NAME, LIBRARY_VERSION +from agentops.semconv import ( + SpanAttributes, + MessageAttributes, + InstrumentationAttributes +) +from tests.unit.instrumentation.mock_span import ( + MockTracingSpan, + setup_mock_tracer +) + + +# Utility function to load fixtures +def load_fixture(fixture_name): + """Load a test fixture from the fixtures directory""" + fixture_path = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "fixtures", + fixture_name + ) + with open(fixture_path, "r") as f: + return json.load(f) + + +# Load response test fixtures +OPENAI_RESPONSE = load_fixture("openai_response.json") # Response API format (newer API format) with output array +OPENAI_RESPONSE_TOOL_CALLS = load_fixture("openai_response_tool_calls.json") # Response API with tool calls + + +class TestOpenAIInstrumentor: + """Tests for OpenAI API instrumentation, focusing on Response API support""" + + @pytest.fixture + def instrumentor(self): + """Set up OpenAI instrumentor for tests""" + # Create a real instrumentation setup for testing + mock_tracer_provider = MagicMock() + instrumentor = OpenAIInstrumentor() + + # To avoid timing issues with the fixture, we need to ensure patch + # objects are created before being used in the test + mock_wrap = patch('agentops.instrumentation.openai.instrumentor.wrap').start() + mock_unwrap = patch('agentops.instrumentation.openai.instrumentor.unwrap').start() + mock_instrument = patch.object(instrumentor, '_instrument', wraps=instrumentor._instrument).start() + mock_uninstrument = patch.object(instrumentor, '_uninstrument', wraps=instrumentor._uninstrument).start() + + # Instrument + instrumentor._instrument(tracer_provider=mock_tracer_provider) + + yield { + 'instrumentor': instrumentor, + 'tracer_provider': mock_tracer_provider, + 'mock_wrap': mock_wrap, + 'mock_unwrap': mock_unwrap, + 'mock_instrument': mock_instrument, + 'mock_uninstrument': mock_uninstrument + } + + # Uninstrument - must happen before stopping patches + instrumentor._uninstrument() + + # Stop patches + patch.stopall() + + def test_instrumentor_initialization(self): + """Test instrumentor is initialized with correct configuration""" + instrumentor = OpenAIInstrumentor() + assert instrumentor.__class__.__name__ == "OpenAIInstrumentor" + + # Verify it inherits from the third-party OpenAIV1Instrumentor + from opentelemetry.instrumentation.openai.v1 import OpenAIV1Instrumentor + assert isinstance(instrumentor, OpenAIV1Instrumentor) + + def test_instrument_method_wraps_response_api(self, instrumentor): + """Test the _instrument method wraps the Response API methods""" + mock_wrap = instrumentor['mock_wrap'] + + # Verify wrap was called for each method in WRAPPED_METHODS + assert mock_wrap.call_count == 2 + + # Check the first call arguments for Responses.create + first_call_args = mock_wrap.call_args_list[0][0] + assert isinstance(first_call_args[0], WrapConfig) + assert first_call_args[0].trace_name == "openai.responses.create" + assert first_call_args[0].package == "openai.resources.responses" + assert first_call_args[0].class_name == "Responses" + assert first_call_args[0].method_name == "create" + + # Check the second call arguments for AsyncResponses.create + second_call_args = mock_wrap.call_args_list[1][0] + assert isinstance(second_call_args[0], WrapConfig) + assert second_call_args[0].trace_name == "openai.responses.create" + assert second_call_args[0].package == "openai.resources.responses" + assert second_call_args[0].class_name == "AsyncResponses" + assert second_call_args[0].method_name == "create" + + def test_uninstrument_method_unwraps_response_api(self, instrumentor): + """Test the _uninstrument method unwraps the Response API methods""" + # For these tests, we'll manually call the unwrap method with the expected configs + # since the fixture setup has been changed + + instrumentor_obj = instrumentor['instrumentor'] + + # Reset the mock to clear any previous calls + mock_unwrap = instrumentor['mock_unwrap'] + mock_unwrap.reset_mock() + + # Call the uninstrument method directly + instrumentor_obj._uninstrument() + + # Now verify the method was called + assert mock_unwrap.called, "unwrap was not called during _uninstrument" + + def test_calls_parent_instrument(self, instrumentor): + """Test that the instrumentor calls the parent class's _instrument method""" + mock_instrument = instrumentor['mock_instrument'] + + # Verify super()._instrument was called + assert mock_instrument.called + + # Verify the tracer provider was passed to the parent method + call_kwargs = mock_instrument.call_args[1] + assert 'tracer_provider' in call_kwargs + assert call_kwargs['tracer_provider'] == instrumentor['tracer_provider'] + + def test_calls_parent_uninstrument(self, instrumentor): + """Test that the instrumentor calls the parent class's _uninstrument method""" + instrumentor_obj = instrumentor['instrumentor'] + mock_uninstrument = instrumentor['mock_uninstrument'] + + # Reset the mock to clear any previous calls + mock_uninstrument.reset_mock() + + # Directly call uninstrument + instrumentor_obj._uninstrument() + + # Now verify the method was called at least once + assert mock_uninstrument.called, "Parent _uninstrument was not called" + + def test_wrapper_error_handling(self): + """Test that the instrumentor handles errors when wrapping methods""" + # Create instrumentor + instrumentor = OpenAIInstrumentor() + + # Mock wrap to raise an exception + with patch('agentops.instrumentation.openai.instrumentor.wrap') as mock_wrap: + mock_wrap.side_effect = AttributeError("Module not found") + + # Mock the parent class's _instrument method + with patch.object(instrumentor, '_instrument') as mock_instrument: + # Instrument should not raise exceptions even if wrapping fails + instrumentor._instrument(tracer_provider=MagicMock()) + + # Verify the parent method was still called + assert mock_instrument.called + + def test_unwrapper_error_handling(self): + """Test that the instrumentor handles errors when unwrapping methods""" + # Create instrumentor + instrumentor = OpenAIInstrumentor() + + # Mock unwrap to raise an exception + with patch('agentops.instrumentation.openai.instrumentor.unwrap') as mock_unwrap: + mock_unwrap.side_effect = Exception("Failed to unwrap") + + # Mock the parent class's _uninstrument method + with patch.object(instrumentor, '_uninstrument') as mock_uninstrument: + # Uninstrument should not raise exceptions even if unwrapping fails + instrumentor._uninstrument() + + # Verify the parent method was still called + assert mock_uninstrument.called + + def test_instrumentation_with_tracer(self): + """Test that the instrumentor gets a tracer with the correct name and version""" + # Create instrumentor + instrumentor = OpenAIInstrumentor() + + # Since get_tracer is now imported at module level in openai/instrumentor.py, + # we can test this through spying on the _instrument method instead + with patch.object(instrumentor, '_instrument', wraps=instrumentor._instrument) as mock_instrument_method: + # Instrument + mock_tracer_provider = MagicMock() + instrumentor._instrument(tracer_provider=mock_tracer_provider) + + # Verify the method was called with the expected parameters + assert mock_instrument_method.called + assert 'tracer_provider' in mock_instrument_method.call_args[1] + assert mock_instrument_method.call_args[1]['tracer_provider'] == mock_tracer_provider \ No newline at end of file diff --git a/tests/unit/instrumentation/openai_core/test_response_attributes.py b/tests/unit/instrumentation/openai_core/test_response_attributes.py new file mode 100644 index 000000000..3a64d29d5 --- /dev/null +++ b/tests/unit/instrumentation/openai_core/test_response_attributes.py @@ -0,0 +1,404 @@ +""" +Tests for OpenAI Response API attribute extraction + +This module contains tests for extracting attributes from the OpenAI Response API format. +It verifies that our instrumentation correctly extracts and transforms data from OpenAI API +responses into the appropriate OpenTelemetry span attributes. +""" + +import json +import os +import pytest +from unittest.mock import MagicMock, patch + +from agentops.instrumentation.openai.attributes.response import ( + get_response_kwarg_attributes, + get_response_response_attributes, + get_response_output_attributes, + get_response_output_message_attributes, + get_response_output_text_attributes, + get_response_output_reasoning_attributes, + get_response_output_tool_attributes, + get_response_tools_attributes, + get_response_usage_attributes, + get_response_reasoning_attributes +) +from agentops.semconv import ( + SpanAttributes, + MessageAttributes, + ToolAttributes, +) + + +# Utility function to load fixtures +def load_fixture(fixture_name): + """Load a test fixture from the fixtures directory""" + fixture_path = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "fixtures", + fixture_name + ) + with open(fixture_path, "r") as f: + return json.load(f) + + +# Load response test fixtures +OPENAI_RESPONSE = load_fixture("openai_response.json") # Response API format with output array +OPENAI_RESPONSE_TOOL_CALLS = load_fixture("openai_response_tool_calls.json") # Response API with tool calls + + +class MockResponse: + """Mock Response object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + + +class MockOutputMessage: + """Mock ResponseOutputMessage object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + + +class MockOutputText: + """Mock ResponseOutputText object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + + +class MockResponseUsage: + """Mock ResponseUsage object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + + +class MockOutputTokensDetails: + """Mock OutputTokensDetails object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + + +class MockReasoning: + """Mock Reasoning object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + + +class MockFunctionTool: + """Mock FunctionTool object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + self.type = "function" + + +class MockFunctionToolCall: + """Mock ResponseFunctionToolCall object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + + +class MockResponseInputParam: + """Mock ResponseInputParam object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + + +class TestResponseAttributes: + """Tests for OpenAI Response API attribute extraction""" + + def test_get_response_kwarg_attributes_with_string_input(self): + """Test extraction of attributes from kwargs with string input""" + kwargs = { + "input": "What is the capital of France?", + "model": "gpt-4o", + "temperature": 0.7, + "top_p": 1.0, + } + + attributes = get_response_kwarg_attributes(kwargs) + + # Check that string input is correctly mapped to prompt attributes + assert MessageAttributes.PROMPT_ROLE.format(i=0) in attributes + assert attributes[MessageAttributes.PROMPT_ROLE.format(i=0)] == "user" + assert MessageAttributes.PROMPT_CONTENT.format(i=0) in attributes + assert attributes[MessageAttributes.PROMPT_CONTENT.format(i=0)] == "What is the capital of France?" + + # Check that model attribute is correctly mapped + assert SpanAttributes.LLM_REQUEST_MODEL in attributes + assert attributes[SpanAttributes.LLM_REQUEST_MODEL] == "gpt-4o" + + def test_get_response_kwarg_attributes_with_list_input(self): + """Test extraction of attributes from kwargs with list input""" + # Create a list of mock message objects + messages = [ + MockResponseInputParam({ + "type": "text", + "role": "system", + "content": "You are a helpful assistant" + }), + MockResponseInputParam({ + "type": "text", + "role": "user", + "content": "What is the capital of France?" + }) + ] + + kwargs = { + "input": messages, + "model": "gpt-4o", + "temperature": 0.7, + "top_p": 1.0, + } + + attributes = get_response_kwarg_attributes(kwargs) + + # Check that list input is correctly mapped to prompt attributes + assert MessageAttributes.PROMPT_ROLE.format(i=0) in attributes + assert attributes[MessageAttributes.PROMPT_ROLE.format(i=0)] == "system" + assert MessageAttributes.PROMPT_CONTENT.format(i=0) in attributes + assert attributes[MessageAttributes.PROMPT_CONTENT.format(i=0)] == "You are a helpful assistant" + + assert MessageAttributes.PROMPT_ROLE.format(i=1) in attributes + assert attributes[MessageAttributes.PROMPT_ROLE.format(i=1)] == "user" + assert MessageAttributes.PROMPT_CONTENT.format(i=1) in attributes + assert attributes[MessageAttributes.PROMPT_CONTENT.format(i=1)] == "What is the capital of France?" + + # Check that model attribute is correctly mapped + assert SpanAttributes.LLM_REQUEST_MODEL in attributes + assert attributes[SpanAttributes.LLM_REQUEST_MODEL] == "gpt-4o" + + def test_get_response_kwarg_attributes_with_unsupported_input(self): + """Test extraction of attributes from kwargs with unsupported input type""" + kwargs = { + "input": 123, # Unsupported input type + "model": "gpt-4o", + } + + # Should not raise an exception but log a debug message + with patch('agentops.instrumentation.openai.attributes.response.logger.debug') as mock_logger: + attributes = get_response_kwarg_attributes(kwargs) + + # Verify the debug message was logged + mock_logger.assert_called_once() + assert "'int'" in mock_logger.call_args[0][0] + + # Check that model attribute is still correctly mapped + assert SpanAttributes.LLM_REQUEST_MODEL in attributes + assert attributes[SpanAttributes.LLM_REQUEST_MODEL] == "gpt-4o" + + def test_get_response_response_attributes(self): + """Test extraction of attributes from Response object""" + # Create a mock Response object using the fixture data + response_data = OPENAI_RESPONSE.copy() + + # We need to convert nested objects to appropriate classes for the code to handle them + output = [] + for item in response_data['output']: + content = [] + for content_item in item['content']: + content.append(MockOutputText(content_item)) + output.append(MockOutputMessage({**item, 'content': content})) + + usage = MockResponseUsage({ + **response_data['usage'], + 'output_tokens_details': MockOutputTokensDetails(response_data['usage']['output_tokens_details']) + }) + + reasoning = MockReasoning(response_data['reasoning']) + + # Set __dict__ to ensure attribute extraction works properly + mock_response = MockResponse({ + **response_data, + 'output': output, + 'usage': usage, + 'reasoning': reasoning, + 'tools': [], + '__dict__': { + **response_data, + 'output': output, + 'usage': usage, + 'reasoning': reasoning, + 'tools': [] + } + }) + + # Patch the Response and other type checks for simpler testing + with patch('agentops.instrumentation.openai.attributes.response.ResponseOutputMessage', MockOutputMessage): + with patch('agentops.instrumentation.openai.attributes.response.ResponseOutputText', MockOutputText): + # Extract attributes + attributes = get_response_response_attributes(mock_response) + + # Check that basic attributes are extracted + assert SpanAttributes.LLM_RESPONSE_ID in attributes + assert attributes[SpanAttributes.LLM_RESPONSE_ID] == response_data['id'] + assert SpanAttributes.LLM_RESPONSE_MODEL in attributes + assert attributes[SpanAttributes.LLM_RESPONSE_MODEL] == response_data['model'] + assert SpanAttributes.LLM_PROMPTS in attributes + assert attributes[SpanAttributes.LLM_PROMPTS] == response_data['instructions'] + + # Check usage attributes + assert SpanAttributes.LLM_USAGE_PROMPT_TOKENS in attributes + assert attributes[SpanAttributes.LLM_USAGE_PROMPT_TOKENS] == response_data['usage']['input_tokens'] + assert SpanAttributes.LLM_USAGE_COMPLETION_TOKENS in attributes + assert attributes[SpanAttributes.LLM_USAGE_COMPLETION_TOKENS] == response_data['usage']['output_tokens'] + assert SpanAttributes.LLM_USAGE_TOTAL_TOKENS in attributes + assert attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == response_data['usage']['total_tokens'] + + def test_get_response_output_attributes(self): + """Test extraction of attributes from output list""" + # Create a simple dictionary for testing + attributes = {} # We'll use an empty dict to simplify the test + + # Now just verify the function exists and doesn't throw an exception + output = [] # Empty list is fine for this test + + # Patch all the type checks to make testing simpler + with patch('agentops.instrumentation.openai.attributes.response.ResponseOutputMessage', MockOutputMessage): + with patch('agentops.instrumentation.openai.attributes.response.ResponseOutputText', MockOutputText): + with patch('agentops.instrumentation.openai.attributes.response.ResponseFunctionToolCall', MockFunctionToolCall): + result = get_response_output_attributes(output) + + # Simply verify it returns a dictionary + assert isinstance(result, dict) + + def test_get_response_output_message_attributes(self): + """Test extraction of attributes from output message""" + # Create a simplest test we can - just verify the function exists + # and can be called without exception + + # Patch the ResponseOutputText class to make testing simpler + with patch('agentops.instrumentation.openai.attributes.response.ResponseOutputText', MockOutputText): + # Create a minimal mock with required attributes + message = MockOutputMessage({ + 'id': 'msg_12345', + 'content': [], # Empty content for simplicity + 'role': 'assistant', + 'status': 'completed', + 'type': 'message' + }) + + # Call the function + result = get_response_output_message_attributes(0, message) + + # Verify basic expected attributes + assert isinstance(result, dict) + + def test_get_response_output_text_attributes(self): + """Test extraction of attributes from output text""" + # Create a mock text content + text = MockOutputText({ + 'annotations': [], + 'text': 'The capital of France is Paris.', + 'type': 'output_text' + }) + + # Extract attributes + attributes = get_response_output_text_attributes(0, text) + + # Check attributes + assert MessageAttributes.COMPLETION_CONTENT.format(i=0) in attributes + assert attributes[MessageAttributes.COMPLETION_CONTENT.format(i=0)] == 'The capital of France is Paris.' + + def test_get_response_output_tool_attributes(self): + """Test extraction of attributes from output tool""" + # Create a mock tool call + tool_call = MockFunctionToolCall({ + 'id': 'call_67890', + 'name': 'get_weather', + 'arguments': '{"location":"Paris"}', + 'type': 'function' + }) + + # Extract attributes + attributes = get_response_output_tool_attributes(0, tool_call) + + # Check attributes + assert MessageAttributes.FUNCTION_CALL_ID.format(i=0) in attributes + assert attributes[MessageAttributes.FUNCTION_CALL_ID.format(i=0)] == 'call_67890' + assert MessageAttributes.FUNCTION_CALL_NAME.format(i=0) in attributes + assert attributes[MessageAttributes.FUNCTION_CALL_NAME.format(i=0)] == 'get_weather' + assert MessageAttributes.FUNCTION_CALL_ARGUMENTS.format(i=0) in attributes + assert attributes[MessageAttributes.FUNCTION_CALL_ARGUMENTS.format(i=0)] == '{"location":"Paris"}' + assert MessageAttributes.FUNCTION_CALL_TYPE.format(i=0) in attributes + assert attributes[MessageAttributes.FUNCTION_CALL_TYPE.format(i=0)] == 'function' + + def test_get_response_tools_attributes(self): + """Test extraction of attributes from tools list""" + # Simplify the test to just verify the function can be called without error + + # Patch the FunctionTool class to make testing simpler + with patch('agentops.instrumentation.openai.attributes.response.FunctionTool', MockFunctionTool): + # Test with empty list for simplicity + tools = [] + + # Call the function + result = get_response_tools_attributes(tools) + + # Verify basic expected attributes + assert isinstance(result, dict) + + def test_get_response_usage_attributes(self): + """Test extraction of attributes from usage data""" + # Simplify test to verify function can be called without error + + # Patch the OutputTokensDetails class to make testing simpler + with patch('agentops.instrumentation.openai.attributes.response.OutputTokensDetails', MockOutputTokensDetails): + # Create a minimal mock usage object with all necessary attributes + usage = MockResponseUsage({ + 'input_tokens': 50, + 'output_tokens': 20, + 'total_tokens': 70, + 'output_tokens_details': MockOutputTokensDetails({ + 'reasoning_tokens': 5 + }), + 'input_tokens_details': { + 'cached_tokens': 10 + }, + '__dict__': { + 'input_tokens': 50, + 'output_tokens': 20, + 'total_tokens': 70, + 'output_tokens_details': MockOutputTokensDetails({ + 'reasoning_tokens': 5 + }), + 'input_tokens_details': { + 'cached_tokens': 10 + } + } + }) + + # Call the function + result = get_response_usage_attributes(usage) + + # Verify it returns a dictionary with at least these basic attributes + assert isinstance(result, dict) + assert SpanAttributes.LLM_USAGE_PROMPT_TOKENS in result + assert result[SpanAttributes.LLM_USAGE_PROMPT_TOKENS] == 50 + assert SpanAttributes.LLM_USAGE_COMPLETION_TOKENS in result + assert result[SpanAttributes.LLM_USAGE_COMPLETION_TOKENS] == 20 + assert SpanAttributes.LLM_USAGE_TOTAL_TOKENS in result + assert result[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 70 + + def test_get_response_reasoning_attributes(self): + """Test extraction of attributes from reasoning data""" + # Create mock reasoning object + reasoning = MockReasoning({ + 'effort': 'medium', + 'generate_summary': True + }) + + # Extract attributes - currently no attributes are mapped for reasoning + attributes = get_response_reasoning_attributes(reasoning) + + # The current implementation returns an empty dictionary because + # there are no defined attributes in RESPONSE_REASONING_ATTRIBUTES + assert isinstance(attributes, dict) + assert len(attributes) == 0 # Currently no attributes are mapped \ No newline at end of file diff --git a/tests/unit/instrumentation/test_common_attributes.py b/tests/unit/instrumentation/test_common_attributes.py deleted file mode 100644 index 34477e6e4..000000000 --- a/tests/unit/instrumentation/test_common_attributes.py +++ /dev/null @@ -1,285 +0,0 @@ -""" -Tests for Common Attributes Module - -This module contains tests for the common attribute processing utilities that are shared -across all instrumentors in the AgentOps package. -""" - -import pytest -from unittest.mock import patch, MagicMock -from typing import Dict, Any, List, Set - -from agentops.instrumentation.common.attributes import ( - AttributeMap, - _extract_attributes_from_mapping, - get_common_attributes, - get_base_trace_attributes, - get_base_span_attributes -) -from agentops.helpers import get_tags_from_config - -from agentops.semconv import ( - CoreAttributes, - InstrumentationAttributes, - WorkflowAttributes, -) - - -class TestCommonAttributes: - """Test suite for common attribute processing utilities""" - - def test_extract_attributes_from_mapping(self): - """Test extraction of attributes based on mapping""" - # Create a simple span data object with attributes - class SpanData: - def __init__(self): - self.trace_id = "trace123" - self.span_id = "span456" - self.parent_id = "parent789" - self.name = "test_span" - - span_data = SpanData() - - # Define a mapping - mapping = { - "target.trace_id": "trace_id", - "target.span_id": "span_id", - "target.parent_id": "parent_id", - "target.name": "name", - "target.missing": "missing_attr" # This attribute doesn't exist - } - - # Extract attributes - attributes = _extract_attributes_from_mapping(span_data, mapping) - - # Verify extracted attributes - assert attributes["target.trace_id"] == "trace123" - assert attributes["target.span_id"] == "span456" - assert attributes["target.parent_id"] == "parent789" - assert attributes["target.name"] == "test_span" - assert "target.missing" not in attributes # Missing attribute should be skipped - - def test_extract_attributes_from_dict(self): - """Test extraction of attributes from a dictionary""" - # Create a dictionary with attributes - span_data = { - "trace_id": "trace123", - "span_id": "span456", - "parent_id": "parent789", - "name": "test_span" - } - - # Define a mapping - mapping = { - "target.trace_id": "trace_id", - "target.span_id": "span_id", - "target.parent_id": "parent_id", - "target.name": "name", - "target.missing": "missing_key" # This key doesn't exist - } - - # Extract attributes - attributes = _extract_attributes_from_mapping(span_data, mapping) - - # Verify extracted attributes - assert attributes["target.trace_id"] == "trace123" - assert attributes["target.span_id"] == "span456" - assert attributes["target.parent_id"] == "parent789" - assert attributes["target.name"] == "test_span" - assert "target.missing" not in attributes # Missing key should be skipped - - def test_extract_attributes_handles_none_empty_values(self): - """Test that the extraction function properly handles None and empty values""" - # Create a span data object with None and empty values - class SpanData: - def __init__(self): - self.none_attr = None - self.empty_str = "" - self.empty_list = [] - self.empty_dict = {} - self.valid_attr = "valid_value" - - span_data = SpanData() - - # Define a mapping - mapping = { - "target.none": "none_attr", - "target.empty_str": "empty_str", - "target.empty_list": "empty_list", - "target.empty_dict": "empty_dict", - "target.valid": "valid_attr" - } - - # Extract attributes - attributes = _extract_attributes_from_mapping(span_data, mapping) - - # Verify that None and empty values are skipped - assert "target.none" not in attributes - assert "target.empty_str" not in attributes - assert "target.empty_list" not in attributes - assert "target.empty_dict" not in attributes - assert attributes["target.valid"] == "valid_value" - - def test_extract_attributes_serializes_complex_objects(self): - """Test that complex objects are properly serialized during extraction""" - # Create a dictionary with complex values - span_data = { - "complex_obj": {"attr1": "value1", "attr2": "value2"}, - "dict_obj": {"key1": "value1", "key2": "value2"}, - "list_obj": ["item1", "item2"] - } - - # Define a mapping - mapping = { - "target.complex": "complex_obj", - "target.dict": "dict_obj", - "target.list": "list_obj" - } - - # Extract attributes with serialization - attributes = _extract_attributes_from_mapping(span_data, mapping) - - # Verify that complex objects are serialized to strings - assert isinstance(attributes["target.complex"], str) - assert isinstance(attributes["target.dict"], str) - assert isinstance(attributes["target.list"], str) - - # Check that serialized values contain expected content - assert "value1" in attributes["target.complex"] - assert "value2" in attributes["target.complex"] - assert "key1" in attributes["target.dict"] - assert "key2" in attributes["target.dict"] - assert "item1" in attributes["target.list"] - assert "item2" in attributes["target.list"] - - def test_get_common_attributes(self): - """Test that common instrumentation attributes are correctly generated""" - # Get common attributes - attributes = get_common_attributes() - - # Verify required keys and values - assert InstrumentationAttributes.NAME in attributes - assert InstrumentationAttributes.VERSION in attributes - assert attributes[InstrumentationAttributes.NAME] == "agentops" - - def test_get_base_trace_attributes(self): - """Test generation of base trace attributes""" - # Create a simple trace object - class TraceObj: - def __init__(self): - self.name = "test_workflow" - self.trace_id = "trace123" - - trace = TraceObj() - - # Get base trace attributes - attributes = get_base_trace_attributes(trace) - - # Verify core trace attributes - assert attributes[WorkflowAttributes.WORKFLOW_NAME] == "test_workflow" - assert attributes[CoreAttributes.TRACE_ID] == "trace123" - assert attributes[WorkflowAttributes.WORKFLOW_STEP_TYPE] == "trace" - assert attributes[InstrumentationAttributes.NAME] == "agentops" - - # Test error case when trace_id is missing - class InvalidTrace: - def __init__(self): - self.name = "invalid_workflow" - # No trace_id - - invalid_trace = InvalidTrace() - invalid_attributes = get_base_trace_attributes(invalid_trace) - assert invalid_attributes == {} - - def test_get_base_span_attributes(self): - """Test generation of base span attributes""" - # Create a simple span object - class SpanObj: - def __init__(self): - self.span_id = "span456" - self.trace_id = "trace123" - self.parent_id = "parent789" - - span = SpanObj() - - # Get base span attributes - attributes = get_base_span_attributes(span) - - # Verify core span attributes - assert attributes[CoreAttributes.SPAN_ID] == "span456" - assert attributes[CoreAttributes.TRACE_ID] == "trace123" - assert attributes[CoreAttributes.PARENT_ID] == "parent789" - assert attributes[InstrumentationAttributes.NAME] == "agentops" - - # Test without parent_id - class SpanWithoutParent: - def __init__(self): - self.span_id = "span456" - self.trace_id = "trace123" - # No parent_id - - span_without_parent = SpanWithoutParent() - attributes_without_parent = get_base_span_attributes(span_without_parent) - - # Verify parent_id is not included - assert CoreAttributes.PARENT_ID not in attributes_without_parent - - def test_get_tags_from_config(self): - """Test retrieval of tags from the configuration""" - # Mock the get_config function - mock_config = MagicMock() - mock_config.default_tags = {"tag1", "tag2", "tag3"} - - with patch("agentops.helpers.config.get_config", return_value=mock_config): - # Get tags from config - tags = get_tags_from_config() - - # Verify tags are returned as a list - assert isinstance(tags, list) - assert set(tags) == {"tag1", "tag2", "tag3"} - - def test_get_tags_from_config_handles_empty_tags(self): - """Test that empty tags are handled correctly""" - # Mock the get_config function with empty tags - mock_config = MagicMock() - mock_config.default_tags = set() - - with patch("agentops.helpers.config.get_config", return_value=mock_config): - # Get tags from config - tags = get_tags_from_config() - - # Verify empty tags returns an empty list - assert tags == [] # This should pass if get_tags_from_config returns [] for empty sets - - # Removed test_get_tags_from_config_handles_error since we're not handling errors anymore - - def test_tags_added_to_trace_attributes(self): - """Test that tags are added to trace attributes but not span attributes""" - # Create test objects - class TraceObj: - def __init__(self): - self.name = "test_workflow" - self.trace_id = "trace123" - - class SpanObj: - def __init__(self): - self.span_id = "span456" - self.trace_id = "trace123" - - trace = TraceObj() - span = SpanObj() - - # Mock the get_tags_from_config function to return test tags - with patch("agentops.instrumentation.common.attributes.get_tags_from_config", - return_value=["test_tag1", "test_tag2"]): - # Get attributes for both trace and span - trace_attributes = get_base_trace_attributes(trace) - span_attributes = get_base_span_attributes(span) - - - # Verify tags are added to trace attributes - assert CoreAttributes.TAGS in trace_attributes - assert trace_attributes[CoreAttributes.TAGS] == ["test_tag1", "test_tag2"] - - # Verify tags are NOT added to span attributes - assert CoreAttributes.TAGS not in span_attributes \ No newline at end of file From 2b28a515d129b4f381d0da8cbe943c9cc9df9294 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Mon, 31 Mar 2025 13:39:50 -0700 Subject: [PATCH 17/26] Async support for wrappers. --- agentops/instrumentation/common/wrappers.py | 47 +++++++++++++++++-- .../instrumentation/openai/instrumentor.py | 1 + 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/agentops/instrumentation/common/wrappers.py b/agentops/instrumentation/common/wrappers.py index 9c33962f4..325fd768b 100644 --- a/agentops/instrumentation/common/wrappers.py +++ b/agentops/instrumentation/common/wrappers.py @@ -5,7 +5,7 @@ a configuration class for wrapping methods, helper functions for updating spans with attributes, and functions for creating and applying wrappers. """ - +import asyncio from typing import Any, Optional, Tuple, Dict, Callable from dataclasses import dataclass from wrapt import wrap_function_wrapper # type: ignore @@ -34,6 +34,9 @@ class WrapConfig: class_name: The name of the class containing the method method_name: The name of the method to wrap handler: A function that extracts attributes from args, kwargs, or return value + is_async: Whether the method is asynchronous (default: False) + We explicitly specify async methods since `asyncio.iscoroutinefunction` + is not reliable in this context. span_kind: The kind of span to create (default: CLIENT) """ trace_name: str @@ -41,6 +44,7 @@ class WrapConfig: class_name: str method_name: str handler: AttributeHandler + is_async: bool = False span_kind: SpanKind = SpanKind.CLIENT def __repr__(self): @@ -78,7 +82,7 @@ def _finish_span_error(span: Span, exception: Exception) -> None: span.set_status(Status(StatusCode.ERROR, str(exception))) -def _create_wrapper(wrap_config: WrapConfig, tracer: Tracer): +def _create_wrapper(wrap_config: WrapConfig, tracer: Tracer) -> Callable: """Create a wrapper function for the specified configuration. This function creates a wrapper that: @@ -97,6 +101,38 @@ def _create_wrapper(wrap_config: WrapConfig, tracer: Tracer): """ handler = wrap_config.handler + async def awrapper(wrapped, instance, args, kwargs): + # Skip instrumentation if it's suppressed in the current context + # TODO I don't understand what this actually does + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return wrapped(*args, **kwargs) + + return_value = None + + with tracer.start_as_current_span( + wrap_config.trace_name, + kind=wrap_config.span_kind, + ) as span: + try: + # Add the input attributes to the span before execution + attributes = handler(args=args, kwargs=kwargs) + _update_span(span, attributes) + + return_value = await wrapped(*args, **kwargs) + + # Add the output attributes to the span after execution + attributes = handler(return_value=return_value) + _update_span(span, attributes) + _finish_span_success(span) + except Exception as e: + # Add everything we have in the case of an error + attributes = handler(args=args, kwargs=kwargs, return_value=return_value) + _update_span(span, attributes) + _finish_span_error(span, e) + raise + + return return_value + def wrapper(wrapped, instance, args, kwargs): # Skip instrumentation if it's suppressed in the current context # TODO I don't understand what this actually does @@ -129,10 +165,13 @@ def wrapper(wrapped, instance, args, kwargs): return return_value - return wrapper + if wrap_config.is_async: + return awrapper + else: + return wrapper -def wrap(wrap_config: WrapConfig, tracer: Tracer): +def wrap(wrap_config: WrapConfig, tracer: Tracer) -> Callable: """Wrap a method with OpenTelemetry instrumentation. This function applies the wrapper created by _create_wrapper diff --git a/agentops/instrumentation/openai/instrumentor.py b/agentops/instrumentation/openai/instrumentor.py index 9d6ea798b..8adf4c096 100644 --- a/agentops/instrumentation/openai/instrumentor.py +++ b/agentops/instrumentation/openai/instrumentor.py @@ -47,6 +47,7 @@ class_name="AsyncResponses", method_name="create", handler=get_response_attributes, + is_async=True, ), ] From 91a1784852d3d556ac721b5f90d531ea04617871 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Mon, 31 Mar 2025 13:40:19 -0700 Subject: [PATCH 18/26] Example openai responses for synchronous and asynchronous calls --- examples/openai_responses/sync_and_async.py | 43 +++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 examples/openai_responses/sync_and_async.py diff --git a/examples/openai_responses/sync_and_async.py b/examples/openai_responses/sync_and_async.py new file mode 100644 index 000000000..3c857e7a4 --- /dev/null +++ b/examples/openai_responses/sync_and_async.py @@ -0,0 +1,43 @@ +# To run this file from project root: AGENTOPS_LOG_LEVEL=debug uv run examples/openai_responses/sync_and_async.py +import asyncio +from dotenv import load_dotenv + +load_dotenv() + +from openai import OpenAI, AsyncOpenAI +import agentops + + +def sync_responses_request(): + client = OpenAI() + response = client.responses.create( + model="gpt-4o", + input="Explain the concept of synchronous Python in one sentence.", + ) + return response + + +async def async_responses_request(): + client = AsyncOpenAI() + response = await client.responses.create( + model="gpt-4o", + input="Explain the concept of async/await in Python in one sentence.", + stream=False, + ) + return response + + +async def main(): + agentops.init() + + # Synchronous request + sync_response = sync_responses_request() + print(f"Synchronous Response:\n {sync_response.output_text}") + + # Asynchronous request + async_response = await async_responses_request() + print(f"Asynchronous Response:\n {async_response.output_text}") + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file From 2af8d96db2d036c54540e26bca28ebfcbb0e0f38 Mon Sep 17 00:00:00 2001 From: Pratyush Shukla Date: Tue, 1 Apr 2025 14:59:29 +0530 Subject: [PATCH 19/26] add more examples that are not Agents SDK --- .../multi_tool_orchestration.ipynb | 614 ++++++++++++++++++ examples/openai_responses/web_search.ipynb | 297 +++++++++ 2 files changed, 911 insertions(+) create mode 100644 examples/openai_responses/multi_tool_orchestration.ipynb create mode 100644 examples/openai_responses/web_search.ipynb diff --git a/examples/openai_responses/multi_tool_orchestration.ipynb b/examples/openai_responses/multi_tool_orchestration.ipynb new file mode 100644 index 000000000..6a36e9199 --- /dev/null +++ b/examples/openai_responses/multi_tool_orchestration.ipynb @@ -0,0 +1,614 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Multi-Tool Orchestration with RAG approach using OpenAI's Responses API" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "This cookbook guides you through building dynamic, multi-tool workflows using OpenAI's Responses API. It demonstrates how to implement a Retrieval-Augmented Generation (RAG) approach that intelligently routes user queries to the appropriate in-built or external tools. Whether your query calls for general knowledge or requires accessing specific internal context from a vector database (like Pinecone), this guide shows you how to integrate function calls, web searches in-built tool, and leverage document retrieval to generate accurate, context-aware responses." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Install required dependencies\n", + "%pip install datasets tqdm pandas pinecone openai" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from dotenv import load_dotenv\n", + "import time\n", + "from tqdm.auto import tqdm\n", + "from pandas import DataFrame\n", + "from datasets import load_dataset\n", + "import random\n", + "import string\n", + "\n", + "# Import Pinecone client and related specifications.\n", + "from pinecone import Pinecone\n", + "from pinecone import ServerlessSpec" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "load_dotenv()\n", + "\n", + "AGENTOPS_API_KEY = os.getenv(\"AGENTOPS_API_KEY\")\n", + "AGENTOPS_API_ENDPOINT = os.getenv(\"AGENTOPS_API_ENDPOINT\")\n", + "AGENTOPS_EXPORTER_ENDPOINT = os.getenv(\"AGENTOPS_EXPORTER_ENDPOINT\")\n", + "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n", + "PINECONE_API_KEY = os.getenv(\"PINECONE_API_KEY\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import AgentOps client and initialize with your API key.\n", + "import agentops\n", + "\n", + "agentops.init(\n", + " api_key=AGENTOPS_API_KEY,\n", + " endpoint=AGENTOPS_API_ENDPOINT,\n", + " exporter_endpoint=AGENTOPS_EXPORTER_ENDPOINT,\n", + " tags=[\"openai\", \"responses\", \"multi-tool\", \"rag\", \"test\"],\n", + ")\n", + "\n", + "# Import OpenAI client and initialize with your API key.\n", + "from openai import OpenAI\n", + "\n", + "client = OpenAI(api_key=OPENAI_API_KEY)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this example we use a sample medical reasoning dataset from Hugging Face. We convert the dataset into a Pandas DataFrame and merge the “Question” and “Response” columns into a single string. This merged text is used for embedding and later stored as metadata." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load the dataset (ensure you're logged in with huggingface-cli if needed)\n", + "ds = load_dataset(\"FreedomIntelligence/medical-o1-reasoning-SFT\", \"en\", split='train[:100]', trust_remote_code=True)\n", + "ds_dataframe = DataFrame(ds)\n", + "\n", + "# Merge the Question and Response columns into a single string.\n", + "ds_dataframe['merged'] = ds_dataframe.apply(\n", + " lambda row: f\"Question: {row['Question']} Answer: {row['Response']}\", axis=1\n", + ")\n", + "print(\"Example merged text:\", ds_dataframe['merged'].iloc[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_dataframe" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a Pinecone Index Based on the Dataset\n", + "Use the dataset itself to determine the embedding dimensionality. For example, compute one embedding from the merged column and then create the index accordingly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "MODEL = \"text-embedding-3-small\" # Replace with your production embedding model if needed\n", + "# Compute an embedding for the first document to obtain the embedding dimension.\n", + "sample_embedding_resp = client.embeddings.create(\n", + " input=[ds_dataframe['merged'].iloc[0]],\n", + " model=MODEL\n", + ")\n", + "embed_dim = len(sample_embedding_resp.data[0].embedding)\n", + "print(f\"Embedding dimension: {embed_dim}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Pinecone using your API key.\n", + "pc = Pinecone(api_key=PINECONE_API_KEY)\n", + "\n", + "# Define the Pinecone serverless specification.\n", + "AWS_REGION = \"us-east-1\"\n", + "spec = ServerlessSpec(cloud=\"aws\", region=AWS_REGION)\n", + "\n", + "# Create a random index name with lower case alphanumeric characters and '-'\n", + "index_name = 'pinecone-index-' + ''.join(random.choices(string.ascii_lowercase + string.digits, k=10))\n", + "\n", + "# Create the index if it doesn't already exist.\n", + "if index_name not in pc.list_indexes().names():\n", + " pc.create_index(\n", + " index_name,\n", + " dimension=embed_dim,\n", + " metric='dotproduct',\n", + " spec=spec\n", + " )\n", + "\n", + "# Connect to the index.\n", + "index = pc.Index(index_name)\n", + "time.sleep(1)\n", + "print(\"Index stats:\", index.describe_index_stats())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Upsert the Dataset into Pinecone index\n", + "\n", + "Process the dataset in batches, generate embeddings for each merged text, prepare metadata (including separate Question and Answer fields), and upsert each batch into the index. You may also update metadata for specific entries if needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 32\n", + "for i in tqdm(range(0, len(ds_dataframe['merged']), batch_size), desc=\"Upserting to Pinecone\"):\n", + " i_end = min(i + batch_size, len(ds_dataframe['merged']))\n", + " lines_batch = ds_dataframe['merged'][i: i_end]\n", + " ids_batch = [str(n) for n in range(i, i_end)]\n", + " \n", + " # Create embeddings for the current batch.\n", + " res = client.embeddings.create(input=[line for line in lines_batch], model=MODEL)\n", + " embeds = [record.embedding for record in res.data]\n", + " \n", + " # Prepare metadata by extracting original Question and Answer.\n", + " meta = []\n", + " for record in ds_dataframe.iloc[i:i_end].to_dict('records'):\n", + " q_text = record['Question']\n", + " a_text = record['Response']\n", + " # Optionally update metadata for specific entries.\n", + " meta.append({\"Question\": q_text, \"Answer\": a_text})\n", + " \n", + " # Upsert the batch into Pinecone.\n", + " vectors = list(zip(ids_batch, embeds, meta))\n", + " index.upsert(vectors=vectors)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "![Pinecone Image](../../images/responses_pinecone_rag.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Query the Pinecone Index\n", + "\n", + "Create a natural language query, compute its embedding, and perform a similarity search on the Pinecone index. The returned results include metadata that provides context for generating answers." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "def query_pinecone_index(client, index, model, query_text):\n", + " # Generate an embedding for the query.\n", + " query_embedding = client.embeddings.create(input=query_text, model=model).data[0].embedding\n", + "\n", + " # Query the index and return top 5 matches.\n", + " res = index.query(vector=[query_embedding], top_k=5, include_metadata=True)\n", + " print(\"Query Results:\")\n", + " for match in res['matches']:\n", + " print(f\"{match['score']:.2f}: {match['metadata'].get('Question', 'N/A')} - {match['metadata'].get('Answer', 'N/A')}\")\n", + " return res" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example usage with a different query from the train/test set\n", + "query = (\n", + " \"A 45-year-old man with a history of alcohol use presents with symptoms including confusion, ataxia, and ophthalmoplegia. \"\n", + " \"What is the most likely diagnosis and the recommended treatment?\"\n", + ")\n", + "query_pinecone_index(client, index, MODEL, query)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generate a Response Using the Retrieved Context\n", + "\n", + "Select the best matching result from your query results and use the OpenAI Responses API to generate a final answer by combining the retrieved context with the original question." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Retrieve and concatenate top 3 match contexts.\n", + "matches = index.query(\n", + " vector=[client.embeddings.create(input=query, model=MODEL).data[0].embedding],\n", + " top_k=3,\n", + " include_metadata=True\n", + ")['matches']\n", + "\n", + "context = \"\\n\\n\".join(\n", + " f\"Question: {m['metadata'].get('Question', '')}\\nAnswer: {m['metadata'].get('Answer', '')}\"\n", + " for m in matches\n", + ")\n", + "# Use the context to generate a final answer.\n", + "response = client.responses.create(\n", + " model=\"gpt-4o\",\n", + " input=f\"Provide the answer based on the context: {context} and the question: {query} as per the internal knowledge base\",\n", + ")\n", + "print(\"\\nFinal Answer:\")\n", + "print(response.output_text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Orchestrate Multi-Tool Calls\n", + "\n", + "Now, we'll define the built-in function available through the Responses API, including the ability to invoke the external Vector Store - Pinecone as an example.\n", + "\n", + "*Web Search Preview Tool*: Enables the model to perform live web searches and preview the results. This is ideal for retrieving real-time or up-to-date information from the internet.\n", + "\n", + "*Pinecone Search Tool*: Allows the model to query a vector database using semantic search. This is especially useful for retrieving relevant documents—such as medical literature or other domain-specific content—that have been stored in a vectorized format." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "# Tools definition: The list of tools includes:\n", + "# - A web search preview tool.\n", + "# - A Pinecone search tool for retrieving medical documents.\n", + "\n", + "# Define available tools.\n", + "tools = [ \n", + " {\"type\": \"web_search_preview\",\n", + " \"user_location\": {\n", + " \"type\": \"approximate\",\n", + " \"country\": \"US\",\n", + " \"region\": \"California\",\n", + " \"city\": \"SF\"\n", + " },\n", + " \"search_context_size\": \"medium\"},\n", + " {\n", + " \"type\": \"function\",\n", + " \"name\": \"PineconeSearchDocuments\",\n", + " \"description\": \"Search for relevant documents based on the medical question asked by the user that is stored within the vector database using a semantic query.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The natural language query to search the vector database.\"\n", + " },\n", + " \"top_k\": {\n", + " \"type\": \"integer\",\n", + " \"description\": \"Number of top results to return.\",\n", + " \"default\": 3\n", + " }\n", + " },\n", + " \"required\": [\"query\"],\n", + " \"additionalProperties\": False\n", + " }\n", + " }\n", + "]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "# Example queries that the model should route appropriately.\n", + "queries = [\n", + " {\"query\": \"Who won the cricket world cup in 1983?\"},\n", + " {\"query\": \"What is the most common cause of death in the United States according to the internet?\"},\n", + " {\"query\": (\"A 7-year-old boy with sickle cell disease is experiencing knee and hip pain, \"\n", + " \"has been admitted for pain crises in the past, and now walks with a limp. \"\n", + " \"His exam shows a normal, cool hip with decreased range of motion and pain with ambulation. \"\n", + " \"What is the most appropriate next step in management according to the internal knowledge base?\")}\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Process each query dynamically.\n", + "for item in queries:\n", + " input_messages = [{\"role\": \"user\", \"content\": item[\"query\"]}]\n", + " print(\"\\n🌟--- Processing Query ---🌟\")\n", + " print(f\"🔍 **User Query:** {item['query']}\")\n", + " \n", + " # Call the Responses API with tools enabled and allow parallel tool calls.\n", + " response = client.responses.create(\n", + " model=\"gpt-4o\",\n", + " input=[\n", + " {\"role\": \"system\", \"content\": \"When prompted with a question, select the right tool to use based on the question.\"\n", + " },\n", + " {\"role\": \"user\", \"content\": item[\"query\"]}\n", + " ],\n", + " tools=tools,\n", + " parallel_tool_calls=True\n", + " )\n", + " \n", + " print(\"\\n✨ **Initial Response Output:**\")\n", + " print(response.output)\n", + " \n", + " # Determine if a tool call is needed and process accordingly.\n", + " if response.output:\n", + " tool_call = response.output[0]\n", + " if tool_call.type in [\"web_search_preview\", \"function_call\"]:\n", + " tool_name = tool_call.name if tool_call.type == \"function_call\" else \"web_search_preview\"\n", + " print(f\"\\n🔧 **Model triggered a tool call:** {tool_name}\")\n", + " \n", + " if tool_name == \"PineconeSearchDocuments\":\n", + " print(\"🔍 **Invoking PineconeSearchDocuments tool...**\")\n", + " res = query_pinecone_index(client, index, MODEL, item[\"query\"])\n", + " if res[\"matches\"]:\n", + " best_match = res[\"matches\"][0][\"metadata\"]\n", + " result = f\"**Question:** {best_match.get('Question', 'N/A')}\\n**Answer:** {best_match.get('Answer', 'N/A')}\"\n", + " else:\n", + " result = \"**No matching documents found in the index.**\"\n", + " print(\"✅ **PineconeSearchDocuments tool invoked successfully.**\")\n", + " else:\n", + " print(\"🔍 **Invoking simulated web search tool...**\")\n", + " result = \"**Simulated web search result.**\"\n", + " print(\"✅ **Simulated web search tool invoked successfully.**\")\n", + " \n", + " # Append the tool call and its output back into the conversation.\n", + " input_messages.append(tool_call)\n", + " input_messages.append({\n", + " \"type\": \"function_call_output\",\n", + " \"call_id\": tool_call.call_id,\n", + " \"output\": str(result)\n", + " })\n", + " \n", + " # Get the final answer incorporating the tool's result.\n", + " final_response = client.responses.create(\n", + " model=\"gpt-4o\",\n", + " input=input_messages,\n", + " tools=tools,\n", + " parallel_tool_calls=True\n", + " )\n", + " print(\"\\n💡 **Final Answer:**\")\n", + " print(final_response.output_text)\n", + " else:\n", + " # If no tool call is triggered, print the response directly.\n", + " print(\"💡 **Final Answer:**\")\n", + " print(response.output_text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As shown above, depending on the query, appropriate tool is invoked in order to determine the optimal response.\n", + "\n", + "For instance, looking at the third example, when the model triggers the tool named \"PineconeSearchDocuments\", the code calls `query_pinecone_index` with the current query and then extracts the best match (or an appropriate context) as the result. For non health related inqueries or queries where explicit internet search is asked, the code calls the web_search_call function and for other queries, it may choose to not call any tool and rather provide a response based on the question under consideration.\n", + "\n", + "Finally, the tool call and its output are appended to the conversation, and the final answer is generated by the Responses API." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Multi-tool orchestration flow\n", + "\n", + "Now let us try to modify the input query and the system instructions to the responses API in order to follow a tool calling sequence and generate the output. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Process one query as an example to understand the tool calls and function calls as part of the response output\n", + "item = \"What is the most common cause of death in the United States\"\n", + "\n", + "# Initialize input messages with the user's query.\n", + "input_messages = [{\"role\": \"user\", \"content\": item}]\n", + "print(\"\\n🌟--- Processing Query ---🌟\")\n", + "print(f\"🔍 **User Query:** {item}\")\n", + " \n", + " # Call the Responses API with tools enabled and allow parallel tool calls.\n", + "print(\"\\n🔧 **Calling Responses API with Tools Enabled**\")\n", + "print(\"\\n🕵️‍♂️ **Step 1: Web Search Call**\")\n", + "print(\" - Initiating web search to gather initial information.\")\n", + "print(\"\\n📚 **Step 2: Pinecone Search Call**\")\n", + "print(\" - Querying Pinecone to find relevant examples from the internal knowledge base.\")\n", + " \n", + "response = client.responses.create(\n", + " model=\"gpt-4o\",\n", + " input=[\n", + " {\"role\": \"system\", \"content\": \"Every time it's prompted with a question, first call the web search tool for results, then call `PineconeSearchDocuments` to find real examples in the internal knowledge base.\"},\n", + " {\"role\": \"user\", \"content\": item}\n", + " ],\n", + " tools=tools,\n", + " parallel_tool_calls=True\n", + " )\n", + " \n", + "# Print the initial response output.\n", + "print(\"input_messages\", input_messages)\n", + "\n", + "print(\"\\n✨ **Initial Response Output:**\")\n", + "print(response.output)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Understand the tool calls and function calls as part of the response output\n", + "\n", + "import pandas as pd\n", + "\n", + "# Create a list to store the tool call and function call details\n", + "tool_calls = []\n", + "\n", + "# Iterate through the response output and collect the details\n", + "for i in response.output:\n", + " tool_calls.append({\n", + " \"Type\": i.type,\n", + " \"Call ID\": i.call_id if hasattr(i, 'call_id') else i.id if hasattr(i, 'id') else \"N/A\",\n", + " \"Output\": str(i.output) if hasattr(i, 'output') else \"N/A\",\n", + " \"Name\": i.name if hasattr(i, 'name') else \"N/A\"\n", + " })\n", + "\n", + "# Convert the list to a DataFrame for tabular display\n", + "df_tool_calls = pd.DataFrame(tool_calls)\n", + "\n", + "# Display the DataFrame\n", + "df_tool_calls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tool_call_1 = response.output[0]\n", + "print(tool_call_1)\n", + "print(tool_call_1.id)\n", + "\n", + "tool_call_2 = response.output[2]\n", + "print(tool_call_2)\n", + "print(tool_call_2.call_id)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# append the tool call and its output back into the conversation.\n", + "input_messages.append(response.output[2])\n", + "input_messages.append({\n", + " \"type\": \"function_call_output\",\n", + " \"call_id\": tool_call_2.call_id,\n", + " \"output\": str(result)\n", + "})\n", + "print(input_messages)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Get the final answer incorporating the tool's result.\n", + "print(\"\\n🔧 **Calling Responses API for Final Answer**\")\n", + "\n", + "response_2 = client.responses.create(\n", + " model=\"gpt-4o\",\n", + " input=input_messages,\n", + ")\n", + "print(response_2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# print the final answer\n", + "print(response_2.output_text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Here, we have seen how to utilize OpenAI's Responses API to implement a Retrieval-Augmented Generation (RAG) approach with multi-tool calling capabilities. It showcases an example where the model selects the appropriate tool based on the input query: general questions may be handled by built-in tools such as web-search, while specific medical inquiries related to internal knowledge are addressed by retrieving context from a vector database (such as Pinecone) via function calls. Additonally, we have showcased how multiple tool calls can be sequentially combined to generate a final response based on our instructions provided to responses API. Happy coding! " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/openai_responses/web_search.ipynb b/examples/openai_responses/web_search.ipynb new file mode 100644 index 000000000..3b50074c8 --- /dev/null +++ b/examples/openai_responses/web_search.ipynb @@ -0,0 +1,297 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## What is the Responses API\n", + "\n", + "The Responses API is a new API that focuses on greater simplicity and greater expressivity when using our APIs. It is designed for multiple tools, multiple turns, and multiple modalities — as opposed to current APIs, which either have these features bolted onto an API designed primarily for text in and out (chat completions) or need a lot bootstrapping to perform simple actions (assistants api).\n", + "\n", + "Here I will show you a couple of new features that the Responses API has to offer and tie it all together at the end.\n", + "`responses` solves for a number of user painpoints with our current set of APIs. During our time with the completions API, we found that folks wanted:\n", + "\n", + "- the ability to easily perform multi-turn model interactions in a single API call\n", + "- to have access to our hosted tools (file_search, web_search, code_interpreter)\n", + "- granular control over the context sent to the model\n", + "\n", + "As models start to develop longer running reasoning and thinking capabilities, users will want an async-friendly and stateful primitive. Response solves for this. \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Basics\n", + "By design, on the surface, the Responses API is very similar to the Completions API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dotenv import load_dotenv\n", + "import os\n", + "\n", + "load_dotenv()\n", + "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n", + "AGENTOPS_API_KEY = os.getenv(\"AGENTOPS_API_KEY\")\n", + "\n", + "import agentops\n", + "agentops.init(api_key=AGENTOPS_API_KEY)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "client = OpenAI(api_key=OPENAI_API_KEY)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.responses.create(\n", + " model=\"gpt-4o-mini\",\n", + " input=\"tell me a joke\",\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(response.output[0].content[0].text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "One key feature of the Response API is that it is stateful. This means that you do not have to manage the state of the conversation by yourself, the API will handle it for you. For example, you can retrieve the response at any time and it will include the full conversation history." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fetched_response = client.responses.retrieve(\n", + "response_id=response.id)\n", + "\n", + "print(fetched_response.output[0].content[0].text)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can continue the conversation by referring to the previous response." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "response_two = client.responses.create(\n", + " model=\"gpt-4o-mini\",\n", + " input=\"tell me another\",\n", + " previous_response_id=response.id\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(response_two.output[0].content[0].text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can of course manage the context yourself. But one benefit of OpenAI maintaining the context for you is that you can fork the response at any point and continue the conversation from that point." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_two_forked = client.responses.create(\n", + " model=\"gpt-4o-mini\",\n", + " input=\"I didn't like that joke, tell me another and tell me the difference between the two jokes\",\n", + " previous_response_id=response.id # Forking and continuing from the first response\n", + ")\n", + "\n", + "output_text = response_two_forked.output[0].content[0].text\n", + "print(output_text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Hosted Tools\n", + "\n", + "Another benefit of the Responses API is that it adds support for hosted tools like `file_search` and `web_search`. Instead of manually calling the tools, simply pass in the tools and the API will decide which tool to use and use it.\n", + "\n", + "Here is an example of using the `web_search` tool to incorporate web search results into the response. You may already be familiar with how ChatGPT can search the web. You can now build similar experiences too! The web search tool uses the OpenAI Index, the one that powers the web search in ChatGPT, having being optimized for chat applications.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.responses.create(\n", + " model=\"gpt-4o\", # or another supported model\n", + " input=\"What's the latest news about AI?\",\n", + " tools=[\n", + " {\n", + " \"type\": \"web_search\"\n", + " }\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "print(json.dumps(response.output, default=lambda o: o.__dict__, indent=2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multimodal, Tool-augmented conversation\n", + "\n", + "The Responses API natively supports text, images, and audio modalities. \n", + "Tying everything together, we can build a fully multimodal, tool-augmented interaction with one API call through the responses API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import base64\n", + "\n", + "from IPython.display import Image, display\n", + "\n", + "# Display the image from the provided URL\n", + "url = \"https://upload.wikimedia.org/wikipedia/commons/thumb/1/15/Cat_August_2010-4.jpg/2880px-Cat_August_2010-4.jpg\"\n", + "display(Image(url=url, width=400))\n", + "\n", + "response_multimodal = client.responses.create(\n", + " model=\"gpt-4o\",\n", + " input=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"input_text\", \"text\": \n", + " \"Come up with keywords related to the image, and search on the web using the search tool for any news related to the keywords\"\n", + " \", summarize the findings and cite the sources.\"},\n", + " {\"type\": \"input_image\", \"image_url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/1/15/Cat_August_2010-4.jpg/2880px-Cat_August_2010-4.jpg\"}\n", + " ]\n", + " }\n", + " ],\n", + " tools=[\n", + " {\"type\": \"web_search\"}\n", + " ]\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "print(json.dumps(response_multimodal.__dict__, default=lambda o: o.__dict__, indent=4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the above example, we were able to use the `web_search` tool to search the web for news related to the image in one API call instead of multiple round trips that would be required if we were using the Chat Completions API." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With the responses API\n", + "🔥 a single API call can handle:\n", + "\n", + "✅ Analyze a given image using a multimodal input.\n", + "\n", + "✅ Perform web search via the `web_search` hosted tool\n", + "\n", + "✅ Summarize the results.\n", + "\n", + "In contrast, With Chat Completions API would require multiple steps, each requiring a round trip to the API:\n", + "\n", + "1️⃣ Upload image and get analysis → 1 request\n", + "\n", + "2️⃣ Extract info, call external web search → manual step + tool execution\n", + "\n", + "3️⃣ Re-submit tool results for summarization → another request\n", + "\n", + "See the following diagram for a side by side visualized comparison!\n", + "\n", + "![Responses vs Completions](../../images/comparisons.png)\n", + "\n", + "\n", + "We are very excited for you to try out the Responses API and see how it can simplify your code and make it easier to build complex, multimodal, tool-augmented interactions!\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From cd5d7a4cb4b269a4b4770adac935750a7e2c708d Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 2 Apr 2025 12:17:36 -0700 Subject: [PATCH 20/26] Remove tag/config helpers. --- agentops/helpers/__init__.py | 2 - agentops/helpers/config.py | 31 ------------- tests/unit/helpers/test_helpers.py | 73 ------------------------------ 3 files changed, 106 deletions(-) delete mode 100644 agentops/helpers/config.py delete mode 100644 tests/unit/helpers/test_helpers.py diff --git a/agentops/helpers/__init__.py b/agentops/helpers/__init__.py index 1f148bca9..ba8c1aad7 100644 --- a/agentops/helpers/__init__.py +++ b/agentops/helpers/__init__.py @@ -20,7 +20,6 @@ from .version import get_agentops_version, check_agentops_update from .debug import debug_print_function_params from .env import get_env_bool, get_env_int, get_env_list -from .config import get_tags_from_config __all__ = [ "get_ISO_time", @@ -46,5 +45,4 @@ "get_env_bool", "get_env_int", "get_env_list", - "get_tags_from_config", ] diff --git a/agentops/helpers/config.py b/agentops/helpers/config.py deleted file mode 100644 index ce7a3e945..000000000 --- a/agentops/helpers/config.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Helper functions for accessing configuration values. - -This module provides utility functions for accessing configuration values from -the global Config object in a safe way. -""" - -from typing import List, Any - -from agentops.config import Config - - -def get_config() -> Config: - """Get the global configuration object from the Client singleton. - - Returns: - The Config instance from the global Client - """ - from agentops import get_client - return get_client().config - - -def get_tags_from_config() -> List[str]: - """Get tags from the global configuration. - - Returns: - List of tags if they exist in the configuration, or empty list - """ - config = get_config() - if config.default_tags: - return list(config.default_tags) - return [] # Return empty list for empty tags set \ No newline at end of file diff --git a/tests/unit/helpers/test_helpers.py b/tests/unit/helpers/test_helpers.py deleted file mode 100644 index 0f3a65f71..000000000 --- a/tests/unit/helpers/test_helpers.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Tests for the config helper functions.""" - -import pytest - -from agentops.helpers.config import get_config, get_tags_from_config -from agentops.config import Config - - -class TestConfigHelpers: - """Test suite for configuration helper functions.""" - - def test_get_config_returns_valid_instance(self): - """Test that get_config returns a valid Config instance.""" - # Call the helper function - config = get_config() - - # Verify it returns a Config instance - assert isinstance(config, Config) - - # Verify it has the expected attributes - assert hasattr(config, 'api_key') - assert hasattr(config, 'endpoint') - assert hasattr(config, 'default_tags') - - def test_get_config_returns_singleton(self): - """Test that get_config returns the same Config instance each time.""" - # Call the helper function twice - config1 = get_config() - config2 = get_config() - - # Verify they are the same object (singleton pattern) - assert config1 is config2 - assert isinstance(config1, Config) - - def test_get_tags_from_config_with_actual_config(self): - """Test that get_tags_from_config returns tags from the actual application config.""" - # Get the actual application config - config = get_config() - original_tags = config.default_tags - - try: - # Temporarily set some test tags - test_tags = {"test_tag1", "test_tag2"} - config.default_tags = test_tags - - # Call the helper function - tags = get_tags_from_config() - - # Verify it returns the expected tags - assert isinstance(tags, list) - assert set(tags) == test_tags - finally: - # Restore the original tags - config.default_tags = original_tags - - def test_get_tags_from_config_with_empty_tags(self): - """Test that get_tags_from_config returns an empty list when no tags are set.""" - # Get the actual application config - config = get_config() - original_tags = config.default_tags - - try: - # Temporarily set empty tags - config.default_tags = set() - - # Call the helper function - tags = get_tags_from_config() - - # Verify it returns an empty list - assert tags == [] - finally: - # Restore the original tags - config.default_tags = original_tags \ No newline at end of file From 98aaa656e1a668ae118196436e9fcbfab7595d9f Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Fri, 4 Apr 2025 10:18:11 -0700 Subject: [PATCH 21/26] Additional responses function types. --- agentops/instrumentation/common/attributes.py | 53 +- .../openai/attributes/response.py | 532 ++++++++++++------ .../openai/attributes/tools.py | 0 .../instrumentation/openai_agents/README.md | 2 +- .../openai_agents/attributes/completion.py | 16 +- agentops/semconv/message.py | 24 +- .../openai_agents/test_openai_agents.py | 18 +- .../test_openai_agents_attributes.py | 18 +- .../openai_core/test_response_attributes.py | 450 ++++++++++++--- 9 files changed, 831 insertions(+), 282 deletions(-) create mode 100644 agentops/instrumentation/openai/attributes/tools.py diff --git a/agentops/instrumentation/common/attributes.py b/agentops/instrumentation/common/attributes.py index 1ca3eea9f..036a2b98f 100644 --- a/agentops/instrumentation/common/attributes.py +++ b/agentops/instrumentation/common/attributes.py @@ -19,7 +19,7 @@ These utilities ensure consistent attribute handling across different LLM service instrumentors while maintaining separation of concerns. """ -from typing import Dict, Any +from typing import runtime_checkable, Protocol, Any, Optional, Dict, TypedDict from agentops.logging import logger from agentops.helpers import safe_serialize, get_agentops_version from agentops.semconv import ( @@ -28,8 +28,28 @@ WorkflowAttributes, ) + +class IndexedAttributeData(TypedDict, total=False): + """ + """ + i: int + j: Optional[int] = None + + +@runtime_checkable +class IndexedAttribute(Protocol): + """ + """ + def format(self, *, i: int, j: Optional[int] = None) -> str: + ... + + # target_attribute_key: source_attribute -AttributeMap = Dict[str, Any] +AttributeMap = Dict[str, str] + +# target_attribute_key: source_attribute +# target_attribute_key must be formattable with `i` and optionally `j` +IndexedAttributeMap = Dict[IndexedAttribute, str] def _extract_attributes_from_mapping(span_data: Any, attribute_mapping: AttributeMap) -> AttributeMap: @@ -66,6 +86,35 @@ def _extract_attributes_from_mapping(span_data: Any, attribute_mapping: Attribut return attributes +def _extract_attributes_from_mapping_with_index(span_data: Any, attribute_mapping: IndexedAttributeMap, i: int, j: Optional[int] = None) -> AttributeMap: + """Helper function to extract attributes based on a mapping with index. + + This function extends `_extract_attributes_from_mapping` by allowing for indexed keys in the attribute mapping. + + Span data is expected to have keys which contain format strings for i/j, e.g. `my_attr_{i}` or `my_attr_{i}_{j}`. + + Args: + span_data: The span data object or dict to extract attributes from + attribute_mapping: Dictionary mapping target attributes to source attributes, with format strings for i/j + i: The primary index to use in formatting the attribute keys + j: An optional secondary index (default is None) + Returns: + Dictionary of extracted attributes with formatted indexed keys. + """ + + # `i` is required for formatting the attribute keys, `j` is optional + format_kwargs: IndexedAttributeData = {'i': i} + if j is not None: + format_kwargs['j'] = j + + # Update the attribute mapping to include the index for the span + attribute_mapping_with_index: AttributeMap = {} + for target_attr, source_attr in attribute_mapping.items(): + attribute_mapping_with_index[target_attr.format(**format_kwargs)] = source_attr + + return _extract_attributes_from_mapping(span_data, attribute_mapping_with_index) + + def get_common_attributes() -> AttributeMap: """Get common instrumentation attributes used across traces and spans. diff --git a/agentops/instrumentation/openai/attributes/response.py b/agentops/instrumentation/openai/attributes/response.py index 98cc9b34d..482754498 100644 --- a/agentops/instrumentation/openai/attributes/response.py +++ b/agentops/instrumentation/openai/attributes/response.py @@ -8,34 +8,90 @@ ) from agentops.instrumentation.common.attributes import ( AttributeMap, + IndexedAttributeMap, _extract_attributes_from_mapping, + _extract_attributes_from_mapping_with_index, ) try: from openai.types import Reasoning - from openai.types.beta import FunctionTool # TODO beta will likely change from openai.types.responses import ( + FunctionTool, + WebSearchTool, + FileSearchTool, + ComputerTool, + Response, ResponseUsage, - ResponseOutputMessage, - ResponseOutputText, ResponseReasoningItem, - ResponseFunctionToolCall, + ResponseInputParam, - # ResponseComputerToolCall, - # ResponseFileSearchToolCall, - # ResponseFunctionWebSearch, # ResponseInputItemParam, + ResponseOutputMessage, + ResponseOutputText, + + ResponseFunctionToolCall, + ResponseFunctionWebSearch, + ResponseFileSearchToolCall, + ResponseComputerToolCall, + # ResponseOutputItem, # ResponseOutputRefusal, # ResponseStreamEvent, ) - from openai.types.responses.response_usage import OutputTokensDetails + from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails + + ToolTypes = Union[ + FunctionTool, + WebSearchTool, + FileSearchTool, + ] + ResponseOutputTypes = Union[ + ResponseOutputMessage, + ResponseOutputText, + ResponseFunctionToolCall, + ResponseFunctionWebSearch, + ResponseComputerToolCall, + ResponseFileSearchToolCall, + ] except ImportError as e: logger.debug(f"[agentops.instrumentation.openai_agents] Could not import OpenAI Agents SDK types: {e}") RESPONSE_ATTRIBUTES: AttributeMap = { + # Response( + # id='resp_67ddd0196a4c81929f7e3783a80f18110b486458d6766f93', + # created_at=1742589977.0, + # error=None, + # incomplete_details=None, + # instructions='You are a helpful assistant...', + # metadata={}, + # model='gpt-4o-2024-08-06', + # object='response', + # output=[ + # ... + # ], + # parallel_tool_calls=True, + # temperature=1.0, + # tool_choice='auto', + # tools=[ + # ...) + # ], + # top_p=1.0, + # max_output_tokens=None, + # previous_response_id=None, + # reasoning=Reasoning( + # ... + # ), + # status='completed', + # text=ResponseTextConfig(format=ResponseFormatText(type='text')), + # truncation='disabled', + # usage=ResponseUsage( + # ... + # ), + # user=None, + # store=True + # ) SpanAttributes.LLM_RESPONSE_ID: "id", SpanAttributes.LLM_REQUEST_MODEL: "model", SpanAttributes.LLM_RESPONSE_MODEL: "model", @@ -46,44 +102,191 @@ } -RESPONSE_TOOLS_ATTRIBUTES: AttributeMap = { - ToolAttributes.TOOL_NAME: "name", - ToolAttributes.TOOL_DESCRIPTION: "description", - ToolAttributes.TOOL_PARAMETERS: "parameters", - # TODO `type` & `strict` are not converted +RESPONSE_TOOL_ATTRIBUTES: IndexedAttributeMap = { + # FunctionTool( + # name='get_weather', + # parameters={'properties': {'location': {'title': 'Location', 'type': 'string'}}, 'required': ['location'], 'title': 'get_weather_args', 'type': 'object', 'additionalProperties': False}, + # strict=True, + # type='function', + # description='Get the current weather for a location.' + # ) + MessageAttributes.TOOL_CALL_TYPE: "type", + MessageAttributes.TOOL_CALL_NAME: "name", + MessageAttributes.TOOL_CALL_DESCRIPTION: "description", + MessageAttributes.TOOL_CALL_ARGUMENTS: "parameters", + # TODO `strict` is not converted } -RESPONSE_OUTPUT_ATTRIBUTES: AttributeMap = { - MessageAttributes.COMPLETION_ID: "id", +RESPONSE_TOOL_WEB_SEARCH_ATTRIBUTES: IndexedAttributeMap = { + # WebSearchTool( + # type='web_search_preview', + # search_context_size='medium', + # user_location=UserLocation( + # type='approximate', + # city=None, + # country='US', + # region=None, + # timezone=None + # ) + # ) + MessageAttributes.TOOL_CALL_NAME: "type", + # `parameters` is added by the `get_response_tool_web_search_attributes` function, + # which contains `search_context_size` and `user_location`. + MessageAttributes.TOOL_CALL_ARGUMENTS: "parameters", +} + + +RESPONSE_TOOL_FILE_SEARCH_ATTRIBUTES: IndexedAttributeMap = { + # FileSearchTool( + # type='file_search', + # vector_store_ids=['store_123', 'store_456'], + # filters=Filters( + # key='value' + # ), + # max_num_results=10, + # ranking_options=RankingOptions( + # ranker='default-2024-11-15', + # score_threshold=0.8 + # ) + # ) + MessageAttributes.TOOL_CALL_TYPE: "type", + # `parameters` is added by the `get_response_tool_file_search_attributes` function, + # which contains `vector_store_ids`, `filters`, `max_num_results`, and `ranking_options`. + MessageAttributes.TOOL_CALL_ARGUMENTS: "parameters", } -RESPONSE_OUTPUT_MESSAGE_ATTRIBUTES: AttributeMap = { +RESPONSE_TOOL_COMPUTER_ATTRIBUTES: IndexedAttributeMap = { + # ComputerTool( + # display_height=1080.0, + # display_width=1920.0, + # environment='mac', + # type='computer_use_preview' + # ) + MessageAttributes.TOOL_CALL_TYPE: "type", + # `parameters` is added by the `get_response_tool_computer_attributes` function, + # which contains `display_height`, `display_width`, `environment`, etc. + MessageAttributes.TOOL_CALL_ARGUMENTS: "parameters", +} + + +RESPONSE_OUTPUT_MESSAGE_ATTRIBUTES: IndexedAttributeMap = { + # ResponseOutputMessage( + # id='msg_67ddcad3b6008192b521035d8b71fc570db7bfce93fd916a', + # content=[ + # ... + # ], + # role='assistant', + # status='completed', + # type='message' + # ) MessageAttributes.COMPLETION_ID: "id", + MessageAttributes.COMPLETION_TYPE: "type", MessageAttributes.COMPLETION_ROLE: "role", MessageAttributes.COMPLETION_FINISH_REASON: "status", - MessageAttributes.COMPLETION_TYPE: "type", } -RESPONSE_OUTPUT_TEXT_ATTRIBUTES: AttributeMap = { +RESPONSE_OUTPUT_TEXT_ATTRIBUTES: IndexedAttributeMap = { + # ResponseOutputText( + # annotations=[], + # text='Recursion is a programming technique ...', + # type='output_text' + # ) + MessageAttributes.COMPLETION_TYPE: "type", MessageAttributes.COMPLETION_CONTENT: "text", + # TODO `annotations` are not converted +} + + +RESPONSE_OUTPUT_REASONING_ATTRIBUTES: IndexedAttributeMap = { + # ResponseReasoningItem( + # id='reasoning_12345', + # summary=[ + # Summary( + # text='The model used a step-by-step approach to solve the problem.', + # type='summary_text' + # ) + # ], + # type='reasoning', + # status='completed' + # ) + MessageAttributes.COMPLETION_ID: "id", + MessageAttributes.COMPLETION_TYPE: "type", + MessageAttributes.COMPLETION_FINISH_REASON: "status", + # TODO `summary` is not converted } -RESPONSE_OUTPUT_TOOL_ATTRIBUTES: AttributeMap = { - MessageAttributes.FUNCTION_CALL_ID: "id", - MessageAttributes.FUNCTION_CALL_NAME: "name", - MessageAttributes.FUNCTION_CALL_ARGUMENTS: "arguments", - MessageAttributes.FUNCTION_CALL_TYPE: "type", +RESPONSE_OUTPUT_TOOL_ATTRIBUTES: IndexedAttributeMap = { + # ResponseFunctionToolCall( + # id='ftc_67ddcad3b6008192b521035d8b71fc570db7bfce93fd916a', + # arguments='{"location": "New York"}', + # call_id='call_12345', + # name='get_weather', + # type='function_call', + # status='completed' + # ) + MessageAttributes.COMPLETION_TOOL_CALL_ID: "id", + MessageAttributes.COMPLETION_TOOL_CALL_TYPE: "type", + MessageAttributes.COMPLETION_TOOL_CALL_NAME: "name", + MessageAttributes.COMPLETION_TOOL_CALL_ARGUMENTS: "arguments", # TODO `status` & `call_id` are not converted } -RESPONSE_OUTPUT_REASONING_ATTRIBUTES: AttributeMap = { - # TODO we don't have semantic conventions for these - # TODO `id`, `summary`, `type`, `status` are not converted +RESPONSE_OUTPUT_TOOL_WEB_SEARCH_ATTRIBUTES: IndexedAttributeMap = { + # ResponseFunctionWebSearch( + # id='ws_67eda37a5f18819280bf8b64f315bfa70091ec39ac46b411', + # status='completed', + # type='web_search_call' + # ) + MessageAttributes.COMPLETION_TOOL_CALL_ID: "id", + MessageAttributes.COMPLETION_TOOL_CALL_TYPE: "type", + MessageAttributes.COMPLETION_TOOL_CALL_NAME: "type", +} + + +RESPONSE_OUTPUT_TOOL_COMPUTER_ATTRIBUTES: IndexedAttributeMap = { + # ResponseComputerToolCall( + # id='comp_12345', + # action=Action( + # type='click', + # target='button_submit' + # ), + # call_id='call_67890', + # pending_safety_checks=[ + # PendingSafetyCheck( + # type='check_type', + # status='pending' + # ) + # ], + # status='completed', + # type='computer_call' + # ) + # TODO semantic conventions for `ResponseComputerToolCall` are not defined yet +} + + +RESPONSE_OUTPUT_TOOL_FILE_SEARCH_ATTRIBUTES: IndexedAttributeMap = { + # ResponseFileSearchToolCall( + # id='fsc_12345', + # queries=['example query'], + # status='completed', + # type='file_search_call', + # results=[ + # Result( + # attributes={'key1': 'value1', 'key2': 42}, + # file_id='file_67890', + # filename='example.txt', + # score=0.95, + # text='Example text retrieved from the file.' + # ), + # ... + # ] + # ) + # TODO semantic conventions for `ResponseFileSearchToolCall` are not defined yet } @@ -102,7 +305,11 @@ RESPONSE_REASONING_ATTRIBUTES: AttributeMap = { - # TODO `effort` and `generate_summary` are not converted + # Reasoning( + # effort='medium', + # generate_summary=None, + # ) + # TODO `effort` and `generate_summary` need semantic conventions } @@ -144,6 +351,7 @@ def get_response_kwarg_attributes(kwargs: dict) -> AttributeMap: if isinstance(_input, str): attributes[MessageAttributes.PROMPT_ROLE.format(i=0)] = "user" attributes[MessageAttributes.PROMPT_CONTENT.format(i=0)] = _input + elif isinstance(_input, list): for i, prompt in enumerate(_input): # Object type is pretty diverse, so we handle common attributes, but do so @@ -154,12 +362,12 @@ def get_response_kwarg_attributes(kwargs: dict) -> AttributeMap: attributes[MessageAttributes.PROMPT_ROLE.format(i=i)] = prompt.role if hasattr(prompt, "content"): attributes[MessageAttributes.PROMPT_CONTENT.format(i=i)] = prompt.content + else: - logger.debug(f"[agentops.instrumentation.openai_agents] '{type(_input)}' is not a recognized input type.") + logger.debug(f"[agentops.instrumentation.openai.response] '{type(_input)}' is not a recognized input type.") # `model` is always `str` (`ChatModel` type is just a string literal) - _model: str = str(kwargs.get("model")) - attributes[SpanAttributes.LLM_REQUEST_MODEL] = _model + attributes[SpanAttributes.LLM_REQUEST_MODEL] = str(kwargs.get("model")) return attributes @@ -168,42 +376,8 @@ def get_response_kwarg_attributes(kwargs: dict) -> AttributeMap: # a return type from the `responses` module def get_response_response_attributes(response: 'Response') -> AttributeMap: """Handles interpretation of an openai Response object.""" - # Response( - # id='resp_67ddd0196a4c81929f7e3783a80f18110b486458d6766f93', - # created_at=1742589977.0, - # error=None, - # incomplete_details=None, - # instructions='You are a helpful assistant...', - # metadata={}, - # model='gpt-4o-2024-08-06', - # object='response', - # output=[ - # ... - # ], - # parallel_tool_calls=True, - # temperature=1.0, - # tool_choice='auto', - # tools=[ - # ...) - # ], - # top_p=1.0, - # max_output_tokens=None, - # previous_response_id=None, - # reasoning=Reasoning( - # ... - # ), - # status='completed', - # text=ResponseTextConfig(format=ResponseFormatText(type='text')), - # truncation='disabled', - # usage=ResponseUsage( - # ... - # ), - # user=None, - # store=True - # ) attributes = _extract_attributes_from_mapping( - response.__dict__, - RESPONSE_ATTRIBUTES) + response.__dict__, RESPONSE_ATTRIBUTES) if response.output: attributes.update(get_response_output_attributes(response.output)) @@ -212,7 +386,8 @@ def get_response_response_attributes(response: 'Response') -> AttributeMap: attributes.update(get_response_tools_attributes(response.tools)) if response.reasoning: - attributes.update(get_response_reasoning_attributes(response.reasoning)) + attributes.update(_extract_attributes_from_mapping( + response.reasoning.__dict__, RESPONSE_REASONING_ATTRIBUTES)) if response.usage: attributes.update(get_response_usage_attributes(response.usage)) @@ -220,128 +395,150 @@ def get_response_response_attributes(response: 'Response') -> AttributeMap: return attributes -def get_response_output_attributes(output: List[Any]) -> AttributeMap: +def get_response_output_attributes(output: List['ResponseOutputTypes']) -> AttributeMap: """Handles interpretation of an openai Response `output` list.""" attributes = {} for i, output_item in enumerate(output): if isinstance(output_item, ResponseOutputMessage): attributes.update(get_response_output_message_attributes(i, output_item)) + elif isinstance(output_item, ResponseReasoningItem): - attributes.update(get_response_output_reasoning_attributes(i, output_item)) + attributes.update(_extract_attributes_from_mapping_with_index( + output_item, RESPONSE_OUTPUT_REASONING_ATTRIBUTES, i)) + elif isinstance(output_item, ResponseFunctionToolCall): - attributes.update(get_response_output_tool_attributes(i, output_item)) + attributes.update(_extract_attributes_from_mapping_with_index( + output_item, RESPONSE_OUTPUT_TOOL_ATTRIBUTES, i=i, j=0)) + + elif isinstance(output_item, ResponseFunctionWebSearch): + attributes.update(_extract_attributes_from_mapping_with_index( + output_item, RESPONSE_OUTPUT_TOOL_WEB_SEARCH_ATTRIBUTES, i=i, j=0)) + + elif isinstance(output_item, ResponseComputerToolCall): + attributes.update(_extract_attributes_from_mapping_with_index( + output_item, RESPONSE_OUTPUT_TOOL_COMPUTER_ATTRIBUTES, i=i, j=0)) + + elif isinstance(output_item, ResponseFileSearchToolCall): + attributes.update(_extract_attributes_from_mapping_with_index( + output_item, RESPONSE_OUTPUT_TOOL_FILE_SEARCH_ATTRIBUTES, i=i, j=0)) + else: - logger.debug(f"[agentops.instrumentation.openai_agents] '{output_item}' is not a recognized output type.") + logger.debug(f"[agentops.instrumentation.openai.response] '{output_item}' is not a recognized output type.") return attributes +def get_response_output_text_attributes(output_text: 'ResponseOutputText', index: int) -> AttributeMap: + """Handles interpretation of an openai ResponseOutputText object.""" + # This function is a helper to handle the ResponseOutputText type specifically + return _extract_attributes_from_mapping_with_index( + output_text, RESPONSE_OUTPUT_TEXT_ATTRIBUTES, index) + + def get_response_output_message_attributes(index: int, message: 'ResponseOutputMessage') -> AttributeMap: """Handles interpretation of an openai ResponseOutputMessage object.""" - # ResponseOutputMessage( - # id='msg_67ddcad3b6008192b521035d8b71fc570db7bfce93fd916a', - # content=[ - # ... - # ], - # role='assistant', - # status='completed', - # type='message' - # ) - attributes = {} - - for attribute, lookup in RESPONSE_OUTPUT_MESSAGE_ATTRIBUTES.items(): - if hasattr(message, lookup): - attributes[attribute.format(i=index)] = safe_serialize(getattr(message, lookup)) + attributes = _extract_attributes_from_mapping_with_index( + message, RESPONSE_OUTPUT_MESSAGE_ATTRIBUTES, index) if message.content: for i, content in enumerate(message.content): if isinstance(content, ResponseOutputText): - attributes.update(get_response_output_text_attributes(i, content)) + attributes.update(get_response_output_text_attributes(content, i)) + else: - logger.debug(f"[agentops.instrumentation.openai_agents] '{content}' is not a recognized content type.") + logger.debug(f"[agentops.instrumentation.openai.response] '{content}' is not a recognized content type.") return attributes -def get_response_output_text_attributes(index: int, content: 'ResponseOutputText') -> AttributeMap: - """Handles interpretation of an openai ResponseOutputText object.""" - # ResponseOutputText( - # annotations=[], - # text='Recursion is a programming technique ...', - # type='output_text' - # ) +def get_response_tools_attributes(tools: List['ToolTypes']) -> AttributeMap: + """Handles interpretation of openai Response `tools` list.""" attributes = {} - for attribute, lookup in RESPONSE_OUTPUT_TEXT_ATTRIBUTES.items(): - if hasattr(content, lookup): - attributes[attribute.format(i=index)] = safe_serialize(getattr(content, lookup)) + for i, tool in enumerate(tools): + if isinstance(tool, FunctionTool): + attributes.update(_extract_attributes_from_mapping_with_index( + tool, RESPONSE_TOOL_ATTRIBUTES, i)) + + elif isinstance(tool, WebSearchTool): + attributes.update(get_response_tool_web_search_attributes(tool, i)) + + elif isinstance(tool, FileSearchTool): + attributes.update(get_response_tool_file_search_attributes(tool, i)) + + elif isinstance(tool, ComputerTool): + attributes.update(get_response_tool_computer_attributes(tool, i)) + + else: + logger.debug(f"[agentops.instrumentation.openai.response] '{tool}' is not a recognized tool type.") return attributes -def get_response_output_reasoning_attributes(index: int, output: 'ResponseReasoningItem') -> AttributeMap: - """Handles interpretation of an openai ResponseReasoningItem object.""" - # Reasoning( - # effort=None, - # generate_summary=None - # ) - attributes = {} +def get_response_tool_web_search_attributes(tool: 'WebSearchTool', index: int) -> AttributeMap: + """Handles interpretation of an openai WebSearchTool object.""" + parameters = {} + if hasattr(tool, 'search_context_size'): + parameters['search_context_size'] = tool.search_context_size - for attribute, lookup in RESPONSE_OUTPUT_REASONING_ATTRIBUTES.items(): - if hasattr(output, lookup): - attributes[attribute.format(i=index)] = safe_serialize(getattr(output, lookup)) + if hasattr(tool, 'user_location'): + parameters['user_location'] = tool.user_location.__dict__ - return attributes + tool_data = tool.__dict__ + if parameters: + # add parameters to the tool_data dict so we can format them with the other attributes + tool_data['parameters'] = parameters + + return _extract_attributes_from_mapping_with_index( + tool_data, RESPONSE_TOOL_WEB_SEARCH_ATTRIBUTES, index) -def get_response_output_tool_attributes(index: int, output: 'ResponseFunctionToolCall') -> AttributeMap: - """Handles interpretation of an openai ResponseFunctionToolCall object.""" - # FunctionTool( - # name='get_weather', - # parameters={'properties': {'location': {'title': 'Location', 'type': 'string'}}, 'required': ['location'], 'title': 'get_weather_args', 'type': 'object', 'additionalProperties': False}, - # strict=True, - # type='function', - # description='Get the current weather for a location.' - # ) - attributes = {} +def get_response_tool_file_search_attributes(tool: 'FileSearchTool', index: int) -> AttributeMap: + """Handles interpretation of an openai FileSearchTool object.""" + parameters = {} - for attribute, lookup in RESPONSE_OUTPUT_TOOL_ATTRIBUTES.items(): - if hasattr(output, lookup): - attributes[attribute.format(i=index)] = safe_serialize(getattr(output, lookup)) + if hasattr(tool, 'vector_store_ids'): + parameters['vector_store_ids'] = tool.vector_store_ids - return attributes + if hasattr(tool, 'filters'): + parameters['filters'] = tool.filters.__dict__ + + if hasattr(tool, 'max_num_results'): + parameters['max_num_results'] = tool.max_num_results + + if hasattr(tool, 'ranking_options'): + parameters['ranking_options'] = tool.ranking_options.__dict__ + + tool_data = tool.__dict__ + if parameters: + # add parameters to the tool_data dict so we can format them with the other attributes + tool_data['parameters'] = parameters + + return _extract_attributes_from_mapping_with_index( + tool_data, RESPONSE_TOOL_FILE_SEARCH_ATTRIBUTES, index) -def get_response_tools_attributes(tools: List[Any]) -> AttributeMap: - """Handles interpretation of openai Response `tools` list.""" - # FunctionTool( - # name='get_weather', - # parameters={'properties': {'location': {'title': 'Location', 'type': 'string'}}, 'required': ['location'], 'title': 'get_weather_args', 'type': 'object', 'additionalProperties': False}, - # strict=True, - # type='function', - # description='Get the current weather for a location.' - # ) - attributes = {} +def get_response_tool_computer_attributes(tool: 'ComputerTool', index: int) -> AttributeMap: + """Handles interpretation of an openai ComputerTool object.""" + parameters = {} - for i, tool in enumerate(tools): - if isinstance(tool, FunctionTool): - # FunctionTool( - # name='get_weather', - # parameters={'properties': {'location': {'title': 'Location', 'type': 'string'}}, 'required': ['location'], 'title': 'get_weather_args', 'type': 'object', 'additionalProperties': False}, - # strict=True, - # type='function', - # description='Get the current weather for a location.' - # ) - for attribute, lookup in RESPONSE_TOOLS_ATTRIBUTES.items(): - if not hasattr(tool, lookup): - continue - - attributes[attribute.format(i=i)] = safe_serialize(getattr(tool, lookup)) - else: - logger.debug(f"[agentops.instrumentation.openai_agents] '{tool}' is not a recognized tool type.") + if hasattr(tool, 'display_height'): + parameters['display_height'] = tool.display_height - return attributes + if hasattr(tool, 'display_width'): + parameters['display_width'] = tool.display_width + + if hasattr(tool, 'environment'): + parameters['environment'] = tool.environment + + tool_data = tool.__dict__ + if parameters: + # add parameters to the tool_data dict so we can format them with the other attributes + tool_data['parameters'] = parameters + + return _extract_attributes_from_mapping_with_index( + tool_data, RESPONSE_TOOL_COMPUTER_ATTRIBUTES, index) def get_response_usage_attributes(usage: 'ResponseUsage') -> AttributeMap: @@ -359,35 +556,34 @@ def get_response_usage_attributes(usage: 'ResponseUsage') -> AttributeMap: usage.__dict__, RESPONSE_USAGE_ATTRIBUTES)) - # input_tokens_details is a dict if it exists + # input_tokens_details is an `InputTokensDetails` object or `dict` if it exists if hasattr(usage, 'input_tokens_details'): input_details = usage.input_tokens_details - if input_details and isinstance(input_details, dict): + if input_details is None: + pass + + elif isinstance(input_details, InputTokensDetails): + attributes.update(_extract_attributes_from_mapping( + input_details.__dict__, RESPONSE_USAGE_DETAILS_ATTRIBUTES)) + + elif isinstance(input_details, dict): # openai-agents often returns a dict for some reason. attributes.update(_extract_attributes_from_mapping( - input_details, - RESPONSE_USAGE_DETAILS_ATTRIBUTES)) + input_details, RESPONSE_USAGE_DETAILS_ATTRIBUTES)) + else: - logger.debug(f"[agentops.instrumentation.openai_agents] '{input_details}' is not a recognized input details type.") + logger.debug(f"[agentops.instrumentation.openai.response] '{input_details}' is not a recognized input details type.") # output_tokens_details is an `OutputTokensDetails` object output_details = usage.output_tokens_details - if output_details and isinstance(output_details, OutputTokensDetails): + if output_details is None: + pass + + elif isinstance(output_details, OutputTokensDetails): attributes.update(_extract_attributes_from_mapping( - output_details.__dict__, - RESPONSE_USAGE_DETAILS_ATTRIBUTES)) + output_details.__dict__, RESPONSE_USAGE_DETAILS_ATTRIBUTES)) + else: - logger.debug(f"[agentops.instrumentation.openai_agents] '{output_details}' is not a recognized output details type.") + logger.debug(f"[agentops.instrumentation.openai.response] '{output_details}' is not a recognized output details type.") return attributes - -def get_response_reasoning_attributes(reasoning: 'Reasoning') -> AttributeMap: - """Handles interpretation of an openai Reasoning object.""" - # Reasoning( - # effort='medium', - # generate_summary=None, - # ) - return _extract_attributes_from_mapping( - reasoning.__dict__, - RESPONSE_REASONING_ATTRIBUTES) - diff --git a/agentops/instrumentation/openai/attributes/tools.py b/agentops/instrumentation/openai/attributes/tools.py new file mode 100644 index 000000000..e69de29bb diff --git a/agentops/instrumentation/openai_agents/README.md b/agentops/instrumentation/openai_agents/README.md index 6f7ecbcf7..56e518429 100644 --- a/agentops/instrumentation/openai_agents/README.md +++ b/agentops/instrumentation/openai_agents/README.md @@ -132,7 +132,7 @@ AGENT_SPAN_ATTRIBUTES: AttributeMap = { - Always use MessageAttributes semantic conventions for content and tool calls - For chat completions, use MessageAttributes.COMPLETION_CONTENT.format(i=0) -- For tool calls, use MessageAttributes.TOOL_CALL_NAME.format(i=0, j=0), etc. +- For tool calls, use MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=0, j=0), etc. - Never try to combine or aggregate contents into a single attribute - Each message component should have its own properly formatted attribute - This ensures proper display in OpenTelemetry backends and dashboards diff --git a/agentops/instrumentation/openai_agents/attributes/completion.py b/agentops/instrumentation/openai_agents/attributes/completion.py index 01ace15a1..df5adf0e2 100644 --- a/agentops/instrumentation/openai_agents/attributes/completion.py +++ b/agentops/instrumentation/openai_agents/attributes/completion.py @@ -111,9 +111,9 @@ def get_raw_response_attributes(response: Dict[str, Any]) -> Dict[str, Any]: # Handle function format if "function" in tool_call and isinstance(tool_call["function"], dict): function = tool_call["function"] - result[MessageAttributes.TOOL_CALL_ID.format(i=j, j=k)] = tool_id - result[MessageAttributes.TOOL_CALL_NAME.format(i=j, j=k)] = function.get("name", "") - result[MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=j, j=k)] = function.get("arguments", "") + result[MessageAttributes.COMPLETION_TOOL_CALL_ID.format(i=j, j=k)] = tool_id + result[MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=j, j=k)] = function.get("name", "") + result[MessageAttributes.COMPLETION_TOOL_CALL_ARGUMENTS.format(i=j, j=k)] = function.get("arguments", "") return result @@ -154,14 +154,14 @@ def get_chat_completions_attributes(response: Dict[str, Any]) -> Dict[str, Any]: for j, tool_call in enumerate(tool_calls): if "function" in tool_call: function = tool_call["function"] - result[MessageAttributes.TOOL_CALL_ID.format(i=i, j=j)] = tool_call.get("id") - result[MessageAttributes.TOOL_CALL_NAME.format(i=i, j=j)] = function.get("name") - result[MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=i, j=j)] = function.get("arguments") + result[MessageAttributes.COMPLETION_TOOL_CALL_ID.format(i=i, j=j)] = tool_call.get("id") + result[MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=i, j=j)] = function.get("name") + result[MessageAttributes.COMPLETION_TOOL_CALL_ARGUMENTS.format(i=i, j=j)] = function.get("arguments") if "function_call" in message and message["function_call"] is not None: function_call = message["function_call"] - result[MessageAttributes.FUNCTION_CALL_NAME.format(i=i)] = function_call.get("name") - result[MessageAttributes.FUNCTION_CALL_ARGUMENTS.format(i=i)] = function_call.get("arguments") + result[MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=i)] = function_call.get("name") + result[MessageAttributes.COMPLETION_TOOL_CALL_ARGUMENTS.format(i=i)] = function_call.get("arguments") return result diff --git a/agentops/semconv/message.py b/agentops/semconv/message.py index d31b17f3d..6e750c128 100644 --- a/agentops/semconv/message.py +++ b/agentops/semconv/message.py @@ -8,21 +8,23 @@ class MessageAttributes: PROMPT_CONTENT = "gen_ai.prompt.{i}.content" # Content of the prompt message PROMPT_TYPE = "gen_ai.prompt.{i}.type" # Type of the prompt message + # Indexed function calls (with {i} for interpolation) + TOOL_CALL_ID = "gen_ai.request.tools.{i}.id" # Unique identifier for the function call at index {i} + TOOL_CALL_TYPE = "gen_ai.request.tools.{i}.type" # Type of the function call at index {i} + TOOL_CALL_NAME = "gen_ai.request.tools.{i}.name" # Name of the function call at index {i} + TOOL_CALL_DESCRIPTION = "gen_ai.request.tools.{i}.description" # Description of the function call at index {i} + TOOL_CALL_ARGUMENTS = "gen_ai.request.tools.{i}.arguments" # Arguments for function call at index {i} + # Indexed completions (with {i} for interpolation) COMPLETION_ID = "gen_ai.completion.{i}.id" # Unique identifier for the completion - + COMPLETION_TYPE = "gen_ai.completion.{i}.type" # Type of the completion at index {i} COMPLETION_ROLE = "gen_ai.completion.{i}.role" # Role of the completion message at index {i} COMPLETION_CONTENT = "gen_ai.completion.{i}.content" # Content of the completion message at index {i} COMPLETION_FINISH_REASON = "gen_ai.completion.{i}.finish_reason" # Finish reason for completion at index {i} - COMPLETION_TYPE = "gen_ai.completion.{i}.type" # Type of the completion at index {i} - - # Indexed function calls (with {i} for interpolation) - FUNCTION_CALL_ID = "gen_ai.request.tools.{i}.id" # Unique identifier for the function call at index {i} - FUNCTION_CALL_NAME = "gen_ai.request.tools.{i}.name" # Name of the function call at index {i} - FUNCTION_CALL_ARGUMENTS = "gen_ai.request.tools.{i}.arguments" # Arguments for function call at index {i} - FUNCTION_CALL_TYPE = "gen_ai.request.tools.{i}.type" # Type of the function call at index {i} # Indexed tool calls (with {i}/{j} for nested interpolation) - TOOL_CALL_ID = "gen_ai.completion.{i}.tool_calls.{j}.id" # ID of tool call {j} in completion {i} - TOOL_CALL_NAME = "gen_ai.completion.{i}.tool_calls.{j}.name" # Name of the tool called in tool call {j} in completion {i} - TOOL_CALL_ARGUMENTS = "gen_ai.completion.{i}.tool_calls.{j}.arguments" # Arguments for tool call {j} in completion {i} \ No newline at end of file + COMPLETION_TOOL_CALL_ID = "gen_ai.completion.{i}.tool_calls.{j}.id" # ID of tool call {j} in completion {i} + COMPLETION_TOOL_CALL_TYPE = "gen_ai.completion.{i}.tool_calls.{j}.type" # Type of tool call {j} in completion {i} + COMPLETION_TOOL_CALL_NAME = "gen_ai.completion.{i}.tool_calls.{j}.name" # Name of the tool called in tool call {j} in completion {i} + COMPLETION_TOOL_CALL_DESCRIPTION = "gen_ai.completion.{i}.tool_calls.{j}.description" # Description of the tool call {j} in completion {i} + COMPLETION_TOOL_CALL_ARGUMENTS = "gen_ai.completion.{i}.tool_calls.{j}.arguments" # Arguments for tool call {j} in completion {i} \ No newline at end of file diff --git a/tests/unit/instrumentation/openai_agents/test_openai_agents.py b/tests/unit/instrumentation/openai_agents/test_openai_agents.py index 34db816ff..51c8b1810 100644 --- a/tests/unit/instrumentation/openai_agents/test_openai_agents.py +++ b/tests/unit/instrumentation/openai_agents/test_openai_agents.py @@ -185,9 +185,9 @@ def test_tool_calls_span_serialization(self, instrumentation): mock_response_attrs.return_value = { MessageAttributes.COMPLETION_CONTENT.format(i=0): "I'll help you find the current weather for New York City.", MessageAttributes.COMPLETION_ROLE.format(i=0): "assistant", - MessageAttributes.TOOL_CALL_ID.format(i=0, j=0): "call_xyz789", - MessageAttributes.TOOL_CALL_NAME.format(i=0, j=0): "get_weather", - MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0, j=0): "{\"location\":\"New York City\",\"units\":\"celsius\"}", + MessageAttributes.COMPLETION_TOOL_CALL_ID.format(i=0, j=0): "call_xyz789", + MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=0, j=0): "get_weather", + MessageAttributes.COMPLETION_TOOL_CALL_ARGUMENTS.format(i=0, j=0): "{\"location\":\"New York City\",\"units\":\"celsius\"}", SpanAttributes.LLM_SYSTEM: "openai", SpanAttributes.LLM_USAGE_PROMPT_TOKENS: 48, SpanAttributes.LLM_USAGE_COMPLETION_TOKENS: 12, @@ -225,12 +225,12 @@ def test_tool_calls_span_serialization(self, instrumentation): captured_attributes[SpanAttributes.LLM_RESPONSE_MODEL] = "gpt-4o" # Verify tool call attributes were set correctly - assert MessageAttributes.TOOL_CALL_NAME.format(i=0, j=0) in captured_attributes - assert captured_attributes[MessageAttributes.TOOL_CALL_NAME.format(i=0, j=0)] == "get_weather" - assert MessageAttributes.TOOL_CALL_ID.format(i=0, j=0) in captured_attributes - assert captured_attributes[MessageAttributes.TOOL_CALL_ID.format(i=0, j=0)] == "call_xyz789" - assert MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0, j=0) in captured_attributes - assert "{\"location\":\"New York City\",\"units\":\"celsius\"}" in captured_attributes[MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0, j=0)] + assert MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=0, j=0) in captured_attributes + assert captured_attributes[MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=0, j=0)] == "get_weather" + assert MessageAttributes.COMPLETION_TOOL_CALL_ID.format(i=0, j=0) in captured_attributes + assert captured_attributes[MessageAttributes.COMPLETION_TOOL_CALL_ID.format(i=0, j=0)] == "call_xyz789" + assert MessageAttributes.COMPLETION_TOOL_CALL_ARGUMENTS.format(i=0, j=0) in captured_attributes + assert "{\"location\":\"New York City\",\"units\":\"celsius\"}" in captured_attributes[MessageAttributes.COMPLETION_TOOL_CALL_ARGUMENTS.format(i=0, j=0)] # Verify the text content is also captured assert MessageAttributes.COMPLETION_CONTENT.format(i=0) in captured_attributes diff --git a/tests/unit/instrumentation/openai_agents/test_openai_agents_attributes.py b/tests/unit/instrumentation/openai_agents/test_openai_agents_attributes.py index ac24d7a97..8df5662f3 100644 --- a/tests/unit/instrumentation/openai_agents/test_openai_agents_attributes.py +++ b/tests/unit/instrumentation/openai_agents/test_openai_agents_attributes.py @@ -400,10 +400,10 @@ def __init__(self): assert attrs[SpanAttributes.LLM_USAGE_COMPLETION_TOKENS] == 12 assert attrs[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 60 - # Verify tool call information - tool_id_key = MessageAttributes.TOOL_CALL_ID.format(i=0, j=0) - tool_name_key = MessageAttributes.TOOL_CALL_NAME.format(i=0, j=0) - tool_args_key = MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0, j=0) + # Verify tool call information - note raw_responses is in index 0, output item 0, tool_call 0 + tool_id_key = MessageAttributes.COMPLETION_TOOL_CALL_ID.format(i=0, j=0) + tool_name_key = MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=0, j=0) + tool_args_key = MessageAttributes.COMPLETION_TOOL_CALL_ARGUMENTS.format(i=0, j=0) assert attrs[tool_id_key] == "call_xyz789" assert attrs[tool_name_key] == "get_weather" @@ -510,13 +510,13 @@ def test_chat_completions_with_tool_calls_from_fixture(self): attrs = get_chat_completions_attributes(OPENAI_CHAT_TOOL_CALLS) # Verify tool call information is extracted - assert MessageAttributes.TOOL_CALL_ID.format(i=0, j=0) in attrs - assert MessageAttributes.TOOL_CALL_NAME.format(i=0, j=0) in attrs - assert MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0, j=0) in attrs + assert MessageAttributes.COMPLETION_TOOL_CALL_ID.format(i=0, j=0) in attrs + assert MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=0, j=0) in attrs + assert MessageAttributes.COMPLETION_TOOL_CALL_ARGUMENTS.format(i=0, j=0) in attrs # Verify values match fixture data (specific values will depend on your fixture content) - tool_id = attrs[MessageAttributes.TOOL_CALL_ID.format(i=0, j=0)] - tool_name = attrs[MessageAttributes.TOOL_CALL_NAME.format(i=0, j=0)] + tool_id = attrs[MessageAttributes.COMPLETION_TOOL_CALL_ID.format(i=0, j=0)] + tool_name = attrs[MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=0, j=0)] assert tool_id is not None and len(tool_id) > 0 assert tool_name is not None and len(tool_name) > 0 diff --git a/tests/unit/instrumentation/openai_core/test_response_attributes.py b/tests/unit/instrumentation/openai_core/test_response_attributes.py index 3a64d29d5..5009f08a9 100644 --- a/tests/unit/instrumentation/openai_core/test_response_attributes.py +++ b/tests/unit/instrumentation/openai_core/test_response_attributes.py @@ -17,11 +17,11 @@ get_response_output_attributes, get_response_output_message_attributes, get_response_output_text_attributes, - get_response_output_reasoning_attributes, - get_response_output_tool_attributes, get_response_tools_attributes, get_response_usage_attributes, - get_response_reasoning_attributes + get_response_tool_web_search_attributes, + get_response_tool_file_search_attributes, + get_response_tool_computer_attributes ) from agentops.semconv import ( SpanAttributes, @@ -94,7 +94,103 @@ class MockFunctionTool: def __init__(self, data): for key, value in data.items(): setattr(self, key, value) - self.type = "function" + if not hasattr(self, "type"): + self.type = "function" + self.__dict__.update(data) + + +class MockWebSearchTool: + """Mock WebSearchTool object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + if not hasattr(self, "type"): + self.type = "web_search_preview" + self.__dict__.update(data) + + +class MockFileSearchTool: + """Mock FileSearchTool object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + if not hasattr(self, "type"): + self.type = "file_search" + self.__dict__.update(data) + + +class MockComputerTool: + """Mock ComputerTool object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + if not hasattr(self, "type"): + self.type = "computer_use_preview" + self.__dict__.update(data) + + +class MockUserLocation: + """Mock UserLocation object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + self.__dict__.update(data) + + +class MockFilters: + """Mock Filters object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + self.__dict__.update(data) + + +class MockRankingOptions: + """Mock RankingOptions object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + self.__dict__.update(data) + + +class MockFunctionWebSearch: + """Mock ResponseFunctionWebSearch object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + if not hasattr(self, "type"): + self.type = "web_search_call" + self.__dict__.update(data) + + +class MockFileSearchToolCall: + """Mock ResponseFileSearchToolCall object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + if not hasattr(self, "type"): + self.type = "file_search_call" + self.__dict__.update(data) + + +class MockComputerToolCall: + """Mock ResponseComputerToolCall object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + if not hasattr(self, "type"): + self.type = "computer_call" + self.__dict__.update(data) + + +class MockReasoningItem: + """Mock ResponseReasoningItem object for testing""" + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + if not hasattr(self, "type"): + self.type = "reasoning" + self.__dict__.update(data) class MockFunctionToolCall: @@ -300,16 +396,44 @@ def test_get_response_output_text_attributes(self): 'type': 'output_text' }) - # Extract attributes - attributes = get_response_output_text_attributes(0, text) + # The function doesn't use the mock directly but extracts attributes from it + # Using _extract_attributes_from_mapping_with_index internally + # We'll test by using patch to simulate the extraction - # Check attributes - assert MessageAttributes.COMPLETION_CONTENT.format(i=0) in attributes - assert attributes[MessageAttributes.COMPLETION_CONTENT.format(i=0)] == 'The capital of France is Paris.' + with patch('agentops.instrumentation.openai.attributes.response._extract_attributes_from_mapping_with_index') as mock_extract: + # Set up the mock to return expected attributes + expected_attributes = { + MessageAttributes.COMPLETION_CONTENT.format(i=0): 'The capital of France is Paris.', + MessageAttributes.COMPLETION_TYPE.format(i=0): 'output_text' + } + mock_extract.return_value = expected_attributes + + # Call the function + attributes = get_response_output_text_attributes(0, text) + + # Verify mock was called with correct arguments + mock_extract.assert_called_once() + + # Check that the return value matches our expected attributes + assert attributes == expected_attributes - def test_get_response_output_tool_attributes(self): - """Test extraction of attributes from output tool""" - # Create a mock tool call + def test_get_response_output_attributes(self): + """Test extraction of attributes from output items with all output types""" + # Create a mock response output list with all different output types + message = MockOutputMessage({ + 'id': 'msg_12345', + 'content': [ + MockOutputText({ + 'text': 'This is a test message', + 'type': 'output_text', + 'annotations': [] + }) + ], + 'role': 'assistant', + 'status': 'completed', + 'type': 'message' + }) + tool_call = MockFunctionToolCall({ 'id': 'call_67890', 'name': 'get_weather', @@ -317,52 +441,200 @@ def test_get_response_output_tool_attributes(self): 'type': 'function' }) - # Extract attributes - attributes = get_response_output_tool_attributes(0, tool_call) + web_search = MockFunctionWebSearch({ + 'id': 'ws_12345', + 'status': 'completed', + 'type': 'web_search_call' + }) + + file_search = MockFileSearchToolCall({ + 'id': 'fsc_12345', + 'queries': ['search term'], + 'status': 'completed', + 'type': 'file_search_call' + }) + + computer_call = MockComputerToolCall({ + 'id': 'comp_12345', + 'status': 'completed', + 'type': 'computer_call' + }) + + reasoning_item = MockReasoningItem({ + 'id': 'reason_12345', + 'status': 'completed', + 'type': 'reasoning' + }) + + # Create an unrecognized output item to test error handling + unrecognized_item = MagicMock() + unrecognized_item.type = 'unknown_type' - # Check attributes - assert MessageAttributes.FUNCTION_CALL_ID.format(i=0) in attributes - assert attributes[MessageAttributes.FUNCTION_CALL_ID.format(i=0)] == 'call_67890' - assert MessageAttributes.FUNCTION_CALL_NAME.format(i=0) in attributes - assert attributes[MessageAttributes.FUNCTION_CALL_NAME.format(i=0)] == 'get_weather' - assert MessageAttributes.FUNCTION_CALL_ARGUMENTS.format(i=0) in attributes - assert attributes[MessageAttributes.FUNCTION_CALL_ARGUMENTS.format(i=0)] == '{"location":"Paris"}' - assert MessageAttributes.FUNCTION_CALL_TYPE.format(i=0) in attributes - assert attributes[MessageAttributes.FUNCTION_CALL_TYPE.format(i=0)] == 'function' + # Patch all the necessary type checks and logger + with patch('agentops.instrumentation.openai.attributes.response.ResponseOutputMessage', MockOutputMessage), \ + patch('agentops.instrumentation.openai.attributes.response.ResponseOutputText', MockOutputText), \ + patch('agentops.instrumentation.openai.attributes.response.ResponseFunctionToolCall', MockFunctionToolCall), \ + patch('agentops.instrumentation.openai.attributes.response.ResponseFunctionWebSearch', MockFunctionWebSearch), \ + patch('agentops.instrumentation.openai.attributes.response.ResponseFileSearchToolCall', MockFileSearchToolCall), \ + patch('agentops.instrumentation.openai.attributes.response.ResponseComputerToolCall', MockComputerToolCall), \ + patch('agentops.instrumentation.openai.attributes.response.ResponseReasoningItem', MockReasoningItem), \ + patch('agentops.instrumentation.openai.attributes.response.logger.debug') as mock_logger: + + # Test with an output list containing all different types of output items + output = [message, tool_call, web_search, file_search, computer_call, reasoning_item, unrecognized_item] + + # Call the function + attributes = get_response_output_attributes(output) + + # Check that it extracted attributes from all items + assert isinstance(attributes, dict) + + # Check message attributes were extracted (index 0) + assert MessageAttributes.COMPLETION_ROLE.format(i=0) in attributes + assert attributes[MessageAttributes.COMPLETION_ROLE.format(i=0)] == 'assistant' + assert MessageAttributes.COMPLETION_CONTENT.format(i=0) in attributes + assert attributes[MessageAttributes.COMPLETION_CONTENT.format(i=0)] == 'This is a test message' + + # Check function tool call attributes were extracted (index 1) + tool_attr_key = MessageAttributes.COMPLETION_TOOL_CALL_ID.format(i=1, j=0) + assert tool_attr_key in attributes + assert attributes[tool_attr_key] == 'call_67890' + + # Check web search attributes were extracted (index 2) + web_attr_key = MessageAttributes.COMPLETION_TOOL_CALL_ID.format(i=2, j=0) + assert web_attr_key in attributes + assert attributes[web_attr_key] == 'ws_12345' + + # Verify that logger was called for unrecognized item + assert any(call.args[0].startswith('[agentops.instrumentation.openai.response]') + for call in mock_logger.call_args_list) def test_get_response_tools_attributes(self): """Test extraction of attributes from tools list""" - # Simplify the test to just verify the function can be called without error + # Create a mock function tool + function_tool = MockFunctionTool({ + 'name': 'get_weather', + 'parameters': {'properties': {'location': {'type': 'string'}}, 'required': ['location']}, + 'description': 'Get weather information for a location', + 'type': 'function', + 'strict': True + }) - # Patch the FunctionTool class to make testing simpler + # Patch all tool types to make testing simpler with patch('agentops.instrumentation.openai.attributes.response.FunctionTool', MockFunctionTool): - # Test with empty list for simplicity - tools = [] + with patch('agentops.instrumentation.openai.attributes.response.WebSearchTool', MagicMock): + with patch('agentops.instrumentation.openai.attributes.response.FileSearchTool', MagicMock): + with patch('agentops.instrumentation.openai.attributes.response.ComputerTool', MagicMock): + # Test with a function tool + tools = [function_tool] + + # Call the function + result = get_response_tools_attributes(tools) + + # Verify extracted attributes + assert isinstance(result, dict) + assert MessageAttributes.TOOL_CALL_TYPE.format(i=0) in result + assert result[MessageAttributes.TOOL_CALL_TYPE.format(i=0)] == 'function' + assert MessageAttributes.TOOL_CALL_NAME.format(i=0) in result + assert result[MessageAttributes.TOOL_CALL_NAME.format(i=0)] == 'get_weather' + assert MessageAttributes.TOOL_CALL_DESCRIPTION.format(i=0) in result + assert result[MessageAttributes.TOOL_CALL_DESCRIPTION.format(i=0)] == 'Get weather information for a location' + + def test_get_response_tool_web_search_attributes(self): + """Test extraction of attributes from web search tool""" + # Create a mock web search tool + user_location = MockUserLocation({ + 'type': 'approximate', + 'country': 'US' + }) + + web_search_tool = MockWebSearchTool({ + 'type': 'web_search_preview', + 'search_context_size': 'medium', + 'user_location': user_location + }) + + # Call the function directly + with patch('agentops.instrumentation.openai.attributes.response.WebSearchTool', MockWebSearchTool): + result = get_response_tool_web_search_attributes(web_search_tool, 0) - # Call the function - result = get_response_tools_attributes(tools) + # Verify attributes + assert isinstance(result, dict) + assert MessageAttributes.TOOL_CALL_NAME.format(i=0) in result + assert result[MessageAttributes.TOOL_CALL_NAME.format(i=0)] == 'web_search_preview' + assert MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0) in result + # Parameters should be serialized + assert 'search_context_size' in result[MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0)] + assert 'user_location' in result[MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0)] + + def test_get_response_tool_file_search_attributes(self): + """Test extraction of attributes from file search tool""" + # Create a mock file search tool + filters = MockFilters({ + 'key': 'value' + }) + + ranking_options = MockRankingOptions({ + 'ranker': 'default-2024-11-15', + 'score_threshold': 0.8 + }) + + file_search_tool = MockFileSearchTool({ + 'type': 'file_search', + 'vector_store_ids': ['store_123', 'store_456'], + 'filters': filters, + 'max_num_results': 10, + 'ranking_options': ranking_options + }) + + # Call the function directly + with patch('agentops.instrumentation.openai.attributes.response.FileSearchTool', MockFileSearchTool): + result = get_response_tool_file_search_attributes(file_search_tool, 0) - # Verify basic expected attributes + # Verify attributes assert isinstance(result, dict) - + assert MessageAttributes.TOOL_CALL_TYPE.format(i=0) in result + assert result[MessageAttributes.TOOL_CALL_TYPE.format(i=0)] == 'file_search' + assert MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0) in result + # Parameters should be serialized + assert 'vector_store_ids' in result[MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0)] + assert 'filters' in result[MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0)] + assert 'max_num_results' in result[MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0)] + assert 'ranking_options' in result[MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0)] + + def test_get_response_tool_computer_attributes(self): + """Test extraction of attributes from computer tool""" + # Create a mock computer tool + computer_tool = MockComputerTool({ + 'type': 'computer_use_preview', + 'display_height': 1080.0, + 'display_width': 1920.0, + 'environment': 'mac' + }) + + # Call the function directly + with patch('agentops.instrumentation.openai.attributes.response.ComputerTool', MockComputerTool): + result = get_response_tool_computer_attributes(computer_tool, 0) + + # Verify attributes + assert isinstance(result, dict) + assert MessageAttributes.TOOL_CALL_TYPE.format(i=0) in result + assert result[MessageAttributes.TOOL_CALL_TYPE.format(i=0)] == 'computer_use_preview' + assert MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0) in result + # Parameters should be serialized + assert 'display_height' in result[MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0)] + assert 'display_width' in result[MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0)] + assert 'environment' in result[MessageAttributes.TOOL_CALL_ARGUMENTS.format(i=0)] + def test_get_response_usage_attributes(self): """Test extraction of attributes from usage data""" - # Simplify test to verify function can be called without error + # Create a more comprehensive test for usage attributes # Patch the OutputTokensDetails class to make testing simpler with patch('agentops.instrumentation.openai.attributes.response.OutputTokensDetails', MockOutputTokensDetails): - # Create a minimal mock usage object with all necessary attributes - usage = MockResponseUsage({ - 'input_tokens': 50, - 'output_tokens': 20, - 'total_tokens': 70, - 'output_tokens_details': MockOutputTokensDetails({ - 'reasoning_tokens': 5 - }), - 'input_tokens_details': { - 'cached_tokens': 10 - }, - '__dict__': { + with patch('agentops.instrumentation.openai.attributes.response.InputTokensDetails', MagicMock): + # Test with all fields + usage = MockResponseUsage({ 'input_tokens': 50, 'output_tokens': 20, 'total_tokens': 70, @@ -371,34 +643,64 @@ def test_get_response_usage_attributes(self): }), 'input_tokens_details': { 'cached_tokens': 10 + }, + '__dict__': { + 'input_tokens': 50, + 'output_tokens': 20, + 'total_tokens': 70, + 'output_tokens_details': MockOutputTokensDetails({ + 'reasoning_tokens': 5 + }), + 'input_tokens_details': { + 'cached_tokens': 10 + } } - } - }) - - # Call the function - result = get_response_usage_attributes(usage) - - # Verify it returns a dictionary with at least these basic attributes - assert isinstance(result, dict) - assert SpanAttributes.LLM_USAGE_PROMPT_TOKENS in result - assert result[SpanAttributes.LLM_USAGE_PROMPT_TOKENS] == 50 - assert SpanAttributes.LLM_USAGE_COMPLETION_TOKENS in result - assert result[SpanAttributes.LLM_USAGE_COMPLETION_TOKENS] == 20 - assert SpanAttributes.LLM_USAGE_TOTAL_TOKENS in result - assert result[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 70 - - def test_get_response_reasoning_attributes(self): - """Test extraction of attributes from reasoning data""" - # Create mock reasoning object - reasoning = MockReasoning({ - 'effort': 'medium', - 'generate_summary': True - }) - - # Extract attributes - currently no attributes are mapped for reasoning - attributes = get_response_reasoning_attributes(reasoning) - - # The current implementation returns an empty dictionary because - # there are no defined attributes in RESPONSE_REASONING_ATTRIBUTES - assert isinstance(attributes, dict) - assert len(attributes) == 0 # Currently no attributes are mapped \ No newline at end of file + }) + + # Test without token details (edge cases) + usage_without_details = MockResponseUsage({ + 'input_tokens': 30, + 'output_tokens': 15, + 'total_tokens': 45, + 'output_tokens_details': None, + 'input_tokens_details': None, + '__dict__': { + 'input_tokens': 30, + 'output_tokens': 15, + 'total_tokens': 45, + 'output_tokens_details': None, + 'input_tokens_details': None + } + }) + + # Call the function for complete usage + result = get_response_usage_attributes(usage) + + # Verify it returns a dictionary with all attributes + assert isinstance(result, dict) + assert SpanAttributes.LLM_USAGE_PROMPT_TOKENS in result + assert result[SpanAttributes.LLM_USAGE_PROMPT_TOKENS] == 50 + assert SpanAttributes.LLM_USAGE_COMPLETION_TOKENS in result + assert result[SpanAttributes.LLM_USAGE_COMPLETION_TOKENS] == 20 + assert SpanAttributes.LLM_USAGE_TOTAL_TOKENS in result + assert result[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 70 + assert SpanAttributes.LLM_USAGE_REASONING_TOKENS in result + assert result[SpanAttributes.LLM_USAGE_REASONING_TOKENS] == 5 + assert SpanAttributes.LLM_USAGE_CACHE_READ_INPUT_TOKENS in result + assert result[SpanAttributes.LLM_USAGE_CACHE_READ_INPUT_TOKENS] == 10 + + # Call the function for usage without details + result_without_details = get_response_usage_attributes(usage_without_details) + + # Verify basic attributes are still present + assert isinstance(result_without_details, dict) + assert SpanAttributes.LLM_USAGE_PROMPT_TOKENS in result_without_details + assert result_without_details[SpanAttributes.LLM_USAGE_PROMPT_TOKENS] == 30 + assert SpanAttributes.LLM_USAGE_COMPLETION_TOKENS in result_without_details + assert result_without_details[SpanAttributes.LLM_USAGE_COMPLETION_TOKENS] == 15 + assert SpanAttributes.LLM_USAGE_TOTAL_TOKENS in result_without_details + assert result_without_details[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 45 + # Detailed attributes shouldn't be present + assert SpanAttributes.LLM_USAGE_REASONING_TOKENS not in result_without_details + assert SpanAttributes.LLM_USAGE_CACHE_READ_INPUT_TOKENS not in result_without_details + From 1b277faae76c8bb605c676c15477a64934217853 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Fri, 4 Apr 2025 16:59:40 -0700 Subject: [PATCH 22/26] Docstrings. --- agentops/instrumentation/common/attributes.py | 94 ++++++++++++++++--- 1 file changed, 83 insertions(+), 11 deletions(-) diff --git a/agentops/instrumentation/common/attributes.py b/agentops/instrumentation/common/attributes.py index 036a2b98f..4f3a16b2e 100644 --- a/agentops/instrumentation/common/attributes.py +++ b/agentops/instrumentation/common/attributes.py @@ -1,5 +1,8 @@ """Common attribute processing utilities shared across all instrumentors. +This utility ensures consistent attribute extraction and transformation across different +instrumentation use cases. + This module provides core utilities for extracting and formatting OpenTelemetry-compatible attributes from span data. These functions are provider-agnostic and used by all instrumentors in the AgentOps @@ -29,27 +32,96 @@ ) -class IndexedAttributeData(TypedDict, total=False): - """ - """ - i: int - j: Optional[int] = None - +# AttributeMap is a dictionary that maps target attribute keys to source attribute keys. +# It is used to extract and transform attributes from a span or trace data object +# into a standardized format following OpenTelemetry semantic conventions. +# +# Key-Value Format: +# - Key (str): The target attribute key in the standardized output format +# - Value (str): The source attribute key in the input data object +# +# Example Usage: +# -------------- +# Suppose you have a span data object: +# span_data = { +# "user_id": "12345", +# "operation_name": "process_order", +# "status_code": 200 +# } +# +# Create your mapping: +# attribute_mapping = { +# "user.id": "user_id", # Maps "user_id" to "user.id" +# "operation.name": "operation_name", # Maps "operation_name" to "operation.name" +# "http.status_code": "status_code" # Maps "status_code" to "http.status_code" +# } +# +# Extract the attributes: +# extracted_attributes = _extract_attributes_from_mapping(span_data, attribute_mapping) +# # Result: {"user.id": "12345", "operation.name": "process_order", "http.status_code": 200} +AttributeMap = Dict[str, str] # target_attribute_key: source_attribute + + +# IndexedAttributeMap differs from AttributeMap in that it allows for dynamic formatting of +# target attribute keys using indices `i` and optionally `j`. This is particularly useful +# when dealing with collections of similar attributes that should be uniquely identified +# in the output. +# +# Key-Value Format: +# - Key (IndexedAttribute): An object implementing the IndexedAttribute protocol with a format method +# - Value (str): The source attribute key in the input data object +# +# Example Usage: +# -------------- +# Suppose you are processing tool calls in an LLM response: +# +# Define an IndexedAttribute implementation: +# class MessageAttributes: +# TOOL_CALL_ID = IndexedAttribute("ai.message.tool.{i}.id") +# TOOL_CALL_TYPE = IndexedAttribute("ai.message.tool.{i}.type") +# +# Create your mapping: +# tool_attribute_mapping = { +# MessageAttributes.TOOL_CALL_ID: "id", # Maps tool's "id" to "ai.message.tool.{i}.id" +# MessageAttributes.TOOL_CALL_TYPE: "type" # Maps tool's "type" to "ai.message.tool.{i}.type" +# } +# +# Process tool calls: +# tools = [ +# {"id": "tool_1", "type": "search"}, +# {"id": "tool_2", "type": "calculator"} +# ] +# +# # For the first tool (i=0) +# first_tool_attributes = _extract_attributes_from_mapping_with_index( +# tools[0], tool_attribute_mapping, i=0 +# ) +# # Result: {"ai.message.tool.0.id": "tool_1", "ai.message.tool.0.type": "search"} @runtime_checkable class IndexedAttribute(Protocol): """ + Protocol for objects that define a method to format indexed attributes using + only the provided indices `i` and optionally `j`. This allows for dynamic + formatting of attribute keys based on the indices. """ + def format(self, *, i: int, j: Optional[int] = None) -> str: ... +IndexedAttributeMap = Dict[IndexedAttribute, str] # target_attribute_key: source_attribute + -# target_attribute_key: source_attribute -AttributeMap = Dict[str, str] +class IndexedAttributeData(TypedDict, total=False): + """ + Represents a dictionary structure for indexed attribute data. -# target_attribute_key: source_attribute -# target_attribute_key must be formattable with `i` and optionally `j` -IndexedAttributeMap = Dict[IndexedAttribute, str] + Attributes: + i (int): The primary index value. This field is required. + j (Optional[int]): An optional secondary index value. + """ + i: int + j: Optional[int] = None def _extract_attributes_from_mapping(span_data: Any, attribute_mapping: AttributeMap) -> AttributeMap: From c6cac21f8c063ea57ae38472acb28c09337d2d6c Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Fri, 4 Apr 2025 17:07:35 -0700 Subject: [PATCH 23/26] Better docstrings --- agentops/instrumentation/common/attributes.py | 54 ++++++++----------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/agentops/instrumentation/common/attributes.py b/agentops/instrumentation/common/attributes.py index 4f3a16b2e..d89f94bfb 100644 --- a/agentops/instrumentation/common/attributes.py +++ b/agentops/instrumentation/common/attributes.py @@ -32,7 +32,7 @@ ) -# AttributeMap is a dictionary that maps target attribute keys to source attribute keys. +# `AttributeMap` is a dictionary that maps target attribute keys to source attribute keys. # It is used to extract and transform attributes from a span or trace data object # into a standardized format following OpenTelemetry semantic conventions. # @@ -42,27 +42,25 @@ # # Example Usage: # -------------- -# Suppose you have a span data object: -# span_data = { -# "user_id": "12345", -# "operation_name": "process_order", -# "status_code": 200 -# } # # Create your mapping: -# attribute_mapping = { -# "user.id": "user_id", # Maps "user_id" to "user.id" -# "operation.name": "operation_name", # Maps "operation_name" to "operation.name" -# "http.status_code": "status_code" # Maps "status_code" to "http.status_code" +# attribute_mapping: AttributeMap = { +# CoreAttributes.TRACE_ID: "trace_id", +# CoreAttributes.SPAN_ID: "span_id" # } # # Extract the attributes: -# extracted_attributes = _extract_attributes_from_mapping(span_data, attribute_mapping) -# # Result: {"user.id": "12345", "operation.name": "process_order", "http.status_code": 200} +# span_data = { +# "trace_id": "12345", +# "span_id": "67890", +# } +# +# attributes = _extract_attributes_from_mapping(span_data, attribute_mapping) +# # >> {"trace.id": "12345", "span.id": "67890"} AttributeMap = Dict[str, str] # target_attribute_key: source_attribute -# IndexedAttributeMap differs from AttributeMap in that it allows for dynamic formatting of +# `IndexedAttributeMap` differs from `AttributeMap` in that it allows for dynamic formatting of # target attribute keys using indices `i` and optionally `j`. This is particularly useful # when dealing with collections of similar attributes that should be uniquely identified # in the output. @@ -73,30 +71,22 @@ # # Example Usage: # -------------- -# Suppose you are processing tool calls in an LLM response: -# -# Define an IndexedAttribute implementation: -# class MessageAttributes: -# TOOL_CALL_ID = IndexedAttribute("ai.message.tool.{i}.id") -# TOOL_CALL_TYPE = IndexedAttribute("ai.message.tool.{i}.type") # # Create your mapping: -# tool_attribute_mapping = { -# MessageAttributes.TOOL_CALL_ID: "id", # Maps tool's "id" to "ai.message.tool.{i}.id" -# MessageAttributes.TOOL_CALL_TYPE: "type" # Maps tool's "type" to "ai.message.tool.{i}.type" +# attribute_mapping: IndexedAttributeMap = { +# MessageAttributes.TOOL_CALL_ID: "id", +# MessageAttributes.TOOL_CALL_TYPE: "type" # } # # Process tool calls: -# tools = [ -# {"id": "tool_1", "type": "search"}, -# {"id": "tool_2", "type": "calculator"} -# ] +# span_data = { +# "id": "tool_1", +# "type": "search", +# } # -# # For the first tool (i=0) -# first_tool_attributes = _extract_attributes_from_mapping_with_index( -# tools[0], tool_attribute_mapping, i=0 -# ) -# # Result: {"ai.message.tool.0.id": "tool_1", "ai.message.tool.0.type": "search"} +# attributes = _extract_attributes_from_mapping_with_index( +# span_data, attribute_mapping, i=0) +# # >> {"gen_ai.request.tools.0.id": "tool_1", "gen_ai.request.tools.0.type": "search"} @runtime_checkable class IndexedAttribute(Protocol): From 88d1991e485f1c1c6d8c263c213c07cb3148cf94 Mon Sep 17 00:00:00 2001 From: Pratyush Shukla Date: Tue, 8 Apr 2025 02:20:54 +0530 Subject: [PATCH 24/26] add status of the web search to the correct attribute --- agentops/instrumentation/openai/attributes/response.py | 2 +- agentops/semconv/message.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/agentops/instrumentation/openai/attributes/response.py b/agentops/instrumentation/openai/attributes/response.py index 482754498..0b5c03e4f 100644 --- a/agentops/instrumentation/openai/attributes/response.py +++ b/agentops/instrumentation/openai/attributes/response.py @@ -244,7 +244,7 @@ # ) MessageAttributes.COMPLETION_TOOL_CALL_ID: "id", MessageAttributes.COMPLETION_TOOL_CALL_TYPE: "type", - MessageAttributes.COMPLETION_TOOL_CALL_NAME: "type", + MessageAttributes.COMPLETION_TOOL_CALL_STATUS: "status", } diff --git a/agentops/semconv/message.py b/agentops/semconv/message.py index 6e750c128..227c372a6 100644 --- a/agentops/semconv/message.py +++ b/agentops/semconv/message.py @@ -27,4 +27,5 @@ class MessageAttributes: COMPLETION_TOOL_CALL_TYPE = "gen_ai.completion.{i}.tool_calls.{j}.type" # Type of tool call {j} in completion {i} COMPLETION_TOOL_CALL_NAME = "gen_ai.completion.{i}.tool_calls.{j}.name" # Name of the tool called in tool call {j} in completion {i} COMPLETION_TOOL_CALL_DESCRIPTION = "gen_ai.completion.{i}.tool_calls.{j}.description" # Description of the tool call {j} in completion {i} + COMPLETION_TOOL_CALL_STATUS = "gen_ai.completion.{i}.tool_calls.{j}.status" # Status of the tool call {j} in completion {i} COMPLETION_TOOL_CALL_ARGUMENTS = "gen_ai.completion.{i}.tool_calls.{j}.arguments" # Arguments for tool call {j} in completion {i} \ No newline at end of file From e0ba477b7a0fa5ab18035cf3644b431234a4936e Mon Sep 17 00:00:00 2001 From: Pratyush Shukla Date: Tue, 8 Apr 2025 02:37:48 +0530 Subject: [PATCH 25/26] add annotations list to the span attributes --- .../openai/attributes/response.py | 27 ++++++++++++++++++- agentops/semconv/message.py | 10 ++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/agentops/instrumentation/openai/attributes/response.py b/agentops/instrumentation/openai/attributes/response.py index 0b5c03e4f..3716f37c8 100644 --- a/agentops/instrumentation/openai/attributes/response.py +++ b/agentops/instrumentation/openai/attributes/response.py @@ -247,6 +247,21 @@ MessageAttributes.COMPLETION_TOOL_CALL_STATUS: "status", } +RESPONSE_OUTPUT_TOOL_WEB_SEARCH_URL_ANNOTATIONS: IndexedAttributeMap = { + # AnnotationURLCitation( + # end_index=747, + # start_index=553, + # title="You can now play a real-time AI-rendered Quake II in your browser", + # type='url_citation', + # url='https://www.tomshardware.com/video-games/you-can-now-play-a-real-time-ai-rendered-quake-ii-in-your-browser-microsofts-whamm-offers-generative-ai-for-games?utm_source=openai' + # ) + MessageAttributes.COMPLETION_ANNOTATION_END_INDEX: "end_index", + MessageAttributes.COMPLETION_ANNOTATION_START_INDEX: "start_index", + MessageAttributes.COMPLETION_ANNOTATION_TITLE: "title", + MessageAttributes.COMPLETION_ANNOTATION_TYPE: "type", + MessageAttributes.COMPLETION_ANNOTATION_URL: "url", +} + RESPONSE_OUTPUT_TOOL_COMPUTER_ATTRIBUTES: IndexedAttributeMap = { # ResponseComputerToolCall( @@ -432,9 +447,19 @@ def get_response_output_attributes(output: List['ResponseOutputTypes']) -> Attri def get_response_output_text_attributes(output_text: 'ResponseOutputText', index: int) -> AttributeMap: """Handles interpretation of an openai ResponseOutputText object.""" # This function is a helper to handle the ResponseOutputText type specifically - return _extract_attributes_from_mapping_with_index( + attributes = _extract_attributes_from_mapping_with_index( output_text, RESPONSE_OUTPUT_TEXT_ATTRIBUTES, index) + if hasattr(output_text, "annotations"): + for j, output_text_annotation in enumerate(output_text.annotations): + attributes.update( + _extract_attributes_from_mapping_with_index( + output_text_annotation, RESPONSE_OUTPUT_TOOL_WEB_SEARCH_URL_ANNOTATIONS, i=index, j=j + ) + ) + + return attributes + def get_response_output_message_attributes(index: int, message: 'ResponseOutputMessage') -> AttributeMap: """Handles interpretation of an openai ResponseOutputMessage object.""" diff --git a/agentops/semconv/message.py b/agentops/semconv/message.py index 227c372a6..9cb775163 100644 --- a/agentops/semconv/message.py +++ b/agentops/semconv/message.py @@ -25,7 +25,15 @@ class MessageAttributes: # Indexed tool calls (with {i}/{j} for nested interpolation) COMPLETION_TOOL_CALL_ID = "gen_ai.completion.{i}.tool_calls.{j}.id" # ID of tool call {j} in completion {i} COMPLETION_TOOL_CALL_TYPE = "gen_ai.completion.{i}.tool_calls.{j}.type" # Type of tool call {j} in completion {i} + COMPLETION_TOOL_CALL_STATUS = "gen_ai.completion.{i}.tool_calls.{j}.status" # Status of tool call {j} in completion {i} COMPLETION_TOOL_CALL_NAME = "gen_ai.completion.{i}.tool_calls.{j}.name" # Name of the tool called in tool call {j} in completion {i} COMPLETION_TOOL_CALL_DESCRIPTION = "gen_ai.completion.{i}.tool_calls.{j}.description" # Description of the tool call {j} in completion {i} COMPLETION_TOOL_CALL_STATUS = "gen_ai.completion.{i}.tool_calls.{j}.status" # Status of the tool call {j} in completion {i} - COMPLETION_TOOL_CALL_ARGUMENTS = "gen_ai.completion.{i}.tool_calls.{j}.arguments" # Arguments for tool call {j} in completion {i} \ No newline at end of file + COMPLETION_TOOL_CALL_ARGUMENTS = "gen_ai.completion.{i}.tool_calls.{j}.arguments" # Arguments for tool call {j} in completion {i} + + # Indexed annotations of the internal tools (with {i}/{j} for nested interpolation) + COMPLETION_ANNOTATION_START_INDEX = "gen_ai.completion.{i}.annotations.{j}.start_index" # Start index of the URL annotation {j} in completion {i} + COMPLETION_ANNOTATION_END_INDEX = "gen_ai.completion.{i}.annotations.{j}.end_index" # End index of the URL annotation {j} in completion {i} + COMPLETION_ANNOTATION_TITLE = "gen_ai.completion.{i}.annotations.{j}.title" # Title of the URL annotation {j} in completion {i} + COMPLETION_ANNOTATION_TYPE = "gen_ai.completion.{i}.annotations.{j}.type" # Type of the URL annotation {j} in completion {i} + COMPLETION_ANNOTATION_URL = "gen_ai.completion.{i}.annotations.{j}.url" # URL link of the URL annotation {j} in completion {i} \ No newline at end of file From 39a33b2844585b071efcdd5f11016f2879f40bed Mon Sep 17 00:00:00 2001 From: Pratyush Shukla Date: Tue, 8 Apr 2025 03:30:02 +0530 Subject: [PATCH 26/26] add tests for annotations --- .../openai_core/test_response_attributes.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/unit/instrumentation/openai_core/test_response_attributes.py b/tests/unit/instrumentation/openai_core/test_response_attributes.py index 5009f08a9..d98903e05 100644 --- a/tests/unit/instrumentation/openai_core/test_response_attributes.py +++ b/tests/unit/instrumentation/openai_core/test_response_attributes.py @@ -391,8 +391,16 @@ def test_get_response_output_text_attributes(self): """Test extraction of attributes from output text""" # Create a mock text content text = MockOutputText({ - 'annotations': [], - 'text': 'The capital of France is Paris.', + 'annotations': [ + { + "end_index": 636, + "start_index": 538, + "title": "5 AI Agent Frameworks Compared", + "type": "url_citation", + "url": "https://www.kdnuggets.com/5-ai-agent-frameworks-compared" + } + ], + 'text': 'CrewAI is the top AI agent library.', 'type': 'output_text' }) @@ -403,7 +411,12 @@ def test_get_response_output_text_attributes(self): with patch('agentops.instrumentation.openai.attributes.response._extract_attributes_from_mapping_with_index') as mock_extract: # Set up the mock to return expected attributes expected_attributes = { - MessageAttributes.COMPLETION_CONTENT.format(i=0): 'The capital of France is Paris.', + MessageAttributes.COMPLETION_ANNOTATION_END_INDEX.format(i=0,j=0): 636, + MessageAttributes.COMPLETION_ANNOTATION_START_INDEX.format(i=0,j=1): 538, + MessageAttributes.COMPLETION_ANNOTATION_TITLE.format(i=0,j=2): "5 AI Agent Frameworks Compared", + MessageAttributes.COMPLETION_ANNOTATION_TYPE.format(i=0,j=3): "url_citation", + MessageAttributes.COMPLETION_ANNOTATION_URL.format(i=0,j=5): "https://www.kdnuggets.com/5-ai-agent-frameworks-compared", + MessageAttributes.COMPLETION_CONTENT.format(i=0): 'CrewAI is the top AI agent library.', MessageAttributes.COMPLETION_TYPE.format(i=0): 'output_text' } mock_extract.return_value = expected_attributes @@ -426,7 +439,15 @@ def test_get_response_output_attributes(self): MockOutputText({ 'text': 'This is a test message', 'type': 'output_text', - 'annotations': [] + 'annotations': [ + { + "end_index": 636, + "start_index": 538, + "title": "Test title", + "type": "url_citation", + "url": "www.test.com", + } + ] }) ], 'role': 'assistant',