feat(agents): improve instrumentation of input messages

sentry_sdk/integrations/langchain.py

@@ -10,7 +10,7 @@
 from sentry_sdk.scope import should_send_default_pii
 from sentry_sdk.tracing import Span
 from sentry_sdk.tracing_utils import _get_value
-from sentry_sdk.utils import logger, capture_internal_exceptions
+from sentry_sdk.utils import logger, capture_internal_exceptions, safe_serialize
 
 from typing import TYPE_CHECKING
 
@@ -51,7 +51,6 @@
     "presence_penalty": SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY,
     "temperature": SPANDATA.GEN_AI_REQUEST_TEMPERATURE,
     "tool_calls": SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
-    "tools": SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
     "top_k": SPANDATA.GEN_AI_REQUEST_TOP_K,
     "top_p": SPANDATA.GEN_AI_REQUEST_TOP_P,
 }
@@ -203,8 +202,10 @@ def on_llm_start(
                 if key in all_params and all_params[key] is not None:
                     set_data_normalized(span, attribute, all_params[key], unpack=False)
 
+            _set_tools_on_span(span, all_params.get("tools"))
+
             if should_send_default_pii() and self.include_prompts:
-                set_data_normalized(span, SPANDATA.GEN_AI_REQUEST_MESSAGES, prompts)
+                span.set_data(SPANDATA.GEN_AI_REQUEST_MESSAGES, safe_serialize(prompts))
 
     def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs):
         # type: (SentryLangchainCallback, Dict[str, Any], List[List[BaseMessage]], UUID, Any) -> Any
@@ -246,14 +247,18 @@ def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs):
                 if key in all_params and all_params[key] is not None:
                     set_data_normalized(span, attribute, all_params[key], unpack=False)
 
+            _set_tools_on_span(span, all_params.get("tools"))
+
             if should_send_default_pii() and self.include_prompts:
-                set_data_normalized(
-                    span,
+                normalized_messages = []
+                for list_ in messages:
+                    for message in list_:
+                        normalized_messages.append(
+                            self._normalize_langchain_message(message)
+                        )
+                span.set_data(
                     SPANDATA.GEN_AI_REQUEST_MESSAGES,
-                    [
-                        [self._normalize_langchain_message(x) for x in list_]
-                        for list_ in messages
-                    ],
+                    safe_serialize(normalized_messages),
                 )
 
     def on_chat_model_end(self, response, *, run_id, **kwargs):
@@ -473,13 +478,11 @@ def _get_token_usage(obj):
             if usage is not None:
                 return usage
 
-    # check for usage in the object itself
     for name in possible_names:
         usage = _get_value(obj, name)
         if usage is not None:
             return usage
 
-    # no usage found anywhere
     return None
 
 
@@ -531,6 +534,85 @@ def _get_request_data(obj, args, kwargs):
     return (agent_name, tools)
 
 
+def _simplify_langchain_tools(tools):
+    # type: (Any) -> Optional[List[Any]]
+    """Parse and simplify tools into a cleaner format."""
+    if not tools:
+        return None
+
+    if not isinstance(tools, (list, tuple)):
+        return None
+
+    simplified_tools = []
+    for tool in tools:
+        try:
+            if isinstance(tool, dict):
+
+                if "function" in tool and isinstance(tool["function"], dict):
+                    func = tool["function"]
+                    simplified_tool = {
+                        "name": func.get("name"),
+                        "description": func.get("description"),
+                    }
+                    if simplified_tool["name"]:
+                        simplified_tools.append(simplified_tool)
+                elif "name" in tool:
+                    simplified_tool = {
+                        "name": tool.get("name"),
+                        "description": tool.get("description"),
+                    }
+                    simplified_tools.append(simplified_tool)
+                else:
+                    name = (
+                        tool.get("name")
+                        or tool.get("tool_name")
+                        or tool.get("function_name")
+                    )
+                    if name:
+                        simplified_tools.append(
+                            {
+                                "name": name,
+                                "description": tool.get("description")
+                                or tool.get("desc"),
+                            }
+                        )
+            elif hasattr(tool, "name"):
+                simplified_tool = {
+                    "name": getattr(tool, "name", None),
+                    "description": getattr(tool, "description", None)
+                    or getattr(tool, "desc", None),
+                }
+                if simplified_tool["name"]:
+                    simplified_tools.append(simplified_tool)
+            elif hasattr(tool, "__name__"):
+                simplified_tools.append(
+                    {
+                        "name": tool.__name__,
+                        "description": getattr(tool, "__doc__", None),
+                    }
+                )
+            else:
+                tool_str = str(tool)
+                if tool_str and tool_str != "":
+                    simplified_tools.append({"name": tool_str, "description": None})
+        except Exception:
+            continue
+
+    return simplified_tools if simplified_tools else None
+
+
+def _set_tools_on_span(span, tools):
+    # type: (Span, Any) -> None
+    """Set available tools data on a span if tools are provided."""
+    if tools is not None:
+        simplified_tools = _simplify_langchain_tools(tools)
+        if simplified_tools:
+            span.set_data(
+                SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
+                safe_serialize(simplified_tools),
+            )
+
+
 def _wrap_configure(f):
     # type: (Callable[..., Any]) -> Callable[..., Any]
 
@@ -601,7 +683,7 @@ def new_configure(
                 ]
             elif isinstance(local_callbacks, BaseCallbackHandler):
                 local_callbacks = [local_callbacks, sentry_handler]
-            else:  # local_callbacks is a list
+            else:
                 local_callbacks = [*local_callbacks, sentry_handler]
 
         return f(
@@ -638,10 +720,7 @@ def new_invoke(self, *args, **kwargs):
             span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent")
             span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False)
 
-            if tools:
-                set_data_normalized(
-                    span, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, tools, unpack=False
-                )
+            _set_tools_on_span(span, tools)
 
             # Run the agent
             result = f(self, *args, **kwargs)
@@ -653,11 +732,7 @@ def new_invoke(self, *args, **kwargs):
                 and integration.include_prompts
             ):
                 set_data_normalized(
-                    span,
-                    SPANDATA.GEN_AI_REQUEST_MESSAGES,
-                    [
-                        input,
-                    ],
+                    span, SPANDATA.GEN_AI_REQUEST_MESSAGES, safe_serialize([input])
                 )
 
             output = result.get("output")
@@ -698,24 +773,15 @@ def new_stream(self, *args, **kwargs):
         span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent")
         span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, True)
 
-        if tools:
-            set_data_normalized(
-                span, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, tools, unpack=False
-            )
+        _set_tools_on_span(span, tools)
 
         input = args[0].get("input") if len(args) >= 1 else None
         if (
             input is not None
             and should_send_default_pii()
             and integration.include_prompts
         ):
-            set_data_normalized(
-                span,
-                SPANDATA.GEN_AI_REQUEST_MESSAGES,
-                [
-                    input,
-                ],
-            )
+            span.set_data(SPANDATA.GEN_AI_REQUEST_MESSAGES, safe_serialize([input]))
 
         # Run the agent
         result = f(self, *args, **kwargs)

tests/integrations/langchain/test_langchain.py

@@ -589,3 +589,76 @@ def test_langchain_callback_list_existing_callback(sentry_init):
 
         [handler] = passed_callbacks
         assert handler is sentry_callback
+
+
+def test_tools_integration_in_spans(sentry_init, capture_events):
+    """Test that tools are properly set on spans in actual LangChain integration."""
+    global llm_type
+    llm_type = "openai-chat"
+
+    sentry_init(
+        integrations=[LangchainIntegration(include_prompts=False)],
+        traces_sample_rate=1.0,
+    )
+    events = capture_events()
+
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", "You are a helpful assistant"),
+            ("user", "{input}"),
+            MessagesPlaceholder(variable_name="agent_scratchpad"),
+        ]
+    )
+
+    global stream_result_mock
+    stream_result_mock = Mock(
+        side_effect=[
+            [
+                ChatGenerationChunk(
+                    type="ChatGenerationChunk",
+                    message=AIMessageChunk(content="Simple response"),
+                ),
+            ]
+        ]
+    )
+
+    llm = MockOpenAI(
+        model_name="gpt-3.5-turbo",
+        temperature=0,
+        openai_api_key="badkey",
+    )
+    agent = create_openai_tools_agent(llm, [get_word_length], prompt)
+    agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True)
+
+    with start_transaction():
+        list(agent_executor.stream({"input": "Hello"}))
+
+    # Check that events were captured and contain tools data
+    if events:
+        tx = events[0]
+        spans = tx.get("spans", [])
+
+        # Look for spans that should have tools data
+        tools_found = False
+        for span in spans:
+            span_data = span.get("data", {})
+            if SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS in span_data:
+                tools_found = True
+                tools_data = span_data[SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS]
+                # Verify tools are in the expected format
+                assert isinstance(tools_data, (str, list))  # Could be serialized
+                if isinstance(tools_data, str):
+                    # If serialized as string, should contain tool name
+                    assert "get_word_length" in tools_data
+                else:
+                    # If still a list, verify structure
+                    assert len(tools_data) >= 1
+                    names = [
+                        tool.get("name")
+                        for tool in tools_data
+                        if isinstance(tool, dict)
+                    ]
+                    assert "get_word_length" in names
+
+        # Ensure we found at least one span with tools data
+        assert tools_found, "No spans found with tools data"