fix: exception serialization

Author: skoob13

posthog/ai/langchain/callbacks.py

@@ -163,7 +163,7 @@ def on_chain_error(
         **kwargs: Any,
     ):
         self._log_debug_event("on_chain_error", run_id, parent_run_id, error=error)
-        self._pop_run_and_capture_trace_or_span(run_id, parent_run_id, str(error))
+        self._pop_run_and_capture_trace_or_span(run_id, parent_run_id, error)
 
     def on_chat_model_start(
         self,
@@ -262,7 +262,7 @@ def on_tool_error(
         **kwargs: Any,
     ) -> Any:
         self._log_debug_event("on_tool_error", run_id, parent_run_id, error=error)
-        self._pop_run_and_capture_trace_or_span(run_id, parent_run_id, str(error))
+        self._pop_run_and_capture_trace_or_span(run_id, parent_run_id, error)
 
     def on_retriever_start(
         self,
@@ -299,7 +299,7 @@ def on_retriever_error(
     ) -> Any:
         """Run when Retriever errors."""
         self._log_debug_event("on_retriever_error", run_id, parent_run_id, error=error)
-        self._pop_run_and_capture_trace_or_span(run_id, parent_run_id, str(error))
+        self._pop_run_and_capture_trace_or_span(run_id, parent_run_id, error)
 
     def on_agent_action(
         self,
@@ -448,8 +448,13 @@ def _capture_trace_or_span(
             event_properties["$ai_trace_name"] = run.name
         if self._properties:
             event_properties.update(self._properties)
-        if outputs is not None:
+
+        if isinstance(outputs, BaseException):
+            event_properties["$ai_error"] = _stringify_exception(outputs)
+            event_properties["$ai_is_error"] = True
+        elif outputs is not None:
             event_properties["$ai_output_state"] = with_privacy_mode(self._client, self._privacy_mode, outputs)
+
         if self._distinct_id is None:
             event_properties["$process_person_profile"] = False
 
@@ -498,7 +503,7 @@ def _capture_generation(
 
         if isinstance(output, BaseException):
             event_properties["$ai_http_status"] = _get_http_status(output)
-            event_properties["$ai_error"] = str(output)
+            event_properties["$ai_error"] = _stringify_exception(output)
             event_properties["$ai_is_error"] = True
         else:
             # Add usage
@@ -687,3 +692,10 @@ def _get_langchain_run_name(serialized: Optional[Dict[str, Any]], **kwargs: Any)
     except (KeyError, TypeError):
         pass
     return None
+
+
+def _stringify_exception(exception: BaseException) -> str:
+    description = str(exception)
+    if description:
+        return f"{exception.__class__.__name__}: {description}"
+    return exception.__class__.__name__

posthog/test/ai/langchain/test_callbacks.py

@@ -1311,3 +1311,35 @@ def span_3(_):
     span2, span1, trace = [call[1]["properties"] for call in mock_client.capture.call_args_list]
     assert span2["$ai_parent_id"] == span1["$ai_span_id"]
     assert span1["$ai_parent_id"] == trace["$ai_trace_id"]
+
+
+def test_captures_error_with_details_in_span(mock_client):
+    def span(_):
+        raise ValueError("test")
+
+    callbacks = [CallbackHandler(mock_client)]
+    chain = RunnableLambda(span) | RunnableLambda(lambda _: "foo")
+    try:
+        chain.invoke({}, config={"callbacks": callbacks})
+    except ValueError:
+        pass
+
+    assert mock_client.capture.call_count == 2
+    assert mock_client.capture.call_args_list[1][1]["properties"]["$ai_error"] == "ValueError: test"
+    assert mock_client.capture.call_args_list[1][1]["properties"]["$ai_is_error"]
+
+
+def test_captures_error_without_details_in_span(mock_client):
+    def span(_):
+        raise ValueError
+
+    callbacks = [CallbackHandler(mock_client)]
+    chain = RunnableLambda(span) | RunnableLambda(lambda _: "foo")
+    try:
+        chain.invoke({}, config={"callbacks": callbacks})
+    except ValueError:
+        pass
+
+    assert mock_client.capture.call_count == 2
+    assert mock_client.capture.call_args_list[1][1]["properties"]["$ai_error"] == "ValueError"
+    assert mock_client.capture.call_args_list[1][1]["properties"]["$ai_is_error"]