feat(llmobs): [MLOB-2662] [MLOB-3100] add agent manifest (#13311)

This PR adds instrumentation for the agent manifest for our LLM Obs
agentic integrations (minus LangGraph which will be covered in
@sabrenner's [PR](#13904)). I
have been using [this
doc](https://docs.google.com/document/d/1k-vHYQZfCdckvoH29Mi_TzXuNGBLf3KpypZDpm4vFio/edit?pli=1&tab=t.0)
to track what information is available in the Agent Manifest for each
integration, but a summary is below:

## Agent Manifest Schema

### Crew AI
- Framework
- Name
- Goal
- Backstory
- Model
- Model Settings
- Tools
- Handoffs
- Code Execution Permissions
- Max Iterations

<img width="1744" height="918" alt="image"
src="https://github.com/user-attachments/assets/d85ac5a7-bbf9-4098-a4a2-93116be6d007"
/>


### Open AI Agents
- Framework
- Name
- Instructions
- Handoff Description
- Model
- Model Settings
- Tools
- Handoffs
- Guardrails

<img width="1650" height="938" alt="image"
src="https://github.com/user-attachments/assets/88f5034c-1846-441d-9495-49ddf7b075ad"
/>


### PydanticAI
- Framework
- Name
- Instructions
- System Prompts
- Model
- Model Settings
- Tools
- Dependencies

<img width="548" height="830" alt="image"
src="https://github.com/user-attachments/assets/4cbf3466-9de0-475f-808a-43c86e3c45cf"
/>




## Next Steps
- Make the current Agent Manifest UI work for the new fields that have
been added

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

---------

Co-authored-by: Kyle Verhoog <[email protected]>

Author: ncybul

ddtrace/contrib/internal/openai_agents/patch.py

@@ -5,6 +5,10 @@
 
 from ddtrace import config
 from ddtrace.contrib.internal.openai_agents.processor import LLMObsTraceProcessor
+from ddtrace.contrib.trace_utils import unwrap
+from ddtrace.contrib.trace_utils import with_traced_module_async
+from ddtrace.contrib.trace_utils import wrap
+from ddtrace.internal.utils.version import parse_version
 from ddtrace.llmobs._integrations.openai_agents import OpenAIAgentsIntegration
 from ddtrace.trace import Pin
 
@@ -22,6 +26,29 @@ def _supported_versions() -> Dict[str, str]:
     return {"agents": ">=0.0.2"}
 
 
+OPENAI_AGENTS_VERSION = parse_version(get_version())
+
+
+@with_traced_module_async
+async def patched_run_single_turn(agents, pin, func, instance, args, kwargs):
+    return await _patched_run_single_turn(agents, pin, func, instance, args, kwargs, agent_index=0)
+
+
+@with_traced_module_async
+async def patched_run_single_turn_streamed(agents, pin, func, instance, args, kwargs):
+    return await _patched_run_single_turn(agents, pin, func, instance, args, kwargs, agent_index=1)
+
+
+async def _patched_run_single_turn(agents, pin, func, instance, args, kwargs, agent_index=0):
+    current_span = pin.tracer.current_span()
+    result = await func(*args, **kwargs)
+
+    integration = agents._datadog_integration
+    integration.tag_agent_manifest(current_span, args, kwargs, agent_index)
+
+    return result
+
+
 def patch():
     """
     Patch the instrumented methods
@@ -33,7 +60,16 @@ def patch():
 
     Pin().onto(agents)
 
-    add_trace_processor(LLMObsTraceProcessor(OpenAIAgentsIntegration(integration_config=config.openai_agents)))
+    integration = OpenAIAgentsIntegration(integration_config=config.openai_agents)
+    add_trace_processor(LLMObsTraceProcessor(integration))
+    agents._datadog_integration = integration
+
+    if OPENAI_AGENTS_VERSION >= (0, 0, 19):
+        wrap(agents.run.AgentRunner, "_run_single_turn", patched_run_single_turn(agents))
+        wrap(agents.run.AgentRunner, "_run_single_turn_streamed", patched_run_single_turn_streamed(agents))
+    else:
+        wrap(agents.run.Runner, "_run_single_turn", patched_run_single_turn(agents))
+        wrap(agents.run.Runner, "_run_single_turn_streamed", patched_run_single_turn_streamed(agents))
 
 
 def unpatch():
@@ -44,3 +80,10 @@ def unpatch():
         return
 
     agents._datadog_patch = False
+
+    if OPENAI_AGENTS_VERSION >= (0, 0, 19):
+        unwrap(agents.run.AgentRunner, "_run_single_turn")
+        unwrap(agents.run.AgentRunner, "_run_single_turn_streamed")
+    else:
+        unwrap(agents.run.Runner, "_run_single_turn")
+        unwrap(agents.run.Runner, "_run_single_turn_streamed")

ddtrace/llmobs/_constants.py

@@ -9,6 +9,7 @@
 PROPAGATED_LLMOBS_TRACE_ID_KEY = "_dd.p.llmobs_trace_id"
 LLMOBS_TRACE_ID = "_ml_obs.llmobs_trace_id"
 TAGS = "_ml_obs.tags"
+AGENT_MANIFEST = "_ml_obs.meta.agent_manifest"
 
 MODEL_NAME = "_ml_obs.meta.model_name"
 MODEL_PROVIDER = "_ml_obs.meta.model_provider"

ddtrace/llmobs/_integrations/crewai.py

@@ -8,6 +8,7 @@
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.utils import get_argument_value
 from ddtrace.internal.utils.formats import format_trace_id
+from ddtrace.llmobs._constants import AGENT_MANIFEST
 from ddtrace.llmobs._constants import INPUT_VALUE
 from ddtrace.llmobs._constants import METADATA
 from ddtrace.llmobs._constants import NAME
@@ -151,9 +152,8 @@ def _llmobs_set_tags_agent(self, span, args, kwargs, response):
         Agent spans are 1:1 with its parent (task/tool) span, so we link them directly here, even on the parent itself.
         """
         agent_instance = kwargs.get("instance")
+        self._tag_agent_manifest(span, agent_instance)
         agent_role = getattr(agent_instance, "role", "")
-        agent_goal = getattr(agent_instance, "goal", "")
-        agent_backstory = getattr(agent_instance, "backstory", "")
         task_description = getattr(kwargs.get("task"), "description", "")
         context = get_argument_value(args, kwargs, 1, "context", optional=True) or ""
 
@@ -174,7 +174,6 @@ def _llmobs_set_tags_agent(self, span, args, kwargs, response):
         span._set_ctx_items(
             {
                 NAME: agent_role if agent_role else "CrewAI Agent",
-                METADATA: {"description": agent_goal, "backstory": agent_backstory},
                 INPUT_VALUE: {"context": context, "input": task_description},
                 SPAN_LINKS: curr_span_links + [span_link],
             }
@@ -198,6 +197,56 @@ def _llmobs_set_tags_tool(self, span, args, kwargs, response):
             return
         span._set_ctx_item(OUTPUT_VALUE, response)
 
+    def _tag_agent_manifest(self, span, agent):
+        if not agent:
+            return
+
+        manifest = {}
+        manifest["framework"] = "CrewAI"
+        manifest["name"] = agent.role if hasattr(agent, "role") and agent.role else "CrewAI Agent"
+        if hasattr(agent, "goal"):
+            manifest["goal"] = agent.goal
+        if hasattr(agent, "backstory"):
+            manifest["backstory"] = agent.backstory
+        if hasattr(agent, "llm"):
+            if hasattr(agent.llm, "model"):
+                manifest["model"] = agent.llm.model
+            model_settings = {}
+            if hasattr(agent.llm, "max_tokens"):
+                model_settings["max_tokens"] = agent.llm.max_tokens
+            if hasattr(agent.llm, "temperature"):
+                model_settings["temperature"] = agent.llm.temperature
+            if model_settings:
+                manifest["model_settings"] = model_settings
+        if hasattr(agent, "allow_delegation"):
+            manifest["handoffs"] = {"allow_delegation": agent.allow_delegation}
+        code_execution_permissions = {}
+        if hasattr(agent, "allow_code_execution"):
+            manifest["code_execution_permissions"] = {"allow_code_execution": agent.allow_code_execution}
+        if hasattr(agent, "code_execution_mode"):
+            manifest["code_execution_permissions"] = {"code_execution_mode": agent.code_execution_mode}
+        if code_execution_permissions:
+            manifest["code_execution_permissions"] = code_execution_permissions
+        if hasattr(agent, "max_iter"):
+            manifest["max_iterations"] = agent.max_iter
+        if hasattr(agent, "tools"):
+            manifest["tools"] = self._get_agent_tools(agent.tools)
+
+        span._set_ctx_item(AGENT_MANIFEST, manifest)
+
+    def _get_agent_tools(self, tools):
+        if not tools or not isinstance(tools, list):
+            return []
+        formatted_tools = []
+        for tool in tools:
+            tool_dict = {}
+            if hasattr(tool, "name"):
+                tool_dict["name"] = tool.name
+            if hasattr(tool, "description"):
+                tool_dict["description"] = tool.description
+            formatted_tools.append(tool_dict)
+        return formatted_tools
+
     def _llmobs_set_span_link_on_task(self, span, args, kwargs):
         """Set span links for the next queued task in a CrewAI workflow.
         This happens between task executions, (the current span is the crew span and the task span hasn't started yet)

ddtrace/llmobs/_integrations/openai_agents.py

@@ -8,7 +8,9 @@
 
 from ddtrace.internal import core
 from ddtrace.internal.logger import get_logger
+from ddtrace.internal.utils import get_argument_value
 from ddtrace.internal.utils.formats import format_trace_id
+from ddtrace.llmobs._constants import AGENT_MANIFEST
 from ddtrace.llmobs._constants import DISPATCH_ON_LLM_TOOL_CHOICE
 from ddtrace.llmobs._constants import DISPATCH_ON_TOOL_CALL
 from ddtrace.llmobs._constants import DISPATCH_ON_TOOL_CALL_OUTPUT_USED
@@ -31,6 +33,7 @@
 from ddtrace.llmobs._integrations.utils import OaiTraceAdapter
 from ddtrace.llmobs._utils import _get_nearest_llmobs_ancestor
 from ddtrace.llmobs._utils import _get_span_name
+from ddtrace.llmobs._utils import load_data_value
 from ddtrace.trace import Pin
 from ddtrace.trace import Span
 
@@ -296,3 +299,145 @@ def _llmobs_get_trace_info(
     def clear_state(self) -> None:
         self.oai_to_llmobs_span.clear()
         self.llmobs_traces.clear()
+
+    def tag_agent_manifest(self, span: Span, args: List[Any], kwargs: Dict[str, Any], agent_index: int) -> None:
+        agent = get_argument_value(args, kwargs, agent_index, "agent", True)
+        if not agent or not self.llmobs_enabled:
+            return
+
+        manifest = {}
+        manifest["framework"] = "OpenAI"
+        if hasattr(agent, "name"):
+            manifest["name"] = agent.name
+        if hasattr(agent, "instructions"):
+            manifest["instructions"] = agent.instructions
+        if hasattr(agent, "handoff_description"):
+            manifest["handoff_description"] = agent.handoff_description
+        if hasattr(agent, "model"):
+            model = agent.model
+            manifest["model"] = model if isinstance(model, str) else getattr(model, "model", "")
+
+        model_settings = self._extract_model_settings_from_agent(agent)
+        if model_settings:
+            manifest["model_settings"] = model_settings
+
+        tools = self._extract_tools_from_agent(agent)
+        if tools:
+            manifest["tools"] = tools
+
+        handoffs = self._extract_handoffs_from_agent(agent)
+        if handoffs:
+            manifest["handoffs"] = handoffs
+
+        guardrails = self._extract_guardrails_from_agent(agent)
+        if guardrails:
+            manifest["guardrails"] = guardrails
+
+        span._set_ctx_item(AGENT_MANIFEST, manifest)
+
+    def _extract_model_settings_from_agent(self, agent):
+        if not hasattr(agent, "model_settings"):
+            return None
+
+        # convert model_settings to dict if it's not already
+        model_settings = agent.model_settings
+        if type(model_settings) != dict:
+            model_settings = getattr(model_settings, "__dict__", None)
+
+        return load_data_value(model_settings)
+
+    def _extract_tools_from_agent(self, agent):
+        if not hasattr(agent, "tools") or not agent.tools:
+            return None
+
+        tools = []
+        for tool in agent.tools:
+            tool_dict = {}
+            tool_name = getattr(tool, "name", None)
+            if tool_name:
+                tool_dict["name"] = tool_name
+            if tool_name == "web_search_preview":
+                if hasattr(tool, "user_location"):
+                    tool_dict["user_location"] = tool.user_location
+                if hasattr(tool, "search_context_size"):
+                    tool_dict["search_context_size"] = tool.search_context_size
+            elif tool_name == "file_search":
+                if hasattr(tool, "vector_store_ids"):
+                    tool_dict["vector_store_ids"] = tool.vector_store_ids
+                if hasattr(tool, "max_num_results"):
+                    tool_dict["max_num_results"] = tool.max_num_results
+                if hasattr(tool, "include_search_results"):
+                    tool_dict["include_search_results"] = tool.include_search_results
+                if hasattr(tool, "ranking_options"):
+                    tool_dict["ranking_options"] = tool.ranking_options
+                if hasattr(tool, "filters"):
+                    tool_dict["filters"] = tool.filters
+            elif tool_name == "computer_use_preview":
+                if hasattr(tool, "computer"):
+                    tool_dict["computer"] = tool.computer
+                if hasattr(tool, "on_safety_check"):
+                    tool_dict["on_safety_check"] = tool.on_safety_check
+            elif tool_name == "code_interpreter":
+                if hasattr(tool, "tool_config"):
+                    tool_dict["tool_config"] = tool.tool_config
+            elif tool_name == "hosted_mcp":
+                if hasattr(tool, "tool_config"):
+                    tool_dict["tool_config"] = tool.tool_config
+                if hasattr(tool, "on_approval_request"):
+                    tool_dict["on_approval_request"] = tool.on_approval_request
+            elif tool_name == "image_generation":
+                if hasattr(tool, "tool_config"):
+                    tool_dict["tool_config"] = tool.tool_config
+            elif tool_name == "local_shell":
+                if hasattr(tool, "executor"):
+                    tool_dict["executor"] = tool.executor
+            else:
+                if hasattr(tool, "description"):
+                    tool_dict["description"] = tool.description
+                if hasattr(tool, "strict_json_schema"):
+                    tool_dict["strict_json_schema"] = tool.strict_json_schema
+                if hasattr(tool, "params_json_schema"):
+                    parameter_schema = tool.params_json_schema
+                    required_params = {param: True for param in parameter_schema.get("required", [])}
+                    parameters = {}
+                    for param, schema in parameter_schema.get("properties", {}).items():
+                        param_dict = {}
+                        if "type" in schema:
+                            param_dict["type"] = schema["type"]
+                        if "title" in schema:
+                            param_dict["title"] = schema["title"]
+                        if param in required_params:
+                            param_dict["required"] = True
+                        parameters[param] = param_dict
+                    tool_dict["parameters"] = parameters
+            tools.append(tool_dict)
+
+        return tools
+
+    def _extract_handoffs_from_agent(self, agent):
+        if not hasattr(agent, "handoffs") or not agent.handoffs:
+            return None
+
+        handoffs = []
+        for handoff in agent.handoffs:
+            handoff_dict = {}
+            if hasattr(handoff, "handoff_description") or hasattr(handoff, "tool_description"):
+                handoff_dict["handoff_description"] = getattr(handoff, "handoff_description", None) or getattr(
+                    handoff, "tool_description", None
+                )
+            if hasattr(handoff, "name") or hasattr(handoff, "agent_name"):
+                handoff_dict["agent_name"] = getattr(handoff, "name", None) or getattr(handoff, "agent_name", None)
+            if hasattr(handoff, "tool_name"):
+                handoff_dict["tool_name"] = handoff.tool_name
+            if handoff_dict:
+                handoffs.append(handoff_dict)
+
+        return handoffs
+
+    def _extract_guardrails_from_agent(self, agent):
+        guardrails = []
+        if hasattr(agent, "input_guardrails"):
+            guardrails.extend([getattr(guardrail, "name", "") for guardrail in agent.input_guardrails])
+        if hasattr(agent, "output_guardrails"):
+            guardrails.extend([getattr(guardrail, "name", "") for guardrail in agent.output_guardrails])
+        return guardrails