|
| 1 | +# Copyright 2010 New Relic, Inc. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | + |
| 16 | +import logging |
| 17 | +import sys |
| 18 | +import uuid |
| 19 | + |
| 20 | +from newrelic.api.function_trace import FunctionTrace |
| 21 | +from newrelic.api.time_trace import get_trace_linking_metadata |
| 22 | +from newrelic.api.transaction import current_transaction |
| 23 | +from newrelic.common.object_names import callable_name |
| 24 | +from newrelic.common.object_wrapper import wrap_function_wrapper |
| 25 | +from newrelic.common.package_version_utils import get_package_version |
| 26 | +from newrelic.common.signature import bind_args |
| 27 | +from newrelic.core.config import global_settings |
| 28 | + |
| 29 | +# Check for the presence of the autogen-core, autogen-agentchat, or autogen-ext package as they should all have the |
| 30 | +# same version and one or multiple could be installed |
| 31 | +AUTOGEN_VERSION = ( |
| 32 | + get_package_version("autogen-core") |
| 33 | + or get_package_version("autogen-agentchat") |
| 34 | + or get_package_version("autogen-ext") |
| 35 | +) |
| 36 | + |
| 37 | + |
| 38 | +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Autogen instrumentation: Failed to record LLM events. Please report this issue to New Relic Support.\n%s" |
| 39 | + |
| 40 | + |
| 41 | +_logger = logging.getLogger(__name__) |
| 42 | + |
| 43 | + |
| 44 | +async def wrap_from_server_params(wrapped, instance, args, kwargs): |
| 45 | + transaction = current_transaction() |
| 46 | + if not transaction: |
| 47 | + return await wrapped(*args, **kwargs) |
| 48 | + |
| 49 | + func_name = callable_name(wrapped) |
| 50 | + bound_args = bind_args(wrapped, args, kwargs) |
| 51 | + tool_name = bound_args.get("tool_name") or "tool" |
| 52 | + function_trace_name = f"{func_name}/{tool_name}" |
| 53 | + with FunctionTrace(name=function_trace_name, group="Llm", source=wrapped): |
| 54 | + return await wrapped(*args, **kwargs) |
| 55 | + |
| 56 | + |
| 57 | +def wrap_on_messages_stream(wrapped, instance, args, kwargs): |
| 58 | + transaction = current_transaction() |
| 59 | + if not transaction: |
| 60 | + return wrapped(*args, **kwargs) |
| 61 | + |
| 62 | + settings = transaction.settings or global_settings() |
| 63 | + if not settings.ai_monitoring.enabled: |
| 64 | + return wrapped(*args, **kwargs) |
| 65 | + |
| 66 | + # Framework metric also used for entity tagging in the UI |
| 67 | + transaction.add_ml_model_info("Autogen", AUTOGEN_VERSION) |
| 68 | + transaction._add_agent_attribute("llm", True) |
| 69 | + |
| 70 | + agent_name = getattr(instance, "name", "agent") |
| 71 | + agent_id = str(uuid.uuid4()) |
| 72 | + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction) |
| 73 | + func_name = callable_name(wrapped) |
| 74 | + function_trace_name = f"{func_name}/{agent_name}" |
| 75 | + |
| 76 | + ft = FunctionTrace(name=function_trace_name, group="Llm/agent/Autogen") |
| 77 | + ft.__enter__() |
| 78 | + |
| 79 | + try: |
| 80 | + return_val = wrapped(*args, **kwargs) |
| 81 | + except Exception: |
| 82 | + ft.notice_error(attributes={"agent_id": agent_id}) |
| 83 | + ft.__exit__(*sys.exc_info()) |
| 84 | + # If we hit an exception, append the error attribute and duration from the exited function trace |
| 85 | + agent_event_dict.update({"duration": ft.duration * 1000, "error": True}) |
| 86 | + transaction.record_custom_event("LlmAgent", agent_event_dict) |
| 87 | + raise |
| 88 | + |
| 89 | + ft.__exit__(None, None, None) |
| 90 | + agent_event_dict.update({"duration": ft.duration * 1000}) |
| 91 | + |
| 92 | + transaction.record_custom_event("LlmAgent", agent_event_dict) |
| 93 | + |
| 94 | + return return_val |
| 95 | + |
| 96 | + |
| 97 | +def _get_llm_metadata(transaction): |
| 98 | + # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events |
| 99 | + custom_attrs_dict = transaction._custom_params |
| 100 | + llm_metadata_dict = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} |
| 101 | + llm_context_attrs = getattr(transaction, "_llm_context_attrs", None) |
| 102 | + if llm_context_attrs: |
| 103 | + llm_metadata_dict.update(llm_context_attrs) |
| 104 | + |
| 105 | + return llm_metadata_dict |
| 106 | + |
| 107 | + |
| 108 | +def _extract_tool_output(return_val, tool_name): |
| 109 | + try: |
| 110 | + output = getattr(return_val[1], "content", None) |
| 111 | + return output |
| 112 | + except Exception: |
| 113 | + _logger.warning("Unable to parse tool output value from %s. Omitting output from LlmTool event.", tool_name) |
| 114 | + return None |
| 115 | + |
| 116 | + |
| 117 | +def _construct_base_tool_event_dict(bound_args, tool_call_data, tool_id, transaction, settings): |
| 118 | + try: |
| 119 | + _input = getattr(tool_call_data, "arguments", None) |
| 120 | + tool_input = str(_input) if _input else None |
| 121 | + run_id = getattr(tool_call_data, "id", None) |
| 122 | + tool_name = getattr(tool_call_data, "name", "tool") |
| 123 | + agent_name = bound_args.get("agent_name") |
| 124 | + linking_metadata = get_trace_linking_metadata() |
| 125 | + |
| 126 | + tool_event_dict = { |
| 127 | + "id": tool_id, |
| 128 | + "run_id": run_id, |
| 129 | + "name": tool_name, |
| 130 | + "span_id": linking_metadata.get("span.id"), |
| 131 | + "trace_id": linking_metadata.get("trace.id"), |
| 132 | + "agent_name": agent_name, |
| 133 | + "vendor": "autogen", |
| 134 | + "ingest_source": "Python", |
| 135 | + } |
| 136 | + if settings.ai_monitoring.record_content.enabled: |
| 137 | + tool_event_dict.update({"input": tool_input}) |
| 138 | + tool_event_dict.update(_get_llm_metadata(transaction)) |
| 139 | + except Exception: |
| 140 | + tool_event_dict = {} |
| 141 | + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) |
| 142 | + |
| 143 | + return tool_event_dict |
| 144 | + |
| 145 | + |
| 146 | +def _construct_base_agent_event_dict(agent_name, agent_id, transaction): |
| 147 | + try: |
| 148 | + linking_metadata = get_trace_linking_metadata() |
| 149 | + |
| 150 | + agent_event_dict = { |
| 151 | + "id": agent_id, |
| 152 | + "name": agent_name, |
| 153 | + "span_id": linking_metadata.get("span.id"), |
| 154 | + "trace_id": linking_metadata.get("trace.id"), |
| 155 | + "vendor": "autogen", |
| 156 | + "ingest_source": "Python", |
| 157 | + } |
| 158 | + agent_event_dict.update(_get_llm_metadata(transaction)) |
| 159 | + except Exception: |
| 160 | + agent_event_dict = {} |
| 161 | + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) |
| 162 | + |
| 163 | + return agent_event_dict |
| 164 | + |
| 165 | + |
| 166 | +async def wrap__execute_tool_call(wrapped, instance, args, kwargs): |
| 167 | + transaction = current_transaction() |
| 168 | + if not transaction: |
| 169 | + return await wrapped(*args, **kwargs) |
| 170 | + |
| 171 | + settings = transaction.settings or global_settings() |
| 172 | + if not settings.ai_monitoring.enabled: |
| 173 | + return await wrapped(*args, **kwargs) |
| 174 | + |
| 175 | + # Framework metric also used for entity tagging in the UI |
| 176 | + transaction.add_ml_model_info("Autogen", AUTOGEN_VERSION) |
| 177 | + transaction._add_agent_attribute("llm", True) |
| 178 | + |
| 179 | + tool_id = str(uuid.uuid4()) |
| 180 | + bound_args = bind_args(wrapped, args, kwargs) |
| 181 | + tool_call_data = bound_args.get("tool_call") |
| 182 | + tool_event_dict = _construct_base_tool_event_dict(bound_args, tool_call_data, tool_id, transaction, settings) |
| 183 | + |
| 184 | + tool_name = getattr(tool_call_data, "name", "tool") |
| 185 | + |
| 186 | + func_name = callable_name(wrapped) |
| 187 | + ft = FunctionTrace(name=f"{func_name}/{tool_name}", group="Llm/tool/Autogen") |
| 188 | + ft.__enter__() |
| 189 | + |
| 190 | + try: |
| 191 | + return_val = await wrapped(*args, **kwargs) |
| 192 | + except Exception: |
| 193 | + ft.notice_error(attributes={"tool_id": tool_id}) |
| 194 | + ft.__exit__(*sys.exc_info()) |
| 195 | + # If we hit an exception, append the error attribute and duration from the exited function trace |
| 196 | + tool_event_dict.update({"duration": ft.duration * 1000, "error": True}) |
| 197 | + transaction.record_custom_event("LlmTool", tool_event_dict) |
| 198 | + raise |
| 199 | + |
| 200 | + ft.__exit__(None, None, None) |
| 201 | + tool_event_dict.update({"duration": ft.duration * 1000}) |
| 202 | + |
| 203 | + # If the tool was executed successfully, we can grab the tool output from the result |
| 204 | + tool_output = _extract_tool_output(return_val, tool_name) |
| 205 | + if settings.ai_monitoring.record_content.enabled: |
| 206 | + tool_event_dict.update({"output": tool_output}) |
| 207 | + |
| 208 | + transaction.record_custom_event("LlmTool", tool_event_dict) |
| 209 | + |
| 210 | + return return_val |
| 211 | + |
| 212 | + |
| 213 | +def instrument_autogen_agentchat_agents__assistant_agent(module): |
| 214 | + if hasattr(module, "AssistantAgent"): |
| 215 | + if hasattr(module.AssistantAgent, "on_messages_stream"): |
| 216 | + wrap_function_wrapper(module, "AssistantAgent.on_messages_stream", wrap_on_messages_stream) |
| 217 | + if hasattr(module.AssistantAgent, "_execute_tool_call"): |
| 218 | + wrap_function_wrapper(module, "AssistantAgent._execute_tool_call", wrap__execute_tool_call) |
| 219 | + |
| 220 | + |
| 221 | +def instrument_autogen_ext_tools_mcp__base(module): |
| 222 | + if hasattr(module, "McpToolAdapter"): |
| 223 | + if hasattr(module.McpToolAdapter, "from_server_params"): |
| 224 | + wrap_function_wrapper(module, "McpToolAdapter.from_server_params", wrap_from_server_params) |
![mergify[bot]](https://avatars.githubusercontent.com/in/10562?v=4&size=40){kind=avatar}
0 commit comments