diff --git a/agentops/__init__.py b/agentops/__init__.py index 0ed1171a8..2f10c1420 100755 --- a/agentops/__init__.py +++ b/agentops/__init__.py @@ -15,7 +15,7 @@ from typing import List, Optional, Union, Dict, Any from agentops.client import Client from agentops.sdk.core import TraceContext, tracer -from agentops.sdk.decorators import trace, session, agent, task, workflow, operation, tool +from agentops.sdk.decorators import trace, session, agent, task, workflow, operation, tool, guardrail from agentops.enums import TraceState, SUCCESS, ERROR, UNSET from opentelemetry.trace.status import StatusCode @@ -265,6 +265,7 @@ def end_trace( "task", "workflow", "operation", + "guardrail", "tracer", "tool", # Trace state enums diff --git a/agentops/sdk/decorators/__init__.py b/agentops/sdk/decorators/__init__.py index f7524fbbd..6881bc19a 100644 --- a/agentops/sdk/decorators/__init__.py +++ b/agentops/sdk/decorators/__init__.py @@ -18,6 +18,7 @@ trace = create_entity_decorator(SpanKind.SESSION) tool = create_entity_decorator(SpanKind.TOOL) operation = task +guardrail = create_entity_decorator(SpanKind.GUARDRAIL) # For backward compatibility: @session decorator calls @trace decorator @@ -37,4 +38,13 @@ def session(*args, **kwargs): # noqa: F811 # For now, keeping the alias as it was, assuming it was intentional for `operation` to be `task`. operation = task -__all__ = ["agent", "task", "workflow", "trace", "session", "operation", "tool"] +__all__ = [ + "agent", + "task", + "workflow", + "trace", + "session", + "operation", + "tool", + "guardrail", +] diff --git a/agentops/sdk/decorators/factory.py b/agentops/sdk/decorators/factory.py index 6951767e4..7be5c26bb 100644 --- a/agentops/sdk/decorators/factory.py +++ b/agentops/sdk/decorators/factory.py @@ -33,9 +33,10 @@ def decorator( version: Optional[Any] = None, tags: Optional[Union[list, dict]] = None, cost=None, + spec=None, ) -> Callable[..., Any]: if wrapped is None: - return functools.partial(decorator, name=name, version=version, tags=tags, cost=cost) + return functools.partial(decorator, name=name, version=version, tags=tags, cost=cost, spec=spec) if inspect.isclass(wrapped): # Class decoration wraps __init__ and aenter/aexit for context management. @@ -168,10 +169,13 @@ async def _wrapped_session_async() -> Any: attributes={CoreAttributes.TAGS: tags} if tags else None, ) try: - _record_entity_input(span, args, kwargs) + _record_entity_input(span, args, kwargs, entity_kind=entity_kind) # Set cost attribute if tool if entity_kind == "tool" and cost is not None: span.set_attribute(SpanAttributes.LLM_USAGE_TOOL_COST, cost) + # Set spec attribute if guardrail + if entity_kind == "guardrail" and (spec == "input" or spec == "output"): + span.set_attribute(SpanAttributes.AGENTOPS_DECORATOR_SPEC.format(entity_kind=entity_kind), spec) except Exception as e: logger.warning(f"Input recording failed for '{operation_name}': {e}") result = wrapped_func(*args, **kwargs) @@ -184,10 +188,13 @@ async def _wrapped_session_async() -> Any: attributes={CoreAttributes.TAGS: tags} if tags else None, ) try: - _record_entity_input(span, args, kwargs) + _record_entity_input(span, args, kwargs, entity_kind=entity_kind) # Set cost attribute if tool if entity_kind == "tool" and cost is not None: span.set_attribute(SpanAttributes.LLM_USAGE_TOOL_COST, cost) + # Set spec attribute if guardrail + if entity_kind == "guardrail" and (spec == "input" or spec == "output"): + span.set_attribute(SpanAttributes.AGENTOPS_DECORATOR_SPEC.format(entity_kind=entity_kind), spec) except Exception as e: logger.warning(f"Input recording failed for '{operation_name}': {e}") result = wrapped_func(*args, **kwargs) @@ -202,16 +209,21 @@ async def _wrapped_async() -> Any: attributes={CoreAttributes.TAGS: tags} if tags else None, ) as span: try: - _record_entity_input(span, args, kwargs) + _record_entity_input(span, args, kwargs, entity_kind=entity_kind) # Set cost attribute if tool if entity_kind == "tool" and cost is not None: span.set_attribute(SpanAttributes.LLM_USAGE_TOOL_COST, cost) + # Set spec attribute if guardrail + if entity_kind == "guardrail" and (spec == "input" or spec == "output"): + span.set_attribute( + SpanAttributes.AGENTOPS_DECORATOR_SPEC.format(entity_kind=entity_kind), spec + ) except Exception as e: logger.warning(f"Input recording failed for '{operation_name}': {e}") try: result = await wrapped_func(*args, **kwargs) try: - _record_entity_output(span, result) + _record_entity_output(span, result, entity_kind=entity_kind) except Exception as e: logger.warning(f"Output recording failed for '{operation_name}': {e}") return result @@ -229,16 +241,21 @@ async def _wrapped_async() -> Any: attributes={CoreAttributes.TAGS: tags} if tags else None, ) as span: try: - _record_entity_input(span, args, kwargs) + _record_entity_input(span, args, kwargs, entity_kind=entity_kind) # Set cost attribute if tool if entity_kind == "tool" and cost is not None: span.set_attribute(SpanAttributes.LLM_USAGE_TOOL_COST, cost) + # Set spec attribute if guardrail + if entity_kind == "guardrail" and (spec == "input" or spec == "output"): + span.set_attribute( + SpanAttributes.AGENTOPS_DECORATOR_SPEC.format(entity_kind=entity_kind), spec + ) except Exception as e: logger.warning(f"Input recording failed for '{operation_name}': {e}") try: result = wrapped_func(*args, **kwargs) try: - _record_entity_output(span, result) + _record_entity_output(span, result, entity_kind=entity_kind) except Exception as e: logger.warning(f"Output recording failed for '{operation_name}': {e}") return result diff --git a/agentops/sdk/decorators/utility.py b/agentops/sdk/decorators/utility.py index 588301490..8abc8820d 100644 --- a/agentops/sdk/decorators/utility.py +++ b/agentops/sdk/decorators/utility.py @@ -135,27 +135,27 @@ def _create_as_current_span( logger.debug(f"[DEBUG] AFTER {operation_name}.{span_kind} - Returned to context: {after_span}") -def _record_entity_input(span: trace.Span, args: tuple, kwargs: Dict[str, Any]) -> None: +def _record_entity_input(span: trace.Span, args: tuple, kwargs: Dict[str, Any], entity_kind: str = "entity") -> None: """Record operation input parameters to span if content tracing is enabled""" try: input_data = {"args": args, "kwargs": kwargs} json_data = safe_serialize(input_data) if _check_content_size(json_data): - span.set_attribute(SpanAttributes.AGENTOPS_ENTITY_INPUT, json_data) + span.set_attribute(SpanAttributes.AGENTOPS_DECORATOR_INPUT.format(entity_kind=entity_kind), json_data) else: logger.debug("Operation input exceeds size limit, not recording") except Exception as err: logger.warning(f"Failed to serialize operation input: {err}") -def _record_entity_output(span: trace.Span, result: Any) -> None: +def _record_entity_output(span: trace.Span, result: Any, entity_kind: str = "entity") -> None: """Record operation output value to span if content tracing is enabled""" try: json_data = safe_serialize(result) if _check_content_size(json_data): - span.set_attribute(SpanAttributes.AGENTOPS_ENTITY_OUTPUT, json_data) + span.set_attribute(SpanAttributes.AGENTOPS_DECORATOR_OUTPUT.format(entity_kind=entity_kind), json_data) else: logger.debug("Operation output exceeds size limit, not recording") except Exception as err: diff --git a/agentops/semconv/span_attributes.py b/agentops/semconv/span_attributes.py index 67320bffd..0daf0ddfc 100644 --- a/agentops/semconv/span_attributes.py +++ b/agentops/semconv/span_attributes.py @@ -88,6 +88,9 @@ class SpanAttributes: AGENTOPS_ENTITY_INPUT = "agentops.entity.input" AGENTOPS_SPAN_KIND = "agentops.span.kind" AGENTOPS_ENTITY_NAME = "agentops.entity.name" + AGENTOPS_DECORATOR_SPEC = "agentops.{entity_kind}.spec" + AGENTOPS_DECORATOR_INPUT = "agentops.{entity_kind}.input" + AGENTOPS_DECORATOR_OUTPUT = "agentops.{entity_kind}.output" # Operation attributes OPERATION_NAME = "operation.name" diff --git a/agentops/semconv/span_kinds.py b/agentops/semconv/span_kinds.py index 0d90a8cc9..5a75f6d76 100644 --- a/agentops/semconv/span_kinds.py +++ b/agentops/semconv/span_kinds.py @@ -27,6 +27,7 @@ class SpanKind: UNKNOWN = "unknown" CHAIN = "chain" TEXT = "text" + GUARDRAIL = "guardrail" class AgentOpsSpanKindValues(Enum): diff --git a/docs/v2/concepts/decorators.mdx b/docs/v2/concepts/decorators.mdx index 4596e4193..7782175c3 100644 --- a/docs/v2/concepts/decorators.mdx +++ b/docs/v2/concepts/decorators.mdx @@ -14,6 +14,7 @@ AgentOps provides the following decorators: | `@workflow` | Track a sequence of operations | WORKFLOW span | | `@task` | Track smaller units of work (similar to operations) | TASK span | | `@tool` | Track tool usage and cost in agent operations | TOOL span | +| `@guardrail` | Track guardrail input and output | GUARDRAIL span | ## Decorator Hierarchy @@ -235,6 +236,27 @@ The tool decorator provides: - Support for all function types (sync, async, generator, async generator) - Cost accumulation in generator and async generator operations +### @guardrail + +The `@guardrail` decorator tracks guardrail input and output. You can specify the guardrail type (`"input"` or `"output"`) with the `spec` parameter. + +```python +from agentops.sdk.decorators import guardrail +import agentops +import re + +# Initialize AgentOps +agentops.init(api_key="YOUR_API_KEY") + +@guardrail(spec="input") +def secret_key_guardrail(input): + pattern = r'\bsk-[a-zA-Z0-9]{10,}\b' + result = True if re.search(pattern, input) else False + return { + "tripwire_triggered" : result + } +``` + ## Decorator Attributes You can pass additional attributes to decorators: diff --git a/examples/google_adk/human_approval.ipynb b/examples/google_adk/human_approval.ipynb index 92a871781..982d80ad2 100644 --- a/examples/google_adk/human_approval.ipynb +++ b/examples/google_adk/human_approval.ipynb @@ -183,7 +183,7 @@ " \"\"\"\n", " Prompts for human approval and returns the decision as a JSON string.\n", " \"\"\"\n", - " print(f\"🔔 HUMAN APPROVAL REQUIRED:\")\n", + " print(\"🔔 HUMAN APPROVAL REQUIRED:\")\n", " print(f\" Amount: ${amount:,.2f}\")\n", " print(f\" Reason: {reason}\")\n", " decision = \"\"\n", diff --git a/examples/openai_agents/agent_guardrails.ipynb b/examples/openai_agents/agent_guardrails.ipynb new file mode 100644 index 000000000..4c1739c30 --- /dev/null +++ b/examples/openai_agents/agent_guardrails.ipynb @@ -0,0 +1,163 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f68ce4af", + "metadata": {}, + "source": [ + "# OpenAI Agents Guardrails Demonstration\n", + "\n", + "This notebook demonstrates guardrails using the Agents SDK and how one can observe them using the AgentOps platform." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10bcf29b", + "metadata": {}, + "outputs": [], + "source": [ + "# Install required packages\n", + "%pip install agentops\n", + "%pip install openai-agents\n", + "%pip install dotenv pydantic" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3be4e68", + "metadata": {}, + "outputs": [], + "source": [ + "# Import dependencies\n", + "from pydantic import BaseModel\n", + "from agents import (\n", + " Agent,\n", + " GuardrailFunctionOutput,\n", + " InputGuardrailTripwireTriggered,\n", + " RunContextWrapper,\n", + " Runner,\n", + " TResponseInputItem,\n", + " input_guardrail,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2d0dddb6", + "metadata": {}, + "outputs": [], + "source": [ + "# Load API keys\n", + "import os\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv()\n", + "\n", + "os.environ[\"AGENTOPS_API_KEY\"] = os.getenv(\"AGENTOPS_API_KEY\", \"your_api_key_here\")\n", + "os.environ[\"OPENAI_API_KEY\"] = os.getenv(\"OPENAI_API_KEY\", \"your_openai_api_key_here\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "114e216b", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize agentops and import the guardrail decorator\n", + "import agentops\n", + "from agentops import guardrail\n", + "\n", + "agentops.init(api_key=os.environ[\"AGENTOPS_API_KEY\"], tags=[\"agentops-example\"], auto_start_session=False)\n", + "tracer = agentops.start_trace(trace_name=\"OpenAI Agents Guardrail Example\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0bf8b54d", + "metadata": {}, + "outputs": [], + "source": [ + "# OpenAI Agents SDK guardrail example with agentops guardrails decorator for observability\n", + "class MathHomeworkOutput(BaseModel):\n", + " is_math_homework: bool\n", + " reasoning: str\n", + "\n", + "\n", + "guardrail_agent = Agent(\n", + " name=\"Guardrail check\",\n", + " instructions=\"Check if the user is asking you to do their math homework.\",\n", + " output_type=MathHomeworkOutput,\n", + ")\n", + "\n", + "\n", + "@input_guardrail\n", + "@guardrail(spec=\"input\") # Specify guardrail type as input or output\n", + "async def math_guardrail(\n", + " ctx: RunContextWrapper[None], agent: Agent, input: str | list[TResponseInputItem]\n", + ") -> GuardrailFunctionOutput:\n", + " result = await Runner.run(guardrail_agent, input, context=ctx.context)\n", + "\n", + " return GuardrailFunctionOutput(\n", + " output_info=result.final_output,\n", + " tripwire_triggered=result.final_output.is_math_homework,\n", + " )\n", + "\n", + "\n", + "agent = Agent(\n", + " name=\"Customer support agent\",\n", + " instructions=\"You are a customer support agent. You help customers with their questions.\",\n", + " input_guardrails=[math_guardrail],\n", + ")\n", + "\n", + "\n", + "async def main():\n", + " # This should trip the guardrail\n", + " try:\n", + " await Runner.run(agent, \"Hello, can you help me solve for x: 2x + 3 = 11?\")\n", + " print(\"Guardrail didn't trip - this is unexpected\")\n", + "\n", + " except InputGuardrailTripwireTriggered:\n", + " print(\"Math homework guardrail tripped\")\n", + "\n", + "\n", + "await main()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "63bf8e09", + "metadata": {}, + "outputs": [], + "source": [ + "agentops.end_trace(tracer, end_state=\"Success\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "agentops (3.11.11)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/openai_agents/agent_guardrails.py b/examples/openai_agents/agent_guardrails.py new file mode 100644 index 000000000..2429dab4b --- /dev/null +++ b/examples/openai_agents/agent_guardrails.py @@ -0,0 +1,83 @@ +# # OpenAI Agents Guardrails Demonstration +# +# This notebook demonstrates guardrails using the Agents SDK and how one can observe them using the AgentOps platform. + +# Install required packages +# %pip install agentops +# %pip install openai-agents +# %pip install dotenv pydantic + +# Import dependencies +from pydantic import BaseModel +from agents import ( + Agent, + GuardrailFunctionOutput, + InputGuardrailTripwireTriggered, + RunContextWrapper, + Runner, + TResponseInputItem, + input_guardrail, +) + +# Initialize agentops and import the guardrail decorator +import agentops +from agentops import guardrail + +# Load API keys +import os +from dotenv import load_dotenv +import asyncio + +load_dotenv() + +os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") +os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") + +agentops.init(api_key=os.environ["AGENTOPS_API_KEY"], tags=["agentops-example"]) + + +# OpenAI Agents SDK guardrail example with agentops guardrails decorator for observability +class MathHomeworkOutput(BaseModel): + is_math_homework: bool + reasoning: str + + +guardrail_agent = Agent( + name="Guardrail check", + instructions="Check if the user is asking you to do their math homework.", + output_type=MathHomeworkOutput, +) + + +@input_guardrail +@guardrail(spec="input") # Specify guardrail type as input or output +async def math_guardrail( + ctx: RunContextWrapper[None], agent: Agent, input: str | list[TResponseInputItem] +) -> GuardrailFunctionOutput: + result = await Runner.run(guardrail_agent, input, context=ctx.context) + + return GuardrailFunctionOutput( + output_info=result.final_output, + tripwire_triggered=result.final_output.is_math_homework, + ) + + +agent = Agent( + name="Customer support agent", + instructions="You are a customer support agent. You help customers with their questions.", + input_guardrails=[math_guardrail], +) + + +async def main(): + # This should trip the guardrail + try: + await Runner.run(agent, "Hello, can you help me solve for x: 2x + 3 = 11?") + print("Guardrail didn't trip - this is unexpected") + + except InputGuardrailTripwireTriggered: + print("Math homework guardrail tripped") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/openai_agents/agent_patterns.ipynb b/examples/openai_agents/agent_patterns.ipynb index d3614c864..1af989db1 100644 --- a/examples/openai_agents/agent_patterns.ipynb +++ b/examples/openai_agents/agent_patterns.ipynb @@ -100,10 +100,9 @@ " OutputGuardrailTripwireTriggered,\n", " input_guardrail,\n", " output_guardrail,\n", - " RawResponsesStreamEvent,\n", ")\n", "\n", - "from openai.types.responses import ResponseContentPartDoneEvent, ResponseTextDeltaEvent" + "from openai.types.responses import ResponseTextDeltaEvent" ] }, { diff --git a/examples/openai_agents/agents_tools.ipynb b/examples/openai_agents/agents_tools.ipynb index a8bf70b68..b27830b06 100755 --- a/examples/openai_agents/agents_tools.ipynb +++ b/examples/openai_agents/agents_tools.ipynb @@ -79,24 +79,17 @@ "metadata": {}, "outputs": [], "source": [ - "import asyncio\n", "import base64\n", "import os\n", "import subprocess\n", "import sys\n", "import tempfile\n", - "from typing import Literal, Union\n", "\n", "from agents import (\n", " Agent,\n", - " AsyncComputer,\n", - " Button,\n", " CodeInterpreterTool,\n", - " ComputerTool,\n", - " Environment,\n", " FileSearchTool,\n", " ImageGenerationTool,\n", - " ModelSettings,\n", " Runner,\n", " WebSearchTool,\n", " trace,\n", diff --git a/examples/openai_agents/customer_service_agent.ipynb b/examples/openai_agents/customer_service_agent.ipynb index a873049a8..6135c44dd 100644 --- a/examples/openai_agents/customer_service_agent.ipynb +++ b/examples/openai_agents/customer_service_agent.ipynb @@ -67,10 +67,8 @@ "source": [ "from __future__ import annotations as _annotations # noqa: F404\n", "\n", - "import asyncio\n", "import random\n", "import uuid\n", - "import sys\n", "\n", "from pydantic import BaseModel\n", "import agentops\n",