From 09425e668a6eea6f66d4a4d9a42bb4af1b997a6d Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 2 Jun 2025 15:07:08 +0200 Subject: [PATCH 01/89] Barebone of new integration --- .github/workflows/test-integrations-ai.yml | 8 ++ scripts/populate_tox/config.py | 3 + scripts/populate_tox/tox.jinja | 1 + .../split_tox_gh_actions.py | 1 + sentry_sdk/integrations/__init__.py | 1 + sentry_sdk/integrations/openai_agents.py | 36 +++++++ .../openai_agents/test_openai_agents.py | 7 ++ tox.ini | 99 ++++++++++--------- 8 files changed, 111 insertions(+), 45 deletions(-) create mode 100644 sentry_sdk/integrations/openai_agents.py create mode 100644 tests/integrations/openai_agents/test_openai_agents.py diff --git a/.github/workflows/test-integrations-ai.yml b/.github/workflows/test-integrations-ai.yml index 4aa0f36b77..e81d507d27 100644 --- a/.github/workflows/test-integrations-ai.yml +++ b/.github/workflows/test-integrations-ai.yml @@ -66,6 +66,10 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-openai-latest" + - name: Test openai_agents latest + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-openai_agents-latest" - name: Test huggingface_hub latest run: | set -x # print commands that are executed @@ -141,6 +145,10 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai" + - name: Test openai_agents pinned + run: | + set -x # print commands that are executed + ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai_agents" - name: Test huggingface_hub pinned run: | set -x # print commands that are executed diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index 4d5d5b14ce..9803d7aec2 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -139,6 +139,9 @@ "loguru": { "package": "loguru", }, + "openai_agents": { + "package": "openai-agents", + }, "openfeature": { "package": "openfeature-sdk", }, diff --git a/scripts/populate_tox/tox.jinja b/scripts/populate_tox/tox.jinja index 2869da275b..67743bc172 100644 --- a/scripts/populate_tox/tox.jinja +++ b/scripts/populate_tox/tox.jinja @@ -397,6 +397,7 @@ setenv = litestar: TESTPATH=tests/integrations/litestar loguru: TESTPATH=tests/integrations/loguru openai: TESTPATH=tests/integrations/openai + openai_agents: TESTPATH=tests/integrations/openai_agents openfeature: TESTPATH=tests/integrations/openfeature opentelemetry: TESTPATH=tests/integrations/opentelemetry potel: TESTPATH=tests/integrations/opentelemetry diff --git a/scripts/split_tox_gh_actions/split_tox_gh_actions.py b/scripts/split_tox_gh_actions/split_tox_gh_actions.py index 293af897c9..69ff02cb40 100755 --- a/scripts/split_tox_gh_actions/split_tox_gh_actions.py +++ b/scripts/split_tox_gh_actions/split_tox_gh_actions.py @@ -59,6 +59,7 @@ "cohere", "langchain", "openai", + "openai_agents", "huggingface_hub", ], "Cloud": [ diff --git a/sentry_sdk/integrations/__init__.py b/sentry_sdk/integrations/__init__.py index 118289950c..4c522cc8b2 100644 --- a/sentry_sdk/integrations/__init__.py +++ b/sentry_sdk/integrations/__init__.py @@ -145,6 +145,7 @@ def iter_default_integrations(with_auto_enabling_integrations): "launchdarkly": (9, 8, 0), "loguru": (0, 7, 0), "openai": (1, 0, 0), + "openai_agents": (0, 0, 16), "openfeature": (0, 7, 1), "quart": (0, 16, 0), "ray": (2, 7, 0), diff --git a/sentry_sdk/integrations/openai_agents.py b/sentry_sdk/integrations/openai_agents.py new file mode 100644 index 0000000000..214c7c0420 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents.py @@ -0,0 +1,36 @@ +import sentry_sdk +from sentry_sdk.integrations import DidNotEnable, Integration +from sentry_sdk.utils import event_from_exception + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +try: + from agents import Agent # noqa: F401 +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + + +class OpenAIAgentsIntegration(Integration): + identifier = "openai_agents" + origin = f"auto.ai.{identifier}" + + def __init__(self): + pass + + @staticmethod + def setup_once(): + # type: () -> None + pass + + +def _capture_exception(exc): + # type: (Any) -> None + event, hint = event_from_exception( + exc, + client_options=sentry_sdk.get_client().options, + mechanism={"type": OpenAIAgentsIntegration.identifier, "handled": False}, + ) + sentry_sdk.capture_event(event, hint=hint) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py new file mode 100644 index 0000000000..d9ae5d2a2a --- /dev/null +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -0,0 +1,7 @@ +from sentry_sdk.integrations.openai_agents import OpenAIAgentsIntegration + + +def test_basic(sentry_init): + sentry_init(integrations=[OpenAIAgentsIntegration()]) + + assert True diff --git a/tox.ini b/tox.ini index d9632b7cb5..523fc029b4 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-05-20T07:21:06.821358+00:00 +# Last generated: 2025-06-02T13:01:14.729052+00:00 [tox] requires = @@ -138,17 +138,19 @@ envlist = {py3.8,py3.11,py3.12}-anthropic-v0.16.0 {py3.8,py3.11,py3.12}-anthropic-v0.28.1 {py3.8,py3.11,py3.12}-anthropic-v0.40.0 - {py3.8,py3.11,py3.12}-anthropic-v0.51.0 + {py3.8,py3.11,py3.12}-anthropic-v0.52.2 {py3.9,py3.10,py3.11}-cohere-v5.4.0 {py3.9,py3.11,py3.12}-cohere-v5.8.1 {py3.9,py3.11,py3.12}-cohere-v5.11.4 {py3.9,py3.11,py3.12}-cohere-v5.15.0 + {py3.9,py3.11,py3.12}-openai_agents-v0.0.16 + {py3.8,py3.10,py3.11}-huggingface_hub-v0.22.2 {py3.8,py3.10,py3.11}-huggingface_hub-v0.25.2 {py3.8,py3.12,py3.13}-huggingface_hub-v0.28.1 - {py3.8,py3.12,py3.13}-huggingface_hub-v0.31.4 + {py3.8,py3.12,py3.13}-huggingface_hub-v0.32.3 # ~~~ DBs ~~~ @@ -172,7 +174,7 @@ envlist = {py3.8,py3.12,py3.13}-launchdarkly-v9.8.1 {py3.8,py3.12,py3.13}-launchdarkly-v9.9.0 {py3.8,py3.12,py3.13}-launchdarkly-v9.10.0 - {py3.8,py3.12,py3.13}-launchdarkly-v9.11.0 + {py3.8,py3.12,py3.13}-launchdarkly-v9.11.1 {py3.8,py3.12,py3.13}-openfeature-v0.7.5 {py3.9,py3.12,py3.13}-openfeature-v0.8.1 @@ -180,10 +182,11 @@ envlist = {py3.7,py3.12,py3.13}-statsig-v0.55.3 {py3.7,py3.12,py3.13}-statsig-v0.56.0 {py3.7,py3.12,py3.13}-statsig-v0.57.3 + {py3.7,py3.12,py3.13}-statsig-v0.58.0 {py3.8,py3.12,py3.13}-unleash-v6.0.1 {py3.8,py3.12,py3.13}-unleash-v6.1.0 - {py3.8,py3.12,py3.13}-unleash-v6.2.0 + {py3.8,py3.12,py3.13}-unleash-v6.2.1 # ~~~ GraphQL ~~~ @@ -193,8 +196,8 @@ envlist = {py3.9,py3.12,py3.13}-ariadne-v0.26.2 {py3.6,py3.9,py3.10}-gql-v3.4.1 - {py3.7,py3.11,py3.12}-gql-v3.5.2 - {py3.9,py3.12,py3.13}-gql-v3.6.0b4 + {py3.7,py3.11,py3.12}-gql-v3.5.3 + {py3.9,py3.12,py3.13}-gql-v4.0.0b0 {py3.6,py3.9,py3.10}-graphene-v3.3 {py3.8,py3.12,py3.13}-graphene-v3.4.3 @@ -202,26 +205,26 @@ envlist = {py3.8,py3.10,py3.11}-strawberry-v0.209.8 {py3.8,py3.11,py3.12}-strawberry-v0.229.2 {py3.8,py3.12,py3.13}-strawberry-v0.249.0 - {py3.9,py3.12,py3.13}-strawberry-v0.269.0 + {py3.9,py3.12,py3.13}-strawberry-v0.270.5 # ~~~ Network ~~~ {py3.7,py3.8}-grpc-v1.32.0 - {py3.7,py3.9,py3.10}-grpc-v1.44.0 - {py3.7,py3.10,py3.11}-grpc-v1.58.3 - {py3.9,py3.12,py3.13}-grpc-v1.71.0 - {py3.9,py3.12,py3.13}-grpc-v1.72.0rc1 + {py3.7,py3.9,py3.10}-grpc-v1.46.5 + {py3.7,py3.11,py3.12}-grpc-v1.60.2 + {py3.9,py3.12,py3.13}-grpc-v1.72.1 + {py3.9,py3.12,py3.13}-grpc-v1.73.0rc1 # ~~~ Tasks ~~~ {py3.6,py3.7,py3.8}-celery-v4.4.7 {py3.6,py3.7,py3.8}-celery-v5.0.5 - {py3.8,py3.12,py3.13}-celery-v5.5.2 + {py3.8,py3.12,py3.13}-celery-v5.5.3 {py3.6,py3.7}-dramatiq-v1.9.0 {py3.6,py3.8,py3.9}-dramatiq-v1.12.3 {py3.7,py3.10,py3.11}-dramatiq-v1.15.0 - {py3.8,py3.12,py3.13}-dramatiq-v1.17.1 + {py3.9,py3.12,py3.13}-dramatiq-v1.18.0 {py3.6,py3.7}-huey-v2.1.3 {py3.6,py3.7}-huey-v2.2.0 @@ -229,9 +232,8 @@ envlist = {py3.6,py3.11,py3.12}-huey-v2.5.3 {py3.8,py3.9}-spark-v3.0.3 - {py3.8,py3.9}-spark-v3.2.4 - {py3.8,py3.10,py3.11}-spark-v3.4.4 - {py3.8,py3.10,py3.11}-spark-v3.5.5 + {py3.8,py3.10,py3.11}-spark-v3.5.6 + {py3.9,py3.12,py3.13}-spark-v4.0.0 # ~~~ Web 1 ~~~ @@ -250,7 +252,7 @@ envlist = {py3.6,py3.9,py3.10}-starlette-v0.16.0 {py3.7,py3.10,py3.11}-starlette-v0.26.1 {py3.8,py3.11,py3.12}-starlette-v0.36.3 - {py3.9,py3.12,py3.13}-starlette-v0.46.2 + {py3.9,py3.12,py3.13}-starlette-v0.47.0 {py3.6,py3.9,py3.10}-fastapi-v0.79.1 {py3.7,py3.10,py3.11}-fastapi-v0.91.0 @@ -260,9 +262,10 @@ envlist = # ~~~ Web 2 ~~~ {py3.7}-aiohttp-v3.4.4 - {py3.7}-aiohttp-v3.6.3 - {py3.7,py3.9,py3.10}-aiohttp-v3.8.6 - {py3.9,py3.12,py3.13}-aiohttp-v3.11.18 + {py3.7,py3.8,py3.9}-aiohttp-v3.7.4 + {py3.8,py3.12,py3.13}-aiohttp-v3.10.11 + {py3.9,py3.12,py3.13}-aiohttp-v3.12.6 + {py3.9,py3.12,py3.13}-aiohttp-v3.12.7rc0 {py3.6,py3.7}-bottle-v0.12.25 {py3.8,py3.12,py3.13}-bottle-v0.13.3 @@ -289,7 +292,7 @@ envlist = {py3.6,py3.7,py3.8}-tornado-v6.0.4 {py3.7,py3.9,py3.10}-tornado-v6.2 {py3.8,py3.10,py3.11}-tornado-v6.4.2 - {py3.9,py3.12,py3.13}-tornado-v6.5 + {py3.9,py3.12,py3.13}-tornado-v6.5.1 # ~~~ Misc ~~~ @@ -303,6 +306,7 @@ envlist = {py3.9,py3.12,py3.13}-trytond-v7.6.1 {py3.7,py3.12,py3.13}-typer-v0.15.4 + {py3.7,py3.12,py3.13}-typer-v0.16.0 @@ -502,7 +506,7 @@ deps = anthropic-v0.16.0: anthropic==0.16.0 anthropic-v0.28.1: anthropic==0.28.1 anthropic-v0.40.0: anthropic==0.40.0 - anthropic-v0.51.0: anthropic==0.51.0 + anthropic-v0.52.2: anthropic==0.52.2 anthropic: pytest-asyncio anthropic-v0.16.0: httpx<0.28.0 anthropic-v0.28.1: httpx<0.28.0 @@ -513,10 +517,12 @@ deps = cohere-v5.11.4: cohere==5.11.4 cohere-v5.15.0: cohere==5.15.0 + openai_agents-v0.0.16: openai-agents==0.0.16 + huggingface_hub-v0.22.2: huggingface_hub==0.22.2 huggingface_hub-v0.25.2: huggingface_hub==0.25.2 huggingface_hub-v0.28.1: huggingface_hub==0.28.1 - huggingface_hub-v0.31.4: huggingface_hub==0.31.4 + huggingface_hub-v0.32.3: huggingface_hub==0.32.3 # ~~~ DBs ~~~ @@ -541,7 +547,7 @@ deps = launchdarkly-v9.8.1: launchdarkly-server-sdk==9.8.1 launchdarkly-v9.9.0: launchdarkly-server-sdk==9.9.0 launchdarkly-v9.10.0: launchdarkly-server-sdk==9.10.0 - launchdarkly-v9.11.0: launchdarkly-server-sdk==9.11.0 + launchdarkly-v9.11.1: launchdarkly-server-sdk==9.11.1 openfeature-v0.7.5: openfeature-sdk==0.7.5 openfeature-v0.8.1: openfeature-sdk==0.8.1 @@ -549,11 +555,12 @@ deps = statsig-v0.55.3: statsig==0.55.3 statsig-v0.56.0: statsig==0.56.0 statsig-v0.57.3: statsig==0.57.3 + statsig-v0.58.0: statsig==0.58.0 statsig: typing_extensions unleash-v6.0.1: UnleashClient==6.0.1 unleash-v6.1.0: UnleashClient==6.1.0 - unleash-v6.2.0: UnleashClient==6.2.0 + unleash-v6.2.1: UnleashClient==6.2.1 # ~~~ GraphQL ~~~ @@ -566,8 +573,8 @@ deps = ariadne: httpx gql-v3.4.1: gql[all]==3.4.1 - gql-v3.5.2: gql[all]==3.5.2 - gql-v3.6.0b4: gql[all]==3.6.0b4 + gql-v3.5.3: gql[all]==3.5.3 + gql-v4.0.0b0: gql[all]==4.0.0b0 graphene-v3.3: graphene==3.3 graphene-v3.4.3: graphene==3.4.3 @@ -580,7 +587,7 @@ deps = strawberry-v0.209.8: strawberry-graphql[fastapi,flask]==0.209.8 strawberry-v0.229.2: strawberry-graphql[fastapi,flask]==0.229.2 strawberry-v0.249.0: strawberry-graphql[fastapi,flask]==0.249.0 - strawberry-v0.269.0: strawberry-graphql[fastapi,flask]==0.269.0 + strawberry-v0.270.5: strawberry-graphql[fastapi,flask]==0.270.5 strawberry: httpx strawberry-v0.209.8: pydantic<2.11 strawberry-v0.229.2: pydantic<2.11 @@ -589,10 +596,10 @@ deps = # ~~~ Network ~~~ grpc-v1.32.0: grpcio==1.32.0 - grpc-v1.44.0: grpcio==1.44.0 - grpc-v1.58.3: grpcio==1.58.3 - grpc-v1.71.0: grpcio==1.71.0 - grpc-v1.72.0rc1: grpcio==1.72.0rc1 + grpc-v1.46.5: grpcio==1.46.5 + grpc-v1.60.2: grpcio==1.60.2 + grpc-v1.72.1: grpcio==1.72.1 + grpc-v1.73.0rc1: grpcio==1.73.0rc1 grpc: protobuf grpc: mypy-protobuf grpc: types-protobuf @@ -602,7 +609,7 @@ deps = # ~~~ Tasks ~~~ celery-v4.4.7: celery==4.4.7 celery-v5.0.5: celery==5.0.5 - celery-v5.5.2: celery==5.5.2 + celery-v5.5.3: celery==5.5.3 celery: newrelic celery: redis py3.7-celery: importlib-metadata<5.0 @@ -610,7 +617,7 @@ deps = dramatiq-v1.9.0: dramatiq==1.9.0 dramatiq-v1.12.3: dramatiq==1.12.3 dramatiq-v1.15.0: dramatiq==1.15.0 - dramatiq-v1.17.1: dramatiq==1.17.1 + dramatiq-v1.18.0: dramatiq==1.18.0 huey-v2.1.3: huey==2.1.3 huey-v2.2.0: huey==2.2.0 @@ -618,9 +625,8 @@ deps = huey-v2.5.3: huey==2.5.3 spark-v3.0.3: pyspark==3.0.3 - spark-v3.2.4: pyspark==3.2.4 - spark-v3.4.4: pyspark==3.4.4 - spark-v3.5.5: pyspark==3.5.5 + spark-v3.5.6: pyspark==3.5.6 + spark-v4.0.0: pyspark==4.0.0 # ~~~ Web 1 ~~~ @@ -665,7 +671,7 @@ deps = starlette-v0.16.0: starlette==0.16.0 starlette-v0.26.1: starlette==0.26.1 starlette-v0.36.3: starlette==0.36.3 - starlette-v0.46.2: starlette==0.46.2 + starlette-v0.47.0: starlette==0.47.0 starlette: pytest-asyncio starlette: python-multipart starlette: requests @@ -694,12 +700,13 @@ deps = # ~~~ Web 2 ~~~ aiohttp-v3.4.4: aiohttp==3.4.4 - aiohttp-v3.6.3: aiohttp==3.6.3 - aiohttp-v3.8.6: aiohttp==3.8.6 - aiohttp-v3.11.18: aiohttp==3.11.18 + aiohttp-v3.7.4: aiohttp==3.7.4 + aiohttp-v3.10.11: aiohttp==3.10.11 + aiohttp-v3.12.6: aiohttp==3.12.6 + aiohttp-v3.12.7rc0: aiohttp==3.12.7rc0 aiohttp: pytest-aiohttp - aiohttp-v3.8.6: pytest-asyncio - aiohttp-v3.11.18: pytest-asyncio + aiohttp-v3.10.11: pytest-asyncio + aiohttp-v3.12.6: pytest-asyncio bottle-v0.12.25: bottle==0.12.25 bottle-v0.13.3: bottle==0.13.3 @@ -740,7 +747,7 @@ deps = tornado-v6.0.4: tornado==6.0.4 tornado-v6.2: tornado==6.2 tornado-v6.4.2: tornado==6.4.2 - tornado-v6.5: tornado==6.5 + tornado-v6.5.1: tornado==6.5.1 tornado: pytest tornado-v6.0.4: pytest<8.2 tornado-v6.2: pytest<8.2 @@ -761,6 +768,7 @@ deps = trytond-v4.8.18: werkzeug<1.0 typer-v0.15.4: typer==0.15.4 + typer-v0.16.0: typer==0.16.0 @@ -806,6 +814,7 @@ setenv = litestar: TESTPATH=tests/integrations/litestar loguru: TESTPATH=tests/integrations/loguru openai: TESTPATH=tests/integrations/openai + openai_agents: TESTPATH=tests/integrations/openai_agents openfeature: TESTPATH=tests/integrations/openfeature opentelemetry: TESTPATH=tests/integrations/opentelemetry potel: TESTPATH=tests/integrations/opentelemetry From 03f7e2406bdfbb313e8e7175ce890175a10d7f6a Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 3 Jun 2025 10:02:42 +0200 Subject: [PATCH 02/89] Creating some spans --- sentry_sdk/integrations/openai_agents.py | 177 ++++++++++++++++-- .../openai_agents/test_openai_agents.py | 4 + 2 files changed, 164 insertions(+), 17 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents.py b/sentry_sdk/integrations/openai_agents.py index 214c7c0420..ce5c30ad9c 100644 --- a/sentry_sdk/integrations/openai_agents.py +++ b/sentry_sdk/integrations/openai_agents.py @@ -1,36 +1,179 @@ +import sys + import sentry_sdk from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.utils import event_from_exception -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from typing import Any +from typing import Any try: - from agents import Agent # noqa: F401 + import agents + from agents import ( + Agent, + RunContextWrapper, + RunHooks, + Tool, + Usage, + ) + from agents.tracing.setup import TraceProvider + from agents.tracing.spans import Span, TSpanData + from agents.tracing.traces import Trace + except ImportError: raise DidNotEnable("OpenAI Agents not installed") +def _capture_exception(exc): + # type: (Any) -> None + event, hint = event_from_exception( + exc, + client_options=sentry_sdk.get_client().options, + mechanism={"type": OpenAIAgentsIntegration.identifier, "handled": False}, + ) + sentry_sdk.capture_event(event, hint=hint) + + +class SentryTraceProvider: + def __init__(self, original: TraceProvider): + self.original = original + + def create_trace( + self, + name: str, + trace_id: str | None = None, + disabled: bool = False, + **kwargs: Any, + ) -> Trace: + print(f"[SentryTraceProvider] create_trace: {name}") + trace = self.original.create_trace( + name, trace_id=trace_id, disabled=disabled, **kwargs + ) + return trace + + def create_span( + self, + span_data: TSpanData, + span_id: str | None = None, + parent: Trace | Span[Any] | None = None, + disabled: bool = False, + ) -> Span[TSpanData]: + print(f"[SentryTraceProvider] create_span: {span_data}") + span = self.original.create_span(span_data, span_id, parent, disabled) + + # current_span = Scope.get_current_span() + # current_trace = Scope.get_current_trace() + + # sentry_span = sentry_sdk.start_span( + # op=AGENTS_TO_OP[span_data.__class__.__name__], + # name=AGENTS_TO_NAME[span_data.__class__.__name__], + # attributes=span_data.export() + # ) + # sentry_span.finish() + return span + + def __getattr__(self, item: Any) -> Any: + return getattr(self.original, item) + + +class SentryRunHooks(RunHooks): + def __init__(self): + self.event_counter = 0 + + def _usage_to_str(self, usage: Usage) -> str: + return f"{usage.requests} requests, {usage.input_tokens} input tokens, {usage.output_tokens} output tokens, {usage.total_tokens} total tokens" + + async def on_agent_start(self, context: RunContextWrapper, agent: Agent) -> None: + self.event_counter += 1 + print( + f"### {self.event_counter}: Agent {agent.name} started. Usage: {self._usage_to_str(context.usage)}" + ) + span = sentry_sdk.start_span(op="gen_ai.agent_start", description=agent.name) + span.__enter__() + self.agent_span = span + + async def on_agent_end( + self, context: RunContextWrapper, agent: Agent, output: Any + ) -> None: + self.event_counter += 1 + print( + f"### {self.event_counter}: Agent '{agent.name}' ended with output {output}. Usage: {self._usage_to_str(context.usage)}" + ) + print(self.agent_span) + if self.agent_span: + print(f"Span exit agent: {self.agent_span}") + self.agent_span.__exit__(None, None, None) + self.agent_span = None + + async def on_tool_start( + self, context: RunContextWrapper, agent: Agent, tool: Tool + ) -> None: + self.event_counter += 1 + print( + f"### {self.event_counter}: Tool {tool.name} started. Usage: {self._usage_to_str(context.usage)}" + ) + span = sentry_sdk.start_span(op="gen_ai.tool_start", description=tool.name) + span.__enter__() + self.tool_span = span + + async def on_tool_end( + self, context: RunContextWrapper, agent: Agent, tool: Tool, result: str + ) -> None: + self.event_counter += 1 + print( + f"### {self.event_counter}: Tool {tool.name} ended with result {result}. Usage: {self._usage_to_str(context.usage)}" + ) + if self.tool_span: + print(f"Span exit tool: {self.tool_span}") + self.tool_span.__exit__(None, None, None) + self.tool_span = None + + async def on_handoff( + self, context: RunContextWrapper, from_agent: Agent, to_agent: Agent + ) -> None: + self.event_counter += 1 + print( + f"### {self.event_counter}: Handoff from '{from_agent.name}' to '{to_agent.name}'. Usage: {self._usage_to_str(context.usage)}" + ) + if self.agent_span: + span = self.agent_span.start_child( + op="gen_ai.handoff", description=f"{from_agent.name} -> {to_agent.name}" + ) + print(f"Span enter handoff: {span}") + span.__enter__() + print(f"Span exit handoff: {span}") + span.__exit__(None, None, None) + + print(f"Span exit agent: {self.agent_span}") + self.agent_span.__exit__(None, None, None) + self.agent_span = None + + +def _patch_tracer_provider(): + # Monkey path trace provider of openai-agents + name = "GLOBAL_TRACE_PROVIDER" + original = getattr(agents.tracing, name) + already_wrapped = isinstance(original, SentryTraceProvider) + if not already_wrapped: + wrapper = SentryTraceProvider(original) + for module_name, mod in sys.modules.items(): + if module_name.startswith("agents"): + try: + if getattr(mod, name, None) is original: + setattr(mod, name, wrapper) + except Exception: # pragma: no cover + pass + + class OpenAIAgentsIntegration(Integration): identifier = "openai_agents" origin = f"auto.ai.{identifier}" - def __init__(self): - pass + # def __init__(self): + # pass @staticmethod def setup_once(): # type: () -> None - pass - + # _patch_tracer_provider() -def _capture_exception(exc): - # type: (Any) -> None - event, hint = event_from_exception( - exc, - client_options=sentry_sdk.get_client().options, - mechanism={"type": OpenAIAgentsIntegration.identifier, "handled": False}, - ) - sentry_sdk.capture_event(event, hint=hint) + pass diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index d9ae5d2a2a..dc9e17e2dd 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -4,4 +4,8 @@ def test_basic(sentry_init): sentry_init(integrations=[OpenAIAgentsIntegration()]) + import ipdb + + ipdb.set_trace() + assert True From b24fe91c1411518116c385b83c358a337078dc8d Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 3 Jun 2025 10:04:04 +0200 Subject: [PATCH 03/89] Removed traceprovider. the hooks work great --- sentry_sdk/integrations/openai_agents.py | 66 ------------------------ 1 file changed, 66 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents.py b/sentry_sdk/integrations/openai_agents.py index ce5c30ad9c..0500381c9d 100644 --- a/sentry_sdk/integrations/openai_agents.py +++ b/sentry_sdk/integrations/openai_agents.py @@ -1,5 +1,3 @@ -import sys - import sentry_sdk from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.utils import event_from_exception @@ -7,7 +5,6 @@ from typing import Any try: - import agents from agents import ( Agent, RunContextWrapper, @@ -15,9 +12,6 @@ Tool, Usage, ) - from agents.tracing.setup import TraceProvider - from agents.tracing.spans import Span, TSpanData - from agents.tracing.traces import Trace except ImportError: raise DidNotEnable("OpenAI Agents not installed") @@ -33,48 +27,6 @@ def _capture_exception(exc): sentry_sdk.capture_event(event, hint=hint) -class SentryTraceProvider: - def __init__(self, original: TraceProvider): - self.original = original - - def create_trace( - self, - name: str, - trace_id: str | None = None, - disabled: bool = False, - **kwargs: Any, - ) -> Trace: - print(f"[SentryTraceProvider] create_trace: {name}") - trace = self.original.create_trace( - name, trace_id=trace_id, disabled=disabled, **kwargs - ) - return trace - - def create_span( - self, - span_data: TSpanData, - span_id: str | None = None, - parent: Trace | Span[Any] | None = None, - disabled: bool = False, - ) -> Span[TSpanData]: - print(f"[SentryTraceProvider] create_span: {span_data}") - span = self.original.create_span(span_data, span_id, parent, disabled) - - # current_span = Scope.get_current_span() - # current_trace = Scope.get_current_trace() - - # sentry_span = sentry_sdk.start_span( - # op=AGENTS_TO_OP[span_data.__class__.__name__], - # name=AGENTS_TO_NAME[span_data.__class__.__name__], - # attributes=span_data.export() - # ) - # sentry_span.finish() - return span - - def __getattr__(self, item: Any) -> Any: - return getattr(self.original, item) - - class SentryRunHooks(RunHooks): def __init__(self): self.event_counter = 0 @@ -148,22 +100,6 @@ async def on_handoff( self.agent_span = None -def _patch_tracer_provider(): - # Monkey path trace provider of openai-agents - name = "GLOBAL_TRACE_PROVIDER" - original = getattr(agents.tracing, name) - already_wrapped = isinstance(original, SentryTraceProvider) - if not already_wrapped: - wrapper = SentryTraceProvider(original) - for module_name, mod in sys.modules.items(): - if module_name.startswith("agents"): - try: - if getattr(mod, name, None) is original: - setattr(mod, name, wrapper) - except Exception: # pragma: no cover - pass - - class OpenAIAgentsIntegration(Integration): identifier = "openai_agents" origin = f"auto.ai.{identifier}" @@ -174,6 +110,4 @@ class OpenAIAgentsIntegration(Integration): @staticmethod def setup_once(): # type: () -> None - # _patch_tracer_provider() - pass From 2558fd11754119d81ebca1e6ff241ec2a91df279 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 3 Jun 2025 10:07:18 +0200 Subject: [PATCH 04/89] cleanup --- sentry_sdk/integrations/openai_agents.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents.py b/sentry_sdk/integrations/openai_agents.py index 0500381c9d..d45e724ad4 100644 --- a/sentry_sdk/integrations/openai_agents.py +++ b/sentry_sdk/integrations/openai_agents.py @@ -50,9 +50,7 @@ async def on_agent_end( print( f"### {self.event_counter}: Agent '{agent.name}' ended with output {output}. Usage: {self._usage_to_str(context.usage)}" ) - print(self.agent_span) if self.agent_span: - print(f"Span exit agent: {self.agent_span}") self.agent_span.__exit__(None, None, None) self.agent_span = None @@ -75,7 +73,6 @@ async def on_tool_end( f"### {self.event_counter}: Tool {tool.name} ended with result {result}. Usage: {self._usage_to_str(context.usage)}" ) if self.tool_span: - print(f"Span exit tool: {self.tool_span}") self.tool_span.__exit__(None, None, None) self.tool_span = None @@ -87,15 +84,11 @@ async def on_handoff( f"### {self.event_counter}: Handoff from '{from_agent.name}' to '{to_agent.name}'. Usage: {self._usage_to_str(context.usage)}" ) if self.agent_span: - span = self.agent_span.start_child( - op="gen_ai.handoff", description=f"{from_agent.name} -> {to_agent.name}" - ) - print(f"Span enter handoff: {span}") - span.__enter__() - print(f"Span exit handoff: {span}") - span.__exit__(None, None, None) - - print(f"Span exit agent: {self.agent_span}") + with self.agent_span.start_child( + op="gen_ai.handoff", description=f"{from_agent.name} > {to_agent.name}" + ): + pass + self.agent_span.__exit__(None, None, None) self.agent_span = None From 5ce67e74090c018709274c47a8ab9968675ec704 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 3 Jun 2025 11:02:40 +0200 Subject: [PATCH 05/89] use scopes, that what they are meant for. --- sentry_sdk/integrations/openai_agents.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents.py b/sentry_sdk/integrations/openai_agents.py index d45e724ad4..d447ec2845 100644 --- a/sentry_sdk/integrations/openai_agents.py +++ b/sentry_sdk/integrations/openai_agents.py @@ -41,7 +41,6 @@ async def on_agent_start(self, context: RunContextWrapper, agent: Agent) -> None ) span = sentry_sdk.start_span(op="gen_ai.agent_start", description=agent.name) span.__enter__() - self.agent_span = span async def on_agent_end( self, context: RunContextWrapper, agent: Agent, output: Any @@ -50,9 +49,9 @@ async def on_agent_end( print( f"### {self.event_counter}: Agent '{agent.name}' ended with output {output}. Usage: {self._usage_to_str(context.usage)}" ) - if self.agent_span: - self.agent_span.__exit__(None, None, None) - self.agent_span = None + current_span = sentry_sdk.get_current_span() + if current_span: + current_span.__exit__(None, None, None) async def on_tool_start( self, context: RunContextWrapper, agent: Agent, tool: Tool @@ -63,7 +62,6 @@ async def on_tool_start( ) span = sentry_sdk.start_span(op="gen_ai.tool_start", description=tool.name) span.__enter__() - self.tool_span = span async def on_tool_end( self, context: RunContextWrapper, agent: Agent, tool: Tool, result: str @@ -72,9 +70,9 @@ async def on_tool_end( print( f"### {self.event_counter}: Tool {tool.name} ended with result {result}. Usage: {self._usage_to_str(context.usage)}" ) - if self.tool_span: - self.tool_span.__exit__(None, None, None) - self.tool_span = None + current_span = sentry_sdk.get_current_span() + if current_span: + current_span.__exit__(None, None, None) async def on_handoff( self, context: RunContextWrapper, from_agent: Agent, to_agent: Agent @@ -83,14 +81,13 @@ async def on_handoff( print( f"### {self.event_counter}: Handoff from '{from_agent.name}' to '{to_agent.name}'. Usage: {self._usage_to_str(context.usage)}" ) - if self.agent_span: - with self.agent_span.start_child( + current_span = sentry_sdk.get_current_span() + if current_span: + with current_span.start_child( op="gen_ai.handoff", description=f"{from_agent.name} > {to_agent.name}" ): pass - - self.agent_span.__exit__(None, None, None) - self.agent_span = None + current_span.__exit__(None, None, None) class OpenAIAgentsIntegration(Integration): From 55f3ea88aa1a19d57bd9ab17e3d83fbfd814b9b5 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 3 Jun 2025 11:04:27 +0200 Subject: [PATCH 06/89] Cleanup --- sentry_sdk/integrations/openai_agents.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents.py b/sentry_sdk/integrations/openai_agents.py index d447ec2845..d08a798bd0 100644 --- a/sentry_sdk/integrations/openai_agents.py +++ b/sentry_sdk/integrations/openai_agents.py @@ -28,16 +28,12 @@ def _capture_exception(exc): class SentryRunHooks(RunHooks): - def __init__(self): - self.event_counter = 0 - def _usage_to_str(self, usage: Usage) -> str: return f"{usage.requests} requests, {usage.input_tokens} input tokens, {usage.output_tokens} output tokens, {usage.total_tokens} total tokens" async def on_agent_start(self, context: RunContextWrapper, agent: Agent) -> None: - self.event_counter += 1 print( - f"### {self.event_counter}: Agent {agent.name} started. Usage: {self._usage_to_str(context.usage)}" + f"### Agent {agent.name} started. Usage: {self._usage_to_str(context.usage)}" ) span = sentry_sdk.start_span(op="gen_ai.agent_start", description=agent.name) span.__enter__() @@ -45,9 +41,8 @@ async def on_agent_start(self, context: RunContextWrapper, agent: Agent) -> None async def on_agent_end( self, context: RunContextWrapper, agent: Agent, output: Any ) -> None: - self.event_counter += 1 print( - f"### {self.event_counter}: Agent '{agent.name}' ended with output {output}. Usage: {self._usage_to_str(context.usage)}" + f"### Agent '{agent.name}' ended with output {output}. Usage: {self._usage_to_str(context.usage)}" ) current_span = sentry_sdk.get_current_span() if current_span: @@ -56,9 +51,8 @@ async def on_agent_end( async def on_tool_start( self, context: RunContextWrapper, agent: Agent, tool: Tool ) -> None: - self.event_counter += 1 print( - f"### {self.event_counter}: Tool {tool.name} started. Usage: {self._usage_to_str(context.usage)}" + f"### Tool {tool.name} started. Usage: {self._usage_to_str(context.usage)}" ) span = sentry_sdk.start_span(op="gen_ai.tool_start", description=tool.name) span.__enter__() @@ -66,9 +60,8 @@ async def on_tool_start( async def on_tool_end( self, context: RunContextWrapper, agent: Agent, tool: Tool, result: str ) -> None: - self.event_counter += 1 print( - f"### {self.event_counter}: Tool {tool.name} ended with result {result}. Usage: {self._usage_to_str(context.usage)}" + f"### Tool {tool.name} ended with result {result}. Usage: {self._usage_to_str(context.usage)}" ) current_span = sentry_sdk.get_current_span() if current_span: @@ -77,9 +70,8 @@ async def on_tool_end( async def on_handoff( self, context: RunContextWrapper, from_agent: Agent, to_agent: Agent ) -> None: - self.event_counter += 1 print( - f"### {self.event_counter}: Handoff from '{from_agent.name}' to '{to_agent.name}'. Usage: {self._usage_to_str(context.usage)}" + f"### Handoff from '{from_agent.name}' to '{to_agent.name}'. Usage: {self._usage_to_str(context.usage)}" ) current_span = sentry_sdk.get_current_span() if current_span: From 03857752f138413f44dde5f6f81950706fc95d53 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 3 Jun 2025 15:44:47 +0200 Subject: [PATCH 07/89] Create transaction for runner.run --- sentry_sdk/integrations/openai_agents.py | 43 ++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents.py b/sentry_sdk/integrations/openai_agents.py index d08a798bd0..a41d476f83 100644 --- a/sentry_sdk/integrations/openai_agents.py +++ b/sentry_sdk/integrations/openai_agents.py @@ -1,10 +1,13 @@ import sentry_sdk from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.utils import event_from_exception +from functools import wraps +import asyncio from typing import Any try: + import agents from agents import ( Agent, RunContextWrapper, @@ -82,14 +85,48 @@ async def on_handoff( current_span.__exit__(None, None, None) +def _get_span_function(): + current_span = sentry_sdk.get_current_span() + is_transaction = ( + current_span is not None and current_span.containing_transaction == current_span + ) + return sentry_sdk.start_span if is_transaction else sentry_sdk.start_transaction + + +def _create_wrapper(original_func): + is_async = asyncio.iscoroutinefunction(original_func) + + @classmethod + @wraps(original_func) + async def async_wrapper(cls, *args, **kwargs): + agent = args[0] + with _get_span_function()(name=agent.name): + result = await original_func(*args, **kwargs) + return result + + @classmethod + @wraps(original_func) + def sync_wrapper(cls, *args, **kwargs): + agent = args[0] + with _get_span_function()(name=agent.name): + result = original_func(*args, **kwargs) + return result + + return async_wrapper if is_async else sync_wrapper + + +def _patch_runner(): + agents.Runner.run = _create_wrapper(agents.Runner.run) + agents.Runner.run_sync = _create_wrapper(agents.Runner.run_sync) + agents.Runner.run_streamed = _create_wrapper(agents.Runner.run_streamed) + + class OpenAIAgentsIntegration(Integration): identifier = "openai_agents" origin = f"auto.ai.{identifier}" - # def __init__(self): - # pass - @staticmethod def setup_once(): # type: () -> None + _patch_runner() pass From 2f38a88e0dcafb819e8a477fd93322f06139f7f5 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 3 Jun 2025 16:34:39 +0200 Subject: [PATCH 08/89] Enable our RunHooks automatically --- sentry_sdk/integrations/openai_agents.py | 93 +++++++++++++++++++----- 1 file changed, 73 insertions(+), 20 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents.py b/sentry_sdk/integrations/openai_agents.py index a41d476f83..2ac216c17c 100644 --- a/sentry_sdk/integrations/openai_agents.py +++ b/sentry_sdk/integrations/openai_agents.py @@ -4,7 +4,11 @@ from functools import wraps import asyncio -from typing import Any +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + from typing import Callable try: import agents @@ -31,19 +35,20 @@ def _capture_exception(exc): class SentryRunHooks(RunHooks): - def _usage_to_str(self, usage: Usage) -> str: + def _usage_to_str(self, usage): + # type: (Usage) -> str return f"{usage.requests} requests, {usage.input_tokens} input tokens, {usage.output_tokens} output tokens, {usage.total_tokens} total tokens" - async def on_agent_start(self, context: RunContextWrapper, agent: Agent) -> None: + async def on_agent_start(self, context, agent): + # type: (RunContextWrapper, Agent) -> None print( f"### Agent {agent.name} started. Usage: {self._usage_to_str(context.usage)}" ) span = sentry_sdk.start_span(op="gen_ai.agent_start", description=agent.name) span.__enter__() - async def on_agent_end( - self, context: RunContextWrapper, agent: Agent, output: Any - ) -> None: + async def on_agent_end(self, context, agent, output): + # type: (RunContextWrapper, Agent, Any) -> None print( f"### Agent '{agent.name}' ended with output {output}. Usage: {self._usage_to_str(context.usage)}" ) @@ -51,18 +56,16 @@ async def on_agent_end( if current_span: current_span.__exit__(None, None, None) - async def on_tool_start( - self, context: RunContextWrapper, agent: Agent, tool: Tool - ) -> None: + async def on_tool_start(self, context, agent, tool): + # type: (RunContextWrapper, Agent, Tool) -> None print( f"### Tool {tool.name} started. Usage: {self._usage_to_str(context.usage)}" ) span = sentry_sdk.start_span(op="gen_ai.tool_start", description=tool.name) span.__enter__() - async def on_tool_end( - self, context: RunContextWrapper, agent: Agent, tool: Tool, result: str - ) -> None: + async def on_tool_end(self, context, agent, tool, result): + # type: (RunContextWrapper, Agent, Tool, str) -> None print( f"### Tool {tool.name} ended with result {result}. Usage: {self._usage_to_str(context.usage)}" ) @@ -70,9 +73,8 @@ async def on_tool_end( if current_span: current_span.__exit__(None, None, None) - async def on_handoff( - self, context: RunContextWrapper, from_agent: Agent, to_agent: Agent - ) -> None: + async def on_handoff(self, context, from_agent, to_agent): + # type: (RunContextWrapper, Agent, Agent) -> None print( f"### Handoff from '{from_agent.name}' to '{to_agent.name}'. Usage: {self._usage_to_str(context.usage)}" ) @@ -86,6 +88,7 @@ async def on_handoff( def _get_span_function(): + # type: () -> Callable current_span = sentry_sdk.get_current_span() is_transaction = ( current_span is not None and current_span.containing_transaction == current_span @@ -93,22 +96,72 @@ def _get_span_function(): return sentry_sdk.start_span if is_transaction else sentry_sdk.start_transaction -def _create_wrapper(original_func): +def _create_hook_wrapper(original_hook, sentry_hook): + # type: (Callable, Callable) -> Callable + @wraps(original_hook) + async def async_wrapper(*args, **kwargs): + await sentry_hook(*args, **kwargs) + return await original_hook(*args, **kwargs) + + return async_wrapper + + +def _wrap_hooks(hooks): + # type: (RunHooks) -> RunHooks + """ + Our integration uses RunHooks to create spans. This function will either enable our SentryRunHooks + or if the users has given custom RunHooks wrap them so the Sentry hooks and the users hooks are both called + """ + sentry_hooks = SentryRunHooks() + + if hooks is None: + return sentry_hooks + + wrapped_hooks = type("SentryWrappedHooks", (hooks.__class__,), {}) + + # Wrap all methods from RunHooks + for method_name in dir(RunHooks): + if method_name.startswith("on_"): + original_method = getattr(hooks, method_name) + # Only wrap if the method exists in SentryRunHooks + try: + sentry_method = getattr(sentry_hooks, method_name) + setattr( + wrapped_hooks, + method_name, + _create_hook_wrapper(original_method, sentry_method), + ) + except AttributeError: + # If method doesn't exist in SentryRunHooks, just use the original method + setattr(wrapped_hooks, method_name, original_method) + + return wrapped_hooks() + + +def _create_run_wrapper(original_func): + # type: (Callable) -> Callable + """ + Wraps the run methods of the Runner class to create a root span for the agent runs. + """ is_async = asyncio.iscoroutinefunction(original_func) @classmethod @wraps(original_func) async def async_wrapper(cls, *args, **kwargs): + # type: (agents.Runner, *Any, **Any) -> Any agent = args[0] with _get_span_function()(name=agent.name): + kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) result = await original_func(*args, **kwargs) return result @classmethod @wraps(original_func) def sync_wrapper(cls, *args, **kwargs): + # type: (agents.Runner, *Any, **Any) -> Any agent = args[0] with _get_span_function()(name=agent.name): + kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) result = original_func(*args, **kwargs) return result @@ -116,9 +169,10 @@ def sync_wrapper(cls, *args, **kwargs): def _patch_runner(): - agents.Runner.run = _create_wrapper(agents.Runner.run) - agents.Runner.run_sync = _create_wrapper(agents.Runner.run_sync) - agents.Runner.run_streamed = _create_wrapper(agents.Runner.run_streamed) + # type: () -> None + agents.Runner.run = _create_run_wrapper(agents.Runner.run) + agents.Runner.run_sync = _create_run_wrapper(agents.Runner.run_sync) + agents.Runner.run_streamed = _create_run_wrapper(agents.Runner.run_streamed) class OpenAIAgentsIntegration(Integration): @@ -129,4 +183,3 @@ class OpenAIAgentsIntegration(Integration): def setup_once(): # type: () -> None _patch_runner() - pass From 4f25c2e5dd6ad4cdc4e1b57d3808d6e2a4e1408d Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 3 Jun 2025 16:41:27 +0200 Subject: [PATCH 09/89] cleanup --- sentry_sdk/integrations/openai_agents.py | 27 +++++++++++++++++------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents.py b/sentry_sdk/integrations/openai_agents.py index 2ac216c17c..a30eff4626 100644 --- a/sentry_sdk/integrations/openai_agents.py +++ b/sentry_sdk/integrations/openai_agents.py @@ -37,12 +37,18 @@ def _capture_exception(exc): class SentryRunHooks(RunHooks): def _usage_to_str(self, usage): # type: (Usage) -> str - return f"{usage.requests} requests, {usage.input_tokens} input tokens, {usage.output_tokens} output tokens, {usage.total_tokens} total tokens" + return ( + f"{usage.requests} requests, " + f"{usage.input_tokens} input tokens, " + f"{usage.output_tokens} output tokens, " + f"{usage.total_tokens} total tokens" + ) async def on_agent_start(self, context, agent): # type: (RunContextWrapper, Agent) -> None print( - f"### Agent {agent.name} started. Usage: {self._usage_to_str(context.usage)}" + f"### Agent {agent.name} started. " + f"Usage: {self._usage_to_str(context.usage)}" ) span = sentry_sdk.start_span(op="gen_ai.agent_start", description=agent.name) span.__enter__() @@ -50,7 +56,8 @@ async def on_agent_start(self, context, agent): async def on_agent_end(self, context, agent, output): # type: (RunContextWrapper, Agent, Any) -> None print( - f"### Agent '{agent.name}' ended with output {output}. Usage: {self._usage_to_str(context.usage)}" + f"### Agent '{agent.name}' ended with output {output}. " + f"Usage: {self._usage_to_str(context.usage)}" ) current_span = sentry_sdk.get_current_span() if current_span: @@ -59,7 +66,8 @@ async def on_agent_end(self, context, agent, output): async def on_tool_start(self, context, agent, tool): # type: (RunContextWrapper, Agent, Tool) -> None print( - f"### Tool {tool.name} started. Usage: {self._usage_to_str(context.usage)}" + f"### Tool {tool.name} started. " + f"Usage: {self._usage_to_str(context.usage)}" ) span = sentry_sdk.start_span(op="gen_ai.tool_start", description=tool.name) span.__enter__() @@ -67,7 +75,8 @@ async def on_tool_start(self, context, agent, tool): async def on_tool_end(self, context, agent, tool, result): # type: (RunContextWrapper, Agent, Tool, str) -> None print( - f"### Tool {tool.name} ended with result {result}. Usage: {self._usage_to_str(context.usage)}" + f"### Tool {tool.name} ended with result {result}. " + f"Usage: {self._usage_to_str(context.usage)}" ) current_span = sentry_sdk.get_current_span() if current_span: @@ -76,7 +85,8 @@ async def on_tool_end(self, context, agent, tool, result): async def on_handoff(self, context, from_agent, to_agent): # type: (RunContextWrapper, Agent, Agent) -> None print( - f"### Handoff from '{from_agent.name}' to '{to_agent.name}'. Usage: {self._usage_to_str(context.usage)}" + f"### Handoff from '{from_agent.name}' to '{to_agent.name}'. " + f"Usage: {self._usage_to_str(context.usage)}" ) current_span = sentry_sdk.get_current_span() if current_span: @@ -109,8 +119,9 @@ async def async_wrapper(*args, **kwargs): def _wrap_hooks(hooks): # type: (RunHooks) -> RunHooks """ - Our integration uses RunHooks to create spans. This function will either enable our SentryRunHooks - or if the users has given custom RunHooks wrap them so the Sentry hooks and the users hooks are both called + Our integration uses RunHooks to create spans. This function will either + enable our SentryRunHooks or if the users has given custom RunHooks wrap + them so the Sentry hooks and the users hooks are both called """ sentry_hooks = SentryRunHooks() From 22ed21c7caad273966b7bc0f0949e90d059ba049 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 4 Jun 2025 08:40:05 +0200 Subject: [PATCH 10/89] Better naming --- sentry_sdk/integrations/openai_agents.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents.py b/sentry_sdk/integrations/openai_agents.py index a30eff4626..a21fe39c6c 100644 --- a/sentry_sdk/integrations/openai_agents.py +++ b/sentry_sdk/integrations/openai_agents.py @@ -50,7 +50,9 @@ async def on_agent_start(self, context, agent): f"### Agent {agent.name} started. " f"Usage: {self._usage_to_str(context.usage)}" ) - span = sentry_sdk.start_span(op="gen_ai.agent_start", description=agent.name) + span = sentry_sdk.start_span( + op="gen_ai.invoke_agent", name=f"invoke_agent {agent.name}" + ) span.__enter__() async def on_agent_end(self, context, agent, output): @@ -69,7 +71,9 @@ async def on_tool_start(self, context, agent, tool): f"### Tool {tool.name} started. " f"Usage: {self._usage_to_str(context.usage)}" ) - span = sentry_sdk.start_span(op="gen_ai.tool_start", description=tool.name) + span = sentry_sdk.start_span( + op="gen_ai.execute_tool", name=f"execute_tool {tool.name}" + ) span.__enter__() async def on_tool_end(self, context, agent, tool, result): @@ -91,13 +95,14 @@ async def on_handoff(self, context, from_agent, to_agent): current_span = sentry_sdk.get_current_span() if current_span: with current_span.start_child( - op="gen_ai.handoff", description=f"{from_agent.name} > {to_agent.name}" + op="gen_ai.handoff", + name=f"handoff from {from_agent.name} to {to_agent.name}", ): pass current_span.__exit__(None, None, None) -def _get_span_function(): +def _get_start_span_function(): # type: () -> Callable current_span = sentry_sdk.get_current_span() is_transaction = ( @@ -161,7 +166,7 @@ def _create_run_wrapper(original_func): async def async_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any agent = args[0] - with _get_span_function()(name=agent.name): + with _get_start_span_function()(name=f"{agent.name} workflow"): kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) result = await original_func(*args, **kwargs) return result @@ -171,7 +176,7 @@ async def async_wrapper(cls, *args, **kwargs): def sync_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any agent = args[0] - with _get_span_function()(name=agent.name): + with _get_start_span_function()(name=f"{agent.name} workflow"): kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) result = original_func(*args, **kwargs) return result From be376796123fda4221fa46884aa38f3a9af255af Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 4 Jun 2025 08:49:44 +0200 Subject: [PATCH 11/89] organized code --- .../integrations/openai_agents/__init__.py | 25 +++++ .../integrations/openai_agents/run_hooks.py | 87 ++++++++++++++++ .../utils.py} | 98 +------------------ 3 files changed, 116 insertions(+), 94 deletions(-) create mode 100644 sentry_sdk/integrations/openai_agents/__init__.py create mode 100644 sentry_sdk/integrations/openai_agents/run_hooks.py rename sentry_sdk/integrations/{openai_agents.py => openai_agents/utils.py} (51%) diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py new file mode 100644 index 0000000000..064b78b63a --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -0,0 +1,25 @@ +from sentry_sdk.integrations import DidNotEnable, Integration +from sentry_sdk.integrations.openai_agents.utils import _create_run_wrapper + +try: + import agents + +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + + +def _patch_runner(): + # type: () -> None + agents.Runner.run = _create_run_wrapper(agents.Runner.run) + agents.Runner.run_sync = _create_run_wrapper(agents.Runner.run_sync) + agents.Runner.run_streamed = _create_run_wrapper(agents.Runner.run_streamed) + + +class OpenAIAgentsIntegration(Integration): + identifier = "openai_agents" + origin = f"auto.ai.{identifier}" + + @staticmethod + def setup_once(): + # type: () -> None + _patch_runner() diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py new file mode 100644 index 0000000000..3f45b00b53 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -0,0 +1,87 @@ +import sentry_sdk +from sentry_sdk.integrations import DidNotEnable + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +try: + from agents import ( + Agent, + RunContextWrapper, + RunHooks, + Tool, + Usage, + ) + +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + + +class SentryRunHooks(RunHooks): + def _usage_to_str(self, usage): + # type: (Usage) -> str + return ( + f"{usage.requests} requests, " + f"{usage.input_tokens} input tokens, " + f"{usage.output_tokens} output tokens, " + f"{usage.total_tokens} total tokens" + ) + + async def on_agent_start(self, context, agent): + # type: (RunContextWrapper, Agent) -> None + print( + f"### Agent {agent.name} started. " + f"Usage: {self._usage_to_str(context.usage)}" + ) + span = sentry_sdk.start_span( + op="gen_ai.invoke_agent", name=f"invoke_agent {agent.name}" + ) + span.__enter__() + + async def on_agent_end(self, context, agent, output): + # type: (RunContextWrapper, Agent, Any) -> None + print( + f"### Agent '{agent.name}' ended with output {output}. " + f"Usage: {self._usage_to_str(context.usage)}" + ) + current_span = sentry_sdk.get_current_span() + if current_span: + current_span.__exit__(None, None, None) + + async def on_tool_start(self, context, agent, tool): + # type: (RunContextWrapper, Agent, Tool) -> None + print( + f"### Tool {tool.name} started. " + f"Usage: {self._usage_to_str(context.usage)}" + ) + span = sentry_sdk.start_span( + op="gen_ai.execute_tool", name=f"execute_tool {tool.name}" + ) + span.__enter__() + + async def on_tool_end(self, context, agent, tool, result): + # type: (RunContextWrapper, Agent, Tool, str) -> None + print( + f"### Tool {tool.name} ended with result {result}. " + f"Usage: {self._usage_to_str(context.usage)}" + ) + current_span = sentry_sdk.get_current_span() + if current_span: + current_span.__exit__(None, None, None) + + async def on_handoff(self, context, from_agent, to_agent): + # type: (RunContextWrapper, Agent, Agent) -> None + print( + f"### Handoff from '{from_agent.name}' to '{to_agent.name}'. " + f"Usage: {self._usage_to_str(context.usage)}" + ) + current_span = sentry_sdk.get_current_span() + if current_span: + with current_span.start_child( + op="gen_ai.handoff", + name=f"handoff from {from_agent.name} to {to_agent.name}", + ): + pass + current_span.__exit__(None, None, None) diff --git a/sentry_sdk/integrations/openai_agents.py b/sentry_sdk/integrations/openai_agents/utils.py similarity index 51% rename from sentry_sdk/integrations/openai_agents.py rename to sentry_sdk/integrations/openai_agents/utils.py index a21fe39c6c..db1ba024ed 100644 --- a/sentry_sdk/integrations/openai_agents.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,5 +1,6 @@ import sentry_sdk -from sentry_sdk.integrations import DidNotEnable, Integration +from sentry_sdk.integrations import DidNotEnable +from sentry_sdk.integrations.openai_agents.run_hooks import SentryRunHooks from sentry_sdk.utils import event_from_exception from functools import wraps import asyncio @@ -12,13 +13,7 @@ try: import agents - from agents import ( - Agent, - RunContextWrapper, - RunHooks, - Tool, - Usage, - ) + from agents import RunHooks except ImportError: raise DidNotEnable("OpenAI Agents not installed") @@ -29,79 +24,11 @@ def _capture_exception(exc): event, hint = event_from_exception( exc, client_options=sentry_sdk.get_client().options, - mechanism={"type": OpenAIAgentsIntegration.identifier, "handled": False}, + mechanism={"type": "openai_agents", "handled": False}, ) sentry_sdk.capture_event(event, hint=hint) -class SentryRunHooks(RunHooks): - def _usage_to_str(self, usage): - # type: (Usage) -> str - return ( - f"{usage.requests} requests, " - f"{usage.input_tokens} input tokens, " - f"{usage.output_tokens} output tokens, " - f"{usage.total_tokens} total tokens" - ) - - async def on_agent_start(self, context, agent): - # type: (RunContextWrapper, Agent) -> None - print( - f"### Agent {agent.name} started. " - f"Usage: {self._usage_to_str(context.usage)}" - ) - span = sentry_sdk.start_span( - op="gen_ai.invoke_agent", name=f"invoke_agent {agent.name}" - ) - span.__enter__() - - async def on_agent_end(self, context, agent, output): - # type: (RunContextWrapper, Agent, Any) -> None - print( - f"### Agent '{agent.name}' ended with output {output}. " - f"Usage: {self._usage_to_str(context.usage)}" - ) - current_span = sentry_sdk.get_current_span() - if current_span: - current_span.__exit__(None, None, None) - - async def on_tool_start(self, context, agent, tool): - # type: (RunContextWrapper, Agent, Tool) -> None - print( - f"### Tool {tool.name} started. " - f"Usage: {self._usage_to_str(context.usage)}" - ) - span = sentry_sdk.start_span( - op="gen_ai.execute_tool", name=f"execute_tool {tool.name}" - ) - span.__enter__() - - async def on_tool_end(self, context, agent, tool, result): - # type: (RunContextWrapper, Agent, Tool, str) -> None - print( - f"### Tool {tool.name} ended with result {result}. " - f"Usage: {self._usage_to_str(context.usage)}" - ) - current_span = sentry_sdk.get_current_span() - if current_span: - current_span.__exit__(None, None, None) - - async def on_handoff(self, context, from_agent, to_agent): - # type: (RunContextWrapper, Agent, Agent) -> None - print( - f"### Handoff from '{from_agent.name}' to '{to_agent.name}'. " - f"Usage: {self._usage_to_str(context.usage)}" - ) - current_span = sentry_sdk.get_current_span() - if current_span: - with current_span.start_child( - op="gen_ai.handoff", - name=f"handoff from {from_agent.name} to {to_agent.name}", - ): - pass - current_span.__exit__(None, None, None) - - def _get_start_span_function(): # type: () -> Callable current_span = sentry_sdk.get_current_span() @@ -182,20 +109,3 @@ def sync_wrapper(cls, *args, **kwargs): return result return async_wrapper if is_async else sync_wrapper - - -def _patch_runner(): - # type: () -> None - agents.Runner.run = _create_run_wrapper(agents.Runner.run) - agents.Runner.run_sync = _create_run_wrapper(agents.Runner.run_sync) - agents.Runner.run_streamed = _create_run_wrapper(agents.Runner.run_streamed) - - -class OpenAIAgentsIntegration(Integration): - identifier = "openai_agents" - origin = f"auto.ai.{identifier}" - - @staticmethod - def setup_once(): - # type: () -> None - _patch_runner() From 1a72a92436b29edb3ed465d77adc9a8b0b3bf293 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 4 Jun 2025 09:40:54 +0200 Subject: [PATCH 12/89] Set some common data --- .../integrations/openai_agents/run_hooks.py | 5 ++ .../integrations/openai_agents/utils.py | 79 ++++++++++++++++++- 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py index 3f45b00b53..c2b3ec8722 100644 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -3,6 +3,9 @@ from typing import TYPE_CHECKING +from .utils import _set_agent_data + + if TYPE_CHECKING: from typing import Any @@ -39,6 +42,7 @@ async def on_agent_start(self, context, agent): op="gen_ai.invoke_agent", name=f"invoke_agent {agent.name}" ) span.__enter__() + _set_agent_data(agent) async def on_agent_end(self, context, agent, output): # type: (RunContextWrapper, Agent, Any) -> None @@ -60,6 +64,7 @@ async def on_tool_start(self, context, agent, tool): op="gen_ai.execute_tool", name=f"execute_tool {tool.name}" ) span.__enter__() + _set_agent_data(agent) async def on_tool_end(self, context, agent, tool, result): # type: (RunContextWrapper, Agent, Tool, str) -> None diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index db1ba024ed..fc0dc651af 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,9 +1,9 @@ +import asyncio +from functools import wraps + import sentry_sdk from sentry_sdk.integrations import DidNotEnable -from sentry_sdk.integrations.openai_agents.run_hooks import SentryRunHooks from sentry_sdk.utils import event_from_exception -from functools import wraps -import asyncio from typing import TYPE_CHECKING @@ -55,6 +55,8 @@ def _wrap_hooks(hooks): enable our SentryRunHooks or if the users has given custom RunHooks wrap them so the Sentry hooks and the users hooks are both called """ + from .run_hooks import SentryRunHooks + sentry_hooks = SentryRunHooks() if hooks is None: @@ -81,6 +83,71 @@ def _wrap_hooks(hooks): return wrapped_hooks() +def _set_agent_data(agent): + # type: (agents.Agent) -> None + span = sentry_sdk.get_current_span() + + """ + Thats the Agent object: + {'handoff_description': None, + 'handoffs': [], + 'hooks': None, + 'input_guardrails': [], + 'instructions': 'You are a helpful research assistant. Given a query, come up ' + 'with a set of web searches to perform to best answer the ' + 'query. Output between 5 and 20 terms to query for.', + 'mcp_config': {}, + 'mcp_servers': [], + 'model': 'gpt-4o', + 'model_settings': ModelSettings(temperature=None, + top_p=None, + frequency_penalty=None, + presence_penalty=None, + tool_choice=None, + parallel_tool_calls=None, + truncation=None, + max_tokens=None, + reasoning=None, + metadata=None, + store=None, + include_usage=None, + extra_query=None, + extra_body=None, + extra_headers=None), + 'name': 'PlannerAgent', + 'output_guardrails': [], + 'output_type': , + 'reset_tool_choice': True, + 'tool_use_behavior': 'run_llm_again', + 'tools': []} + """ + span.set_data("gen_ai.agent.name", agent.name) + + if agent.model_settings.max_tokens: + span.set_data("gen_ai.request.max_tokens", agent.model_settings.max_tokens) + + if agent.model: + span.set_data("gen_ai.request.model", agent.model) + + if agent.model_settings.presence_penalty: + span.set_data( + "gen_ai.request.presence_penalty", agent.model_settings.presence_penalty + ) + + if agent.model_settings.temperature: + span.set_data("gen_ai.request.temperature", agent.model_settings.temperature) + + if agent.model_settings.top_p: + span.set_data("gen_ai.request.top_p", agent.model_settings.top_p) + + if agent.model_settings.frequency_penalty: + span.set_data( + "gen_ai.request.frequency_penalty", agent.model_settings.frequency_penalty + ) + + # span.set_data("", ) + + def _create_run_wrapper(original_func): # type: (Callable) -> Callable """ @@ -95,7 +162,10 @@ async def async_wrapper(cls, *args, **kwargs): agent = args[0] with _get_start_span_function()(name=f"{agent.name} workflow"): kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) + _set_agent_data(agent) + result = await original_func(*args, **kwargs) + return result @classmethod @@ -105,7 +175,10 @@ def sync_wrapper(cls, *args, **kwargs): agent = args[0] with _get_start_span_function()(name=f"{agent.name} workflow"): kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) + _set_agent_data(agent) + result = original_func(*args, **kwargs) + return result return async_wrapper if is_async else sync_wrapper From f28fc61d7bb96100e50e8d1df8d6b858c6e3a0f0 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 4 Jun 2025 10:39:44 +0200 Subject: [PATCH 13/89] work on tool span --- .../integrations/openai_agents/run_hooks.py | 61 ++++++++++++------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py index c2b3ec8722..8f608b0480 100644 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -22,21 +22,41 @@ raise DidNotEnable("OpenAI Agents not installed") -class SentryRunHooks(RunHooks): - def _usage_to_str(self, usage): - # type: (Usage) -> str - return ( - f"{usage.requests} requests, " - f"{usage.input_tokens} input tokens, " - f"{usage.output_tokens} output tokens, " - f"{usage.total_tokens} total tokens" - ) +def _usage_to_str(usage): + # type: (Usage) -> str + return ( + f"{usage.requests} requests, " + f"{usage.input_tokens} input tokens, " + f"{usage.output_tokens} output tokens, " + f"{usage.total_tokens} total tokens" + ) + + +def execute_tool_span(context, agent, tool): + print(f"### Tool {tool.name} started. " f"Usage: {_usage_to_str(context.usage)}") + span = sentry_sdk.start_span( + op="gen_ai.execute_tool", name=f"execute_tool {tool.name}" + ) + span.__enter__() + import ipdb + + ipdb.set_trace() + span.set_data("gen_ai.operation.name", "execute_tool") + + if tool.__class__.__name__ == "FunctionTool": + span.set_data("gen_ai.tool.type", "function") + + span.set_data("gen_ai.tool.name", tool.name) + span.set_data("gen_ai.tool.description", tool.description) + # span.set_data("gen_ai.tool.message", ) + _set_agent_data(agent) + +class SentryRunHooks(RunHooks): async def on_agent_start(self, context, agent): # type: (RunContextWrapper, Agent) -> None print( - f"### Agent {agent.name} started. " - f"Usage: {self._usage_to_str(context.usage)}" + f"### Agent {agent.name} started. " f"Usage: {_usage_to_str(context.usage)}" ) span = sentry_sdk.start_span( op="gen_ai.invoke_agent", name=f"invoke_agent {agent.name}" @@ -48,7 +68,7 @@ async def on_agent_end(self, context, agent, output): # type: (RunContextWrapper, Agent, Any) -> None print( f"### Agent '{agent.name}' ended with output {output}. " - f"Usage: {self._usage_to_str(context.usage)}" + f"Usage: {_usage_to_str(context.usage)}" ) current_span = sentry_sdk.get_current_span() if current_span: @@ -56,31 +76,26 @@ async def on_agent_end(self, context, agent, output): async def on_tool_start(self, context, agent, tool): # type: (RunContextWrapper, Agent, Tool) -> None - print( - f"### Tool {tool.name} started. " - f"Usage: {self._usage_to_str(context.usage)}" - ) - span = sentry_sdk.start_span( - op="gen_ai.execute_tool", name=f"execute_tool {tool.name}" - ) - span.__enter__() - _set_agent_data(agent) + execute_tool_span(context, agent, tool) async def on_tool_end(self, context, agent, tool, result): # type: (RunContextWrapper, Agent, Tool, str) -> None print( f"### Tool {tool.name} ended with result {result}. " - f"Usage: {self._usage_to_str(context.usage)}" + f"Usage: {_usage_to_str(context.usage)}" ) current_span = sentry_sdk.get_current_span() if current_span: + import ipdb + + ipdb.set_trace() current_span.__exit__(None, None, None) async def on_handoff(self, context, from_agent, to_agent): # type: (RunContextWrapper, Agent, Agent) -> None print( f"### Handoff from '{from_agent.name}' to '{to_agent.name}'. " - f"Usage: {self._usage_to_str(context.usage)}" + f"Usage: {_usage_to_str(context.usage)}" ) current_span = sentry_sdk.get_current_span() if current_span: From 2e249972b11e2a3b70eba9cd3ffc8cadc4f1fdf7 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 4 Jun 2025 10:49:06 +0200 Subject: [PATCH 14/89] organize code --- .../integrations/openai_agents/run_hooks.py | 69 ++++--------------- .../integrations/openai_agents/utils.py | 11 +++ 2 files changed, 24 insertions(+), 56 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py index 8f608b0480..d3969544f3 100644 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -3,7 +3,13 @@ from typing import TYPE_CHECKING -from .utils import _set_agent_data +from sentry_sdk.integrations.openai_agents.spans import ( + execute_tool_span, + invoke_agent_span, + handoff_span, +) + +from .utils import _usage_to_str if TYPE_CHECKING: @@ -15,54 +21,15 @@ RunContextWrapper, RunHooks, Tool, - Usage, ) except ImportError: raise DidNotEnable("OpenAI Agents not installed") -def _usage_to_str(usage): - # type: (Usage) -> str - return ( - f"{usage.requests} requests, " - f"{usage.input_tokens} input tokens, " - f"{usage.output_tokens} output tokens, " - f"{usage.total_tokens} total tokens" - ) - - -def execute_tool_span(context, agent, tool): - print(f"### Tool {tool.name} started. " f"Usage: {_usage_to_str(context.usage)}") - span = sentry_sdk.start_span( - op="gen_ai.execute_tool", name=f"execute_tool {tool.name}" - ) - span.__enter__() - import ipdb - - ipdb.set_trace() - span.set_data("gen_ai.operation.name", "execute_tool") - - if tool.__class__.__name__ == "FunctionTool": - span.set_data("gen_ai.tool.type", "function") - - span.set_data("gen_ai.tool.name", tool.name) - span.set_data("gen_ai.tool.description", tool.description) - # span.set_data("gen_ai.tool.message", ) - _set_agent_data(agent) - - class SentryRunHooks(RunHooks): - async def on_agent_start(self, context, agent): - # type: (RunContextWrapper, Agent) -> None - print( - f"### Agent {agent.name} started. " f"Usage: {_usage_to_str(context.usage)}" - ) - span = sentry_sdk.start_span( - op="gen_ai.invoke_agent", name=f"invoke_agent {agent.name}" - ) - span.__enter__() - _set_agent_data(agent) + async def on_agent_start(self, context: RunContextWrapper, agent: Agent) -> None: + invoke_agent_span(context, agent) async def on_agent_end(self, context, agent, output): # type: (RunContextWrapper, Agent, Any) -> None @@ -91,17 +58,7 @@ async def on_tool_end(self, context, agent, tool, result): ipdb.set_trace() current_span.__exit__(None, None, None) - async def on_handoff(self, context, from_agent, to_agent): - # type: (RunContextWrapper, Agent, Agent) -> None - print( - f"### Handoff from '{from_agent.name}' to '{to_agent.name}'. " - f"Usage: {_usage_to_str(context.usage)}" - ) - current_span = sentry_sdk.get_current_span() - if current_span: - with current_span.start_child( - op="gen_ai.handoff", - name=f"handoff from {from_agent.name} to {to_agent.name}", - ): - pass - current_span.__exit__(None, None, None) + async def on_handoff( + self, context: RunContextWrapper, from_agent: Agent, to_agent: Agent + ) -> None: + handoff_span(context, from_agent, to_agent) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index fc0dc651af..c928890148 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from typing import Any from typing import Callable + from agents import Usage try: import agents @@ -29,6 +30,16 @@ def _capture_exception(exc): sentry_sdk.capture_event(event, hint=hint) +def _usage_to_str(usage): + # type: (Usage) -> str + return ( + f"{usage.requests} requests, " + f"{usage.input_tokens} input tokens, " + f"{usage.output_tokens} output tokens, " + f"{usage.total_tokens} total tokens" + ) + + def _get_start_span_function(): # type: () -> Callable current_span = sentry_sdk.get_current_span() From bba3ef24ae2ee8cac134d6467efcc2bf99525d0d Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 4 Jun 2025 10:53:53 +0200 Subject: [PATCH 15/89] organize code --- .../integrations/openai_agents/run_hooks.py | 26 ++++--------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py index d3969544f3..761dbae327 100644 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -1,16 +1,15 @@ -import sentry_sdk from sentry_sdk.integrations import DidNotEnable from typing import TYPE_CHECKING from sentry_sdk.integrations.openai_agents.spans import ( execute_tool_span, - invoke_agent_span, handoff_span, + invoke_agent_span, + update_execute_tool_span, + update_invoke_agent_span, ) -from .utils import _usage_to_str - if TYPE_CHECKING: from typing import Any @@ -33,13 +32,7 @@ async def on_agent_start(self, context: RunContextWrapper, agent: Agent) -> None async def on_agent_end(self, context, agent, output): # type: (RunContextWrapper, Agent, Any) -> None - print( - f"### Agent '{agent.name}' ended with output {output}. " - f"Usage: {_usage_to_str(context.usage)}" - ) - current_span = sentry_sdk.get_current_span() - if current_span: - current_span.__exit__(None, None, None) + update_invoke_agent_span(context, agent, output) async def on_tool_start(self, context, agent, tool): # type: (RunContextWrapper, Agent, Tool) -> None @@ -47,16 +40,7 @@ async def on_tool_start(self, context, agent, tool): async def on_tool_end(self, context, agent, tool, result): # type: (RunContextWrapper, Agent, Tool, str) -> None - print( - f"### Tool {tool.name} ended with result {result}. " - f"Usage: {_usage_to_str(context.usage)}" - ) - current_span = sentry_sdk.get_current_span() - if current_span: - import ipdb - - ipdb.set_trace() - current_span.__exit__(None, None, None) + update_execute_tool_span(context, agent, tool, result) async def on_handoff( self, context: RunContextWrapper, from_agent: Agent, to_agent: Agent From cf06427dd0152502c35273e1c54f0329f2632b49 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 4 Jun 2025 11:01:13 +0200 Subject: [PATCH 16/89] organize code --- sentry_sdk/integrations/openai_agents/run_hooks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py index 761dbae327..e9c94cbe9f 100644 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -6,8 +6,8 @@ execute_tool_span, handoff_span, invoke_agent_span, - update_execute_tool_span, - update_invoke_agent_span, + finish_execute_tool_span, + finish_invoke_agent_span, ) @@ -32,7 +32,7 @@ async def on_agent_start(self, context: RunContextWrapper, agent: Agent) -> None async def on_agent_end(self, context, agent, output): # type: (RunContextWrapper, Agent, Any) -> None - update_invoke_agent_span(context, agent, output) + finish_invoke_agent_span(context, agent, output) async def on_tool_start(self, context, agent, tool): # type: (RunContextWrapper, Agent, Tool) -> None @@ -40,7 +40,7 @@ async def on_tool_start(self, context, agent, tool): async def on_tool_end(self, context, agent, tool, result): # type: (RunContextWrapper, Agent, Tool, str) -> None - update_execute_tool_span(context, agent, tool, result) + finish_execute_tool_span(context, agent, tool, result) async def on_handoff( self, context: RunContextWrapper, from_agent: Agent, to_agent: Agent From d9acb1e8fc5454002c29dc7efc6f99f0df147eed Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 4 Jun 2025 11:06:36 +0200 Subject: [PATCH 17/89] organize code and set operation name --- .../openai_agents/spans/__init__.py | 3 ++ .../openai_agents/spans/execute_tool.py | 37 +++++++++++++++++++ .../openai_agents/spans/handoff.py | 24 ++++++++++++ .../openai_agents/spans/invoke_agent.py | 32 ++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 sentry_sdk/integrations/openai_agents/spans/__init__.py create mode 100644 sentry_sdk/integrations/openai_agents/spans/execute_tool.py create mode 100644 sentry_sdk/integrations/openai_agents/spans/handoff.py create mode 100644 sentry_sdk/integrations/openai_agents/spans/invoke_agent.py diff --git a/sentry_sdk/integrations/openai_agents/spans/__init__.py b/sentry_sdk/integrations/openai_agents/spans/__init__.py new file mode 100644 index 0000000000..acc453e5e6 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/spans/__init__.py @@ -0,0 +1,3 @@ +from .execute_tool import execute_tool_span, finish_execute_tool_span # noqa: F401 +from .invoke_agent import invoke_agent_span, finish_invoke_agent_span # noqa: F401 +from .handoff import handoff_span # noqa: F401 diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py new file mode 100644 index 0000000000..ff2ff85968 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -0,0 +1,37 @@ +import sentry_sdk +from sentry_sdk.integrations.openai_agents.utils import _set_agent_data, _usage_to_str + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from agents import Agent, RunContextWrapper, Tool + + +def execute_tool_span(context, agent, tool): + # type: (RunContextWrapper, Agent, Tool) -> None + + print(f"### Tool {tool.name} started. " f"Usage: {_usage_to_str(context.usage)}") + span = sentry_sdk.start_span( + op="gen_ai.execute_tool", name=f"execute_tool {tool.name}" + ) + span.__enter__() + + span.set_data("gen_ai.operation.name", "execute_tool") + span.set_data("gen_ai.tool.name", tool.name) + span.set_data("gen_ai.tool.description", tool.description) + + if tool.__class__.__name__ == "FunctionTool": + span.set_data("gen_ai.tool.type", "function") + + _set_agent_data(agent) + + +def finish_execute_tool_span(context, agent, tool, result): + # type: (RunContextWrapper, Agent, Tool, str) -> None + print( + f"### Tool {tool.name} ended with result {result}. " + f"Usage: {_usage_to_str(context.usage)}" + ) + current_span = sentry_sdk.get_current_span() + if current_span: + current_span.__exit__(None, None, None) diff --git a/sentry_sdk/integrations/openai_agents/spans/handoff.py b/sentry_sdk/integrations/openai_agents/spans/handoff.py new file mode 100644 index 0000000000..7efd5bb876 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/spans/handoff.py @@ -0,0 +1,24 @@ +import sentry_sdk +from sentry_sdk.integrations.openai_agents.utils import _usage_to_str + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from agents import Agent, RunContextWrapper + + +def handoff_span(context, from_agent, to_agent): + # type: (RunContextWrapper, Agent, Agent) -> None + print( + f"### Handoff from '{from_agent.name}' to '{to_agent.name}'. " + f"Usage: {_usage_to_str(context.usage)}" + ) + current_span = sentry_sdk.get_current_span() + if current_span: + with current_span.start_child( + op="gen_ai.handoff", + name=f"handoff from {from_agent.name} to {to_agent.name}", + ) as span: + span.set_data("gen_ai.operation.name", "handoff") + + current_span.__exit__(None, None, None) diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py new file mode 100644 index 0000000000..51bc88ef78 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -0,0 +1,32 @@ +import sentry_sdk +from sentry_sdk.integrations.openai_agents.utils import _usage_to_str, _set_agent_data + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + from agents import Agent, RunContextWrapper + + +def invoke_agent_span(context, agent): + # type: (RunContextWrapper, Agent) -> None + print(f"### Agent {agent.name} started. " f"Usage: {_usage_to_str(context.usage)}") + span = sentry_sdk.start_span( + op="gen_ai.invoke_agent", name=f"invoke_agent {agent.name}" + ) + span.__enter__() + + span.set_data("gen_ai.operation.name", "invoke_agent") + + _set_agent_data(agent) + + +def finish_invoke_agent_span(context, agent, output): + # type: (RunContextWrapper, Agent, Any) -> None + print( + f"### Agent '{agent.name}' ended with output {output}. " + f"Usage: {_usage_to_str(context.usage)}" + ) + current_span = sentry_sdk.get_current_span() + if current_span: + current_span.__exit__(None, None, None) From 87f656244f09605272dbf78f7a7ed90dc9ed3e39 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 4 Jun 2025 12:36:22 +0200 Subject: [PATCH 18/89] set gen_ai.system --- sentry_sdk/integrations/openai_agents/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index c928890148..a88b748120 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -132,6 +132,10 @@ def _set_agent_data(agent): 'tool_use_behavior': 'run_llm_again', 'tools': []} """ + span.set_data( + "gen_ai.system", "openai" + ) # See footnote for https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/#gen-ai-system for explanation why. + span.set_data("gen_ai.agent.name", agent.name) if agent.model_settings.max_tokens: From c8a89db43ccf298d80b177336ce08aed7194f7e7 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 5 Jun 2025 11:36:16 +0200 Subject: [PATCH 19/89] ai client spans --- .../integrations/openai_agents/__init__.py | 11 ++- .../openai_agents/spans/__init__.py | 3 +- .../openai_agents/spans/ai_client.py | 70 +++++++++++++++++++ .../integrations/openai_agents/utils.py | 27 ++++++- 4 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 sentry_sdk/integrations/openai_agents/spans/ai_client.py diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py index 064b78b63a..b3c7bbccbd 100644 --- a/sentry_sdk/integrations/openai_agents/__init__.py +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -1,5 +1,9 @@ from sentry_sdk.integrations import DidNotEnable, Integration -from sentry_sdk.integrations.openai_agents.utils import _create_run_wrapper +from sentry_sdk.integrations.openai_agents.utils import ( + _create_run_wrapper, + _create_get_model_wrapper, +) + try: import agents @@ -15,6 +19,10 @@ def _patch_runner(): agents.Runner.run_streamed = _create_run_wrapper(agents.Runner.run_streamed) +def _patch_model(): + agents.Runner._get_model = _create_get_model_wrapper(agents.Runner._get_model) + + class OpenAIAgentsIntegration(Integration): identifier = "openai_agents" origin = f"auto.ai.{identifier}" @@ -23,3 +31,4 @@ class OpenAIAgentsIntegration(Integration): def setup_once(): # type: () -> None _patch_runner() + _patch_model() diff --git a/sentry_sdk/integrations/openai_agents/spans/__init__.py b/sentry_sdk/integrations/openai_agents/spans/__init__.py index acc453e5e6..2fbc4c9122 100644 --- a/sentry_sdk/integrations/openai_agents/spans/__init__.py +++ b/sentry_sdk/integrations/openai_agents/spans/__init__.py @@ -1,3 +1,4 @@ +from .ai_client import ai_client_span, finish_ai_client_span # noqa: F401 from .execute_tool import execute_tool_span, finish_execute_tool_span # noqa: F401 -from .invoke_agent import invoke_agent_span, finish_invoke_agent_span # noqa: F401 from .handoff import handoff_span # noqa: F401 +from .invoke_agent import invoke_agent_span, finish_invoke_agent_span # noqa: F401 diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py new file mode 100644 index 0000000000..bc4e16a0dd --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -0,0 +1,70 @@ +import sentry_sdk + +from ..utils import _set_agent_data + + +def ai_client_span(agent, model, run_config, get_response_kwargs): + # import ipdb; ipdb.set_trace() + + """ + https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#inference + + gen_ai.operation.name # chat|generate_content|text_completion|embeddings + gen_ai.system + gen_ai.conversation.id + gen_ai.output.type + gen_ai.request.choice.count + gen_ai.request.model + gen_ai.request.seed + gen_ai.request.frequency_penalty + gen_ai.request.max_tokens + gen_ai.request.presence_penalty + gen_ai.request.stop_sequences + gen_ai.request.temperature + gen_ai.request.top_k + gen_ai.request.top_p + gen_ai.response.finish_reasons + gen_ai.response.id + gen_ai.response.model + gen_ai.usage.input_tokens + gen_ai.usage.output_tokens + server.address + server.port + error.type + + + INPUT: + gen_ai.system.message + gen_ai.user.message + gen_ai.assistant.message + gen_ai.tool.message (response from tool call as input) + + OUTPUT: + gen_ai.choice + + """ + # TODO-anton: implement other types of operations + return sentry_sdk.start_span( + op="gen_ai.chat", + description=f"*chat* (TODO: remove hardcoded stuff) {agent.model}", + ) + + +def finish_ai_client_span(agent, model, run_config, get_response_kwargs, result): + _set_agent_data(agent) + span = sentry_sdk.get_current_span() + # import ipdb; ipdb.set_trace() + + if get_response_kwargs.get("system_instructions"): + span.set_data( + "gen_ai.system.message", get_response_kwargs.get("system_instructions") + ) + + for message in get_response_kwargs.get("input", []): + if message.get("role") == "user": + span.set_data("gen_ai.user.message", message.get("content")) + break + + for index, message in enumerate(result.output): + span.set_data(f"output.{index}", message) + # if message.get("type") == "function_call": diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index a88b748120..99f2ee4672 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -160,8 +160,6 @@ def _set_agent_data(agent): "gen_ai.request.frequency_penalty", agent.model_settings.frequency_penalty ) - # span.set_data("", ) - def _create_run_wrapper(original_func): # type: (Callable) -> Callable @@ -197,3 +195,28 @@ def sync_wrapper(cls, *args, **kwargs): return result return async_wrapper if is_async else sync_wrapper + + +def _create_get_model_wrapper(original_get_model): + """ + Wraps the get_response method of the model to create a AI client span. + """ + from .spans import ai_client_span, finish_ai_client_span + + @classmethod + @wraps(original_get_model) + def wrapped_get_model(cls, agent, run_config): + model = original_get_model(agent, run_config) + original_get_response = model.get_response + + @wraps(original_get_response) + async def wrapped_get_response(*args, **kwargs): + with ai_client_span(agent, model, run_config, kwargs): + result = await original_get_response(*args, **kwargs) + finish_ai_client_span(agent, model, run_config, kwargs, result) + return result + + model.get_response = wrapped_get_response + return model + + return wrapped_get_model From 1f21a3b973de82f6fd67e03983bcf38ebfe53f0a Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 5 Jun 2025 11:42:53 +0200 Subject: [PATCH 20/89] Added token usage to ai client spans --- .../openai_agents/spans/ai_client.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index bc4e16a0dd..fed9b6eb46 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -4,8 +4,6 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): - # import ipdb; ipdb.set_trace() - """ https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#inference @@ -41,7 +39,6 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): OUTPUT: gen_ai.choice - """ # TODO-anton: implement other types of operations return sentry_sdk.start_span( @@ -52,8 +49,8 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): def finish_ai_client_span(agent, model, run_config, get_response_kwargs, result): _set_agent_data(agent) + span = sentry_sdk.get_current_span() - # import ipdb; ipdb.set_trace() if get_response_kwargs.get("system_instructions"): span.set_data( @@ -68,3 +65,15 @@ def finish_ai_client_span(agent, model, run_config, get_response_kwargs, result) for index, message in enumerate(result.output): span.set_data(f"output.{index}", message) # if message.get("type") == "function_call": + + span.set_data("gen_ai.usage.input_tokens", result.usage.input_tokens) + span.set_data( + "gen_ai.usage.input_tokens.cached", + result.usage.input_tokens_details.cached_tokens, + ) + span.set_data("gen_ai.usage.output_tokens", result.usage.output_tokens) + span.set_data( + "gen_ai.usage.output_tokens.reasoning", + result.usage.output_tokens_details.reasoning_tokens, + ) + span.set_data("gen_ai.usage.total_tokens", result.usage.total_tokens) From 1e24c1064d48f6e5d356ba6a6e6a6815009bd558 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 5 Jun 2025 11:56:01 +0200 Subject: [PATCH 21/89] refactoring --- sentry_sdk/consts.py | 30 +++++++++++++++++++ .../openai_agents/spans/ai_client.py | 18 ++++++----- .../openai_agents/spans/execute_tool.py | 11 +++---- .../openai_agents/spans/handoff.py | 5 ++-- .../openai_agents/spans/invoke_agent.py | 5 ++-- .../integrations/openai_agents/utils.py | 23 +++++++++----- 6 files changed, 67 insertions(+), 25 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 241d197699..988bf7b40c 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -458,6 +458,30 @@ class SPANDATA: Example: "5249fbada8d5416482c2f6e47e337372" """ + # OpenAI Agents specific constants + GEN_AI_SYSTEM = "gen_ai.system" + GEN_AI_AGENT_NAME = "gen_ai.agent.name" + GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens" + GEN_AI_REQUEST_MODEL = "gen_ai.request.model" + GEN_AI_REQUEST_PRESENCE_PENALTY = "gen_ai.request.presence_penalty" + GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature" + GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p" + GEN_AI_REQUEST_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty" + GEN_AI_OPERATION_NAME = "gen_ai.operation.name" + GEN_AI_TOOL_NAME = "gen_ai.tool.name" + GEN_AI_TOOL_DESCRIPTION = "gen_ai.tool.description" + GEN_AI_TOOL_TYPE = "gen_ai.tool.type" + GEN_AI_SYSTEM_MESSAGE = "gen_ai.system.message" + GEN_AI_USER_MESSAGE = "gen_ai.user.message" + GEN_AI_ASSISTANT_MESSAGE = "gen_ai.assistant.message" + GEN_AI_TOOL_MESSAGE = "gen_ai.tool.message" + GEN_AI_CHOICE = "gen_ai.choice" + GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens" + GEN_AI_USAGE_INPUT_TOKENS_CACHED = "gen_ai.usage.input_tokens.cached" + GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens" + GEN_AI_USAGE_OUTPUT_TOKENS_REASONING = "gen_ai.usage.output_tokens.reasoning" + GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens" + class SPANSTATUS: """ @@ -550,6 +574,12 @@ class OP: SOCKET_CONNECTION = "socket.connection" SOCKET_DNS = "socket.dns" + # OpenAI Agents specific operations + GEN_AI_CHAT = "gen_ai.chat" + GEN_AI_EXECUTE_TOOL = "gen_ai.execute_tool" + GEN_AI_INVOKE_AGENT = "gen_ai.invoke_agent" + GEN_AI_HANDOFF = "gen_ai.handoff" + # This type exists to trick mypy and PyCharm into thinking `init` and `Client` # take these arguments (even though they take opaque **kwargs) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index fed9b6eb46..b959d78b21 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -1,4 +1,5 @@ import sentry_sdk +from sentry_sdk.consts import OP, SPANDATA from ..utils import _set_agent_data @@ -42,7 +43,7 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): """ # TODO-anton: implement other types of operations return sentry_sdk.start_span( - op="gen_ai.chat", + op=OP.GEN_AI_CHAT, description=f"*chat* (TODO: remove hardcoded stuff) {agent.model}", ) @@ -54,26 +55,27 @@ def finish_ai_client_span(agent, model, run_config, get_response_kwargs, result) if get_response_kwargs.get("system_instructions"): span.set_data( - "gen_ai.system.message", get_response_kwargs.get("system_instructions") + SPANDATA.GEN_AI_SYSTEM_MESSAGE, + get_response_kwargs.get("system_instructions"), ) for message in get_response_kwargs.get("input", []): if message.get("role") == "user": - span.set_data("gen_ai.user.message", message.get("content")) + span.set_data(SPANDATA.GEN_AI_USER_MESSAGE, message.get("content")) break for index, message in enumerate(result.output): span.set_data(f"output.{index}", message) # if message.get("type") == "function_call": - span.set_data("gen_ai.usage.input_tokens", result.usage.input_tokens) + span.set_data(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, result.usage.input_tokens) span.set_data( - "gen_ai.usage.input_tokens.cached", + SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, result.usage.input_tokens_details.cached_tokens, ) - span.set_data("gen_ai.usage.output_tokens", result.usage.output_tokens) + span.set_data(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS, result.usage.output_tokens) span.set_data( - "gen_ai.usage.output_tokens.reasoning", + SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, result.usage.output_tokens_details.reasoning_tokens, ) - span.set_data("gen_ai.usage.total_tokens", result.usage.total_tokens) + span.set_data(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, result.usage.total_tokens) diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index ff2ff85968..82f0d2844d 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -1,5 +1,6 @@ import sentry_sdk from sentry_sdk.integrations.openai_agents.utils import _set_agent_data, _usage_to_str +from sentry_sdk.consts import OP, SPANDATA from typing import TYPE_CHECKING @@ -12,16 +13,16 @@ def execute_tool_span(context, agent, tool): print(f"### Tool {tool.name} started. " f"Usage: {_usage_to_str(context.usage)}") span = sentry_sdk.start_span( - op="gen_ai.execute_tool", name=f"execute_tool {tool.name}" + op=OP.GEN_AI_EXECUTE_TOOL, name=f"execute_tool {tool.name}" ) span.__enter__() - span.set_data("gen_ai.operation.name", "execute_tool") - span.set_data("gen_ai.tool.name", tool.name) - span.set_data("gen_ai.tool.description", tool.description) + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "execute_tool") + span.set_data(SPANDATA.GEN_AI_TOOL_NAME, tool.name) + span.set_data(SPANDATA.GEN_AI_TOOL_DESCRIPTION, tool.description) if tool.__class__.__name__ == "FunctionTool": - span.set_data("gen_ai.tool.type", "function") + span.set_data(SPANDATA.GEN_AI_TOOL_TYPE, "function") _set_agent_data(agent) diff --git a/sentry_sdk/integrations/openai_agents/spans/handoff.py b/sentry_sdk/integrations/openai_agents/spans/handoff.py index 7efd5bb876..881c08537b 100644 --- a/sentry_sdk/integrations/openai_agents/spans/handoff.py +++ b/sentry_sdk/integrations/openai_agents/spans/handoff.py @@ -1,5 +1,6 @@ import sentry_sdk from sentry_sdk.integrations.openai_agents.utils import _usage_to_str +from sentry_sdk.consts import OP, SPANDATA from typing import TYPE_CHECKING @@ -16,9 +17,9 @@ def handoff_span(context, from_agent, to_agent): current_span = sentry_sdk.get_current_span() if current_span: with current_span.start_child( - op="gen_ai.handoff", + op=OP.GEN_AI_HANDOFF, name=f"handoff from {from_agent.name} to {to_agent.name}", ) as span: - span.set_data("gen_ai.operation.name", "handoff") + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "handoff") current_span.__exit__(None, None, None) diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index 51bc88ef78..3997eb32f1 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -1,5 +1,6 @@ import sentry_sdk from sentry_sdk.integrations.openai_agents.utils import _usage_to_str, _set_agent_data +from sentry_sdk.consts import OP, SPANDATA from typing import TYPE_CHECKING @@ -12,11 +13,11 @@ def invoke_agent_span(context, agent): # type: (RunContextWrapper, Agent) -> None print(f"### Agent {agent.name} started. " f"Usage: {_usage_to_str(context.usage)}") span = sentry_sdk.start_span( - op="gen_ai.invoke_agent", name=f"invoke_agent {agent.name}" + op=OP.GEN_AI_INVOKE_AGENT, name=f"invoke_agent {agent.name}" ) span.__enter__() - span.set_data("gen_ai.operation.name", "invoke_agent") + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") _set_agent_data(agent) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 99f2ee4672..66ba0ec56c 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -4,6 +4,7 @@ import sentry_sdk from sentry_sdk.integrations import DidNotEnable from sentry_sdk.utils import event_from_exception +from sentry_sdk.consts import SPANDATA from typing import TYPE_CHECKING @@ -133,31 +134,37 @@ def _set_agent_data(agent): 'tools': []} """ span.set_data( - "gen_ai.system", "openai" + SPANDATA.GEN_AI_SYSTEM, "openai" ) # See footnote for https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/#gen-ai-system for explanation why. - span.set_data("gen_ai.agent.name", agent.name) + span.set_data(SPANDATA.GEN_AI_AGENT_NAME, agent.name) if agent.model_settings.max_tokens: - span.set_data("gen_ai.request.max_tokens", agent.model_settings.max_tokens) + span.set_data( + SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, agent.model_settings.max_tokens + ) if agent.model: - span.set_data("gen_ai.request.model", agent.model) + span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, agent.model) if agent.model_settings.presence_penalty: span.set_data( - "gen_ai.request.presence_penalty", agent.model_settings.presence_penalty + SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, + agent.model_settings.presence_penalty, ) if agent.model_settings.temperature: - span.set_data("gen_ai.request.temperature", agent.model_settings.temperature) + span.set_data( + SPANDATA.GEN_AI_REQUEST_TEMPERATURE, agent.model_settings.temperature + ) if agent.model_settings.top_p: - span.set_data("gen_ai.request.top_p", agent.model_settings.top_p) + span.set_data(SPANDATA.GEN_AI_REQUEST_TOP_P, agent.model_settings.top_p) if agent.model_settings.frequency_penalty: span.set_data( - "gen_ai.request.frequency_penalty", agent.model_settings.frequency_penalty + SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, + agent.model_settings.frequency_penalty, ) From a1763ec575b3a7ddd22f40c86934cdb923c7e1cb Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 5 Jun 2025 12:09:14 +0200 Subject: [PATCH 22/89] refacotring --- sentry_sdk/consts.py | 109 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 988bf7b40c..a41f198874 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -460,27 +460,136 @@ class SPANDATA: # OpenAI Agents specific constants GEN_AI_SYSTEM = "gen_ai.system" + """ + The name of the AI system being used. + Example: "openai" + """ + GEN_AI_AGENT_NAME = "gen_ai.agent.name" + """ + The name of the agent being used. + Example: "ResearchAssistant" + """ + GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens" + """ + The maximum number of tokens to generate in the response. + Example: 2048 + """ + GEN_AI_REQUEST_MODEL = "gen_ai.request.model" + """ + The model identifier being used for the request. + Example: "gpt-4-turbo-preview" + """ + GEN_AI_REQUEST_PRESENCE_PENALTY = "gen_ai.request.presence_penalty" + """ + The presence penalty parameter used to reduce repetitiveness of generated tokens. + Example: 0.1 + """ + GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature" + """ + The temperature parameter used to control randomness in the output. + Example: 0.7 + """ + GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p" + """ + The top_p parameter used to control diversity via nucleus sampling. + Example: 1.0 + """ + GEN_AI_REQUEST_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty" + """ + The frequency penalty parameter used to reduce repetitiveness of generated tokens. + Example: 0.1 + """ + GEN_AI_OPERATION_NAME = "gen_ai.operation.name" + """ + The name of the operation being performed. + Example: "chat" + """ + GEN_AI_TOOL_NAME = "gen_ai.tool.name" + """ + The name of the tool being used. + Example: "web_search" + """ + GEN_AI_TOOL_DESCRIPTION = "gen_ai.tool.description" + """ + The description of the tool being used. + Example: "Searches the web for current information about a topic" + """ + GEN_AI_TOOL_TYPE = "gen_ai.tool.type" + """ + The type of tool being used. + Example: "function" + """ + GEN_AI_SYSTEM_MESSAGE = "gen_ai.system.message" + """ + The system instructions passed to the model. + Example: "You are a helpful assistant" + """ + GEN_AI_USER_MESSAGE = "gen_ai.user.message" + """ + The user message passed to the model. + Example: "What's the weather in Paris?" + """ + GEN_AI_ASSISTANT_MESSAGE = "gen_ai.assistant.message" + """ + The assistant message passed to the model. + Example: "get_weather tool call" + """ + GEN_AI_TOOL_MESSAGE = "gen_ai.tool.message" + """ + The response from a tool or function call passed to the model. + Example: "rainy, 57°F" + """ + GEN_AI_CHOICE = "gen_ai.choice" + """ + The model's response message. + Example: "The weather in Paris is rainy and overcast, with temperatures around 57°F" + """ + GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens" + """ + The number of tokens in the input. + Example: 150 + """ + GEN_AI_USAGE_INPUT_TOKENS_CACHED = "gen_ai.usage.input_tokens.cached" + """ + The number of cached tokens in the input. + Example: 50 + """ + GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens" + """ + The number of tokens in the output. + Example: 250 + """ + GEN_AI_USAGE_OUTPUT_TOKENS_REASONING = "gen_ai.usage.output_tokens.reasoning" + """ + The number of tokens used for reasoning in the output. + Example: 75 + """ + GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens" + """ + The total number of tokens used (input + output). + Example: 400 + """ class SPANSTATUS: From 5d392f626cf645da0b288273ed59c996e602b1a9 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 5 Jun 2025 12:11:16 +0200 Subject: [PATCH 23/89] order --- sentry_sdk/consts.py | 468 +++++++++++++++++++++---------------------- 1 file changed, 234 insertions(+), 234 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index a41f198874..1532d2a522 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -108,89 +108,68 @@ class SPANDATA: See: https://develop.sentry.dev/sdk/performance/span-data-conventions/ """ - AI_FREQUENCY_PENALTY = "ai.frequency_penalty" - """ - Used to reduce repetitiveness of generated tokens. - Example: 0.5 - """ - - AI_PRESENCE_PENALTY = "ai.presence_penalty" - """ - Used to reduce repetitiveness of generated tokens. - Example: 0.5 - """ - - AI_INPUT_MESSAGES = "ai.input_messages" - """ - The input messages to an LLM call. - Example: [{"role": "user", "message": "hello"}] - """ - - AI_MODEL_ID = "ai.model_id" - """ - The unique descriptor of the model being execugted - Example: gpt-4 - """ - - AI_METADATA = "ai.metadata" + AI_CITATIONS = "ai.citations" """ - Extra metadata passed to an AI pipeline step. - Example: {"executed_function": "add_integers"} + References or sources cited by the AI model in its response. + Example: ["Smith et al. 2020", "Jones 2019"] """ - AI_TAGS = "ai.tags" + AI_DOCUMENTS = "ai.documents" """ - Tags that describe an AI pipeline step. - Example: {"executed_function": "add_integers"} + Documents or content chunks used as context for the AI model. + Example: ["doc1.txt", "doc2.pdf"] """ - AI_STREAMING = "ai.streaming" + AI_FINISH_REASON = "ai.finish_reason" """ - Whether or not the AI model call's repsonse was streamed back asynchronously - Example: true + The reason why the model stopped generating. + Example: "length" """ - AI_TEMPERATURE = "ai.temperature" + AI_FREQUENCY_PENALTY = "ai.frequency_penalty" """ - For an AI model call, the temperature parameter. Temperature essentially means how random the output will be. + Used to reduce repetitiveness of generated tokens. Example: 0.5 """ - AI_TOP_P = "ai.top_p" + AI_FUNCTION_CALL = "ai.function_call" """ - For an AI model call, the top_p parameter. Top_p essentially controls how random the output will be. - Example: 0.5 + For an AI model call, the function that was called. This is deprecated for OpenAI, and replaced by tool_calls """ - AI_TOP_K = "ai.top_k" + AI_GENERATION_ID = "ai.generation_id" """ - For an AI model call, the top_k parameter. Top_k essentially controls how random the output will be. - Example: 35 + Unique identifier for the completion. + Example: "gen_123abc" """ - AI_FUNCTION_CALL = "ai.function_call" + AI_INPUT_MESSAGES = "ai.input_messages" """ - For an AI model call, the function that was called. This is deprecated for OpenAI, and replaced by tool_calls + The input messages to an LLM call. + Example: [{"role": "user", "message": "hello"}] """ - AI_TOOL_CALLS = "ai.tool_calls" + AI_LOGIT_BIAS = "ai.logit_bias" """ - For an AI model call, the function that was called. This is deprecated for OpenAI, and replaced by tool_calls + For an AI model call, the logit bias """ - AI_TOOLS = "ai.tools" + AI_METADATA = "ai.metadata" """ - For an AI model call, the functions that are available + Extra metadata passed to an AI pipeline step. + Example: {"executed_function": "add_integers"} """ - AI_RESPONSE_FORMAT = "ai.response_format" + AI_MODEL_ID = "ai.model_id" """ - For an AI model call, the format of the response + The unique descriptor of the model being execugted + Example: gpt-4 """ - AI_LOGIT_BIAS = "ai.logit_bias" + AI_PIPELINE_NAME = "ai.pipeline.name" """ - For an AI model call, the logit bias + Name of the AI pipeline or chain being executed. + Example: "qa-pipeline" """ AI_PREAMBLE = "ai.preamble" @@ -200,33 +179,27 @@ class SPANDATA: Example: "You are now a clown." """ - AI_RAW_PROMPTING = "ai.raw_prompting" - """ - Minimize pre-processing done to the prompt sent to the LLM. - Example: true - """ - AI_RESPONSES = "ai.responses" + AI_PRESENCE_PENALTY = "ai.presence_penalty" """ - The responses to an AI model call. Always as a list. - Example: ["hello", "world"] + Used to reduce repetitiveness of generated tokens. + Example: 0.5 """ - AI_SEED = "ai.seed" + AI_RAW_PROMPTING = "ai.raw_prompting" """ - The seed, ideally models given the same seed and same other parameters will produce the exact same output. - Example: 123.45 + Minimize pre-processing done to the prompt sent to the LLM. + Example: true """ - AI_CITATIONS = "ai.citations" + AI_RESPONSE_FORMAT = "ai.response_format" """ - References or sources cited by the AI model in its response. - Example: ["Smith et al. 2020", "Jones 2019"] + For an AI model call, the format of the response """ - AI_DOCUMENTS = "ai.documents" + AI_RESPONSES = "ai.responses" """ - Documents or content chunks used as context for the AI model. - Example: ["doc1.txt", "doc2.pdf"] + The responses to an AI model call. Always as a list. + Example: ["hello", "world"] """ AI_SEARCH_QUERIES = "ai.search_queries" @@ -235,34 +208,40 @@ class SPANDATA: Example: ["climate change effects", "renewable energy"] """ + AI_SEARCH_REQUIRED = "ai.is_search_required" + """ + Boolean indicating if the model needs to perform a search. + Example: true + """ + AI_SEARCH_RESULTS = "ai.search_results" """ Results returned from search queries for context. Example: ["Result 1", "Result 2"] """ - AI_GENERATION_ID = "ai.generation_id" + AI_SEED = "ai.seed" """ - Unique identifier for the completion. - Example: "gen_123abc" + The seed, ideally models given the same seed and same other parameters will produce the exact same output. + Example: 123.45 """ - AI_SEARCH_REQUIRED = "ai.is_search_required" + AI_STREAMING = "ai.streaming" """ - Boolean indicating if the model needs to perform a search. + Whether or not the AI model call's response was streamed back asynchronously Example: true """ - AI_FINISH_REASON = "ai.finish_reason" + AI_TAGS = "ai.tags" """ - The reason why the model stopped generating. - Example: "length" + Tags that describe an AI pipeline step. + Example: {"executed_function": "add_integers"} """ - AI_PIPELINE_NAME = "ai.pipeline.name" + AI_TEMPERATURE = "ai.temperature" """ - Name of the AI pipeline or chain being executed. - Example: "qa-pipeline" + For an AI model call, the temperature parameter. Temperature essentially means how random the output will be. + Example: 0.5 """ AI_TEXTS = "ai.texts" @@ -271,44 +250,32 @@ class SPANDATA: Example: ["What is machine learning?"] """ - AI_WARNINGS = "ai.warnings" - """ - Warning messages generated during model execution. - Example: ["Token limit exceeded"] - """ - - DB_NAME = "db.name" + AI_TOP_K = "ai.top_k" """ - The name of the database being accessed. For commands that switch the database, this should be set to the target database (even if the command fails). - Example: myDatabase + For an AI model call, the top_k parameter. Top_k essentially controls how random the output will be. + Example: 35 """ - DB_USER = "db.user" + AI_TOP_P = "ai.top_p" """ - The name of the database user used for connecting to the database. - See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/database.md - Example: my_user + For an AI model call, the top_p parameter. Top_p essentially controls how random the output will be. + Example: 0.5 """ - DB_OPERATION = "db.operation" + AI_TOOL_CALLS = "ai.tool_calls" """ - The name of the operation being executed, e.g. the MongoDB command name such as findAndModify, or the SQL keyword. - See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/database.md - Example: findAndModify, HMSET, SELECT + For an AI model call, the function that was called. This is deprecated for OpenAI, and replaced by tool_calls """ - DB_SYSTEM = "db.system" + AI_TOOLS = "ai.tools" """ - An identifier for the database management system (DBMS) product being used. - See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/database.md - Example: postgresql + For an AI model call, the functions that are available """ - DB_MONGODB_COLLECTION = "db.mongodb.collection" + AI_WARNINGS = "ai.warnings" """ - The MongoDB collection being accessed within the database. - See: https://github.com/open-telemetry/semantic-conventions/blob/main/docs/database/mongodb.md#attributes - Example: public.users; customers + Warning messages generated during model execution. + Example: ["Token limit exceeded"] """ CACHE_HIT = "cache.hit" @@ -329,266 +296,299 @@ class SPANDATA: Example: template.cache.some_item.867da7e2af8e6b2f3aa7213a4080edb3 """ - NETWORK_PEER_ADDRESS = "network.peer.address" + CODE_FILEPATH = "code.filepath" """ - Peer address of the network connection - IP address or Unix domain socket name. - Example: 10.1.2.80, /tmp/my.sock, localhost + The source code file name that identifies the code unit as uniquely as possible (preferably an absolute file path). + Example: "/app/myapplication/http/handler/server.py" """ - NETWORK_PEER_PORT = "network.peer.port" + CODE_FUNCTION = "code.function" """ - Peer port number of the network connection. - Example: 6379 + The method or function name, or equivalent (usually rightmost part of the code unit's name). + Example: "server_request" """ - HTTP_QUERY = "http.query" + CODE_LINENO = "code.lineno" """ - The Query string present in the URL. - Example: ?foo=bar&bar=baz + The line number in `code.filepath` best representing the operation. It SHOULD point within the code unit named in `code.function`. + Example: 42 """ - HTTP_FRAGMENT = "http.fragment" + CODE_NAMESPACE = "code.namespace" """ - The Fragments present in the URL. - Example: #foo=bar + The "namespace" within which `code.function` is defined. Usually the qualified class or module name, such that `code.namespace` + some separator + `code.function` form a unique identifier for the code unit. + Example: "http.handler" """ - HTTP_METHOD = "http.method" + DB_MONGODB_COLLECTION = "db.mongodb.collection" """ - The HTTP method used. - Example: GET + The MongoDB collection being accessed within the database. + See: https://github.com/open-telemetry/semantic-conventions/blob/main/docs/database/mongodb.md#attributes + Example: public.users; customers """ - HTTP_STATUS_CODE = "http.response.status_code" + DB_NAME = "db.name" """ - The HTTP status code as an integer. - Example: 418 + The name of the database being accessed. For commands that switch the database, this should be set to the target database (even if the command fails). + Example: myDatabase """ - MESSAGING_DESTINATION_NAME = "messaging.destination.name" + DB_OPERATION = "db.operation" """ - The destination name where the message is being consumed from, - e.g. the queue name or topic. + The name of the operation being executed, e.g. the MongoDB command name such as findAndModify, or the SQL keyword. + See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/database.md + Example: findAndModify, HMSET, SELECT """ - MESSAGING_MESSAGE_ID = "messaging.message.id" + DB_SYSTEM = "db.system" """ - The message's identifier. + An identifier for the database management system (DBMS) product being used. + See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/database.md + Example: postgresql """ - MESSAGING_MESSAGE_RETRY_COUNT = "messaging.message.retry.count" + DB_USER = "db.user" """ - Number of retries/attempts to process a message. + The name of the database user used for connecting to the database. + See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/database.md + Example: my_user """ - MESSAGING_MESSAGE_RECEIVE_LATENCY = "messaging.message.receive.latency" + GEN_AI_AGENT_NAME = "gen_ai.agent.name" """ - The latency between when the task was enqueued and when it was started to be processed. + The name of the agent being used. + Example: "ResearchAssistant" """ - MESSAGING_SYSTEM = "messaging.system" + GEN_AI_ASSISTANT_MESSAGE = "gen_ai.assistant.message" """ - The messaging system's name, e.g. `kafka`, `aws_sqs` + The assistant message passed to the model. + Example: "get_weather tool call" """ - SERVER_ADDRESS = "server.address" + GEN_AI_CHOICE = "gen_ai.choice" """ - Name of the database host. - Example: example.com + The model's response message. + Example: "The weather in Paris is rainy and overcast, with temperatures around 57°F" """ - SERVER_PORT = "server.port" + GEN_AI_OPERATION_NAME = "gen_ai.operation.name" """ - Logical server port number - Example: 80; 8080; 443 + The name of the operation being performed. + Example: "chat" """ - SERVER_SOCKET_ADDRESS = "server.socket.address" + GEN_AI_REQUEST_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty" """ - Physical server IP address or Unix socket address. - Example: 10.5.3.2 + The frequency penalty parameter used to reduce repetitiveness of generated tokens. + Example: 0.1 """ - SERVER_SOCKET_PORT = "server.socket.port" + GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens" """ - Physical server port. - Recommended: If different than server.port. - Example: 16456 + The maximum number of tokens to generate in the response. + Example: 2048 """ - CODE_FILEPATH = "code.filepath" + GEN_AI_REQUEST_MODEL = "gen_ai.request.model" """ - The source code file name that identifies the code unit as uniquely as possible (preferably an absolute file path). - Example: "/app/myapplication/http/handler/server.py" + The model identifier being used for the request. + Example: "gpt-4-turbo-preview" """ - CODE_LINENO = "code.lineno" + GEN_AI_REQUEST_PRESENCE_PENALTY = "gen_ai.request.presence_penalty" """ - The line number in `code.filepath` best representing the operation. It SHOULD point within the code unit named in `code.function`. - Example: 42 + The presence penalty parameter used to reduce repetitiveness of generated tokens. + Example: 0.1 """ - CODE_FUNCTION = "code.function" + GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature" """ - The method or function name, or equivalent (usually rightmost part of the code unit's name). - Example: "server_request" + The temperature parameter used to control randomness in the output. + Example: 0.7 """ - CODE_NAMESPACE = "code.namespace" + GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p" """ - The "namespace" within which `code.function` is defined. Usually the qualified class or module name, such that `code.namespace` + some separator + `code.function` form a unique identifier for the code unit. - Example: "http.handler" + The top_p parameter used to control diversity via nucleus sampling. + Example: 1.0 """ - THREAD_ID = "thread.id" + GEN_AI_SYSTEM = "gen_ai.system" """ - Identifier of a thread from where the span originated. This should be a string. - Example: "7972576320" + The name of the AI system being used. + Example: "openai" """ - THREAD_NAME = "thread.name" + GEN_AI_SYSTEM_MESSAGE = "gen_ai.system.message" """ - Label identifying a thread from where the span originated. This should be a string. - Example: "MainThread" + The system instructions passed to the model. + Example: "You are a helpful assistant" """ - PROFILER_ID = "profiler_id" + GEN_AI_TOOL_DESCRIPTION = "gen_ai.tool.description" """ - Label identifying the profiler id that the span occurred in. This should be a string. - Example: "5249fbada8d5416482c2f6e47e337372" + The description of the tool being used. + Example: "Searches the web for current information about a topic" """ - # OpenAI Agents specific constants - GEN_AI_SYSTEM = "gen_ai.system" + GEN_AI_TOOL_MESSAGE = "gen_ai.tool.message" """ - The name of the AI system being used. - Example: "openai" + The response from a tool or function call passed to the model. + Example: "rainy, 57°F" """ - GEN_AI_AGENT_NAME = "gen_ai.agent.name" + GEN_AI_TOOL_NAME = "gen_ai.tool.name" """ - The name of the agent being used. - Example: "ResearchAssistant" + The name of the tool being used. + Example: "web_search" """ - GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens" + GEN_AI_TOOL_TYPE = "gen_ai.tool.type" """ - The maximum number of tokens to generate in the response. - Example: 2048 + The type of tool being used. + Example: "function" """ - GEN_AI_REQUEST_MODEL = "gen_ai.request.model" + GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens" """ - The model identifier being used for the request. - Example: "gpt-4-turbo-preview" + The number of tokens in the input. + Example: 150 """ - GEN_AI_REQUEST_PRESENCE_PENALTY = "gen_ai.request.presence_penalty" + GEN_AI_USAGE_INPUT_TOKENS_CACHED = "gen_ai.usage.input_tokens.cached" """ - The presence penalty parameter used to reduce repetitiveness of generated tokens. - Example: 0.1 + The number of cached tokens in the input. + Example: 50 """ - GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature" + GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens" """ - The temperature parameter used to control randomness in the output. - Example: 0.7 + The number of tokens in the output. + Example: 250 """ - GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p" + GEN_AI_USAGE_OUTPUT_TOKENS_REASONING = "gen_ai.usage.output_tokens.reasoning" """ - The top_p parameter used to control diversity via nucleus sampling. - Example: 1.0 + The number of tokens used for reasoning in the output. + Example: 75 """ - GEN_AI_REQUEST_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty" + GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens" """ - The frequency penalty parameter used to reduce repetitiveness of generated tokens. - Example: 0.1 + The total number of tokens used (input + output). + Example: 400 """ - GEN_AI_OPERATION_NAME = "gen_ai.operation.name" + GEN_AI_USER_MESSAGE = "gen_ai.user.message" """ - The name of the operation being performed. - Example: "chat" + The user message passed to the model. + Example: "What's the weather in Paris?" """ - GEN_AI_TOOL_NAME = "gen_ai.tool.name" + HTTP_FRAGMENT = "http.fragment" """ - The name of the tool being used. - Example: "web_search" + The Fragments present in the URL. + Example: #foo=bar """ - GEN_AI_TOOL_DESCRIPTION = "gen_ai.tool.description" + HTTP_METHOD = "http.method" """ - The description of the tool being used. - Example: "Searches the web for current information about a topic" + The HTTP method used. + Example: GET """ - GEN_AI_TOOL_TYPE = "gen_ai.tool.type" + HTTP_QUERY = "http.query" """ - The type of tool being used. - Example: "function" + The Query string present in the URL. + Example: ?foo=bar&bar=baz """ - GEN_AI_SYSTEM_MESSAGE = "gen_ai.system.message" + HTTP_STATUS_CODE = "http.response.status_code" """ - The system instructions passed to the model. - Example: "You are a helpful assistant" + The HTTP status code as an integer. + Example: 418 """ - GEN_AI_USER_MESSAGE = "gen_ai.user.message" + MESSAGING_DESTINATION_NAME = "messaging.destination.name" """ - The user message passed to the model. - Example: "What's the weather in Paris?" + The destination name where the message is being consumed from, + e.g. the queue name or topic. """ - GEN_AI_ASSISTANT_MESSAGE = "gen_ai.assistant.message" + MESSAGING_MESSAGE_ID = "messaging.message.id" """ - The assistant message passed to the model. - Example: "get_weather tool call" + The message's identifier. """ - GEN_AI_TOOL_MESSAGE = "gen_ai.tool.message" + MESSAGING_MESSAGE_RECEIVE_LATENCY = "messaging.message.receive.latency" """ - The response from a tool or function call passed to the model. - Example: "rainy, 57°F" + The latency between when the task was enqueued and when it was started to be processed. """ - GEN_AI_CHOICE = "gen_ai.choice" + MESSAGING_MESSAGE_RETRY_COUNT = "messaging.message.retry.count" """ - The model's response message. - Example: "The weather in Paris is rainy and overcast, with temperatures around 57°F" + Number of retries/attempts to process a message. """ - GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens" + MESSAGING_SYSTEM = "messaging.system" """ - The number of tokens in the input. - Example: 150 + The messaging system's name, e.g. `kafka`, `aws_sqs` """ - GEN_AI_USAGE_INPUT_TOKENS_CACHED = "gen_ai.usage.input_tokens.cached" + NETWORK_PEER_ADDRESS = "network.peer.address" """ - The number of cached tokens in the input. - Example: 50 + Peer address of the network connection - IP address or Unix domain socket name. + Example: 10.1.2.80, /tmp/my.sock, localhost """ - GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens" + NETWORK_PEER_PORT = "network.peer.port" """ - The number of tokens in the output. - Example: 250 + Peer port number of the network connection. + Example: 6379 """ - GEN_AI_USAGE_OUTPUT_TOKENS_REASONING = "gen_ai.usage.output_tokens.reasoning" + PROFILER_ID = "profiler_id" """ - The number of tokens used for reasoning in the output. - Example: 75 + Label identifying the profiler id that the span occurred in. This should be a string. + Example: "5249fbada8d5416482c2f6e47e337372" """ - GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens" + SERVER_ADDRESS = "server.address" """ - The total number of tokens used (input + output). - Example: 400 + Name of the database host. + Example: example.com + """ + + SERVER_PORT = "server.port" + """ + Logical server port number + Example: 80; 8080; 443 + """ + + SERVER_SOCKET_ADDRESS = "server.socket.address" + """ + Physical server IP address or Unix socket address. + Example: 10.5.3.2 + """ + + SERVER_SOCKET_PORT = "server.socket.port" + """ + Physical server port. + Recommended: If different than server.port. + Example: 16456 + """ + + THREAD_ID = "thread.id" + """ + Identifier of a thread from where the span originated. This should be a string. + Example: "7972576320" + """ + + THREAD_NAME = "thread.name" + """ + Label identifying a thread from where the span originated. This should be a string. + Example: "MainThread" """ From e7710bcc844bdbc384c1c4f97ff493f542607cc2 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 5 Jun 2025 12:11:39 +0200 Subject: [PATCH 24/89] order --- sentry_sdk/consts.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 1532d2a522..61d59a3971 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -630,6 +630,10 @@ class OP: FUNCTION = "function" FUNCTION_AWS = "function.aws" FUNCTION_GCP = "function.gcp" + GEN_AI_CHAT = "gen_ai.chat" + GEN_AI_EXECUTE_TOOL = "gen_ai.execute_tool" + GEN_AI_HANDOFF = "gen_ai.handoff" + GEN_AI_INVOKE_AGENT = "gen_ai.invoke_agent" GRAPHQL_EXECUTE = "graphql.execute" GRAPHQL_MUTATION = "graphql.mutation" GRAPHQL_PARSE = "graphql.parse" @@ -683,12 +687,6 @@ class OP: SOCKET_CONNECTION = "socket.connection" SOCKET_DNS = "socket.dns" - # OpenAI Agents specific operations - GEN_AI_CHAT = "gen_ai.chat" - GEN_AI_EXECUTE_TOOL = "gen_ai.execute_tool" - GEN_AI_INVOKE_AGENT = "gen_ai.invoke_agent" - GEN_AI_HANDOFF = "gen_ai.handoff" - # This type exists to trick mypy and PyCharm into thinking `init` and `Client` # take these arguments (even though they take opaque **kwargs) From 990fae0ec073c23c7bf27e5937723ee41ef7d2d5 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Fri, 6 Jun 2025 11:06:11 +0200 Subject: [PATCH 25/89] better ai client spans --- .../openai_agents/spans/ai_client.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index b959d78b21..8cfc0f700d 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -42,10 +42,14 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): gen_ai.choice """ # TODO-anton: implement other types of operations - return sentry_sdk.start_span( + span = sentry_sdk.start_span( op=OP.GEN_AI_CHAT, description=f"*chat* (TODO: remove hardcoded stuff) {agent.model}", ) + # TODO-anton: remove hardcoded stuff and replace something that also works for embedding and so on + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "chat") + + return span def finish_ai_client_span(agent, model, run_config, get_response_kwargs, result): @@ -64,10 +68,6 @@ def finish_ai_client_span(agent, model, run_config, get_response_kwargs, result) span.set_data(SPANDATA.GEN_AI_USER_MESSAGE, message.get("content")) break - for index, message in enumerate(result.output): - span.set_data(f"output.{index}", message) - # if message.get("type") == "function_call": - span.set_data(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, result.usage.input_tokens) span.set_data( SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, @@ -79,3 +79,11 @@ def finish_ai_client_span(agent, model, run_config, get_response_kwargs, result) result.usage.output_tokens_details.reasoning_tokens, ) span.set_data(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, result.usage.total_tokens) + + output = [item.to_json() for item in result.output] + span.set_data(SPANDATA.GEN_AI_CHOICE, output) + + # TODO-anton: for debugging, remove this + for index, message in enumerate(result.output): + span.set_data(f"DEBUG.output.{index}", message) + # if message.get("type") == "function_call": From 1ac43375c32cc818aa6ffacc1ac2f9792302c276 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Fri, 6 Jun 2025 11:31:18 +0200 Subject: [PATCH 26/89] cleanup --- sentry_sdk/integrations/openai_agents/run_hooks.py | 3 ++- sentry_sdk/integrations/openai_agents/spans/execute_tool.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py index e9c94cbe9f..150a0402f7 100644 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -27,7 +27,8 @@ class SentryRunHooks(RunHooks): - async def on_agent_start(self, context: RunContextWrapper, agent: Agent) -> None: + async def on_agent_start(self, context, agent) -> None: + # type: (RunContextWrapper, Agent) -> None invoke_agent_span(context, agent) async def on_agent_end(self, context, agent, output): diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index 82f0d2844d..a005bdba07 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -10,7 +10,6 @@ def execute_tool_span(context, agent, tool): # type: (RunContextWrapper, Agent, Tool) -> None - print(f"### Tool {tool.name} started. " f"Usage: {_usage_to_str(context.usage)}") span = sentry_sdk.start_span( op=OP.GEN_AI_EXECUTE_TOOL, name=f"execute_tool {tool.name}" From 08c47e5a01686a7b463b90a037c389bd40cd13e7 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Fri, 6 Jun 2025 14:59:13 +0200 Subject: [PATCH 27/89] moving stuff around --- sentry_sdk/integrations/__init__.py | 2 +- .../integrations/openai_agents/__init__.py | 18 ++++-- .../openai_agents/patches/__init__.py | 3 + .../openai_agents/patches/models.py | 38 +++++++++++ .../openai_agents/patches/runner.py | 55 ++++++++++++++++ .../openai_agents/patches/tools.py | 64 +++++++++++++++++++ .../integrations/openai_agents/run_hooks.py | 39 ++++------- .../openai_agents/spans/__init__.py | 10 ++- .../openai_agents/spans/agent_workflow.py | 20 ++++++ .../openai_agents/spans/ai_client.py | 43 ++----------- .../openai_agents/spans/execute_tool.py | 32 +++++----- .../openai_agents/spans/handoff.py | 4 +- .../openai_agents/spans/invoke_agent.py | 8 +-- .../integrations/openai_agents/utils.py | 62 ------------------ 14 files changed, 243 insertions(+), 155 deletions(-) create mode 100644 sentry_sdk/integrations/openai_agents/patches/__init__.py create mode 100644 sentry_sdk/integrations/openai_agents/patches/models.py create mode 100644 sentry_sdk/integrations/openai_agents/patches/runner.py create mode 100644 sentry_sdk/integrations/openai_agents/patches/tools.py create mode 100644 sentry_sdk/integrations/openai_agents/spans/agent_workflow.py diff --git a/sentry_sdk/integrations/__init__.py b/sentry_sdk/integrations/__init__.py index 4c522cc8b2..d63352ac28 100644 --- a/sentry_sdk/integrations/__init__.py +++ b/sentry_sdk/integrations/__init__.py @@ -145,7 +145,7 @@ def iter_default_integrations(with_auto_enabling_integrations): "launchdarkly": (9, 8, 0), "loguru": (0, 7, 0), "openai": (1, 0, 0), - "openai_agents": (0, 0, 16), + "openai_agents": (0, 0, 17), "openfeature": (0, 7, 1), "quart": (0, 16, 0), "ray": (2, 7, 0), diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py index b3c7bbccbd..bea5217ed7 100644 --- a/sentry_sdk/integrations/openai_agents/__init__.py +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -1,10 +1,11 @@ from sentry_sdk.integrations import DidNotEnable, Integration -from sentry_sdk.integrations.openai_agents.utils import ( - _create_run_wrapper, + +from .patches import ( _create_get_model_wrapper, + _create_get_all_tools_wrapper, + _create_run_wrapper, ) - try: import agents @@ -20,9 +21,17 @@ def _patch_runner(): def _patch_model(): + # type: () -> None agents.Runner._get_model = _create_get_model_wrapper(agents.Runner._get_model) +def _patch_tools(): + # type: () -> None + agents.Runner._get_all_tools = _create_get_all_tools_wrapper( + agents.Runner._get_all_tools + ) + + class OpenAIAgentsIntegration(Integration): identifier = "openai_agents" origin = f"auto.ai.{identifier}" @@ -30,5 +39,6 @@ class OpenAIAgentsIntegration(Integration): @staticmethod def setup_once(): # type: () -> None - _patch_runner() + _patch_tools() _patch_model() + _patch_runner() diff --git a/sentry_sdk/integrations/openai_agents/patches/__init__.py b/sentry_sdk/integrations/openai_agents/patches/__init__.py new file mode 100644 index 0000000000..29e0fd6245 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/patches/__init__.py @@ -0,0 +1,3 @@ +from .models import _create_get_model_wrapper # noqa: F401 +from .tools import _create_get_all_tools_wrapper # noqa: F401 +from .runner import _create_run_wrapper # noqa: F401 diff --git a/sentry_sdk/integrations/openai_agents/patches/models.py b/sentry_sdk/integrations/openai_agents/patches/models.py new file mode 100644 index 0000000000..5902590dae --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/patches/models.py @@ -0,0 +1,38 @@ +from functools import wraps + +from sentry_sdk.integrations import DidNotEnable + +from ..spans import ai_client_span, update_ai_client_span + + +try: + import agents +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + + +def _create_get_model_wrapper(original_get_model): + """ + Wraps the agents.Runner._get_model method to wrap the get_response method of the model to create a AI client span. + """ + + @classmethod + @wraps(original_get_model) + def wrapped_get_model(cls, agent, run_config): + # type: (agents.Runner, agents.Agent, agents.RunConfig) -> agents.Model + + model = original_get_model(agent, run_config) + original_get_response = model.get_response + + @wraps(original_get_response) + async def wrapped_get_response(*args, **kwargs): + with ai_client_span(agent, model, run_config, kwargs): + result = await original_get_response(*args, **kwargs) + update_ai_client_span(agent, model, run_config, kwargs, result) + return result + + model.get_response = wrapped_get_response + + return model + + return wrapped_get_model diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py new file mode 100644 index 0000000000..9107fafe94 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -0,0 +1,55 @@ +import asyncio + +from functools import wraps + +from sentry_sdk.integrations import DidNotEnable + +from ..spans import agent_workflow_span, update_agent_workflow_span +from ..utils import _wrap_hooks + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + from typing import Callable + +try: + import agents +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + + +def _create_run_wrapper(original_func): + # type: (Callable) -> Callable + """ + Wraps the agents.Runner.run* methods to create a root span for the agent workflow runs. + """ + is_async = asyncio.iscoroutinefunction(original_func) + + @classmethod + @wraps(original_func) + async def async_wrapper(cls, *args, **kwargs): + # type: (agents.Runner, *Any, **Any) -> Any + agent = args[0] + with agent_workflow_span(agent) as span: + kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) + + result = await original_func(*args, **kwargs) + update_agent_workflow_span(span, agent, result) + + return result + + @classmethod + @wraps(original_func) + def sync_wrapper(cls, *args, **kwargs): + # type: (agents.Runner, *Any, **Any) -> Any + agent = args[0] + with agent_workflow_span(agent) as span: + kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) + + result = original_func(*args, **kwargs) + update_agent_workflow_span(span, agent, result) + + return result + + return async_wrapper if is_async else sync_wrapper diff --git a/sentry_sdk/integrations/openai_agents/patches/tools.py b/sentry_sdk/integrations/openai_agents/patches/tools.py new file mode 100644 index 0000000000..2cc4d923b1 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/patches/tools.py @@ -0,0 +1,64 @@ +from functools import wraps + +from sentry_sdk.integrations import DidNotEnable + +from ..spans import execute_tool_span, update_execute_tool_span + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +try: + import agents +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + + +def _create_get_all_tools_wrapper(original_get_all_tools): + """ + Wraps the agents.Runner._get_all_tools method of the Runner class to wrap all function tools with Sentry instrumentation. + """ + + @classmethod + @wraps(original_get_all_tools) + async def wrapped_get_all_tools(cls, agent, context_wrapper): + # type: (agents.Runner, agents.Agent, agents.RunContextWrapper) -> list[agents.Tool] + + # Get the original tools + tools = await original_get_all_tools(agent, context_wrapper) + + wrapped_tools = [] + for tool in tools: + # Wrap only the function tools (for now) + if not isinstance(tool, agents.Tool): + wrapped_tools.append(tool) + continue + + # Create a new FunctionTool with our wrapped invoke method + original_on_invoke = tool.on_invoke_tool + + def create_wrapped_invoke(current_tool, current_on_invoke): + @wraps(current_on_invoke) + async def sentry_wrapped_on_invoke_tool(*args, **kwargs): + # type: (*Any, **Any) -> Any + with execute_tool_span(current_tool, *args, **kwargs) as span: + result = await current_on_invoke(*args, **kwargs) + update_execute_tool_span(span, agent, result) + return result + + return sentry_wrapped_on_invoke_tool + + wrapped_tool = agents.FunctionTool( + name=tool.name, + description=tool.description, + params_json_schema=tool.params_json_schema, + on_invoke_tool=create_wrapped_invoke(tool, original_on_invoke), + strict_json_schema=tool.strict_json_schema, + is_enabled=tool.is_enabled, + ) + wrapped_tools.append(wrapped_tool) + + return wrapped_tools + + return wrapped_get_all_tools diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py index 150a0402f7..49c0d32cd8 100644 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -2,48 +2,33 @@ from typing import TYPE_CHECKING -from sentry_sdk.integrations.openai_agents.spans import ( - execute_tool_span, - handoff_span, - invoke_agent_span, - finish_execute_tool_span, - finish_invoke_agent_span, -) - +from .spans import invoke_agent_span, update_invoke_agent_span, handoff_span if TYPE_CHECKING: from typing import Any try: - from agents import ( - Agent, - RunContextWrapper, - RunHooks, - Tool, - ) + import agents + except ImportError: raise DidNotEnable("OpenAI Agents not installed") -class SentryRunHooks(RunHooks): +class SentryRunHooks(agents.RunHooks): async def on_agent_start(self, context, agent) -> None: - # type: (RunContextWrapper, Agent) -> None + # type: (agents.RunContextWrapper, agents.Agent) -> None invoke_agent_span(context, agent) async def on_agent_end(self, context, agent, output): - # type: (RunContextWrapper, Agent, Any) -> None - finish_invoke_agent_span(context, agent, output) - - async def on_tool_start(self, context, agent, tool): - # type: (RunContextWrapper, Agent, Tool) -> None - execute_tool_span(context, agent, tool) - - async def on_tool_end(self, context, agent, tool, result): - # type: (RunContextWrapper, Agent, Tool, str) -> None - finish_execute_tool_span(context, agent, tool, result) + # type: (agents.RunContextWrapper, agents.Agent, Any) -> None + update_invoke_agent_span(context, agent, output) async def on_handoff( - self, context: RunContextWrapper, from_agent: Agent, to_agent: Agent + self, + context: agents.RunContextWrapper, + from_agent: agents.Agent, + to_agent: agents.Agent, ) -> None: handoff_span(context, from_agent, to_agent) + pass diff --git a/sentry_sdk/integrations/openai_agents/spans/__init__.py b/sentry_sdk/integrations/openai_agents/spans/__init__.py index 2fbc4c9122..81e8145cab 100644 --- a/sentry_sdk/integrations/openai_agents/spans/__init__.py +++ b/sentry_sdk/integrations/openai_agents/spans/__init__.py @@ -1,4 +1,8 @@ -from .ai_client import ai_client_span, finish_ai_client_span # noqa: F401 -from .execute_tool import execute_tool_span, finish_execute_tool_span # noqa: F401 +from .agent_workflow import ( # noqa: F401 + agent_workflow_span, + update_agent_workflow_span, +) +from .ai_client import ai_client_span, update_ai_client_span # noqa: F401 +from .execute_tool import execute_tool_span, update_execute_tool_span # noqa: F401 from .handoff import handoff_span # noqa: F401 -from .invoke_agent import invoke_agent_span, finish_invoke_agent_span # noqa: F401 +from .invoke_agent import invoke_agent_span, update_invoke_agent_span # noqa: F401 diff --git a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py new file mode 100644 index 0000000000..7718fd4e88 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py @@ -0,0 +1,20 @@ +from ..utils import _get_start_span_function + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from agents import Agent + from sentry_sdk import Span + from typing import Any + + +def agent_workflow_span(*args, **kwargs): + # type: (*Any, **Any) -> Span + agent = args[0] + span = _get_start_span_function()(name=f"{agent.name} workflow") + return span + + +def update_agent_workflow_span(span, agent, result): + # type: (Span, Agent, Any) -> None + pass diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 8cfc0f700d..95ef10709f 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -3,44 +3,15 @@ from ..utils import _set_agent_data +from typing import TYPE_CHECKING -def ai_client_span(agent, model, run_config, get_response_kwargs): - """ - https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#inference - - gen_ai.operation.name # chat|generate_content|text_completion|embeddings - gen_ai.system - gen_ai.conversation.id - gen_ai.output.type - gen_ai.request.choice.count - gen_ai.request.model - gen_ai.request.seed - gen_ai.request.frequency_penalty - gen_ai.request.max_tokens - gen_ai.request.presence_penalty - gen_ai.request.stop_sequences - gen_ai.request.temperature - gen_ai.request.top_k - gen_ai.request.top_p - gen_ai.response.finish_reasons - gen_ai.response.id - gen_ai.response.model - gen_ai.usage.input_tokens - gen_ai.usage.output_tokens - server.address - server.port - error.type +if TYPE_CHECKING: + from agents import Agent, Model, RunConfig + from sentry_sdk import Span - INPUT: - gen_ai.system.message - gen_ai.user.message - gen_ai.assistant.message - gen_ai.tool.message (response from tool call as input) - - OUTPUT: - gen_ai.choice - """ +def ai_client_span(agent, model, run_config, get_response_kwargs): + # type: (Agent, Model, RunConfig, dict) -> Span # TODO-anton: implement other types of operations span = sentry_sdk.start_span( op=OP.GEN_AI_CHAT, @@ -52,7 +23,7 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): return span -def finish_ai_client_span(agent, model, run_config, get_response_kwargs, result): +def update_ai_client_span(agent, model, run_config, get_response_kwargs, result): _set_agent_data(agent) span = sentry_sdk.get_current_span() diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index a005bdba07..d6f56a8a11 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -1,37 +1,37 @@ import sentry_sdk -from sentry_sdk.integrations.openai_agents.utils import _set_agent_data, _usage_to_str from sentry_sdk.consts import OP, SPANDATA from typing import TYPE_CHECKING if TYPE_CHECKING: - from agents import Agent, RunContextWrapper, Tool + import agents + from sentry_sdk import Span + from typing import Any -def execute_tool_span(context, agent, tool): - # type: (RunContextWrapper, Agent, Tool) -> None - print(f"### Tool {tool.name} started. " f"Usage: {_usage_to_str(context.usage)}") +def execute_tool_span(tool, *args, **kwargs): + # type: (agents.Tool, *Any, **Any) -> Span span = sentry_sdk.start_span( op=OP.GEN_AI_EXECUTE_TOOL, name=f"execute_tool {tool.name}" ) - span.__enter__() span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "execute_tool") span.set_data(SPANDATA.GEN_AI_TOOL_NAME, tool.name) span.set_data(SPANDATA.GEN_AI_TOOL_DESCRIPTION, tool.description) + # Set tool input + input = args[1] + span.set_data("XXX.gen_ai.tool_input", input) + if tool.__class__.__name__ == "FunctionTool": span.set_data(SPANDATA.GEN_AI_TOOL_TYPE, "function") - _set_agent_data(agent) + return span -def finish_execute_tool_span(context, agent, tool, result): - # type: (RunContextWrapper, Agent, Tool, str) -> None - print( - f"### Tool {tool.name} ended with result {result}. " - f"Usage: {_usage_to_str(context.usage)}" - ) - current_span = sentry_sdk.get_current_span() - if current_span: - current_span.__exit__(None, None, None) +def update_execute_tool_span(span, agent, result): + # type: (Span, agents.Agent, Any) -> None + from sentry_sdk.integrations.openai_agents.utils import _set_agent_data + + _set_agent_data(agent) + pass diff --git a/sentry_sdk/integrations/openai_agents/spans/handoff.py b/sentry_sdk/integrations/openai_agents/spans/handoff.py index 881c08537b..deefacd2ba 100644 --- a/sentry_sdk/integrations/openai_agents/spans/handoff.py +++ b/sentry_sdk/integrations/openai_agents/spans/handoff.py @@ -5,11 +5,11 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from agents import Agent, RunContextWrapper + import agents def handoff_span(context, from_agent, to_agent): - # type: (RunContextWrapper, Agent, Agent) -> None + # type: (agents.RunContextWrapper, agents.Agent, agents.Agent) -> None print( f"### Handoff from '{from_agent.name}' to '{to_agent.name}'. " f"Usage: {_usage_to_str(context.usage)}" diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index 3997eb32f1..3e19789f2a 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -5,12 +5,12 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: + import agents from typing import Any - from agents import Agent, RunContextWrapper def invoke_agent_span(context, agent): - # type: (RunContextWrapper, Agent) -> None + # type: (agents.RunContextWrapper, agents.Agent) -> None print(f"### Agent {agent.name} started. " f"Usage: {_usage_to_str(context.usage)}") span = sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, name=f"invoke_agent {agent.name}" @@ -22,8 +22,8 @@ def invoke_agent_span(context, agent): _set_agent_data(agent) -def finish_invoke_agent_span(context, agent, output): - # type: (RunContextWrapper, Agent, Any) -> None +def update_invoke_agent_span(context, agent, output): + # type: (agents.RunContextWrapper, agents.Agent, Any) -> None print( f"### Agent '{agent.name}' ended with output {output}. " f"Usage: {_usage_to_str(context.usage)}" diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 66ba0ec56c..a28ee07a94 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,4 +1,3 @@ -import asyncio from functools import wraps import sentry_sdk @@ -166,64 +165,3 @@ def _set_agent_data(agent): SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, agent.model_settings.frequency_penalty, ) - - -def _create_run_wrapper(original_func): - # type: (Callable) -> Callable - """ - Wraps the run methods of the Runner class to create a root span for the agent runs. - """ - is_async = asyncio.iscoroutinefunction(original_func) - - @classmethod - @wraps(original_func) - async def async_wrapper(cls, *args, **kwargs): - # type: (agents.Runner, *Any, **Any) -> Any - agent = args[0] - with _get_start_span_function()(name=f"{agent.name} workflow"): - kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) - _set_agent_data(agent) - - result = await original_func(*args, **kwargs) - - return result - - @classmethod - @wraps(original_func) - def sync_wrapper(cls, *args, **kwargs): - # type: (agents.Runner, *Any, **Any) -> Any - agent = args[0] - with _get_start_span_function()(name=f"{agent.name} workflow"): - kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) - _set_agent_data(agent) - - result = original_func(*args, **kwargs) - - return result - - return async_wrapper if is_async else sync_wrapper - - -def _create_get_model_wrapper(original_get_model): - """ - Wraps the get_response method of the model to create a AI client span. - """ - from .spans import ai_client_span, finish_ai_client_span - - @classmethod - @wraps(original_get_model) - def wrapped_get_model(cls, agent, run_config): - model = original_get_model(agent, run_config) - original_get_response = model.get_response - - @wraps(original_get_response) - async def wrapped_get_response(*args, **kwargs): - with ai_client_span(agent, model, run_config, kwargs): - result = await original_get_response(*args, **kwargs) - finish_ai_client_span(agent, model, run_config, kwargs, result) - return result - - model.get_response = wrapped_get_response - return model - - return wrapped_get_model From 6db1476725450e67de3b928f9d2457eb2ca71dbf Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Fri, 6 Jun 2025 15:06:28 +0200 Subject: [PATCH 28/89] moving stuff around --- .../openai_agents/patches/models.py | 4 +- .../openai_agents/spans/agent_workflow.py | 1 + .../openai_agents/spans/ai_client.py | 6 +-- .../openai_agents/spans/execute_tool.py | 7 ++-- .../openai_agents/spans/invoke_agent.py | 7 +++- .../integrations/openai_agents/utils.py | 40 +------------------ 6 files changed, 15 insertions(+), 50 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/models.py b/sentry_sdk/integrations/openai_agents/patches/models.py index 5902590dae..fd4b21cae6 100644 --- a/sentry_sdk/integrations/openai_agents/patches/models.py +++ b/sentry_sdk/integrations/openai_agents/patches/models.py @@ -26,9 +26,9 @@ def wrapped_get_model(cls, agent, run_config): @wraps(original_get_response) async def wrapped_get_response(*args, **kwargs): - with ai_client_span(agent, model, run_config, kwargs): + with ai_client_span(agent, model, run_config, kwargs) as span: result = await original_get_response(*args, **kwargs) - update_ai_client_span(agent, model, run_config, kwargs, result) + update_ai_client_span(span, agent, model, run_config, kwargs, result) return result model.get_response = wrapped_get_response diff --git a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py index 7718fd4e88..83c4c3655a 100644 --- a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py +++ b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py @@ -12,6 +12,7 @@ def agent_workflow_span(*args, **kwargs): # type: (*Any, **Any) -> Span agent = args[0] span = _get_start_span_function()(name=f"{agent.name} workflow") + return span diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 95ef10709f..96c4c51c44 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -23,10 +23,8 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): return span -def update_ai_client_span(agent, model, run_config, get_response_kwargs, result): - _set_agent_data(agent) - - span = sentry_sdk.get_current_span() +def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, result): + _set_agent_data(span, agent) if get_response_kwargs.get("system_instructions"): span.set_data( diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index d6f56a8a11..54664a15b0 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -1,6 +1,8 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA +from ..utils import _set_agent_data + from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -31,7 +33,4 @@ def execute_tool_span(tool, *args, **kwargs): def update_execute_tool_span(span, agent, result): # type: (Span, agents.Agent, Any) -> None - from sentry_sdk.integrations.openai_agents.utils import _set_agent_data - - _set_agent_data(agent) - pass + _set_agent_data(span, agent) diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index 3e19789f2a..ef05b0494b 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -1,7 +1,8 @@ import sentry_sdk -from sentry_sdk.integrations.openai_agents.utils import _usage_to_str, _set_agent_data from sentry_sdk.consts import OP, SPANDATA +from ..utils import _set_agent_data, _usage_to_str + from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -19,7 +20,9 @@ def invoke_agent_span(context, agent): span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") - _set_agent_data(agent) + _set_agent_data(span, agent) + + return span def update_invoke_agent_span(context, agent, output): diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index a28ee07a94..d59a1ffb7a 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -94,44 +94,8 @@ def _wrap_hooks(hooks): return wrapped_hooks() -def _set_agent_data(agent): - # type: (agents.Agent) -> None - span = sentry_sdk.get_current_span() - - """ - Thats the Agent object: - {'handoff_description': None, - 'handoffs': [], - 'hooks': None, - 'input_guardrails': [], - 'instructions': 'You are a helpful research assistant. Given a query, come up ' - 'with a set of web searches to perform to best answer the ' - 'query. Output between 5 and 20 terms to query for.', - 'mcp_config': {}, - 'mcp_servers': [], - 'model': 'gpt-4o', - 'model_settings': ModelSettings(temperature=None, - top_p=None, - frequency_penalty=None, - presence_penalty=None, - tool_choice=None, - parallel_tool_calls=None, - truncation=None, - max_tokens=None, - reasoning=None, - metadata=None, - store=None, - include_usage=None, - extra_query=None, - extra_body=None, - extra_headers=None), - 'name': 'PlannerAgent', - 'output_guardrails': [], - 'output_type': , - 'reset_tool_choice': True, - 'tool_use_behavior': 'run_llm_again', - 'tools': []} - """ +def _set_agent_data(span, agent): + # type: (sentry_sdk.Span, agents.Agent) -> None span.set_data( SPANDATA.GEN_AI_SYSTEM, "openai" ) # See footnote for https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/#gen-ai-system for explanation why. From 5ca575c288d1fa6cfca3e4964c38a82d2d451361 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Fri, 6 Jun 2025 15:09:54 +0200 Subject: [PATCH 29/89] some consistency --- sentry_sdk/integrations/openai_agents/patches/models.py | 1 + sentry_sdk/integrations/openai_agents/patches/runner.py | 6 ++---- sentry_sdk/integrations/openai_agents/patches/tools.py | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/models.py b/sentry_sdk/integrations/openai_agents/patches/models.py index fd4b21cae6..58251a203c 100644 --- a/sentry_sdk/integrations/openai_agents/patches/models.py +++ b/sentry_sdk/integrations/openai_agents/patches/models.py @@ -29,6 +29,7 @@ async def wrapped_get_response(*args, **kwargs): with ai_client_span(agent, model, run_config, kwargs) as span: result = await original_get_response(*args, **kwargs) update_ai_client_span(span, agent, model, run_config, kwargs, result) + return result model.get_response = wrapped_get_response diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index 9107fafe94..fdf158bf6b 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -33,11 +33,10 @@ async def async_wrapper(cls, *args, **kwargs): agent = args[0] with agent_workflow_span(agent) as span: kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) - result = await original_func(*args, **kwargs) update_agent_workflow_span(span, agent, result) - return result + return result @classmethod @wraps(original_func) @@ -46,10 +45,9 @@ def sync_wrapper(cls, *args, **kwargs): agent = args[0] with agent_workflow_span(agent) as span: kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) - result = original_func(*args, **kwargs) update_agent_workflow_span(span, agent, result) - return result + return result return async_wrapper if is_async else sync_wrapper diff --git a/sentry_sdk/integrations/openai_agents/patches/tools.py b/sentry_sdk/integrations/openai_agents/patches/tools.py index 2cc4d923b1..1bdf660bcc 100644 --- a/sentry_sdk/integrations/openai_agents/patches/tools.py +++ b/sentry_sdk/integrations/openai_agents/patches/tools.py @@ -45,7 +45,8 @@ async def sentry_wrapped_on_invoke_tool(*args, **kwargs): with execute_tool_span(current_tool, *args, **kwargs) as span: result = await current_on_invoke(*args, **kwargs) update_execute_tool_span(span, agent, result) - return result + + return result return sentry_wrapped_on_invoke_tool From c6dbe47b183c6b665503aebde00b50f73fb587ad Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Fri, 6 Jun 2025 15:19:54 +0200 Subject: [PATCH 30/89] Tool input and output --- sentry_sdk/integrations/openai_agents/patches/tools.py | 2 +- .../integrations/openai_agents/spans/execute_tool.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/tools.py b/sentry_sdk/integrations/openai_agents/patches/tools.py index 1bdf660bcc..898de6ce25 100644 --- a/sentry_sdk/integrations/openai_agents/patches/tools.py +++ b/sentry_sdk/integrations/openai_agents/patches/tools.py @@ -44,7 +44,7 @@ async def sentry_wrapped_on_invoke_tool(*args, **kwargs): # type: (*Any, **Any) -> Any with execute_tool_span(current_tool, *args, **kwargs) as span: result = await current_on_invoke(*args, **kwargs) - update_execute_tool_span(span, agent, result) + update_execute_tool_span(span, agent, current_tool, result) return result diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index 54664a15b0..c35ae79a17 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -21,9 +21,8 @@ def execute_tool_span(tool, *args, **kwargs): span.set_data(SPANDATA.GEN_AI_TOOL_NAME, tool.name) span.set_data(SPANDATA.GEN_AI_TOOL_DESCRIPTION, tool.description) - # Set tool input input = args[1] - span.set_data("XXX.gen_ai.tool_input", input) + span.set_data("gen_ai.tool.input", input) if tool.__class__.__name__ == "FunctionTool": span.set_data(SPANDATA.GEN_AI_TOOL_TYPE, "function") @@ -31,6 +30,7 @@ def execute_tool_span(tool, *args, **kwargs): return span -def update_execute_tool_span(span, agent, result): - # type: (Span, agents.Agent, Any) -> None +def update_execute_tool_span(span, agent, tool, result): + # type: (Span, agents.Agent, agents.Tool, Any) -> None _set_agent_data(span, agent) + span.set_data("gen_ai.tool.output", result) From fd028ef8dae8b949d5ed93408695c8c4a41ce4bd Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Fri, 6 Jun 2025 15:20:58 +0200 Subject: [PATCH 31/89] removed debug output --- sentry_sdk/integrations/openai_agents/spans/ai_client.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 96c4c51c44..c79eaede52 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -51,8 +51,3 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r output = [item.to_json() for item in result.output] span.set_data(SPANDATA.GEN_AI_CHOICE, output) - - # TODO-anton: for debugging, remove this - for index, message in enumerate(result.output): - span.set_data(f"DEBUG.output.{index}", message) - # if message.get("type") == "function_call": From 18f3b41d49f070eceba7b8ba83cc6c35c061aa4b Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 11 Jun 2025 13:46:18 +0200 Subject: [PATCH 32/89] Using deprecated attr names to make ui work --- .../openai_agents/spans/agent_workflow.py | 1 + .../integrations/openai_agents/spans/ai_client.py | 13 +++++++++++++ .../openai_agents/spans/execute_tool.py | 4 ++++ sentry_sdk/integrations/openai_agents/utils.py | 4 ++++ 4 files changed, 22 insertions(+) diff --git a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py index 83c4c3655a..5934a8723c 100644 --- a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py +++ b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py @@ -11,6 +11,7 @@ def agent_workflow_span(*args, **kwargs): # type: (*Any, **Any) -> Span agent = args[0] + # Create a transaction or a span if an transaction is already active span = _get_start_span_function()(name=f"{agent.name} workflow") return span diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index c79eaede52..6a38df68d2 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -32,6 +32,10 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r get_response_kwargs.get("system_instructions"), ) + # Deprecated name just for first iteration. + # TODO-anton: define how to set input message and document in sentry-conventions. + span.set_data("ai.prompt.messages", get_response_kwargs.get("input", [])) + for message in get_response_kwargs.get("input", []): if message.get("role") == "user": span.set_data(SPANDATA.GEN_AI_USER_MESSAGE, message.get("content")) @@ -49,5 +53,14 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r ) span.set_data(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, result.usage.total_tokens) + # Deprecated name just for first iteration. + # TODO-anton: define how to set tool response messages and document in sentry-conventions. + tool_calls = [ + item.to_json() + for item in result.output + if item.__class__.__name__ == "ResponseFunctionToolCall" + ] + span.set_data("ai.response.toolCalls", tool_calls) + output = [item.to_json() for item in result.output] span.set_data(SPANDATA.GEN_AI_CHOICE, output) diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index c35ae79a17..5e5b80c9e4 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -21,6 +21,10 @@ def execute_tool_span(tool, *args, **kwargs): span.set_data(SPANDATA.GEN_AI_TOOL_NAME, tool.name) span.set_data(SPANDATA.GEN_AI_TOOL_DESCRIPTION, tool.description) + # Deprecated name just for first iteration. + # TODO-anton: remove this if the Sentry.io UI is using GEN_AI_TOOL_NAME + span.set_data("ai.toolCall.name", tool.name) + input = args[1] span.set_data("gen_ai.tool.input", input) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index d59a1ffb7a..766abd247a 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -110,6 +110,10 @@ def _set_agent_data(span, agent): if agent.model: span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, agent.model) + # Deprecated name just for first iteration. + # TODO-anton: remove this if the Sentry.io UI is using GEN_AI_REQUEST_MODEL + span.set_data("ai.model.id", agent.model) + if agent.model_settings.presence_penalty: span.set_data( SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, From d64a1de833c5cb7b8e51e8b9f735dbc04a8862e9 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 12 Jun 2025 08:24:56 +0200 Subject: [PATCH 33/89] Updated prompt messages format. --- .../openai_agents/spans/ai_client.py | 50 +++++++++++++++---- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 6a38df68d2..f8b9a2bdb4 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -12,10 +12,10 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): # type: (Agent, Model, RunConfig, dict) -> Span - # TODO-anton: implement other types of operations + # TODO-anton: implement other types of operations. Now "chat" is hardcoded. span = sentry_sdk.start_span( op=OP.GEN_AI_CHAT, - description=f"*chat* (TODO: remove hardcoded stuff) {agent.model}", + description=f"chat {agent.model}", ) # TODO-anton: remove hardcoded stuff and replace something that also works for embedding and so on span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "chat") @@ -26,15 +26,40 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, result): _set_agent_data(span, agent) - if get_response_kwargs.get("system_instructions"): - span.set_data( - SPANDATA.GEN_AI_SYSTEM_MESSAGE, - get_response_kwargs.get("system_instructions"), - ) + system_instructions = get_response_kwargs.get("system_instructions") + if system_instructions: + span.set_data(SPANDATA.GEN_AI_SYSTEM_MESSAGE, system_instructions) + + # "system" messages have a content of type string. + prompt_messages = [] + if system_instructions: + prompt_messages.append({"role": "system", "content": system_instructions}) + + # "user", "assistant", "tool" messages have a content of type array of objects. + other_messages = { + "user": [], + "assistant": [], + "tool": [], + } + for message in get_response_kwargs.get("input", []): + if "role" in message: + other_messages[message.get("role")].append( + {"type": "text", "text": message.get("content")} + ) + else: + if message.get("type") == "function_call": + other_messages["assistant"].append(message) + elif message.get("type") == "function_call_output": + other_messages["tool"].append(message) + + for role, messages in other_messages.items(): + if len(messages) > 0: + prompt_messages.append({"role": role, "content": messages}) # Deprecated name just for first iteration. # TODO-anton: define how to set input message and document in sentry-conventions. - span.set_data("ai.prompt.messages", get_response_kwargs.get("input", [])) + span.set_data("ai.prompt.format", "messages") + span.set_data("ai.prompt.messages", prompt_messages) for message in get_response_kwargs.get("input", []): if message.get("role") == "user": @@ -55,12 +80,17 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r # Deprecated name just for first iteration. # TODO-anton: define how to set tool response messages and document in sentry-conventions. - tool_calls = [ + assistant_responses = [ item.to_json() for item in result.output if item.__class__.__name__ == "ResponseFunctionToolCall" ] - span.set_data("ai.response.toolCalls", tool_calls) + if len(assistant_responses) > 0: + span.set_data("ai.response.toolCalls", assistant_responses) output = [item.to_json() for item in result.output] span.set_data(SPANDATA.GEN_AI_CHOICE, output) + + # Deprecated name just for first iteration. + # TODO-anton: define how to set tool response messages and document in sentry-conventions. + span.set_data("ai.response.text", output) From 8c7a3dc81e0b242bbe7d59066e94b2d8d62e672b Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 12 Jun 2025 12:55:40 +0200 Subject: [PATCH 34/89] better input and output for ai client span --- .../openai_agents/spans/ai_client.py | 39 ++++++++++++------- .../openai_agents/spans/handoff.py | 5 --- .../openai_agents/spans/invoke_agent.py | 7 +--- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index f8b9a2bdb4..76bf947cf5 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -30,6 +30,8 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r if system_instructions: span.set_data(SPANDATA.GEN_AI_SYSTEM_MESSAGE, system_instructions) + # LLM Request INPUT + # "system" messages have a content of type string. prompt_messages = [] if system_instructions: @@ -66,6 +68,8 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r span.set_data(SPANDATA.GEN_AI_USER_MESSAGE, message.get("content")) break + # LLM Request USAGE + span.set_data(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, result.usage.input_tokens) span.set_data( SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, @@ -78,19 +82,28 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r ) span.set_data(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, result.usage.total_tokens) - # Deprecated name just for first iteration. - # TODO-anton: define how to set tool response messages and document in sentry-conventions. - assistant_responses = [ - item.to_json() - for item in result.output - if item.__class__.__name__ == "ResponseFunctionToolCall" - ] - if len(assistant_responses) > 0: - span.set_data("ai.response.toolCalls", assistant_responses) - - output = [item.to_json() for item in result.output] - span.set_data(SPANDATA.GEN_AI_CHOICE, output) + # LLM Response OUTPUT # Deprecated name just for first iteration. # TODO-anton: define how to set tool response messages and document in sentry-conventions. - span.set_data("ai.response.text", output) + + output_messages = { + "response": [], + "tool": [], + } + for output in result.output: + if output.type == "function_call": + output_messages["tool"].append(output.to_json()) + elif output.type == "message": + for output_message in output.content: + output_messages["response"].append(output_message.to_json()) + + if len(output_messages["tool"]) > 0: + span.set_data("ai.response.toolCalls", output_messages["tool"]) + + if len(output_messages["response"]) > 0: + span.set_data(SPANDATA.GEN_AI_CHOICE, output_messages["response"]) + + # Deprecated name just for first iteration. + # TODO-anton: define how to set tool response messages and document in sentry-conventions. + span.set_data("ai.response.text", output_messages["response"]) diff --git a/sentry_sdk/integrations/openai_agents/spans/handoff.py b/sentry_sdk/integrations/openai_agents/spans/handoff.py index deefacd2ba..b1f4dbda1e 100644 --- a/sentry_sdk/integrations/openai_agents/spans/handoff.py +++ b/sentry_sdk/integrations/openai_agents/spans/handoff.py @@ -1,5 +1,4 @@ import sentry_sdk -from sentry_sdk.integrations.openai_agents.utils import _usage_to_str from sentry_sdk.consts import OP, SPANDATA from typing import TYPE_CHECKING @@ -10,10 +9,6 @@ def handoff_span(context, from_agent, to_agent): # type: (agents.RunContextWrapper, agents.Agent, agents.Agent) -> None - print( - f"### Handoff from '{from_agent.name}' to '{to_agent.name}'. " - f"Usage: {_usage_to_str(context.usage)}" - ) current_span = sentry_sdk.get_current_span() if current_span: with current_span.start_child( diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index ef05b0494b..8bc6cf3fd3 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -1,7 +1,7 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA -from ..utils import _set_agent_data, _usage_to_str +from ..utils import _set_agent_data from typing import TYPE_CHECKING @@ -12,7 +12,6 @@ def invoke_agent_span(context, agent): # type: (agents.RunContextWrapper, agents.Agent) -> None - print(f"### Agent {agent.name} started. " f"Usage: {_usage_to_str(context.usage)}") span = sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, name=f"invoke_agent {agent.name}" ) @@ -27,10 +26,6 @@ def invoke_agent_span(context, agent): def update_invoke_agent_span(context, agent, output): # type: (agents.RunContextWrapper, agents.Agent, Any) -> None - print( - f"### Agent '{agent.name}' ended with output {output}. " - f"Usage: {_usage_to_str(context.usage)}" - ) current_span = sentry_sdk.get_current_span() if current_span: current_span.__exit__(None, None, None) From c41a63babd2d650cb73ee6c6a7ccbec138d3172a Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Fri, 13 Jun 2025 13:01:04 +0200 Subject: [PATCH 35/89] renamed some attributes --- sentry_sdk/integrations/openai_agents/spans/ai_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 76bf947cf5..2143f688ff 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -61,7 +61,7 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r # Deprecated name just for first iteration. # TODO-anton: define how to set input message and document in sentry-conventions. span.set_data("ai.prompt.format", "messages") - span.set_data("ai.prompt.messages", prompt_messages) + span.set_data("gen_ai.request.messages", prompt_messages) for message in get_response_kwargs.get("input", []): if message.get("role") == "user": @@ -99,11 +99,11 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r output_messages["response"].append(output_message.to_json()) if len(output_messages["tool"]) > 0: - span.set_data("ai.response.toolCalls", output_messages["tool"]) + span.set_data("gen_ai.response.tool_calls", output_messages["tool"]) if len(output_messages["response"]) > 0: span.set_data(SPANDATA.GEN_AI_CHOICE, output_messages["response"]) # Deprecated name just for first iteration. # TODO-anton: define how to set tool response messages and document in sentry-conventions. - span.set_data("ai.response.text", output_messages["response"]) + span.set_data("gen_ai.response.text", output_messages["response"]) From 2e53aa75756b95a99c4d2a22483a214dd45d8494 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Fri, 13 Jun 2025 13:23:20 +0200 Subject: [PATCH 36/89] Add available tools to client span --- sentry_sdk/integrations/openai_agents/spans/ai_client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 2143f688ff..6c488d873e 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -30,6 +30,11 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r if system_instructions: span.set_data(SPANDATA.GEN_AI_SYSTEM_MESSAGE, system_instructions) + # Available tools + span.set_data( + "gen_ai.request.available_tools", [vars(tool) for tool in agent.tools] + ) + # LLM Request INPUT # "system" messages have a content of type string. From 5331aceb4b0c3d405ba7a8a30d73c6ee1067cb85 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Fri, 13 Jun 2025 13:25:13 +0200 Subject: [PATCH 37/89] add it everywhere where an agent is available --- sentry_sdk/integrations/openai_agents/spans/ai_client.py | 5 ----- sentry_sdk/integrations/openai_agents/utils.py | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 6c488d873e..2143f688ff 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -30,11 +30,6 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r if system_instructions: span.set_data(SPANDATA.GEN_AI_SYSTEM_MESSAGE, system_instructions) - # Available tools - span.set_data( - "gen_ai.request.available_tools", [vars(tool) for tool in agent.tools] - ) - # LLM Request INPUT # "system" messages have a content of type string. diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 766abd247a..a8bdfd55e3 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -133,3 +133,8 @@ def _set_agent_data(span, agent): SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, agent.model_settings.frequency_penalty, ) + + if len(agent.tools) > 0: + span.set_data( + "gen_ai.request.available_tools", [vars(tool) for tool in agent.tools] + ) From cd1f4eead4e3a7810b92a515ac6f8633963b4942 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 16 Jun 2025 10:10:52 +0200 Subject: [PATCH 38/89] made tool_calls array of object --- sentry_sdk/integrations/openai_agents/spans/ai_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 2143f688ff..052710adb4 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -93,7 +93,7 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r } for output in result.output: if output.type == "function_call": - output_messages["tool"].append(output.to_json()) + output_messages["tool"].append(output.dict()) elif output.type == "message": for output_message in output.content: output_messages["response"].append(output_message.to_json()) From c9b06e413781a854cc20841e85ba97ce6ff123ec Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 16 Jun 2025 10:16:18 +0200 Subject: [PATCH 39/89] cleanup --- sentry_sdk/consts.py | 18 ------------------ .../openai_agents/spans/ai_client.py | 12 ++---------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 61d59a3971..d6382c5d2f 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -360,12 +360,6 @@ class SPANDATA: Example: "ResearchAssistant" """ - GEN_AI_ASSISTANT_MESSAGE = "gen_ai.assistant.message" - """ - The assistant message passed to the model. - Example: "get_weather tool call" - """ - GEN_AI_CHOICE = "gen_ai.choice" """ The model's response message. @@ -420,24 +414,12 @@ class SPANDATA: Example: "openai" """ - GEN_AI_SYSTEM_MESSAGE = "gen_ai.system.message" - """ - The system instructions passed to the model. - Example: "You are a helpful assistant" - """ - GEN_AI_TOOL_DESCRIPTION = "gen_ai.tool.description" """ The description of the tool being used. Example: "Searches the web for current information about a topic" """ - GEN_AI_TOOL_MESSAGE = "gen_ai.tool.message" - """ - The response from a tool or function call passed to the model. - Example: "rainy, 57°F" - """ - GEN_AI_TOOL_NAME = "gen_ai.tool.name" """ The name of the tool being used. diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 052710adb4..bef389137c 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -26,14 +26,11 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, result): _set_agent_data(span, agent) - system_instructions = get_response_kwargs.get("system_instructions") - if system_instructions: - span.set_data(SPANDATA.GEN_AI_SYSTEM_MESSAGE, system_instructions) - # LLM Request INPUT - # "system" messages have a content of type string. prompt_messages = [] + + system_instructions = get_response_kwargs.get("system_instructions") if system_instructions: prompt_messages.append({"role": "system", "content": system_instructions}) @@ -63,11 +60,6 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r span.set_data("ai.prompt.format", "messages") span.set_data("gen_ai.request.messages", prompt_messages) - for message in get_response_kwargs.get("input", []): - if message.get("role") == "user": - span.set_data(SPANDATA.GEN_AI_USER_MESSAGE, message.get("content")) - break - # LLM Request USAGE span.set_data(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, result.usage.input_tokens) From 0942b5607d14d81489f34217b23ed9e4c56c53e3 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 16 Jun 2025 10:43:21 +0200 Subject: [PATCH 40/89] cleanup --- sentry_sdk/consts.py | 30 +++++++++++++++++++ .../openai_agents/spans/ai_client.py | 7 ++--- .../openai_agents/spans/execute_tool.py | 8 ++--- .../integrations/openai_agents/utils.py | 7 ++--- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index d6382c5d2f..2c6ddfbeda 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -372,6 +372,18 @@ class SPANDATA: Example: "chat" """ + GEN_AI_RESPONSE_TOOL_CALLS = "gen_ai.response.tool_calls" + """ + The tool calls in the model's response. + Example: [{"name": "get_weather", "arguments": {"location": "Paris"}}] + """ + + GEN_AI_REQUEST_AVAILABLE_TOOLS = "gen_ai.request.available_tools" + """ + The available tools for the model. + Example: [{"name": "get_weather", "description": "Get the weather for a given location"}, {"name": "get_news", "description": "Get the news for a given topic"}] + """ + GEN_AI_REQUEST_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty" """ The frequency penalty parameter used to reduce repetitiveness of generated tokens. @@ -384,6 +396,12 @@ class SPANDATA: Example: 2048 """ + GEN_AI_REQUEST_MESSAGES = "gen_ai.request.messages" + """ + The messages passed to the model. The "content" can be a string or an array of objects. + Example: [{role: "system", "content: "Generate a random number."}, {"role": "user", "content": [{"text": "Generate a random number between 0 and 10.", "type": "text"}]}] + """ + GEN_AI_REQUEST_MODEL = "gen_ai.request.model" """ The model identifier being used for the request. @@ -420,12 +438,24 @@ class SPANDATA: Example: "Searches the web for current information about a topic" """ + GEN_AI_TOOL_INPUT = "gen_ai.tool.input" + """ + The input of the tool being used. + Example: {"location": "Paris"} + """ + GEN_AI_TOOL_NAME = "gen_ai.tool.name" """ The name of the tool being used. Example: "web_search" """ + GEN_AI_TOOL_OUTPUT = "gen_ai.tool.output" + """ + The output of the tool being used. + Example: "rainy, 57°F" + """ + GEN_AI_TOOL_TYPE = "gen_ai.tool.type" """ The type of tool being used. diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index bef389137c..ce7e314dac 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -55,10 +55,7 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r if len(messages) > 0: prompt_messages.append({"role": role, "content": messages}) - # Deprecated name just for first iteration. - # TODO-anton: define how to set input message and document in sentry-conventions. - span.set_data("ai.prompt.format", "messages") - span.set_data("gen_ai.request.messages", prompt_messages) + span.set_data(SPANDATA.GEN_AI_REQUEST_MESSAGES, prompt_messages) # LLM Request USAGE @@ -91,7 +88,7 @@ def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, r output_messages["response"].append(output_message.to_json()) if len(output_messages["tool"]) > 0: - span.set_data("gen_ai.response.tool_calls", output_messages["tool"]) + span.set_data(SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, output_messages["tool"]) if len(output_messages["response"]) > 0: span.set_data(SPANDATA.GEN_AI_CHOICE, output_messages["response"]) diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index 5e5b80c9e4..3a05bab773 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -21,12 +21,8 @@ def execute_tool_span(tool, *args, **kwargs): span.set_data(SPANDATA.GEN_AI_TOOL_NAME, tool.name) span.set_data(SPANDATA.GEN_AI_TOOL_DESCRIPTION, tool.description) - # Deprecated name just for first iteration. - # TODO-anton: remove this if the Sentry.io UI is using GEN_AI_TOOL_NAME - span.set_data("ai.toolCall.name", tool.name) - input = args[1] - span.set_data("gen_ai.tool.input", input) + span.set_data(SPANDATA.GEN_AI_TOOL_INPUT, input) if tool.__class__.__name__ == "FunctionTool": span.set_data(SPANDATA.GEN_AI_TOOL_TYPE, "function") @@ -37,4 +33,4 @@ def execute_tool_span(tool, *args, **kwargs): def update_execute_tool_span(span, agent, tool, result): # type: (Span, agents.Agent, agents.Tool, Any) -> None _set_agent_data(span, agent) - span.set_data("gen_ai.tool.output", result) + span.set_data(SPANDATA.GEN_AI_TOOL_OUTPUT, result) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index a8bdfd55e3..8b99a2290b 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -110,10 +110,6 @@ def _set_agent_data(span, agent): if agent.model: span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, agent.model) - # Deprecated name just for first iteration. - # TODO-anton: remove this if the Sentry.io UI is using GEN_AI_REQUEST_MODEL - span.set_data("ai.model.id", agent.model) - if agent.model_settings.presence_penalty: span.set_data( SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, @@ -136,5 +132,6 @@ def _set_agent_data(span, agent): if len(agent.tools) > 0: span.set_data( - "gen_ai.request.available_tools", [vars(tool) for tool in agent.tools] + SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, + [vars(tool) for tool in agent.tools], ) From 92f015d0a03b4df60346ffb691136964800f1927 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 16 Jun 2025 12:12:43 +0200 Subject: [PATCH 41/89] cleanup --- .../openai_agents/spans/ai_client.py | 51 ++----------------- .../integrations/openai_agents/utils.py | 46 +++++++++++++++++ 2 files changed, 49 insertions(+), 48 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index ce7e314dac..81fd2e6d5c 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -1,7 +1,7 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA -from ..utils import _set_agent_data +from ..utils import _set_agent_data, _set_input_data, _set_usage_data from typing import TYPE_CHECKING @@ -26,56 +26,11 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, result): _set_agent_data(span, agent) - # LLM Request INPUT + _set_usage_data(span, result.usage) - prompt_messages = [] - - system_instructions = get_response_kwargs.get("system_instructions") - if system_instructions: - prompt_messages.append({"role": "system", "content": system_instructions}) - - # "user", "assistant", "tool" messages have a content of type array of objects. - other_messages = { - "user": [], - "assistant": [], - "tool": [], - } - for message in get_response_kwargs.get("input", []): - if "role" in message: - other_messages[message.get("role")].append( - {"type": "text", "text": message.get("content")} - ) - else: - if message.get("type") == "function_call": - other_messages["assistant"].append(message) - elif message.get("type") == "function_call_output": - other_messages["tool"].append(message) - - for role, messages in other_messages.items(): - if len(messages) > 0: - prompt_messages.append({"role": role, "content": messages}) - - span.set_data(SPANDATA.GEN_AI_REQUEST_MESSAGES, prompt_messages) - - # LLM Request USAGE - - span.set_data(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, result.usage.input_tokens) - span.set_data( - SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, - result.usage.input_tokens_details.cached_tokens, - ) - span.set_data(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS, result.usage.output_tokens) - span.set_data( - SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, - result.usage.output_tokens_details.reasoning_tokens, - ) - span.set_data(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, result.usage.total_tokens) + _set_input_data(span, get_response_kwargs) # LLM Response OUTPUT - - # Deprecated name just for first iteration. - # TODO-anton: define how to set tool response messages and document in sentry-conventions. - output_messages = { "response": [], "tool": [], diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 8b99a2290b..b94086f5b9 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -135,3 +135,49 @@ def _set_agent_data(span, agent): SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, [vars(tool) for tool in agent.tools], ) + + +def _set_usage_data(span, usage): + # type: (sentry_sdk.Span, Usage) -> None + span.set_data(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, usage.input_tokens) + span.set_data( + SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, + usage.input_tokens_details.cached_tokens, + ) + span.set_data(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS, usage.output_tokens) + span.set_data( + SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, + usage.output_tokens_details.reasoning_tokens, + ) + span.set_data(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, usage.total_tokens) + + +def _set_input_data(span, get_response_kwargs): + # type: (sentry_sdk.Span, dict[str, Any]) -> None + messages_by_role = { + "system": [], + "user": [], + "assistant": [], + "tool": [], + } + system_instructions = get_response_kwargs.get("system_instructions") + if system_instructions: + messages_by_role["system"].append({"type": "text", "text": system_instructions}) + + for message in get_response_kwargs.get("input", []): + if "role" in message: + messages_by_role[message.get("role")].append( + {"type": "text", "text": message.get("content")} + ) + else: + if message.get("type") == "function_call": + messages_by_role["assistant"].append(message) + elif message.get("type") == "function_call_output": + messages_by_role["tool"].append(message) + + request_messages = [] + for role, messages in messages_by_role.items(): + if len(messages) > 0: + messages.append({"role": role, "content": messages_by_role}) + + span.set_data(SPANDATA.GEN_AI_REQUEST_MESSAGES, request_messages) From 78064e7195f9d8082076cb707304bef5e856bf1a Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 16 Jun 2025 12:21:34 +0200 Subject: [PATCH 42/89] cleanup --- .../openai_agents/spans/ai_client.py | 27 ++----------------- .../integrations/openai_agents/utils.py | 27 ++++++++++++++++++- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 81fd2e6d5c..d5385e6baf 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -1,7 +1,7 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA -from ..utils import _set_agent_data, _set_input_data, _set_usage_data +from ..utils import _set_agent_data, _set_input_data, _set_output_data, _set_usage_data from typing import TYPE_CHECKING @@ -25,29 +25,6 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, result): _set_agent_data(span, agent) - _set_usage_data(span, result.usage) - _set_input_data(span, get_response_kwargs) - - # LLM Response OUTPUT - output_messages = { - "response": [], - "tool": [], - } - for output in result.output: - if output.type == "function_call": - output_messages["tool"].append(output.dict()) - elif output.type == "message": - for output_message in output.content: - output_messages["response"].append(output_message.to_json()) - - if len(output_messages["tool"]) > 0: - span.set_data(SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, output_messages["tool"]) - - if len(output_messages["response"]) > 0: - span.set_data(SPANDATA.GEN_AI_CHOICE, output_messages["response"]) - - # Deprecated name just for first iteration. - # TODO-anton: define how to set tool response messages and document in sentry-conventions. - span.set_data("gen_ai.response.text", output_messages["response"]) + _set_output_data(span, result) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index b94086f5b9..3592097f99 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -178,6 +178,31 @@ def _set_input_data(span, get_response_kwargs): request_messages = [] for role, messages in messages_by_role.items(): if len(messages) > 0: - messages.append({"role": role, "content": messages_by_role}) + request_messages.append({"role": role, "content": messages}) span.set_data(SPANDATA.GEN_AI_REQUEST_MESSAGES, request_messages) + + +def _set_output_data(span, result): + # type: (sentry_sdk.Span, Any) -> None + output_messages = { + "response": [], + "tool": [], + } + + for output in result.output: + if output.type == "function_call": + output_messages["tool"].append(output.dict()) + elif output.type == "message": + for output_message in output.content: + output_messages["response"].append(output_message.to_json()) + + if len(output_messages["tool"]) > 0: + span.set_data(SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, output_messages["tool"]) + + if len(output_messages["response"]) > 0: + span.set_data(SPANDATA.GEN_AI_CHOICE, output_messages["response"]) + + # Deprecated name just for first iteration. + # TODO-anton: define how to set tool response messages and document in sentry-conventions. + span.set_data("gen_ai.response.text", output_messages["response"]) From e79d3c7a4b5542b66e724100c7a6849486c69302 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 16 Jun 2025 12:44:13 +0200 Subject: [PATCH 43/89] handle pii --- .../openai_agents/patches/models.py | 4 ++-- .../openai_agents/spans/ai_client.py | 10 ++++++---- .../openai_agents/spans/execute_tool.py | 17 +++++++++++------ sentry_sdk/integrations/openai_agents/utils.py | 9 ++++++++- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/models.py b/sentry_sdk/integrations/openai_agents/patches/models.py index 58251a203c..406871eab0 100644 --- a/sentry_sdk/integrations/openai_agents/patches/models.py +++ b/sentry_sdk/integrations/openai_agents/patches/models.py @@ -26,9 +26,9 @@ def wrapped_get_model(cls, agent, run_config): @wraps(original_get_response) async def wrapped_get_response(*args, **kwargs): - with ai_client_span(agent, model, run_config, kwargs) as span: + with ai_client_span(agent, kwargs) as span: result = await original_get_response(*args, **kwargs) - update_ai_client_span(span, agent, model, run_config, kwargs, result) + update_ai_client_span(span, agent, kwargs, result) return result diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index d5385e6baf..f9d746afec 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -6,12 +6,13 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from agents import Agent, Model, RunConfig + from agents import Agent + from typing import Any from sentry_sdk import Span -def ai_client_span(agent, model, run_config, get_response_kwargs): - # type: (Agent, Model, RunConfig, dict) -> Span +def ai_client_span(agent, get_response_kwargs): + # type: (Agent, dict[str, Any]) -> Span # TODO-anton: implement other types of operations. Now "chat" is hardcoded. span = sentry_sdk.start_span( op=OP.GEN_AI_CHAT, @@ -23,7 +24,8 @@ def ai_client_span(agent, model, run_config, get_response_kwargs): return span -def update_ai_client_span(span, agent, model, run_config, get_response_kwargs, result): +def update_ai_client_span(span, agent, get_response_kwargs, result): + # type: (Span, Agent, dict[str, Any], Any) -> None _set_agent_data(span, agent) _set_usage_data(span, result.usage) _set_input_data(span, get_response_kwargs) diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index 3a05bab773..c9ce8e41a2 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -1,5 +1,6 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA +from sentry_sdk.scope import should_send_default_pii from ..utils import _set_agent_data @@ -18,19 +19,23 @@ def execute_tool_span(tool, *args, **kwargs): ) span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "execute_tool") - span.set_data(SPANDATA.GEN_AI_TOOL_NAME, tool.name) - span.set_data(SPANDATA.GEN_AI_TOOL_DESCRIPTION, tool.description) - - input = args[1] - span.set_data(SPANDATA.GEN_AI_TOOL_INPUT, input) if tool.__class__.__name__ == "FunctionTool": span.set_data(SPANDATA.GEN_AI_TOOL_TYPE, "function") + span.set_data(SPANDATA.GEN_AI_TOOL_NAME, tool.name) + span.set_data(SPANDATA.GEN_AI_TOOL_DESCRIPTION, tool.description) + + if should_send_default_pii(): + input = args[1] + span.set_data(SPANDATA.GEN_AI_TOOL_INPUT, input) + return span def update_execute_tool_span(span, agent, tool, result): # type: (Span, agents.Agent, agents.Tool, Any) -> None _set_agent_data(span, agent) - span.set_data(SPANDATA.GEN_AI_TOOL_OUTPUT, result) + + if should_send_default_pii(): + span.set_data(SPANDATA.GEN_AI_TOOL_OUTPUT, result) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 3592097f99..43cbf92e26 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,9 +1,10 @@ from functools import wraps import sentry_sdk +from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable +from sentry_sdk.scope import should_send_default_pii from sentry_sdk.utils import event_from_exception -from sentry_sdk.consts import SPANDATA from typing import TYPE_CHECKING @@ -154,6 +155,9 @@ def _set_usage_data(span, usage): def _set_input_data(span, get_response_kwargs): # type: (sentry_sdk.Span, dict[str, Any]) -> None + if not should_send_default_pii(): + return + messages_by_role = { "system": [], "user": [], @@ -185,6 +189,9 @@ def _set_input_data(span, get_response_kwargs): def _set_output_data(span, result): # type: (sentry_sdk.Span, Any) -> None + if not should_send_default_pii(): + return + output_messages = { "response": [], "tool": [], From 3ff92425c94c38fb37129cbd6b7af419b1b85625 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 16 Jun 2025 16:08:23 +0200 Subject: [PATCH 44/89] first version of vibe coded test suite --- scripts/populate_tox/config.py | 3 + .../openai_agents/test_openai_agents.py | 227 +++++++++++++++++- tox.ini | 37 +-- 3 files changed, 244 insertions(+), 23 deletions(-) diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index dd22ffb768..411d7fe666 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -141,6 +141,9 @@ }, "openai_agents": { "package": "openai-agents", + "deps": { + "*": ["pytest-asyncio"], + }, }, "openfeature": { "package": "openfeature-sdk", diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index dc9e17e2dd..4d48a13320 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -1,11 +1,228 @@ +import pytest +from unittest.mock import AsyncMock, MagicMock, patch + +import sentry_sdk from sentry_sdk.integrations.openai_agents import OpenAIAgentsIntegration +import agents +from agents import ( + Agent, + RunResult, + ModelResponse, + Usage, + MessageOutputItem, + ToolCallItem, + ToolCallOutputItem, +) +from agents.items import ( + ResponseOutputMessage, + ResponseOutputText, + ResponseFunctionToolCall, + FunctionCallOutput, +) +from agents.exceptions import AgentsException + + +@pytest.fixture +def mock_usage(): + return Usage( + requests=1, + input_tokens=10, + output_tokens=20, + total_tokens=30, + input_tokens_details=MagicMock(cached_tokens=0), + output_tokens_details=MagicMock(reasoning_tokens=5), + ) + + +@pytest.fixture +def mock_model_response(mock_usage): + return ModelResponse( + output=[ + ResponseOutputMessage( + content=[ + ResponseOutputText( + text="Hello, how can I help you?", type="text", annotations={} + ) + ], + role="assistant", + ) + ], + usage=mock_usage, + response_id="resp_123", + ) + + +@pytest.fixture +def mock_tool_call(): + return ResponseFunctionToolCall( + id="call_123", + function=MagicMock(name="test_tool", arguments='{"arg1": "value1"}'), + ) + + +@pytest.fixture +def mock_tool_output(): + return FunctionCallOutput(tool_call_id="call_123", output="Tool execution result") + + +@pytest.fixture +def mock_run_result(mock_model_response, mock_tool_call, mock_tool_output): + return RunResult( + input="Test input", + new_items=[ + MessageOutputItem( + agent=MagicMock(), + raw_item=mock_model_response.output[0], + type="message_output_item", + ), + ToolCallItem( + agent=MagicMock(), raw_item=mock_tool_call, type="tool_call_item" + ), + ToolCallOutputItem( + agent=MagicMock(), + raw_item=mock_tool_output, + output="Tool execution result", + type="tool_call_output_item", + ), + ], + raw_responses=[mock_model_response], + final_output="Final result", + input_guardrail_results=[], + output_guardrail_results=[], + context_wrapper=MagicMock(), + _last_agent=MagicMock(), + ) + + +@pytest.fixture +def mock_agent(): + agent = MagicMock(spec=Agent) + agent.name = "test_agent" + agent.model = "gpt-4" + agent.model_settings = MagicMock( + max_tokens=100, + temperature=0.7, + top_p=1.0, + presence_penalty=0.0, + frequency_penalty=0.0, + ) + agent.tools = [] + return agent + + +def test_integration_initialization(): + integration = OpenAIAgentsIntegration() + assert integration is not None + + +@pytest.mark.asyncio +async def test_agent_invocation_span( + sentry_init, capture_events, mock_agent, mock_run_result +): + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + ) + events = capture_events() + + with sentry_sdk.start_transaction(name="test_transaction"): + with patch("agents.Runner.run", new_callable=AsyncMock) as mock_run: + mock_run.return_value = mock_run_result + + await agents.Runner.run(mock_agent, "Test input") + + (transaction,) = events + spans = transaction["spans"] + + assert len(spans) > 0 + agent_span = next(span for span in spans if span["op"] == "agent.run") + assert agent_span is not None + assert agent_span["data"]["gen_ai.system"] == "openai" + assert agent_span["data"]["gen_ai.agent.name"] == "test_agent" + assert agent_span["data"]["gen_ai.request.model"] == "gpt-4" + assert agent_span["data"]["gen_ai.request.max_tokens"] == 100 + assert agent_span["data"]["gen_ai.request.temperature"] == 0.7 + + +@pytest.mark.asyncio +async def test_tool_execution_span( + sentry_init, capture_events, mock_agent, mock_run_result +): + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + ) + + events = capture_events() + + with sentry_sdk.start_transaction(name="test_transaction"): + with patch("agents.Runner.run", new_callable=AsyncMock) as mock_run: + mock_run.return_value = mock_run_result + + await agents.Runner.run(mock_agent, "Test input") + + (transaction,) = events + spans = transaction["spans"] + + tool_spans = [span for span in spans if span["op"] == "agent.tool"] + assert len(tool_spans) > 0 + tool_span = tool_spans[0] # type: ignore + assert tool_span["data"]["gen_ai.tool.name"] == "test_tool" + assert tool_span["data"]["gen_ai.tool.input"] == '{"arg1": "value1"}' + assert tool_span["data"]["gen_ai.tool.output"] == "Tool execution result" + + +@pytest.mark.asyncio +async def test_llm_request_span( + sentry_init, capture_events, mock_agent, mock_run_result +): + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + ) + + events = capture_events() + + with sentry_sdk.start_transaction(name="test_transaction"): + with patch("agents.Runner.run", new_callable=AsyncMock) as mock_run: + mock_run.return_value = mock_run_result + + await agents.Runner.run(mock_agent, "Test input") + + (transaction,) = events + spans = transaction["spans"] + + llm_spans = [span for span in spans if span["op"] == "llm.request"] + assert len(llm_spans) > 0 + llm_span = llm_spans[0] # type: ignore + assert llm_span["data"]["gen_ai.request.model"] == "gpt-4" + assert llm_span["data"]["gen_ai.usage.input_tokens"] == 10 + assert llm_span["data"]["gen_ai.usage.output_tokens"] == 20 + assert llm_span["data"]["gen_ai.usage.total_tokens"] == 30 + + +@pytest.mark.asyncio +async def test_error_handling(sentry_init, capture_events, mock_agent): + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + ) + + events = capture_events() -def test_basic(sentry_init): - sentry_init(integrations=[OpenAIAgentsIntegration()]) + with sentry_sdk.start_transaction(name="test_transaction"): + with patch("agents.Runner.run", new_callable=AsyncMock) as mock_run: + mock_run.side_effect = AgentsException("Test error") - import ipdb + with pytest.raises(AgentsException): + await agents.Runner.run(mock_agent, "Test input") - ipdb.set_trace() + (transaction,) = events + spans = transaction["spans"] - assert True + assert len(spans) > 0 + error_span = next(span for span in spans if span["status"] == "internal_error") + assert error_span is not None + assert error_span["data"]["error.type"] == "AgentsException" + assert error_span["data"]["error.value"] == "Test error" diff --git a/tox.ini b/tox.ini index bc893414aa..31be13f882 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-06-11T12:20:52.494394+00:00 +# Last generated: 2025-06-16T13:51:07.677014+00:00 [tox] requires = @@ -145,12 +145,12 @@ envlist = {py3.9,py3.11,py3.12}-cohere-v5.11.4 {py3.9,py3.11,py3.12}-cohere-v5.15.0 - {py3.9,py3.11,py3.12}-openai_agents-v0.0.16 + {py3.9,py3.11,py3.12}-openai_agents-v0.0.17 {py3.8,py3.10,py3.11}-huggingface_hub-v0.22.2 - {py3.8,py3.10,py3.11}-huggingface_hub-v0.25.2 - {py3.8,py3.12,py3.13}-huggingface_hub-v0.28.1 - {py3.8,py3.12,py3.13}-huggingface_hub-v0.32.6 + {py3.8,py3.11,py3.12}-huggingface_hub-v0.26.5 + {py3.8,py3.12,py3.13}-huggingface_hub-v0.30.2 + {py3.8,py3.12,py3.13}-huggingface_hub-v0.33.0 # ~~~ DBs ~~~ @@ -159,7 +159,7 @@ envlist = {py3.6}-pymongo-v3.5.1 {py3.6,py3.10,py3.11}-pymongo-v3.13.0 {py3.6,py3.9,py3.10}-pymongo-v4.0.2 - {py3.9,py3.12,py3.13}-pymongo-v4.13.0 + {py3.9,py3.12,py3.13}-pymongo-v4.13.1 {py3.6}-redis_py_cluster_legacy-v1.3.6 {py3.6,py3.7}-redis_py_cluster_legacy-v2.0.0 @@ -205,7 +205,7 @@ envlist = {py3.8,py3.10,py3.11}-strawberry-v0.209.8 {py3.8,py3.11,py3.12}-strawberry-v0.230.0 {py3.8,py3.12,py3.13}-strawberry-v0.251.0 - {py3.9,py3.12,py3.13}-strawberry-v0.273.0 + {py3.9,py3.12,py3.13}-strawberry-v0.273.2 # ~~~ Network ~~~ @@ -263,10 +263,10 @@ envlist = {py3.7}-aiohttp-v3.4.4 {py3.7,py3.8,py3.9}-aiohttp-v3.7.4 {py3.8,py3.12,py3.13}-aiohttp-v3.10.11 - {py3.9,py3.12,py3.13}-aiohttp-v3.12.12 + {py3.9,py3.12,py3.13}-aiohttp-v3.12.13 {py3.6,py3.7}-bottle-v0.12.25 - {py3.8,py3.12,py3.13}-bottle-v0.13.3 + {py3.8,py3.12,py3.13}-bottle-v0.13.4 {py3.6}-falcon-v1.4.1 {py3.6,py3.7}-falcon-v2.0.0 @@ -516,12 +516,13 @@ deps = cohere-v5.11.4: cohere==5.11.4 cohere-v5.15.0: cohere==5.15.0 - openai_agents-v0.0.16: openai-agents==0.0.16 + openai_agents-v0.0.17: openai-agents==0.0.17 + openai_agents: pytest-asyncio huggingface_hub-v0.22.2: huggingface_hub==0.22.2 - huggingface_hub-v0.25.2: huggingface_hub==0.25.2 - huggingface_hub-v0.28.1: huggingface_hub==0.28.1 - huggingface_hub-v0.32.6: huggingface_hub==0.32.6 + huggingface_hub-v0.26.5: huggingface_hub==0.26.5 + huggingface_hub-v0.30.2: huggingface_hub==0.30.2 + huggingface_hub-v0.33.0: huggingface_hub==0.33.0 # ~~~ DBs ~~~ @@ -530,7 +531,7 @@ deps = pymongo-v3.5.1: pymongo==3.5.1 pymongo-v3.13.0: pymongo==3.13.0 pymongo-v4.0.2: pymongo==4.0.2 - pymongo-v4.13.0: pymongo==4.13.0 + pymongo-v4.13.1: pymongo==4.13.1 pymongo: mockupdb redis_py_cluster_legacy-v1.3.6: redis-py-cluster==1.3.6 @@ -586,7 +587,7 @@ deps = strawberry-v0.209.8: strawberry-graphql[fastapi,flask]==0.209.8 strawberry-v0.230.0: strawberry-graphql[fastapi,flask]==0.230.0 strawberry-v0.251.0: strawberry-graphql[fastapi,flask]==0.251.0 - strawberry-v0.273.0: strawberry-graphql[fastapi,flask]==0.273.0 + strawberry-v0.273.2: strawberry-graphql[fastapi,flask]==0.273.2 strawberry: httpx strawberry-v0.209.8: pydantic<2.11 strawberry-v0.230.0: pydantic<2.11 @@ -700,13 +701,13 @@ deps = aiohttp-v3.4.4: aiohttp==3.4.4 aiohttp-v3.7.4: aiohttp==3.7.4 aiohttp-v3.10.11: aiohttp==3.10.11 - aiohttp-v3.12.12: aiohttp==3.12.12 + aiohttp-v3.12.13: aiohttp==3.12.13 aiohttp: pytest-aiohttp aiohttp-v3.10.11: pytest-asyncio - aiohttp-v3.12.12: pytest-asyncio + aiohttp-v3.12.13: pytest-asyncio bottle-v0.12.25: bottle==0.12.25 - bottle-v0.13.3: bottle==0.13.3 + bottle-v0.13.4: bottle==0.13.4 bottle: werkzeug<2.1.0 falcon-v1.4.1: falcon==1.4.1 From 39c124de9d7cbf6d6562a05bcdfa86b12fc3b1bf Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 16 Jun 2025 16:22:48 +0200 Subject: [PATCH 45/89] move stuff --- .../openai_agents/test_openai_agents.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 4d48a13320..ea05dcba28 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -40,9 +40,14 @@ def mock_model_response(mock_usage): return ModelResponse( output=[ ResponseOutputMessage( + id="msg_123", + type="message", + status="completed", content=[ ResponseOutputText( - text="Hello, how can I help you?", type="text", annotations={} + text="Hello, how can I help you?", + type="output_text", + annotations=[], ) ], role="assistant", @@ -57,6 +62,10 @@ def mock_model_response(mock_usage): def mock_tool_call(): return ResponseFunctionToolCall( id="call_123", + call_id="call_123", + name="test_tool", + type="function_call", + arguments='{"arg1": "value1"}', function=MagicMock(name="test_tool", arguments='{"arg1": "value1"}'), ) @@ -211,13 +220,16 @@ async def test_error_handling(sentry_init, capture_events, mock_agent): events = capture_events() - with sentry_sdk.start_transaction(name="test_transaction"): + with sentry_sdk.start_transaction(name="test_transaction") as transaction: with patch("agents.Runner.run", new_callable=AsyncMock) as mock_run: mock_run.side_effect = AgentsException("Test error") with pytest.raises(AgentsException): await agents.Runner.run(mock_agent, "Test input") + # Ensure the transaction is finished and error is captured + transaction.finish() + (transaction,) = events spans = transaction["spans"] From 20c5343c255719c134a5973e5ea20d27fec27fe1 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 09:04:13 +0200 Subject: [PATCH 46/89] tests --- .../openai_agents/test_openai_agents.py | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index ea05dcba28..a465be3966 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -117,6 +117,8 @@ def mock_agent(): frequency_penalty=0.0, ) agent.tools = [] + agent.handoffs = [] + agent.output_type = str return agent @@ -133,11 +135,14 @@ async def test_agent_invocation_span( integrations=[OpenAIAgentsIntegration()], traces_sample_rate=1.0, ) + events = capture_events() with sentry_sdk.start_transaction(name="test_transaction"): - with patch("agents.Runner.run", new_callable=AsyncMock) as mock_run: - mock_run.return_value = mock_run_result + with patch( + "sentry_sdk.integrations.openai_agents.patches.runner._create_run_wrapper" + ) as mock_run: + mock_run.return_value = AsyncMock(return_value=mock_run_result) await agents.Runner.run(mock_agent, "Test input") @@ -156,7 +161,7 @@ async def test_agent_invocation_span( @pytest.mark.asyncio async def test_tool_execution_span( - sentry_init, capture_events, mock_agent, mock_run_result + sentry_init, capture_events, mock_agent, mock_run_result, mock_tool_call ): sentry_init( integrations=[OpenAIAgentsIntegration()], @@ -166,8 +171,13 @@ async def test_tool_execution_span( events = capture_events() with sentry_sdk.start_transaction(name="test_transaction"): - with patch("agents.Runner.run", new_callable=AsyncMock) as mock_run: - mock_run.return_value = mock_run_result + with patch( + "sentry_sdk.integrations.openai_agents.patches.runner._create_run_wrapper" + ) as mock_run, patch( + "sentry_sdk.integrations.openai_agents.patches.tools._create_get_all_tools_wrapper" + ) as mock_tools: + mock_run.return_value = AsyncMock(return_value=mock_run_result) + mock_tools.return_value = AsyncMock(return_value=[mock_tool_call]) await agents.Runner.run(mock_agent, "Test input") @@ -194,8 +204,13 @@ async def test_llm_request_span( events = capture_events() with sentry_sdk.start_transaction(name="test_transaction"): - with patch("agents.Runner.run", new_callable=AsyncMock) as mock_run: - mock_run.return_value = mock_run_result + with patch( + "sentry_sdk.integrations.openai_agents.patches.runner._create_run_wrapper" + ) as mock_run, patch( + "sentry_sdk.integrations.openai_agents.patches.models._create_get_model_wrapper" + ) as mock_model: + mock_run.return_value = AsyncMock(return_value=mock_run_result) + mock_model.return_value = AsyncMock(return_value=mock_model_response) await agents.Runner.run(mock_agent, "Test input") @@ -221,8 +236,10 @@ async def test_error_handling(sentry_init, capture_events, mock_agent): events = capture_events() with sentry_sdk.start_transaction(name="test_transaction") as transaction: - with patch("agents.Runner.run", new_callable=AsyncMock) as mock_run: - mock_run.side_effect = AgentsException("Test error") + with patch( + "sentry_sdk.integrations.openai_agents.patches.runner._create_run_wrapper" + ) as mock_run: + mock_run.return_value = AsyncMock(side_effect=AgentsException("Test error")) with pytest.raises(AgentsException): await agents.Runner.run(mock_agent, "Test input") From c31d4a4526601ceb51db845b99e046adca241de2 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 09:17:19 +0200 Subject: [PATCH 47/89] Better output --- sentry_sdk/consts.py | 6 ++++++ sentry_sdk/integrations/openai_agents/utils.py | 12 ++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index d9a028fee9..53148a36df 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -372,6 +372,12 @@ class SPANDATA: Example: "chat" """ + GEN_AI_RESPONSE_TEXT = "gen_ai.response.text" + """ + The model's response text messages. + Example: ["The weather in Paris is rainy and overcast, with temperatures around 57°F", "The weather in London is sunny and warm, with temperatures around 65°F"] + """ + GEN_AI_RESPONSE_TOOL_CALLS = "gen_ai.response.tool_calls" """ The tool calls in the model's response. diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 43cbf92e26..7bb233de2d 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -202,14 +202,14 @@ def _set_output_data(span, result): output_messages["tool"].append(output.dict()) elif output.type == "message": for output_message in output.content: - output_messages["response"].append(output_message.to_json()) + try: + output_messages["response"].append(output_message.text) + except AttributeError: + # Unknown output message type, just return the json + output_messages["response"].append(output_message.to_json()) if len(output_messages["tool"]) > 0: span.set_data(SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, output_messages["tool"]) if len(output_messages["response"]) > 0: - span.set_data(SPANDATA.GEN_AI_CHOICE, output_messages["response"]) - - # Deprecated name just for first iteration. - # TODO-anton: define how to set tool response messages and document in sentry-conventions. - span.set_data("gen_ai.response.text", output_messages["response"]) + span.set_data(SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"]) From 5aa30891d9b58df9f80b4602d1b027f755ebdd94 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 10:49:06 +0200 Subject: [PATCH 48/89] a working test --- .../openai_agents/test_openai_agents.py | 280 +++++++++++++----- 1 file changed, 203 insertions(+), 77 deletions(-) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index a465be3966..d60338ef26 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -1,5 +1,6 @@ import pytest -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock, patch +import os import sentry_sdk from sentry_sdk.integrations.openai_agents import OpenAIAgentsIntegration @@ -13,6 +14,7 @@ MessageOutputItem, ToolCallItem, ToolCallOutputItem, + ModelSettings, ) from agents.items import ( ResponseOutputMessage, @@ -20,7 +22,6 @@ ResponseFunctionToolCall, FunctionCallOutput, ) -from agents.exceptions import AgentsException @pytest.fixture @@ -122,6 +123,23 @@ def mock_agent(): return agent +@pytest.fixture +def test_agent(): + """Create a real Agent instance for testing.""" + return Agent( + name="test_agent", + instructions="You are a helpful test assistant.", + model="gpt-4", + model_settings=ModelSettings( + max_tokens=100, + temperature=0.7, + top_p=1.0, + presence_penalty=0.0, + frequency_penalty=0.0, + ), + ) + + def test_integration_initialization(): integration = OpenAIAgentsIntegration() assert integration is not None @@ -129,40 +147,57 @@ def test_integration_initialization(): @pytest.mark.asyncio async def test_agent_invocation_span( - sentry_init, capture_events, mock_agent, mock_run_result + sentry_init, capture_events, test_agent, mock_model_response ): - sentry_init( - integrations=[OpenAIAgentsIntegration()], - traces_sample_rate=1.0, - ) + """Test that the integration creates spans for agent invocations.""" - events = capture_events() - - with sentry_sdk.start_transaction(name="test_transaction"): + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): with patch( - "sentry_sdk.integrations.openai_agents.patches.runner._create_run_wrapper" - ) as mock_run: - mock_run.return_value = AsyncMock(return_value=mock_run_result) + "agents.models.openai_responses.OpenAIResponsesModel.get_response" + ) as mock_get_response: + mock_get_response.return_value = mock_model_response - await agents.Runner.run(mock_agent, "Test input") + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + ) + + events = capture_events() + + with sentry_sdk.start_transaction(name="test_transaction"): + result = await agents.Runner.run(test_agent, "Test input") + + assert result is not None + assert result.final_output == "Hello, how can I help you?" (transaction,) = events spans = transaction["spans"] + agent_workflow_span, invoke_agent_span, ai_client_span = spans + + assert agent_workflow_span["description"] == "test_agent workflow" + + assert invoke_agent_span["description"] == "invoke_agent test_agent" + assert invoke_agent_span["data"]["gen_ai.operation.name"] == "invoke_agent" + assert invoke_agent_span["data"]["gen_ai.system"] == "openai" + assert invoke_agent_span["data"]["gen_ai.agent.name"] == "test_agent" + assert invoke_agent_span["data"]["gen_ai.request.max_tokens"] == 100 + assert invoke_agent_span["data"]["gen_ai.request.model"] == "gpt-4" + assert invoke_agent_span["data"]["gen_ai.request.temperature"] == 0.7 + assert invoke_agent_span["data"]["gen_ai.request.top_p"] == 1.0 - assert len(spans) > 0 - agent_span = next(span for span in spans if span["op"] == "agent.run") - assert agent_span is not None - assert agent_span["data"]["gen_ai.system"] == "openai" - assert agent_span["data"]["gen_ai.agent.name"] == "test_agent" - assert agent_span["data"]["gen_ai.request.model"] == "gpt-4" - assert agent_span["data"]["gen_ai.request.max_tokens"] == 100 - assert agent_span["data"]["gen_ai.request.temperature"] == 0.7 + assert ai_client_span["description"] == "chat gpt-4" + assert ai_client_span["data"]["gen_ai.operation.name"] == "chat" + assert ai_client_span["data"]["gen_ai.system"] == "openai" + assert ai_client_span["data"]["gen_ai.agent.name"] == "test_agent" + assert ai_client_span["data"]["gen_ai.request.max_tokens"] == 100 + assert ai_client_span["data"]["gen_ai.request.model"] == "gpt-4" + assert ai_client_span["data"]["gen_ai.request.temperature"] == 0.7 + assert ai_client_span["data"]["gen_ai.request.top_p"] == 1.0 @pytest.mark.asyncio -async def test_tool_execution_span( - sentry_init, capture_events, mock_agent, mock_run_result, mock_tool_call -): +async def test_tool_execution_span(sentry_init, capture_events, test_agent): + """Test tool execution span creation.""" sentry_init( integrations=[OpenAIAgentsIntegration()], traces_sample_rate=1.0, @@ -170,32 +205,102 @@ async def test_tool_execution_span( events = capture_events() - with sentry_sdk.start_transaction(name="test_transaction"): + # Create a simple test tool + from agents import function_tool + + @function_tool + def simple_test_tool(message: str) -> str: + """A simple test tool that returns a message.""" + return f"Tool executed with: {message}" + + # Create agent with the tool + agent_with_tool = test_agent.clone(tools=[simple_test_tool]) + + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): with patch( - "sentry_sdk.integrations.openai_agents.patches.runner._create_run_wrapper" - ) as mock_run, patch( - "sentry_sdk.integrations.openai_agents.patches.tools._create_get_all_tools_wrapper" - ) as mock_tools: - mock_run.return_value = AsyncMock(return_value=mock_run_result) - mock_tools.return_value = AsyncMock(return_value=[mock_tool_call]) + "agents.models.openai_responses.OpenAIResponsesModel.get_response" + ) as mock_get_response: + # Create a mock response that includes tool calls + from agents.items import ResponseFunctionToolCall + + tool_call = ResponseFunctionToolCall( + id="call_123", + call_id="call_123", + name="simple_test_tool", + type="function_call", + arguments='{"message": "hello"}', + function=MagicMock( + name="simple_test_tool", arguments='{"message": "hello"}' + ), + ) - await agents.Runner.run(mock_agent, "Test input") + # First response with tool call + tool_response = ModelResponse( + output=[tool_call], + usage=Usage( + requests=1, input_tokens=10, output_tokens=5, total_tokens=15 + ), + response_id="resp_tool_123", + ) - (transaction,) = events - spans = transaction["spans"] + # Second response with final answer + final_response = ModelResponse( + output=[ + ResponseOutputMessage( + id="msg_final", + type="message", + status="completed", + content=[ + ResponseOutputText( + text="Task completed using the tool", + type="output_text", + annotations=[], + ) + ], + role="assistant", + ) + ], + usage=Usage( + requests=1, input_tokens=15, output_tokens=10, total_tokens=25 + ), + response_id="resp_final_123", + ) + + # Return different responses on successive calls + mock_get_response.side_effect = [tool_response, final_response] + + with sentry_sdk.start_transaction(name="test_transaction"): + await agents.Runner.run( + agent_with_tool, "Please use the simple test tool" + ) + + # Verify spans were created + assert len(events) > 0 + transaction = events[0] + spans = transaction.get("spans", []) + + # Should have workflow span + workflow_spans = [ + span for span in spans if "workflow" in span.get("description", "") + ] + assert len(workflow_spans) > 0 + + # Should have AI client spans + ai_spans = [span for span in spans if span.get("op") == "llm.request"] + assert len(ai_spans) > 0 - tool_spans = [span for span in spans if span["op"] == "agent.tool"] + # Look for tool execution spans + tool_spans = [span for span in spans if span.get("op") == "agent.tool"] assert len(tool_spans) > 0 - tool_span = tool_spans[0] # type: ignore - assert tool_span["data"]["gen_ai.tool.name"] == "test_tool" - assert tool_span["data"]["gen_ai.tool.input"] == '{"arg1": "value1"}' - assert tool_span["data"]["gen_ai.tool.output"] == "Tool execution result" + + tool_span = tool_spans[0] + tool_data = tool_span.get("data", {}) + assert tool_data.get("gen_ai.tool.name") == "simple_test_tool" @pytest.mark.asyncio -async def test_llm_request_span( - sentry_init, capture_events, mock_agent, mock_run_result -): +async def test_error_handling(sentry_init, capture_events, test_agent): + """Test error handling in agent execution.""" sentry_init( integrations=[OpenAIAgentsIntegration()], traces_sample_rate=1.0, @@ -203,31 +308,38 @@ async def test_llm_request_span( events = capture_events() - with sentry_sdk.start_transaction(name="test_transaction"): + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): with patch( - "sentry_sdk.integrations.openai_agents.patches.runner._create_run_wrapper" - ) as mock_run, patch( - "sentry_sdk.integrations.openai_agents.patches.models._create_get_model_wrapper" - ) as mock_model: - mock_run.return_value = AsyncMock(return_value=mock_run_result) - mock_model.return_value = AsyncMock(return_value=mock_model_response) + "agents.models.openai_responses.OpenAIResponsesModel.get_response" + ) as mock_get_response: + # Make the model call fail + mock_get_response.side_effect = Exception("Model Error") - await agents.Runner.run(mock_agent, "Test input") + with sentry_sdk.start_transaction(name="test_transaction"): + with pytest.raises(Exception, match="Model Error"): + await agents.Runner.run(test_agent, "Test input") - (transaction,) = events - spans = transaction["spans"] + # Verify spans were created even with errors + if events: + transaction = events[0] + spans = transaction.get("spans", []) + + # Should have workflow span + workflow_spans = [ + span for span in spans if "workflow" in span.get("description", "") + ] + assert len(workflow_spans) > 0 - llm_spans = [span for span in spans if span["op"] == "llm.request"] - assert len(llm_spans) > 0 - llm_span = llm_spans[0] # type: ignore - assert llm_span["data"]["gen_ai.request.model"] == "gpt-4" - assert llm_span["data"]["gen_ai.usage.input_tokens"] == 10 - assert llm_span["data"]["gen_ai.usage.output_tokens"] == 20 - assert llm_span["data"]["gen_ai.usage.total_tokens"] == 30 + # Check for error spans + error_spans = [span for span in spans if span.get("status") == "internal_error"] + assert len(error_spans) > 0 @pytest.mark.asyncio -async def test_error_handling(sentry_init, capture_events, mock_agent): +async def test_integration_with_hooks( + sentry_init, capture_events, test_agent, mock_model_response +): + """Test that the integration properly wraps hooks.""" sentry_init( integrations=[OpenAIAgentsIntegration()], traces_sample_rate=1.0, @@ -235,23 +347,37 @@ async def test_error_handling(sentry_init, capture_events, mock_agent): events = capture_events() - with sentry_sdk.start_transaction(name="test_transaction") as transaction: - with patch( - "sentry_sdk.integrations.openai_agents.patches.runner._create_run_wrapper" - ) as mock_run: - mock_run.return_value = AsyncMock(side_effect=AgentsException("Test error")) + # Create a simple hook to test with + hook_calls = [] - with pytest.raises(AgentsException): - await agents.Runner.run(mock_agent, "Test input") + from agents.lifecycle import AgentHooks - # Ensure the transaction is finished and error is captured - transaction.finish() + class TestHooks(AgentHooks): + async def on_agent_start(self, context, agent): + hook_calls.append("start") - (transaction,) = events - spans = transaction["spans"] + async def on_agent_end(self, context, agent, output): + hook_calls.append("end") - assert len(spans) > 0 - error_span = next(span for span in spans if span["status"] == "internal_error") - assert error_span is not None - assert error_span["data"]["error.type"] == "AgentsException" - assert error_span["data"]["error.value"] == "Test error" + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): + with patch( + "agents.models.openai_responses.OpenAIResponsesModel.get_response" + ) as mock_get_response: + mock_get_response.return_value = mock_model_response + + with sentry_sdk.start_transaction(name="test_transaction"): + await agents.Runner.run(test_agent, "Test input", hooks=TestHooks()) + + # Verify hooks were called (integration shouldn't break them) + assert "start" in hook_calls + assert "end" in hook_calls + + # Verify spans were still created + assert len(events) > 0 + transaction = events[0] + spans = transaction.get("spans", []) + + workflow_spans = [ + span for span in spans if "workflow" in span.get("description", "") + ] + assert len(workflow_spans) > 0 From 4ef332f95c4eedd5a86feb286c4522005802281c Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 11:23:54 +0200 Subject: [PATCH 49/89] another test --- .../openai_agents/test_openai_agents.py | 270 ++++++++++++++---- 1 file changed, 209 insertions(+), 61 deletions(-) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index d60338ef26..32c430ac6f 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -1,3 +1,4 @@ +from unittest import mock import pytest from unittest.mock import MagicMock, patch import os @@ -198,19 +199,10 @@ async def test_agent_invocation_span( @pytest.mark.asyncio async def test_tool_execution_span(sentry_init, capture_events, test_agent): """Test tool execution span creation.""" - sentry_init( - integrations=[OpenAIAgentsIntegration()], - traces_sample_rate=1.0, - ) - - events = capture_events() - - # Create a simple test tool - from agents import function_tool - @function_tool + @agents.function_tool def simple_test_tool(message: str) -> str: - """A simple test tool that returns a message.""" + """A simple tool""" return f"Tool executed with: {message}" # Create agent with the tool @@ -221,8 +213,6 @@ def simple_test_tool(message: str) -> str: "agents.models.openai_responses.OpenAIResponsesModel.get_response" ) as mock_get_response: # Create a mock response that includes tool calls - from agents.items import ResponseFunctionToolCall - tool_call = ResponseFunctionToolCall( id="call_123", call_id="call_123", @@ -269,44 +259,164 @@ def simple_test_tool(message: str) -> str: # Return different responses on successive calls mock_get_response.side_effect = [tool_response, final_response] + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + send_default_pii=True, + ) + + events = capture_events() + with sentry_sdk.start_transaction(name="test_transaction"): await agents.Runner.run( agent_with_tool, "Please use the simple test tool" ) - # Verify spans were created - assert len(events) > 0 - transaction = events[0] - spans = transaction.get("spans", []) - - # Should have workflow span - workflow_spans = [ - span for span in spans if "workflow" in span.get("description", "") + (transaction,) = events + spans = transaction["spans"] + ( + agent_workflow_span, + agent_span, + ai_client_span1, + tool_span, + ai_client_span2, + ) = spans + + available_tools = [ + { + "name": "simple_test_tool", + "description": "A simple tool", + "params_json_schema": { + "properties": {"message": {"title": "Message", "type": "string"}}, + "required": ["message"], + "title": "simple_test_tool_args", + "type": "object", + "additionalProperties": False, + }, + "on_invoke_tool": mock.ANY, + "strict_json_schema": True, + "is_enabled": True, + } ] - assert len(workflow_spans) > 0 - # Should have AI client spans - ai_spans = [span for span in spans if span.get("op") == "llm.request"] - assert len(ai_spans) > 0 + assert agent_workflow_span["description"] == "test_agent workflow" - # Look for tool execution spans - tool_spans = [span for span in spans if span.get("op") == "agent.tool"] - assert len(tool_spans) > 0 + assert agent_span["description"] == "invoke_agent test_agent" + assert agent_span["data"]["gen_ai.agent.name"] == "test_agent" + assert agent_span["data"]["gen_ai.operation.name"] == "invoke_agent" + assert agent_span["data"]["gen_ai.request.available_tools"] == available_tools + assert agent_span["data"]["gen_ai.request.max_tokens"] == 100 + assert agent_span["data"]["gen_ai.request.model"] == "gpt-4" + assert agent_span["data"]["gen_ai.request.temperature"] == 0.7 + assert agent_span["data"]["gen_ai.request.top_p"] == 1.0 + assert agent_span["data"]["gen_ai.system"] == "openai" + + assert ai_client_span1["description"] == "chat gpt-4" + assert ai_client_span1["data"]["gen_ai.operation.name"] == "chat" + assert ai_client_span1["data"]["gen_ai.system"] == "openai" + assert ai_client_span1["data"]["gen_ai.agent.name"] == "test_agent" + assert ai_client_span1["data"]["gen_ai.request.available_tools"] == available_tools + assert ai_client_span1["data"]["gen_ai.request.max_tokens"] == 100 + assert ai_client_span1["data"]["gen_ai.request.messages"] == [ + { + "role": "system", + "content": [{"type": "text", "text": "You are a helpful test assistant."}], + }, + { + "role": "user", + "content": [{"type": "text", "text": "Please use the simple test tool"}], + }, + ] + assert ai_client_span1["data"]["gen_ai.request.model"] == "gpt-4" + assert ai_client_span1["data"]["gen_ai.request.temperature"] == 0.7 + assert ai_client_span1["data"]["gen_ai.request.top_p"] == 1.0 + assert ai_client_span1["data"]["gen_ai.usage.input_tokens"] == 10 + assert ai_client_span1["data"]["gen_ai.usage.input_tokens.cached"] == 0 + assert ai_client_span1["data"]["gen_ai.usage.output_tokens"] == 5 + assert ai_client_span1["data"]["gen_ai.usage.output_tokens.reasoning"] == 0 + assert ai_client_span1["data"]["gen_ai.usage.total_tokens"] == 15 + assert ai_client_span1["data"]["gen_ai.response.tool_calls"] == [ + { + "arguments": '{"message": "hello"}', + "call_id": "call_123", + "name": "simple_test_tool", + "type": "function_call", + "id": "call_123", + "status": None, + "function": mock.ANY, + } + ] - tool_span = tool_spans[0] - tool_data = tool_span.get("data", {}) - assert tool_data.get("gen_ai.tool.name") == "simple_test_tool" + assert tool_span["description"] == "execute_tool simple_test_tool" + assert tool_span["data"]["gen_ai.agent.name"] == "test_agent" + assert tool_span["data"]["gen_ai.operation.name"] == "execute_tool" + assert tool_span["data"]["gen_ai.request.available_tools"] == available_tools + assert tool_span["data"]["gen_ai.request.max_tokens"] == 100 + assert tool_span["data"]["gen_ai.request.model"] == "gpt-4" + assert tool_span["data"]["gen_ai.request.temperature"] == 0.7 + assert tool_span["data"]["gen_ai.request.top_p"] == 1.0 + assert tool_span["data"]["gen_ai.system"] == "openai" + assert tool_span["data"]["gen_ai.tool.description"] == "A simple tool" + assert tool_span["data"]["gen_ai.tool.input"] == '{"message": "hello"}' + assert tool_span["data"]["gen_ai.tool.name"] == "simple_test_tool" + assert tool_span["data"]["gen_ai.tool.output"] == "Tool executed with: hello" + assert tool_span["data"]["gen_ai.tool.type"] == "function" + + assert ai_client_span2["description"] == "chat gpt-4" + assert ai_client_span2["data"]["gen_ai.agent.name"] == "test_agent" + assert ai_client_span2["data"]["gen_ai.operation.name"] == "chat" + assert ai_client_span2["data"]["gen_ai.request.available_tools"] == available_tools + assert ai_client_span2["data"]["gen_ai.request.max_tokens"] == 100 + assert ai_client_span2["data"]["gen_ai.request.messages"] == [ + { + "role": "system", + "content": [{"type": "text", "text": "You are a helpful test assistant."}], + }, + { + "role": "user", + "content": [{"type": "text", "text": "Please use the simple test tool"}], + }, + { + "role": "assistant", + "content": [ + { + "arguments": '{"message": "hello"}', + "call_id": "call_123", + "name": "simple_test_tool", + "type": "function_call", + "id": "call_123", + "function": mock.ANY, + } + ], + }, + { + "role": "tool", + "content": [ + { + "call_id": "call_123", + "output": "Tool executed with: hello", + "type": "function_call_output", + } + ], + }, + ] + assert ai_client_span2["data"]["gen_ai.request.model"] == "gpt-4" + assert ai_client_span2["data"]["gen_ai.request.temperature"] == 0.7 + assert ai_client_span2["data"]["gen_ai.request.top_p"] == 1.0 + assert ai_client_span2["data"]["gen_ai.response.text"] == [ + "Task completed using the tool" + ] + assert ai_client_span2["data"]["gen_ai.system"] == "openai" + assert ai_client_span2["data"]["gen_ai.usage.input_tokens.cached"] == 0 + assert ai_client_span2["data"]["gen_ai.usage.input_tokens"] == 15 + assert ai_client_span2["data"]["gen_ai.usage.output_tokens.reasoning"] == 0 + assert ai_client_span2["data"]["gen_ai.usage.output_tokens"] == 10 + assert ai_client_span2["data"]["gen_ai.usage.total_tokens"] == 25 @pytest.mark.asyncio async def test_error_handling(sentry_init, capture_events, test_agent): """Test error handling in agent execution.""" - sentry_init( - integrations=[OpenAIAgentsIntegration()], - traces_sample_rate=1.0, - ) - - events = capture_events() with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): with patch( @@ -315,24 +425,46 @@ async def test_error_handling(sentry_init, capture_events, test_agent): # Make the model call fail mock_get_response.side_effect = Exception("Model Error") + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + ) + + events = capture_events() + with sentry_sdk.start_transaction(name="test_transaction"): with pytest.raises(Exception, match="Model Error"): await agents.Runner.run(test_agent, "Test input") # Verify spans were created even with errors - if events: - transaction = events[0] - spans = transaction.get("spans", []) + (transaction,) = events + spans = transaction["spans"] + + # Should have workflow span and invoke agent span at minimum + assert len(spans) >= 2 + + # Check workflow span + workflow_spans = [ + span for span in spans if "workflow" in span.get("description", "") + ] + assert len(workflow_spans) == 1 + workflow_span = workflow_spans[0] + assert workflow_span["description"] == "test_agent workflow" - # Should have workflow span - workflow_spans = [ - span for span in spans if "workflow" in span.get("description", "") - ] - assert len(workflow_spans) > 0 + # Check invoke agent span + invoke_spans = [ + span for span in spans if span.get("description", "").startswith("invoke_agent") + ] + assert len(invoke_spans) == 1 + invoke_span = invoke_spans[0] + assert invoke_span["description"] == "invoke_agent test_agent" + assert invoke_span["data"]["gen_ai.operation.name"] == "invoke_agent" + assert invoke_span["data"]["gen_ai.system"] == "openai" + assert invoke_span["data"]["gen_ai.agent.name"] == "test_agent" - # Check for error spans - error_spans = [span for span in spans if span.get("status") == "internal_error"] - assert len(error_spans) > 0 + # Check for error spans + error_spans = [span for span in spans if span.get("status") == "internal_error"] + assert len(error_spans) >= 1 @pytest.mark.asyncio @@ -340,12 +472,6 @@ async def test_integration_with_hooks( sentry_init, capture_events, test_agent, mock_model_response ): """Test that the integration properly wraps hooks.""" - sentry_init( - integrations=[OpenAIAgentsIntegration()], - traces_sample_rate=1.0, - ) - - events = capture_events() # Create a simple hook to test with hook_calls = [] @@ -365,6 +491,13 @@ async def on_agent_end(self, context, agent, output): ) as mock_get_response: mock_get_response.return_value = mock_model_response + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + ) + + events = capture_events() + with sentry_sdk.start_transaction(name="test_transaction"): await agents.Runner.run(test_agent, "Test input", hooks=TestHooks()) @@ -373,11 +506,26 @@ async def on_agent_end(self, context, agent, output): assert "end" in hook_calls # Verify spans were still created - assert len(events) > 0 - transaction = events[0] - spans = transaction.get("spans", []) + (transaction,) = events + spans = transaction["spans"] + agent_workflow_span, invoke_agent_span, ai_client_span = spans - workflow_spans = [ - span for span in spans if "workflow" in span.get("description", "") - ] - assert len(workflow_spans) > 0 + assert agent_workflow_span["description"] == "test_agent workflow" + + assert invoke_agent_span["description"] == "invoke_agent test_agent" + assert invoke_agent_span["data"]["gen_ai.operation.name"] == "invoke_agent" + assert invoke_agent_span["data"]["gen_ai.system"] == "openai" + assert invoke_agent_span["data"]["gen_ai.agent.name"] == "test_agent" + assert invoke_agent_span["data"]["gen_ai.request.max_tokens"] == 100 + assert invoke_agent_span["data"]["gen_ai.request.model"] == "gpt-4" + assert invoke_agent_span["data"]["gen_ai.request.temperature"] == 0.7 + assert invoke_agent_span["data"]["gen_ai.request.top_p"] == 1.0 + + assert ai_client_span["description"] == "chat gpt-4" + assert ai_client_span["data"]["gen_ai.operation.name"] == "chat" + assert ai_client_span["data"]["gen_ai.system"] == "openai" + assert ai_client_span["data"]["gen_ai.agent.name"] == "test_agent" + assert ai_client_span["data"]["gen_ai.request.max_tokens"] == 100 + assert ai_client_span["data"]["gen_ai.request.model"] == "gpt-4" + assert ai_client_span["data"]["gen_ai.request.temperature"] == 0.7 + assert ai_client_span["data"]["gen_ai.request.top_p"] == 1.0 From abe5d05967233451be358465b985533426d610f0 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 11:25:29 +0200 Subject: [PATCH 50/89] another test --- .../openai_agents/test_openai_agents.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 32c430ac6f..6c8113fb53 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -150,7 +150,9 @@ def test_integration_initialization(): async def test_agent_invocation_span( sentry_init, capture_events, test_agent, mock_model_response ): - """Test that the integration creates spans for agent invocations.""" + """ + Test that the integration creates spans for agent invocations. + """ with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): with patch( @@ -198,7 +200,9 @@ async def test_agent_invocation_span( @pytest.mark.asyncio async def test_tool_execution_span(sentry_init, capture_events, test_agent): - """Test tool execution span creation.""" + """ + Test tool execution span creation. + """ @agents.function_tool def simple_test_tool(message: str) -> str: @@ -416,7 +420,9 @@ def simple_test_tool(message: str) -> str: @pytest.mark.asyncio async def test_error_handling(sentry_init, capture_events, test_agent): - """Test error handling in agent execution.""" + """ + Test error handling in agent execution. + """ with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): with patch( @@ -471,7 +477,9 @@ async def test_error_handling(sentry_init, capture_events, test_agent): async def test_integration_with_hooks( sentry_init, capture_events, test_agent, mock_model_response ): - """Test that the integration properly wraps hooks.""" + """ + Test that the integration properly wraps hooks. + """ # Create a simple hook to test with hook_calls = [] From 3a2230d1aa2f499ec8af7892711930d219ed10cc Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 11:47:41 +0200 Subject: [PATCH 51/89] Setting span origin --- .../openai_agents/spans/agent_workflow.py | 4 ++- .../openai_agents/spans/ai_client.py | 11 ++++-- .../openai_agents/spans/execute_tool.py | 8 ++--- .../openai_agents/spans/invoke_agent.py | 7 ++-- .../integrations/openai_agents/utils.py | 10 ++++-- .../openai_agents/test_openai_agents.py | 36 +++++-------------- 6 files changed, 36 insertions(+), 40 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py index 5934a8723c..c02b3583c5 100644 --- a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py +++ b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py @@ -12,7 +12,9 @@ def agent_workflow_span(*args, **kwargs): # type: (*Any, **Any) -> Span agent = args[0] # Create a transaction or a span if an transaction is already active - span = _get_start_span_function()(name=f"{agent.name} workflow") + span = _get_start_span_function()( + name=f"{agent.name} workflow", + ) return span diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index f9d746afec..90c69cbe9a 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -1,7 +1,12 @@ -import sentry_sdk from sentry_sdk.consts import OP, SPANDATA -from ..utils import _set_agent_data, _set_input_data, _set_output_data, _set_usage_data +from ..utils import ( + _get_start_span_function, + _set_agent_data, + _set_input_data, + _set_output_data, + _set_usage_data, +) from typing import TYPE_CHECKING @@ -14,7 +19,7 @@ def ai_client_span(agent, get_response_kwargs): # type: (Agent, dict[str, Any]) -> Span # TODO-anton: implement other types of operations. Now "chat" is hardcoded. - span = sentry_sdk.start_span( + span = _get_start_span_function()( op=OP.GEN_AI_CHAT, description=f"chat {agent.model}", ) diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index c9ce8e41a2..60a3d91014 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -1,8 +1,7 @@ -import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.scope import should_send_default_pii -from ..utils import _set_agent_data +from ..utils import _get_start_span_function, _set_agent_data from typing import TYPE_CHECKING @@ -14,8 +13,9 @@ def execute_tool_span(tool, *args, **kwargs): # type: (agents.Tool, *Any, **Any) -> Span - span = sentry_sdk.start_span( - op=OP.GEN_AI_EXECUTE_TOOL, name=f"execute_tool {tool.name}" + span = _get_start_span_function()( + op=OP.GEN_AI_EXECUTE_TOOL, + name=f"execute_tool {tool.name}", ) span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "execute_tool") diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index 8bc6cf3fd3..97ee2e3f82 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -1,7 +1,7 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA -from ..utils import _set_agent_data +from ..utils import _get_start_span_function, _set_agent_data from typing import TYPE_CHECKING @@ -12,8 +12,9 @@ def invoke_agent_span(context, agent): # type: (agents.RunContextWrapper, agents.Agent) -> None - span = sentry_sdk.start_span( - op=OP.GEN_AI_INVOKE_AGENT, name=f"invoke_agent {agent.name}" + span = _get_start_span_function()( + op=OP.GEN_AI_INVOKE_AGENT, + name=f"invoke_agent {agent.name}", ) span.__enter__() diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 7bb233de2d..844631a2c1 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,4 +1,4 @@ -from functools import wraps +from functools import partial, wraps import sentry_sdk from sentry_sdk.consts import SPANDATA @@ -47,7 +47,13 @@ def _get_start_span_function(): is_transaction = ( current_span is not None and current_span.containing_transaction == current_span ) - return sentry_sdk.start_span if is_transaction else sentry_sdk.start_transaction + start_span_function = ( + sentry_sdk.start_span if is_transaction else sentry_sdk.start_transaction + ) + return partial( + start_span_function, + origin=sentry_sdk.integrations.openai_agents.OpenAIAgentsIntegration.origin, + ) def _create_hook_wrapper(original_hook, sentry_hook): diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 6c8113fb53..1f35f58451 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -304,8 +304,10 @@ def simple_test_tool(message: str) -> str: ] assert agent_workflow_span["description"] == "test_agent workflow" + assert agent_workflow_span["origin"] == "auto.ai.openai_agents" assert agent_span["description"] == "invoke_agent test_agent" + assert agent_span["origin"] == "auto.ai.openai_agents" assert agent_span["data"]["gen_ai.agent.name"] == "test_agent" assert agent_span["data"]["gen_ai.operation.name"] == "invoke_agent" assert agent_span["data"]["gen_ai.request.available_tools"] == available_tools @@ -316,6 +318,7 @@ def simple_test_tool(message: str) -> str: assert agent_span["data"]["gen_ai.system"] == "openai" assert ai_client_span1["description"] == "chat gpt-4" + assert ai_client_span1["tags"]["status"] == "internal_error" assert ai_client_span1["data"]["gen_ai.operation.name"] == "chat" assert ai_client_span1["data"]["gen_ai.system"] == "openai" assert ai_client_span1["data"]["gen_ai.agent.name"] == "test_agent" @@ -352,6 +355,7 @@ def simple_test_tool(message: str) -> str: ] assert tool_span["description"] == "execute_tool simple_test_tool" + assert tool_span["tags"]["status"] == "internal_error" assert tool_span["data"]["gen_ai.agent.name"] == "test_agent" assert tool_span["data"]["gen_ai.operation.name"] == "execute_tool" assert tool_span["data"]["gen_ai.request.available_tools"] == available_tools @@ -367,6 +371,7 @@ def simple_test_tool(message: str) -> str: assert tool_span["data"]["gen_ai.tool.type"] == "function" assert ai_client_span2["description"] == "chat gpt-4" + assert ai_client_span2["tags"]["status"] == "internal_error" assert ai_client_span2["data"]["gen_ai.agent.name"] == "test_agent" assert ai_client_span2["data"]["gen_ai.operation.name"] == "chat" assert ai_client_span2["data"]["gen_ai.request.available_tools"] == available_tools @@ -428,7 +433,6 @@ async def test_error_handling(sentry_init, capture_events, test_agent): with patch( "agents.models.openai_responses.OpenAIResponsesModel.get_response" ) as mock_get_response: - # Make the model call fail mock_get_response.side_effect = Exception("Model Error") sentry_init( @@ -442,35 +446,13 @@ async def test_error_handling(sentry_init, capture_events, test_agent): with pytest.raises(Exception, match="Model Error"): await agents.Runner.run(test_agent, "Test input") - # Verify spans were created even with errors (transaction,) = events spans = transaction["spans"] + (agent_workflow_span,) = spans - # Should have workflow span and invoke agent span at minimum - assert len(spans) >= 2 - - # Check workflow span - workflow_spans = [ - span for span in spans if "workflow" in span.get("description", "") - ] - assert len(workflow_spans) == 1 - workflow_span = workflow_spans[0] - assert workflow_span["description"] == "test_agent workflow" - - # Check invoke agent span - invoke_spans = [ - span for span in spans if span.get("description", "").startswith("invoke_agent") - ] - assert len(invoke_spans) == 1 - invoke_span = invoke_spans[0] - assert invoke_span["description"] == "invoke_agent test_agent" - assert invoke_span["data"]["gen_ai.operation.name"] == "invoke_agent" - assert invoke_span["data"]["gen_ai.system"] == "openai" - assert invoke_span["data"]["gen_ai.agent.name"] == "test_agent" - - # Check for error spans - error_spans = [span for span in spans if span.get("status") == "internal_error"] - assert len(error_spans) >= 1 + assert agent_workflow_span["description"] == "test_agent workflow" + assert agent_workflow_span["origin"] == "auto.ai.openai_agents" + assert agent_workflow_span["tags"]["status"] == "internal_error" @pytest.mark.asyncio From 90d2cb4668e1c56f9e6cd29a3f86d4a6246c1586 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 11:47:48 +0200 Subject: [PATCH 52/89] Setting span origin --- tests/integrations/openai_agents/test_openai_agents.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 1f35f58451..7026ef9cec 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -173,6 +173,10 @@ async def test_agent_invocation_span( assert result is not None assert result.final_output == "Hello, how can I help you?" + import ipdb + + ipdb.set_trace() + (transaction,) = events spans = transaction["spans"] agent_workflow_span, invoke_agent_span, ai_client_span = spans From 005e6d0cb527f38c683b7954b6ac9d54777094c1 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 11:52:27 +0200 Subject: [PATCH 53/89] fixed tests --- .../openai_agents/spans/agent_workflow.py | 2 ++ .../integrations/openai_agents/spans/ai_client.py | 5 +++-- .../openai_agents/spans/execute_tool.py | 6 ++++-- .../openai_agents/spans/invoke_agent.py | 5 +++-- sentry_sdk/integrations/openai_agents/utils.py | 10 ++-------- .../openai_agents/test_openai_agents.py | 13 +++++-------- 6 files changed, 19 insertions(+), 22 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py index c02b3583c5..8d15dfdc22 100644 --- a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py +++ b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py @@ -1,3 +1,4 @@ +import sentry_sdk from ..utils import _get_start_span_function from typing import TYPE_CHECKING @@ -14,6 +15,7 @@ def agent_workflow_span(*args, **kwargs): # Create a transaction or a span if an transaction is already active span = _get_start_span_function()( name=f"{agent.name} workflow", + origin=sentry_sdk.integrations.openai_agents.OpenAIAgentsIntegration.origin, ) return span diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 90c69cbe9a..446b2451ee 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -1,7 +1,7 @@ +import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from ..utils import ( - _get_start_span_function, _set_agent_data, _set_input_data, _set_output_data, @@ -19,9 +19,10 @@ def ai_client_span(agent, get_response_kwargs): # type: (Agent, dict[str, Any]) -> Span # TODO-anton: implement other types of operations. Now "chat" is hardcoded. - span = _get_start_span_function()( + span = sentry_sdk.start_span( op=OP.GEN_AI_CHAT, description=f"chat {agent.model}", + origin=sentry_sdk.integrations.openai_agents.OpenAIAgentsIntegration.origin, ) # TODO-anton: remove hardcoded stuff and replace something that also works for embedding and so on span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "chat") diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index 60a3d91014..f7cf89ec04 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -1,7 +1,8 @@ +import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.scope import should_send_default_pii -from ..utils import _get_start_span_function, _set_agent_data +from ..utils import _set_agent_data from typing import TYPE_CHECKING @@ -13,9 +14,10 @@ def execute_tool_span(tool, *args, **kwargs): # type: (agents.Tool, *Any, **Any) -> Span - span = _get_start_span_function()( + span = sentry_sdk.start_span( op=OP.GEN_AI_EXECUTE_TOOL, name=f"execute_tool {tool.name}", + origin=sentry_sdk.integrations.openai_agents.OpenAIAgentsIntegration.origin, ) span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "execute_tool") diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index 97ee2e3f82..874fefe02f 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -1,7 +1,7 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA -from ..utils import _get_start_span_function, _set_agent_data +from ..utils import _set_agent_data from typing import TYPE_CHECKING @@ -12,9 +12,10 @@ def invoke_agent_span(context, agent): # type: (agents.RunContextWrapper, agents.Agent) -> None - span = _get_start_span_function()( + span = sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, name=f"invoke_agent {agent.name}", + origin=sentry_sdk.integrations.openai_agents.OpenAIAgentsIntegration.origin, ) span.__enter__() diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 844631a2c1..7bb233de2d 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,4 +1,4 @@ -from functools import partial, wraps +from functools import wraps import sentry_sdk from sentry_sdk.consts import SPANDATA @@ -47,13 +47,7 @@ def _get_start_span_function(): is_transaction = ( current_span is not None and current_span.containing_transaction == current_span ) - start_span_function = ( - sentry_sdk.start_span if is_transaction else sentry_sdk.start_transaction - ) - return partial( - start_span_function, - origin=sentry_sdk.integrations.openai_agents.OpenAIAgentsIntegration.origin, - ) + return sentry_sdk.start_span if is_transaction else sentry_sdk.start_transaction def _create_hook_wrapper(original_hook, sentry_hook): diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 7026ef9cec..e65b481b46 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -173,10 +173,6 @@ async def test_agent_invocation_span( assert result is not None assert result.final_output == "Hello, how can I help you?" - import ipdb - - ipdb.set_trace() - (transaction,) = events spans = transaction["spans"] agent_workflow_span, invoke_agent_span, ai_client_span = spans @@ -322,7 +318,6 @@ def simple_test_tool(message: str) -> str: assert agent_span["data"]["gen_ai.system"] == "openai" assert ai_client_span1["description"] == "chat gpt-4" - assert ai_client_span1["tags"]["status"] == "internal_error" assert ai_client_span1["data"]["gen_ai.operation.name"] == "chat" assert ai_client_span1["data"]["gen_ai.system"] == "openai" assert ai_client_span1["data"]["gen_ai.agent.name"] == "test_agent" @@ -359,7 +354,6 @@ def simple_test_tool(message: str) -> str: ] assert tool_span["description"] == "execute_tool simple_test_tool" - assert tool_span["tags"]["status"] == "internal_error" assert tool_span["data"]["gen_ai.agent.name"] == "test_agent" assert tool_span["data"]["gen_ai.operation.name"] == "execute_tool" assert tool_span["data"]["gen_ai.request.available_tools"] == available_tools @@ -375,7 +369,6 @@ def simple_test_tool(message: str) -> str: assert tool_span["data"]["gen_ai.tool.type"] == "function" assert ai_client_span2["description"] == "chat gpt-4" - assert ai_client_span2["tags"]["status"] == "internal_error" assert ai_client_span2["data"]["gen_ai.agent.name"] == "test_agent" assert ai_client_span2["data"]["gen_ai.operation.name"] == "chat" assert ai_client_span2["data"]["gen_ai.request.available_tools"] == available_tools @@ -452,12 +445,16 @@ async def test_error_handling(sentry_init, capture_events, test_agent): (transaction,) = events spans = transaction["spans"] - (agent_workflow_span,) = spans + (agent_workflow_span, ai_client_span) = spans assert agent_workflow_span["description"] == "test_agent workflow" assert agent_workflow_span["origin"] == "auto.ai.openai_agents" assert agent_workflow_span["tags"]["status"] == "internal_error" + assert ai_client_span["description"] == "chat gpt-4" + assert ai_client_span["origin"] == "auto.ai.openai_agents" + assert ai_client_span["tags"]["status"] == "internal_error" + @pytest.mark.asyncio async def test_integration_with_hooks( From c5dd40b34c0457ab29e83d2c7201fa580581a6d9 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 11:58:23 +0200 Subject: [PATCH 54/89] cleanup --- .../openai_agents/test_openai_agents.py | 66 ------------------- 1 file changed, 66 deletions(-) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index e65b481b46..255daf63c8 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -454,69 +454,3 @@ async def test_error_handling(sentry_init, capture_events, test_agent): assert ai_client_span["description"] == "chat gpt-4" assert ai_client_span["origin"] == "auto.ai.openai_agents" assert ai_client_span["tags"]["status"] == "internal_error" - - -@pytest.mark.asyncio -async def test_integration_with_hooks( - sentry_init, capture_events, test_agent, mock_model_response -): - """ - Test that the integration properly wraps hooks. - """ - - # Create a simple hook to test with - hook_calls = [] - - from agents.lifecycle import AgentHooks - - class TestHooks(AgentHooks): - async def on_agent_start(self, context, agent): - hook_calls.append("start") - - async def on_agent_end(self, context, agent, output): - hook_calls.append("end") - - with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): - with patch( - "agents.models.openai_responses.OpenAIResponsesModel.get_response" - ) as mock_get_response: - mock_get_response.return_value = mock_model_response - - sentry_init( - integrations=[OpenAIAgentsIntegration()], - traces_sample_rate=1.0, - ) - - events = capture_events() - - with sentry_sdk.start_transaction(name="test_transaction"): - await agents.Runner.run(test_agent, "Test input", hooks=TestHooks()) - - # Verify hooks were called (integration shouldn't break them) - assert "start" in hook_calls - assert "end" in hook_calls - - # Verify spans were still created - (transaction,) = events - spans = transaction["spans"] - agent_workflow_span, invoke_agent_span, ai_client_span = spans - - assert agent_workflow_span["description"] == "test_agent workflow" - - assert invoke_agent_span["description"] == "invoke_agent test_agent" - assert invoke_agent_span["data"]["gen_ai.operation.name"] == "invoke_agent" - assert invoke_agent_span["data"]["gen_ai.system"] == "openai" - assert invoke_agent_span["data"]["gen_ai.agent.name"] == "test_agent" - assert invoke_agent_span["data"]["gen_ai.request.max_tokens"] == 100 - assert invoke_agent_span["data"]["gen_ai.request.model"] == "gpt-4" - assert invoke_agent_span["data"]["gen_ai.request.temperature"] == 0.7 - assert invoke_agent_span["data"]["gen_ai.request.top_p"] == 1.0 - - assert ai_client_span["description"] == "chat gpt-4" - assert ai_client_span["data"]["gen_ai.operation.name"] == "chat" - assert ai_client_span["data"]["gen_ai.system"] == "openai" - assert ai_client_span["data"]["gen_ai.agent.name"] == "test_agent" - assert ai_client_span["data"]["gen_ai.request.max_tokens"] == 100 - assert ai_client_span["data"]["gen_ai.request.model"] == "gpt-4" - assert ai_client_span["data"]["gen_ai.request.temperature"] == 0.7 - assert ai_client_span["data"]["gen_ai.request.top_p"] == 1.0 From c6fcd9d06e546adfbd1e819c207b907552c0810c Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 12:27:02 +0200 Subject: [PATCH 55/89] disable openai tracing because it emits a warning when no api_key is set --- .../openai_agents/test_openai_agents.py | 88 +++---------------- 1 file changed, 11 insertions(+), 77 deletions(-) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 255daf63c8..d4ebf9cd6e 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -9,21 +9,18 @@ import agents from agents import ( Agent, - RunResult, ModelResponse, Usage, - MessageOutputItem, - ToolCallItem, - ToolCallOutputItem, ModelSettings, ) from agents.items import ( ResponseOutputMessage, ResponseOutputText, ResponseFunctionToolCall, - FunctionCallOutput, ) +test_run_config = agents.RunConfig(tracing_disabled=True) + @pytest.fixture def mock_usage(): @@ -60,70 +57,6 @@ def mock_model_response(mock_usage): ) -@pytest.fixture -def mock_tool_call(): - return ResponseFunctionToolCall( - id="call_123", - call_id="call_123", - name="test_tool", - type="function_call", - arguments='{"arg1": "value1"}', - function=MagicMock(name="test_tool", arguments='{"arg1": "value1"}'), - ) - - -@pytest.fixture -def mock_tool_output(): - return FunctionCallOutput(tool_call_id="call_123", output="Tool execution result") - - -@pytest.fixture -def mock_run_result(mock_model_response, mock_tool_call, mock_tool_output): - return RunResult( - input="Test input", - new_items=[ - MessageOutputItem( - agent=MagicMock(), - raw_item=mock_model_response.output[0], - type="message_output_item", - ), - ToolCallItem( - agent=MagicMock(), raw_item=mock_tool_call, type="tool_call_item" - ), - ToolCallOutputItem( - agent=MagicMock(), - raw_item=mock_tool_output, - output="Tool execution result", - type="tool_call_output_item", - ), - ], - raw_responses=[mock_model_response], - final_output="Final result", - input_guardrail_results=[], - output_guardrail_results=[], - context_wrapper=MagicMock(), - _last_agent=MagicMock(), - ) - - -@pytest.fixture -def mock_agent(): - agent = MagicMock(spec=Agent) - agent.name = "test_agent" - agent.model = "gpt-4" - agent.model_settings = MagicMock( - max_tokens=100, - temperature=0.7, - top_p=1.0, - presence_penalty=0.0, - frequency_penalty=0.0, - ) - agent.tools = [] - agent.handoffs = [] - agent.output_type = str - return agent - - @pytest.fixture def test_agent(): """Create a real Agent instance for testing.""" @@ -141,11 +74,6 @@ def test_agent(): ) -def test_integration_initialization(): - integration = OpenAIAgentsIntegration() - assert integration is not None - - @pytest.mark.asyncio async def test_agent_invocation_span( sentry_init, capture_events, test_agent, mock_model_response @@ -168,7 +96,9 @@ async def test_agent_invocation_span( events = capture_events() with sentry_sdk.start_transaction(name="test_transaction"): - result = await agents.Runner.run(test_agent, "Test input") + result = await agents.Runner.run( + test_agent, "Test input", run_config=test_run_config + ) assert result is not None assert result.final_output == "Hello, how can I help you?" @@ -273,7 +203,9 @@ def simple_test_tool(message: str) -> str: with sentry_sdk.start_transaction(name="test_transaction"): await agents.Runner.run( - agent_with_tool, "Please use the simple test tool" + agent_with_tool, + "Please use the simple test tool", + run_config=test_run_config, ) (transaction,) = events @@ -441,7 +373,9 @@ async def test_error_handling(sentry_init, capture_events, test_agent): with sentry_sdk.start_transaction(name="test_transaction"): with pytest.raises(Exception, match="Model Error"): - await agents.Runner.run(test_agent, "Test input") + await agents.Runner.run( + test_agent, "Test input", run_config=test_run_config + ) (transaction,) = events spans = transaction["spans"] From 1a1aa26850595343db3bae84386aabca38eddf26 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 12:31:15 +0200 Subject: [PATCH 56/89] updated test matrix --- tox.ini | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tox.ini b/tox.ini index 31be13f882..575e47628c 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-06-16T13:51:07.677014+00:00 +# Last generated: 2025-06-17T10:29:38.027366+00:00 [tox] requires = @@ -145,7 +145,7 @@ envlist = {py3.9,py3.11,py3.12}-cohere-v5.11.4 {py3.9,py3.11,py3.12}-cohere-v5.15.0 - {py3.9,py3.11,py3.12}-openai_agents-v0.0.17 + {py3.9,py3.11,py3.12}-openai_agents-v0.0.18 {py3.8,py3.10,py3.11}-huggingface_hub-v0.22.2 {py3.8,py3.11,py3.12}-huggingface_hub-v0.26.5 @@ -159,7 +159,7 @@ envlist = {py3.6}-pymongo-v3.5.1 {py3.6,py3.10,py3.11}-pymongo-v3.13.0 {py3.6,py3.9,py3.10}-pymongo-v4.0.2 - {py3.9,py3.12,py3.13}-pymongo-v4.13.1 + {py3.9,py3.12,py3.13}-pymongo-v4.13.2 {py3.6}-redis_py_cluster_legacy-v1.3.6 {py3.6,py3.7}-redis_py_cluster_legacy-v2.0.0 @@ -182,7 +182,7 @@ envlist = {py3.7,py3.12,py3.13}-statsig-v0.55.3 {py3.7,py3.12,py3.13}-statsig-v0.56.0 {py3.7,py3.12,py3.13}-statsig-v0.57.3 - {py3.7,py3.12,py3.13}-statsig-v0.58.1 + {py3.7,py3.12,py3.13}-statsig-v0.58.2 {py3.8,py3.12,py3.13}-unleash-v6.0.1 {py3.8,py3.12,py3.13}-unleash-v6.1.0 @@ -203,9 +203,9 @@ envlist = {py3.8,py3.12,py3.13}-graphene-v3.4.3 {py3.8,py3.10,py3.11}-strawberry-v0.209.8 - {py3.8,py3.11,py3.12}-strawberry-v0.230.0 - {py3.8,py3.12,py3.13}-strawberry-v0.251.0 - {py3.9,py3.12,py3.13}-strawberry-v0.273.2 + {py3.8,py3.11,py3.12}-strawberry-v0.231.1 + {py3.8,py3.12,py3.13}-strawberry-v0.253.1 + {py3.9,py3.12,py3.13}-strawberry-v0.274.0 # ~~~ Network ~~~ @@ -516,7 +516,7 @@ deps = cohere-v5.11.4: cohere==5.11.4 cohere-v5.15.0: cohere==5.15.0 - openai_agents-v0.0.17: openai-agents==0.0.17 + openai_agents-v0.0.18: openai-agents==0.0.18 openai_agents: pytest-asyncio huggingface_hub-v0.22.2: huggingface_hub==0.22.2 @@ -531,7 +531,7 @@ deps = pymongo-v3.5.1: pymongo==3.5.1 pymongo-v3.13.0: pymongo==3.13.0 pymongo-v4.0.2: pymongo==4.0.2 - pymongo-v4.13.1: pymongo==4.13.1 + pymongo-v4.13.2: pymongo==4.13.2 pymongo: mockupdb redis_py_cluster_legacy-v1.3.6: redis-py-cluster==1.3.6 @@ -555,7 +555,7 @@ deps = statsig-v0.55.3: statsig==0.55.3 statsig-v0.56.0: statsig==0.56.0 statsig-v0.57.3: statsig==0.57.3 - statsig-v0.58.1: statsig==0.58.1 + statsig-v0.58.2: statsig==0.58.2 statsig: typing_extensions unleash-v6.0.1: UnleashClient==6.0.1 @@ -585,13 +585,13 @@ deps = py3.6-graphene: aiocontextvars strawberry-v0.209.8: strawberry-graphql[fastapi,flask]==0.209.8 - strawberry-v0.230.0: strawberry-graphql[fastapi,flask]==0.230.0 - strawberry-v0.251.0: strawberry-graphql[fastapi,flask]==0.251.0 - strawberry-v0.273.2: strawberry-graphql[fastapi,flask]==0.273.2 + strawberry-v0.231.1: strawberry-graphql[fastapi,flask]==0.231.1 + strawberry-v0.253.1: strawberry-graphql[fastapi,flask]==0.253.1 + strawberry-v0.274.0: strawberry-graphql[fastapi,flask]==0.274.0 strawberry: httpx strawberry-v0.209.8: pydantic<2.11 - strawberry-v0.230.0: pydantic<2.11 - strawberry-v0.251.0: pydantic<2.11 + strawberry-v0.231.1: pydantic<2.11 + strawberry-v0.253.1: pydantic<2.11 # ~~~ Network ~~~ From e0fdbcf58f9a4681601a62c924eddbf718423baa Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 12:40:08 +0200 Subject: [PATCH 57/89] cleanup --- sentry_sdk/integrations/openai_agents/utils.py | 10 ---------- tests/integrations/openai_agents/__init__.py | 3 +++ 2 files changed, 3 insertions(+), 10 deletions(-) create mode 100644 tests/integrations/openai_agents/__init__.py diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 7bb233de2d..677a6527bf 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -31,16 +31,6 @@ def _capture_exception(exc): sentry_sdk.capture_event(event, hint=hint) -def _usage_to_str(usage): - # type: (Usage) -> str - return ( - f"{usage.requests} requests, " - f"{usage.input_tokens} input tokens, " - f"{usage.output_tokens} output tokens, " - f"{usage.total_tokens} total tokens" - ) - - def _get_start_span_function(): # type: () -> Callable current_span = sentry_sdk.get_current_span() diff --git a/tests/integrations/openai_agents/__init__.py b/tests/integrations/openai_agents/__init__.py new file mode 100644 index 0000000000..420b9127a1 --- /dev/null +++ b/tests/integrations/openai_agents/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.importorskip("openai-agents") From ed4997bbc882f04b14fb2570d5dcffede071535a Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 12:54:47 +0200 Subject: [PATCH 58/89] linting --- sentry_sdk/integrations/openai_agents/__init__.py | 10 +++++----- sentry_sdk/integrations/openai_agents/run_hooks.py | 2 +- .../integrations/openai_agents/spans/ai_client.py | 5 ++--- .../integrations/openai_agents/spans/execute_tool.py | 5 ++--- sentry_sdk/integrations/openai_agents/utils.py | 12 ++++++------ tests/integrations/openai_agents/__init__.py | 2 +- 6 files changed, 17 insertions(+), 19 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py index bea5217ed7..411fa74715 100644 --- a/sentry_sdk/integrations/openai_agents/__init__.py +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -15,19 +15,19 @@ def _patch_runner(): # type: () -> None - agents.Runner.run = _create_run_wrapper(agents.Runner.run) - agents.Runner.run_sync = _create_run_wrapper(agents.Runner.run_sync) - agents.Runner.run_streamed = _create_run_wrapper(agents.Runner.run_streamed) + agents.Runner.run = _create_run_wrapper(agents.Runner.run) # type: ignore[method-assign] + agents.Runner.run_sync = _create_run_wrapper(agents.Runner.run_sync) # type: ignore[method-assign] + agents.Runner.run_streamed = _create_run_wrapper(agents.Runner.run_streamed) # type: ignore[method-assign] def _patch_model(): # type: () -> None - agents.Runner._get_model = _create_get_model_wrapper(agents.Runner._get_model) + agents.Runner._get_model = _create_get_model_wrapper(agents.Runner._get_model) # type: ignore[method-assign] def _patch_tools(): # type: () -> None - agents.Runner._get_all_tools = _create_get_all_tools_wrapper( + agents.Runner._get_all_tools = _create_get_all_tools_wrapper( # type: ignore[method-assign] agents.Runner._get_all_tools ) diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py index 49c0d32cd8..71b7c1542b 100644 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -16,7 +16,7 @@ class SentryRunHooks(agents.RunHooks): - async def on_agent_start(self, context, agent) -> None: + async def on_agent_start(self, context, agent): # type: (agents.RunContextWrapper, agents.Agent) -> None invoke_agent_span(context, agent) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 446b2451ee..07e5bd3559 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -13,11 +13,10 @@ if TYPE_CHECKING: from agents import Agent from typing import Any - from sentry_sdk import Span def ai_client_span(agent, get_response_kwargs): - # type: (Agent, dict[str, Any]) -> Span + # type: (Agent, dict[str, Any]) -> sentry_sdk.tracing.Span # TODO-anton: implement other types of operations. Now "chat" is hardcoded. span = sentry_sdk.start_span( op=OP.GEN_AI_CHAT, @@ -31,7 +30,7 @@ def ai_client_span(agent, get_response_kwargs): def update_ai_client_span(span, agent, get_response_kwargs, result): - # type: (Span, Agent, dict[str, Any], Any) -> None + # type: (sentry_sdk.tracing.Span, Agent, dict[str, Any], Any) -> None _set_agent_data(span, agent) _set_usage_data(span, result.usage) _set_input_data(span, get_response_kwargs) diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index f7cf89ec04..90d025a5e9 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -8,12 +8,11 @@ if TYPE_CHECKING: import agents - from sentry_sdk import Span from typing import Any def execute_tool_span(tool, *args, **kwargs): - # type: (agents.Tool, *Any, **Any) -> Span + # type: (agents.Tool, *Any, **Any) -> sentry_sdk.tracing.Span span = sentry_sdk.start_span( op=OP.GEN_AI_EXECUTE_TOOL, name=f"execute_tool {tool.name}", @@ -36,7 +35,7 @@ def execute_tool_span(tool, *args, **kwargs): def update_execute_tool_span(span, agent, tool, result): - # type: (Span, agents.Agent, agents.Tool, Any) -> None + # type: (sentry_sdk.tracing.Span, agents.Agent, agents.Tool, Any) -> None _set_agent_data(span, agent) if should_send_default_pii(): diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 677a6527bf..edae709930 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -32,7 +32,7 @@ def _capture_exception(exc): def _get_start_span_function(): - # type: () -> Callable + # type: () -> Callable[..., Any] current_span = sentry_sdk.get_current_span() is_transaction = ( current_span is not None and current_span.containing_transaction == current_span @@ -41,7 +41,7 @@ def _get_start_span_function(): def _create_hook_wrapper(original_hook, sentry_hook): - # type: (Callable, Callable) -> Callable + # type: (Callable[..., Any], Callable[..., Any]) -> Callable[..., Any] @wraps(original_hook) async def async_wrapper(*args, **kwargs): await sentry_hook(*args, **kwargs) @@ -86,7 +86,7 @@ def _wrap_hooks(hooks): def _set_agent_data(span, agent): - # type: (sentry_sdk.Span, agents.Agent) -> None + # type: (sentry_sdk.tracing.Span, agents.Agent) -> None span.set_data( SPANDATA.GEN_AI_SYSTEM, "openai" ) # See footnote for https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/#gen-ai-system for explanation why. @@ -129,7 +129,7 @@ def _set_agent_data(span, agent): def _set_usage_data(span, usage): - # type: (sentry_sdk.Span, Usage) -> None + # type: (sentry_sdk.tracing.Span, Usage) -> None span.set_data(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, usage.input_tokens) span.set_data( SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, @@ -144,7 +144,7 @@ def _set_usage_data(span, usage): def _set_input_data(span, get_response_kwargs): - # type: (sentry_sdk.Span, dict[str, Any]) -> None + # type: (sentry_sdk.tracing.Span, dict[str, Any]) -> None if not should_send_default_pii(): return @@ -178,7 +178,7 @@ def _set_input_data(span, get_response_kwargs): def _set_output_data(span, result): - # type: (sentry_sdk.Span, Any) -> None + # type: (sentry_sdk.tracing.Span, Any) -> None if not should_send_default_pii(): return diff --git a/tests/integrations/openai_agents/__init__.py b/tests/integrations/openai_agents/__init__.py index 420b9127a1..6940e2bbbe 100644 --- a/tests/integrations/openai_agents/__init__.py +++ b/tests/integrations/openai_agents/__init__.py @@ -1,3 +1,3 @@ import pytest -pytest.importorskip("openai-agents") +pytest.importorskip("agents") From 634ddce673eddf2ba4b0ec9c33ca9d5f9f5eee81 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 13:43:51 +0200 Subject: [PATCH 59/89] linting --- pyproject.toml | 4 ++++ sentry_sdk/integrations/openai_agents/__init__.py | 10 +++++----- .../integrations/openai_agents/patches/models.py | 7 +++++++ .../integrations/openai_agents/patches/runner.py | 5 ++--- sentry_sdk/integrations/openai_agents/patches/tools.py | 4 +++- .../integrations/openai_agents/spans/agent_workflow.py | 5 ++--- .../integrations/openai_agents/spans/invoke_agent.py | 2 +- sentry_sdk/integrations/openai_agents/utils.py | 5 +++-- 8 files changed, 27 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5e16b30793..e5eae2c21f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -183,6 +183,10 @@ ignore_missing_imports = true module = "grpc.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "agents.*" +ignore_missing_imports = true + # # Tool: Flake8 # diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py index 411fa74715..bea5217ed7 100644 --- a/sentry_sdk/integrations/openai_agents/__init__.py +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -15,19 +15,19 @@ def _patch_runner(): # type: () -> None - agents.Runner.run = _create_run_wrapper(agents.Runner.run) # type: ignore[method-assign] - agents.Runner.run_sync = _create_run_wrapper(agents.Runner.run_sync) # type: ignore[method-assign] - agents.Runner.run_streamed = _create_run_wrapper(agents.Runner.run_streamed) # type: ignore[method-assign] + agents.Runner.run = _create_run_wrapper(agents.Runner.run) + agents.Runner.run_sync = _create_run_wrapper(agents.Runner.run_sync) + agents.Runner.run_streamed = _create_run_wrapper(agents.Runner.run_streamed) def _patch_model(): # type: () -> None - agents.Runner._get_model = _create_get_model_wrapper(agents.Runner._get_model) # type: ignore[method-assign] + agents.Runner._get_model = _create_get_model_wrapper(agents.Runner._get_model) def _patch_tools(): # type: () -> None - agents.Runner._get_all_tools = _create_get_all_tools_wrapper( # type: ignore[method-assign] + agents.Runner._get_all_tools = _create_get_all_tools_wrapper( agents.Runner._get_all_tools ) diff --git a/sentry_sdk/integrations/openai_agents/patches/models.py b/sentry_sdk/integrations/openai_agents/patches/models.py index 406871eab0..571787e3ae 100644 --- a/sentry_sdk/integrations/openai_agents/patches/models.py +++ b/sentry_sdk/integrations/openai_agents/patches/models.py @@ -4,6 +4,11 @@ from ..spans import ai_client_span, update_ai_client_span +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Callable + try: import agents @@ -12,6 +17,7 @@ def _create_get_model_wrapper(original_get_model): + # type: (Callable[..., Any]) -> Callable[..., Any] """ Wraps the agents.Runner._get_model method to wrap the get_response method of the model to create a AI client span. """ @@ -26,6 +32,7 @@ def wrapped_get_model(cls, agent, run_config): @wraps(original_get_response) async def wrapped_get_response(*args, **kwargs): + # type: (*Any, **Any) -> Any with ai_client_span(agent, kwargs) as span: result = await original_get_response(*args, **kwargs) update_ai_client_span(span, agent, kwargs, result) diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index fdf158bf6b..1d34e0b45b 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -10,8 +10,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Any - from typing import Callable + from typing import Any, Callable try: import agents @@ -20,7 +19,7 @@ def _create_run_wrapper(original_func): - # type: (Callable) -> Callable + # type: (Callable[..., Any]) -> Callable[..., Any] """ Wraps the agents.Runner.run* methods to create a root span for the agent workflow runs. """ diff --git a/sentry_sdk/integrations/openai_agents/patches/tools.py b/sentry_sdk/integrations/openai_agents/patches/tools.py index 898de6ce25..684259a20d 100644 --- a/sentry_sdk/integrations/openai_agents/patches/tools.py +++ b/sentry_sdk/integrations/openai_agents/patches/tools.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Any + from typing import Any, Callable try: import agents @@ -16,6 +16,7 @@ def _create_get_all_tools_wrapper(original_get_all_tools): + # type: (Callable[..., Any]) -> Callable[..., Any] """ Wraps the agents.Runner._get_all_tools method of the Runner class to wrap all function tools with Sentry instrumentation. """ @@ -39,6 +40,7 @@ async def wrapped_get_all_tools(cls, agent, context_wrapper): original_on_invoke = tool.on_invoke_tool def create_wrapped_invoke(current_tool, current_on_invoke): + # type: (agents.Tool, Callable[..., Any]) -> Callable[..., Any] @wraps(current_on_invoke) async def sentry_wrapped_on_invoke_tool(*args, **kwargs): # type: (*Any, **Any) -> Any diff --git a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py index 8d15dfdc22..679ab51e8a 100644 --- a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py +++ b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py @@ -5,12 +5,11 @@ if TYPE_CHECKING: from agents import Agent - from sentry_sdk import Span from typing import Any def agent_workflow_span(*args, **kwargs): - # type: (*Any, **Any) -> Span + # type: (*Any, **Any) -> sentry_sdk.tracing.Span agent = args[0] # Create a transaction or a span if an transaction is already active span = _get_start_span_function()( @@ -22,5 +21,5 @@ def agent_workflow_span(*args, **kwargs): def update_agent_workflow_span(span, agent, result): - # type: (Span, Agent, Any) -> None + # type: (sentry_sdk.tracing.Span, Agent, Any) -> None pass diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index 874fefe02f..0b786582f7 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -11,7 +11,7 @@ def invoke_agent_span(context, agent): - # type: (agents.RunContextWrapper, agents.Agent) -> None + # type: (agents.RunContextWrapper, agents.Agent) -> sentry_sdk.tracing.Span span = sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, name=f"invoke_agent {agent.name}", diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index edae709930..31212c0580 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -44,6 +44,7 @@ def _create_hook_wrapper(original_hook, sentry_hook): # type: (Callable[..., Any], Callable[..., Any]) -> Callable[..., Any] @wraps(original_hook) async def async_wrapper(*args, **kwargs): + # type: (*Any, **Any) -> Any await sentry_hook(*args, **kwargs) return await original_hook(*args, **kwargs) @@ -153,7 +154,7 @@ def _set_input_data(span, get_response_kwargs): "user": [], "assistant": [], "tool": [], - } + } # type: (dict[str, list[Any]]) system_instructions = get_response_kwargs.get("system_instructions") if system_instructions: messages_by_role["system"].append({"type": "text", "text": system_instructions}) @@ -185,7 +186,7 @@ def _set_output_data(span, result): output_messages = { "response": [], "tool": [], - } + } # type: (dict[str, list[Any]]) for output in result.output: if output.type == "function_call": From 68cdbc43db70c5c2b9aff9ecf907cb867ac7d4fe Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 13:53:08 +0200 Subject: [PATCH 60/89] improved check --- sentry_sdk/integrations/openai_agents/patches/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/tools.py b/sentry_sdk/integrations/openai_agents/patches/tools.py index 684259a20d..602ea30b06 100644 --- a/sentry_sdk/integrations/openai_agents/patches/tools.py +++ b/sentry_sdk/integrations/openai_agents/patches/tools.py @@ -32,7 +32,7 @@ async def wrapped_get_all_tools(cls, agent, context_wrapper): wrapped_tools = [] for tool in tools: # Wrap only the function tools (for now) - if not isinstance(tool, agents.Tool): + if tool.__class__.__name__ != "FunctionTool": wrapped_tools.append(tool) continue From 82e8a02391fabf2c9488f2d8686d89c97345ec4a Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 14:02:03 +0200 Subject: [PATCH 61/89] better span origin --- sentry_sdk/integrations/openai_agents/__init__.py | 1 - sentry_sdk/integrations/openai_agents/consts.py | 1 + sentry_sdk/integrations/openai_agents/spans/agent_workflow.py | 4 +++- sentry_sdk/integrations/openai_agents/spans/ai_client.py | 3 ++- sentry_sdk/integrations/openai_agents/spans/execute_tool.py | 3 ++- sentry_sdk/integrations/openai_agents/spans/invoke_agent.py | 3 ++- 6 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 sentry_sdk/integrations/openai_agents/consts.py diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py index bea5217ed7..0c246bd6dd 100644 --- a/sentry_sdk/integrations/openai_agents/__init__.py +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -34,7 +34,6 @@ def _patch_tools(): class OpenAIAgentsIntegration(Integration): identifier = "openai_agents" - origin = f"auto.ai.{identifier}" @staticmethod def setup_once(): diff --git a/sentry_sdk/integrations/openai_agents/consts.py b/sentry_sdk/integrations/openai_agents/consts.py new file mode 100644 index 0000000000..f5de978be0 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/consts.py @@ -0,0 +1 @@ +SPAN_ORIGIN = "auto.ai.openai_agents" diff --git a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py index 679ab51e8a..67e0895aa5 100644 --- a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py +++ b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py @@ -1,4 +1,6 @@ import sentry_sdk + +from ..consts import SPAN_ORIGIN from ..utils import _get_start_span_function from typing import TYPE_CHECKING @@ -14,7 +16,7 @@ def agent_workflow_span(*args, **kwargs): # Create a transaction or a span if an transaction is already active span = _get_start_span_function()( name=f"{agent.name} workflow", - origin=sentry_sdk.integrations.openai_agents.OpenAIAgentsIntegration.origin, + origin=SPAN_ORIGIN, ) return span diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 07e5bd3559..30c5fd1dac 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -1,6 +1,7 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA +from ..consts import SPAN_ORIGIN from ..utils import ( _set_agent_data, _set_input_data, @@ -21,7 +22,7 @@ def ai_client_span(agent, get_response_kwargs): span = sentry_sdk.start_span( op=OP.GEN_AI_CHAT, description=f"chat {agent.model}", - origin=sentry_sdk.integrations.openai_agents.OpenAIAgentsIntegration.origin, + origin=SPAN_ORIGIN, ) # TODO-anton: remove hardcoded stuff and replace something that also works for embedding and so on span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "chat") diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index 90d025a5e9..e6e880b64c 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -2,6 +2,7 @@ from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.scope import should_send_default_pii +from ..consts import SPAN_ORIGIN from ..utils import _set_agent_data from typing import TYPE_CHECKING @@ -16,7 +17,7 @@ def execute_tool_span(tool, *args, **kwargs): span = sentry_sdk.start_span( op=OP.GEN_AI_EXECUTE_TOOL, name=f"execute_tool {tool.name}", - origin=sentry_sdk.integrations.openai_agents.OpenAIAgentsIntegration.origin, + origin=SPAN_ORIGIN, ) span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "execute_tool") diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index 0b786582f7..549ade1246 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -1,6 +1,7 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA +from ..consts import SPAN_ORIGIN from ..utils import _set_agent_data from typing import TYPE_CHECKING @@ -15,7 +16,7 @@ def invoke_agent_span(context, agent): span = sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, name=f"invoke_agent {agent.name}", - origin=sentry_sdk.integrations.openai_agents.OpenAIAgentsIntegration.origin, + origin=SPAN_ORIGIN, ) span.__enter__() From 1aecf4f730e139932fa4e6c37eaa2307ee13bc92 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 14:08:52 +0200 Subject: [PATCH 62/89] linting --- sentry_sdk/integrations/openai_agents/patches/models.py | 1 - sentry_sdk/integrations/openai_agents/patches/runner.py | 2 -- sentry_sdk/integrations/openai_agents/patches/tools.py | 1 - sentry_sdk/integrations/openai_agents/run_hooks.py | 2 +- 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/models.py b/sentry_sdk/integrations/openai_agents/patches/models.py index 571787e3ae..91858eaecc 100644 --- a/sentry_sdk/integrations/openai_agents/patches/models.py +++ b/sentry_sdk/integrations/openai_agents/patches/models.py @@ -22,7 +22,6 @@ def _create_get_model_wrapper(original_get_model): Wraps the agents.Runner._get_model method to wrap the get_response method of the model to create a AI client span. """ - @classmethod @wraps(original_get_model) def wrapped_get_model(cls, agent, run_config): # type: (agents.Runner, agents.Agent, agents.RunConfig) -> agents.Model diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index 1d34e0b45b..9b5df20b66 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -25,7 +25,6 @@ def _create_run_wrapper(original_func): """ is_async = asyncio.iscoroutinefunction(original_func) - @classmethod @wraps(original_func) async def async_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any @@ -37,7 +36,6 @@ async def async_wrapper(cls, *args, **kwargs): return result - @classmethod @wraps(original_func) def sync_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any diff --git a/sentry_sdk/integrations/openai_agents/patches/tools.py b/sentry_sdk/integrations/openai_agents/patches/tools.py index 602ea30b06..efdbe00788 100644 --- a/sentry_sdk/integrations/openai_agents/patches/tools.py +++ b/sentry_sdk/integrations/openai_agents/patches/tools.py @@ -21,7 +21,6 @@ def _create_get_all_tools_wrapper(original_get_all_tools): Wraps the agents.Runner._get_all_tools method of the Runner class to wrap all function tools with Sentry instrumentation. """ - @classmethod @wraps(original_get_all_tools) async def wrapped_get_all_tools(cls, agent, context_wrapper): # type: (agents.Runner, agents.Agent, agents.RunContextWrapper) -> list[agents.Tool] diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py index 71b7c1542b..012bec594c 100644 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -15,7 +15,7 @@ raise DidNotEnable("OpenAI Agents not installed") -class SentryRunHooks(agents.RunHooks): +class SentryRunHooks(agents.RunHooks): # type: ignore[misc] async def on_agent_start(self, context, agent): # type: (agents.RunContextWrapper, agents.Agent) -> None invoke_agent_span(context, agent) From 66a296028d6f1baf68a695d6329ebc152bead67c Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 14:44:40 +0200 Subject: [PATCH 63/89] . --- sentry_sdk/integrations/openai_agents/patches/models.py | 1 + sentry_sdk/integrations/openai_agents/patches/runner.py | 2 ++ sentry_sdk/integrations/openai_agents/patches/tools.py | 1 + .../integrations/openai_agents/spans/agent_workflow.py | 6 +++--- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/models.py b/sentry_sdk/integrations/openai_agents/patches/models.py index 91858eaecc..058bf2694f 100644 --- a/sentry_sdk/integrations/openai_agents/patches/models.py +++ b/sentry_sdk/integrations/openai_agents/patches/models.py @@ -22,6 +22,7 @@ def _create_get_model_wrapper(original_get_model): Wraps the agents.Runner._get_model method to wrap the get_response method of the model to create a AI client span. """ + @classmethod # type: ignore[misc] @wraps(original_get_model) def wrapped_get_model(cls, agent, run_config): # type: (agents.Runner, agents.Agent, agents.RunConfig) -> agents.Model diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index 9b5df20b66..551fc5a0a9 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -25,6 +25,7 @@ def _create_run_wrapper(original_func): """ is_async = asyncio.iscoroutinefunction(original_func) + @classmethod # type: ignore[misc] @wraps(original_func) async def async_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any @@ -36,6 +37,7 @@ async def async_wrapper(cls, *args, **kwargs): return result + @classmethod # type: ignore[misc] @wraps(original_func) def sync_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any diff --git a/sentry_sdk/integrations/openai_agents/patches/tools.py b/sentry_sdk/integrations/openai_agents/patches/tools.py index efdbe00788..c42e9452f7 100644 --- a/sentry_sdk/integrations/openai_agents/patches/tools.py +++ b/sentry_sdk/integrations/openai_agents/patches/tools.py @@ -21,6 +21,7 @@ def _create_get_all_tools_wrapper(original_get_all_tools): Wraps the agents.Runner._get_all_tools method of the Runner class to wrap all function tools with Sentry instrumentation. """ + @classmethod # type: ignore[misc] @wraps(original_get_all_tools) async def wrapped_get_all_tools(cls, agent, context_wrapper): # type: (agents.Runner, agents.Agent, agents.RunContextWrapper) -> list[agents.Tool] diff --git a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py index 67e0895aa5..05e1a75a1c 100644 --- a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py +++ b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py @@ -10,9 +10,9 @@ from typing import Any -def agent_workflow_span(*args, **kwargs): - # type: (*Any, **Any) -> sentry_sdk.tracing.Span - agent = args[0] +def agent_workflow_span(agent): + # type: (Agent) -> sentry_sdk.tracing.Span + # Create a transaction or a span if an transaction is already active span = _get_start_span_function()( name=f"{agent.name} workflow", From 69cd9f376139636aea76ee62f94c3a5dc60d65ef Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 15:06:10 +0200 Subject: [PATCH 64/89] proper classmethod patching --- .../integrations/openai_agents/__init__.py | 20 +++++++++++++------ .../openai_agents/patches/models.py | 7 +++++-- .../openai_agents/patches/runner.py | 10 ++++++---- .../openai_agents/patches/tools.py | 7 +++++-- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py index 0c246bd6dd..69ceba81f6 100644 --- a/sentry_sdk/integrations/openai_agents/__init__.py +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -15,20 +15,28 @@ def _patch_runner(): # type: () -> None - agents.Runner.run = _create_run_wrapper(agents.Runner.run) - agents.Runner.run_sync = _create_run_wrapper(agents.Runner.run_sync) - agents.Runner.run_streamed = _create_run_wrapper(agents.Runner.run_streamed) + agents.Runner.run = classmethod( + _create_run_wrapper(agents.Runner.run), + ) + agents.Runner.run_sync = classmethod( + _create_run_wrapper(agents.Runner.run_sync), + ) + agents.Runner.run_streamed = classmethod( + _create_run_wrapper(agents.Runner.run_streamed), + ) def _patch_model(): # type: () -> None - agents.Runner._get_model = _create_get_model_wrapper(agents.Runner._get_model) + agents.Runner._get_model = classmethod( + _create_get_model_wrapper(agents.Runner._get_model), + ) def _patch_tools(): # type: () -> None - agents.Runner._get_all_tools = _create_get_all_tools_wrapper( - agents.Runner._get_all_tools + agents.Runner._get_all_tools = classmethod( + _create_get_all_tools_wrapper(agents.Runner._get_all_tools), ) diff --git a/sentry_sdk/integrations/openai_agents/patches/models.py b/sentry_sdk/integrations/openai_agents/patches/models.py index 058bf2694f..3125fcfa8c 100644 --- a/sentry_sdk/integrations/openai_agents/patches/models.py +++ b/sentry_sdk/integrations/openai_agents/patches/models.py @@ -22,8 +22,11 @@ def _create_get_model_wrapper(original_get_model): Wraps the agents.Runner._get_model method to wrap the get_response method of the model to create a AI client span. """ - @classmethod # type: ignore[misc] - @wraps(original_get_model) + @wraps( + original_get_model.__func__ + if hasattr(original_get_model, "__func__") + else original_get_model + ) def wrapped_get_model(cls, agent, run_config): # type: (agents.Runner, agents.Agent, agents.RunConfig) -> agents.Model diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index 551fc5a0a9..04cdb54dd0 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -25,8 +25,9 @@ def _create_run_wrapper(original_func): """ is_async = asyncio.iscoroutinefunction(original_func) - @classmethod # type: ignore[misc] - @wraps(original_func) + @wraps( + original_func.__func__ if hasattr(original_func, "__func__") else original_func + ) async def async_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any agent = args[0] @@ -37,8 +38,9 @@ async def async_wrapper(cls, *args, **kwargs): return result - @classmethod # type: ignore[misc] - @wraps(original_func) + @wraps( + original_func.__func__ if hasattr(original_func, "__func__") else original_func + ) def sync_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any agent = args[0] diff --git a/sentry_sdk/integrations/openai_agents/patches/tools.py b/sentry_sdk/integrations/openai_agents/patches/tools.py index c42e9452f7..c0534dd30d 100644 --- a/sentry_sdk/integrations/openai_agents/patches/tools.py +++ b/sentry_sdk/integrations/openai_agents/patches/tools.py @@ -21,8 +21,11 @@ def _create_get_all_tools_wrapper(original_get_all_tools): Wraps the agents.Runner._get_all_tools method of the Runner class to wrap all function tools with Sentry instrumentation. """ - @classmethod # type: ignore[misc] - @wraps(original_get_all_tools) + @wraps( + original_get_all_tools.__func__ + if hasattr(original_get_all_tools, "__func__") + else original_get_all_tools + ) async def wrapped_get_all_tools(cls, agent, context_wrapper): # type: (agents.Runner, agents.Agent, agents.RunContextWrapper) -> list[agents.Tool] From a8c6415cfee3cc17b1d98c40fd32c255b13ea31c Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 16:45:00 +0200 Subject: [PATCH 65/89] better invocation spans --- .../integrations/openai_agents/__init__.py | 7 ++ .../openai_agents/patches/__init__.py | 2 +- .../openai_agents/patches/runner.py | 34 +++++- .../openai_agents/spans/invoke_agent.py | 13 +-- .../openai_agents/test_openai_agents.py | 105 +++++++++++++----- 5 files changed, 123 insertions(+), 38 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py index 69ceba81f6..3f08a2e01d 100644 --- a/sentry_sdk/integrations/openai_agents/__init__.py +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -4,6 +4,7 @@ _create_get_model_wrapper, _create_get_all_tools_wrapper, _create_run_wrapper, + _create_run_single_turn_wrapper, ) try: @@ -15,6 +16,7 @@ def _patch_runner(): # type: () -> None + # Agent workflow spans agents.Runner.run = classmethod( _create_run_wrapper(agents.Runner.run), ) @@ -25,6 +27,11 @@ def _patch_runner(): _create_run_wrapper(agents.Runner.run_streamed), ) + # Agent invocation spans + agents.Runner._run_single_turn = classmethod( + _create_run_single_turn_wrapper(agents.Runner._run_single_turn), + ) + def _patch_model(): # type: () -> None diff --git a/sentry_sdk/integrations/openai_agents/patches/__init__.py b/sentry_sdk/integrations/openai_agents/patches/__init__.py index 29e0fd6245..a703cf307d 100644 --- a/sentry_sdk/integrations/openai_agents/patches/__init__.py +++ b/sentry_sdk/integrations/openai_agents/patches/__init__.py @@ -1,3 +1,3 @@ from .models import _create_get_model_wrapper # noqa: F401 from .tools import _create_get_all_tools_wrapper # noqa: F401 -from .runner import _create_run_wrapper # noqa: F401 +from .runner import _create_run_wrapper, _create_run_single_turn_wrapper # noqa: F401 diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index 04cdb54dd0..e7688543d3 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -4,8 +4,12 @@ from sentry_sdk.integrations import DidNotEnable -from ..spans import agent_workflow_span, update_agent_workflow_span -from ..utils import _wrap_hooks +from ..spans import ( + agent_workflow_span, + update_agent_workflow_span, + invoke_agent_span, + update_invoke_agent_span, +) from typing import TYPE_CHECKING @@ -32,7 +36,6 @@ async def async_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any agent = args[0] with agent_workflow_span(agent) as span: - kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) result = await original_func(*args, **kwargs) update_agent_workflow_span(span, agent, result) @@ -45,10 +48,33 @@ def sync_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any agent = args[0] with agent_workflow_span(agent) as span: - kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) result = original_func(*args, **kwargs) update_agent_workflow_span(span, agent, result) return result return async_wrapper if is_async else sync_wrapper + + +def _create_run_single_turn_wrapper(original_func): + # type: (Callable[..., Any]) -> Callable[..., Any] + """ + Wraps the agents.Runner._run_single_turn method to create a span for the agent invocation. + """ + + @wraps( + original_func.__func__ if hasattr(original_func, "__func__") else original_func + ) + async def async_wrapper(cls, *args, **kwargs): + # type: (agents.Runner, *Any, **Any) -> Any + agent = kwargs.get("agent") + if agent is None or not kwargs.get("should_run_agent_start_hooks", False): + return await original_func(*args, **kwargs) + + with invoke_agent_span(agent) as span: + result = await original_func(*args, **kwargs) + update_invoke_agent_span(span, agent, result) + + return result + + return async_wrapper diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index 549ade1246..742cacfee4 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -11,14 +11,13 @@ from typing import Any -def invoke_agent_span(context, agent): - # type: (agents.RunContextWrapper, agents.Agent) -> sentry_sdk.tracing.Span +def invoke_agent_span(agent): + # type: (agents.Agent) -> sentry_sdk.tracing.Span span = sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, name=f"invoke_agent {agent.name}", origin=SPAN_ORIGIN, ) - span.__enter__() span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") @@ -27,8 +26,6 @@ def invoke_agent_span(context, agent): return span -def update_invoke_agent_span(context, agent, output): - # type: (agents.RunContextWrapper, agents.Agent, Any) -> None - current_span = sentry_sdk.get_current_span() - if current_span: - current_span.__exit__(None, None, None) +def update_invoke_agent_span(span, agent, output): + # type: (sentry_sdk.tracing.span, agents.Agent, Any) -> None + pass diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index d4ebf9cd6e..3278362b6e 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -3,7 +3,6 @@ from unittest.mock import MagicMock, patch import os -import sentry_sdk from sentry_sdk.integrations.openai_agents import OpenAIAgentsIntegration import agents @@ -95,17 +94,73 @@ async def test_agent_invocation_span( events = capture_events() - with sentry_sdk.start_transaction(name="test_transaction"): - result = await agents.Runner.run( - test_agent, "Test input", run_config=test_run_config - ) + result = await agents.Runner.run( + test_agent, "Test input", run_config=test_run_config + ) + + assert result is not None + assert result.final_output == "Hello, how can I help you?" + + (transaction,) = events + spans = transaction["spans"] + invoke_agent_span, ai_client_span = spans + + assert transaction["transaction"] == "test_agent workflow" + assert transaction["contexts"]["trace"]["origin"] == "auto.ai.openai_agents" + + assert invoke_agent_span["description"] == "invoke_agent test_agent" + assert invoke_agent_span["data"]["gen_ai.operation.name"] == "invoke_agent" + assert invoke_agent_span["data"]["gen_ai.system"] == "openai" + assert invoke_agent_span["data"]["gen_ai.agent.name"] == "test_agent" + assert invoke_agent_span["data"]["gen_ai.request.max_tokens"] == 100 + assert invoke_agent_span["data"]["gen_ai.request.model"] == "gpt-4" + assert invoke_agent_span["data"]["gen_ai.request.temperature"] == 0.7 + assert invoke_agent_span["data"]["gen_ai.request.top_p"] == 1.0 + + assert ai_client_span["description"] == "chat gpt-4" + assert ai_client_span["data"]["gen_ai.operation.name"] == "chat" + assert ai_client_span["data"]["gen_ai.system"] == "openai" + assert ai_client_span["data"]["gen_ai.agent.name"] == "test_agent" + assert ai_client_span["data"]["gen_ai.request.max_tokens"] == 100 + assert ai_client_span["data"]["gen_ai.request.model"] == "gpt-4" + assert ai_client_span["data"]["gen_ai.request.temperature"] == 0.7 + assert ai_client_span["data"]["gen_ai.request.top_p"] == 1.0 + + +def test_agent_invocation_span_sync( + sentry_init, capture_events, test_agent, mock_model_response +): + """ + Test that the integration creates spans for agent invocations. + """ + + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): + with patch( + "agents.models.openai_responses.OpenAIResponsesModel.get_response" + ) as mock_get_response: + mock_get_response.return_value = mock_model_response + + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + ) + + events = capture_events() + + result = agents.Runner.run_sync( + test_agent, "Test input", run_config=test_run_config + ) - assert result is not None - assert result.final_output == "Hello, how can I help you?" + assert result is not None + assert result.final_output == "Hello, how can I help you?" (transaction,) = events spans = transaction["spans"] agent_workflow_span, invoke_agent_span, ai_client_span = spans + # TODO: why is there a workflow span? + + assert transaction["transaction"] == "test_agent workflow" + assert transaction["contexts"]["trace"]["origin"] == "auto.ai.openai_agents" assert agent_workflow_span["description"] == "test_agent workflow" @@ -201,17 +256,15 @@ def simple_test_tool(message: str) -> str: events = capture_events() - with sentry_sdk.start_transaction(name="test_transaction"): - await agents.Runner.run( - agent_with_tool, - "Please use the simple test tool", - run_config=test_run_config, - ) + await agents.Runner.run( + agent_with_tool, + "Please use the simple test tool", + run_config=test_run_config, + ) (transaction,) = events spans = transaction["spans"] ( - agent_workflow_span, agent_span, ai_client_span1, tool_span, @@ -235,8 +288,8 @@ def simple_test_tool(message: str) -> str: } ] - assert agent_workflow_span["description"] == "test_agent workflow" - assert agent_workflow_span["origin"] == "auto.ai.openai_agents" + assert transaction["transaction"] == "test_agent workflow" + assert transaction["contexts"]["trace"]["origin"] == "auto.ai.openai_agents" assert agent_span["description"] == "invoke_agent test_agent" assert agent_span["origin"] == "auto.ai.openai_agents" @@ -371,19 +424,21 @@ async def test_error_handling(sentry_init, capture_events, test_agent): events = capture_events() - with sentry_sdk.start_transaction(name="test_transaction"): - with pytest.raises(Exception, match="Model Error"): - await agents.Runner.run( - test_agent, "Test input", run_config=test_run_config - ) + with pytest.raises(Exception, match="Model Error"): + await agents.Runner.run( + test_agent, "Test input", run_config=test_run_config + ) (transaction,) = events spans = transaction["spans"] - (agent_workflow_span, ai_client_span) = spans + (invoke_agent_span, ai_client_span) = spans - assert agent_workflow_span["description"] == "test_agent workflow" - assert agent_workflow_span["origin"] == "auto.ai.openai_agents" - assert agent_workflow_span["tags"]["status"] == "internal_error" + assert transaction["transaction"] == "test_agent workflow" + assert transaction["contexts"]["trace"]["origin"] == "auto.ai.openai_agents" + + assert invoke_agent_span["description"] == "invoke_agent test_agent" + assert invoke_agent_span["origin"] == "auto.ai.openai_agents" + assert invoke_agent_span["tags"]["status"] == "internal_error" assert ai_client_span["description"] == "chat gpt-4" assert ai_client_span["origin"] == "auto.ai.openai_agents" From 0f97113629d941ba2e1280e0ac286c7a4661d36b Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 16:56:22 +0200 Subject: [PATCH 66/89] proper invocation spans --- sentry_sdk/integrations/openai_agents/__init__.py | 4 +--- sentry_sdk/integrations/openai_agents/patches/runner.py | 8 ++------ tests/integrations/openai_agents/test_openai_agents.py | 5 +---- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py index 3f08a2e01d..c95e92742b 100644 --- a/sentry_sdk/integrations/openai_agents/__init__.py +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -23,9 +23,7 @@ def _patch_runner(): agents.Runner.run_sync = classmethod( _create_run_wrapper(agents.Runner.run_sync), ) - agents.Runner.run_streamed = classmethod( - _create_run_wrapper(agents.Runner.run_streamed), - ) + # TODO-anton: Also patch streaming runner: agents.Runner.run_streamed # Agent invocation spans agents.Runner._run_single_turn = classmethod( diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index e7688543d3..b4020bff8b 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -46,12 +46,8 @@ async def async_wrapper(cls, *args, **kwargs): ) def sync_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any - agent = args[0] - with agent_workflow_span(agent) as span: - result = original_func(*args, **kwargs) - update_agent_workflow_span(span, agent, result) - - return result + # The sync version (.run_sync()) is just a wrapper around the async version (.run()) + return original_func(*args, **kwargs) return async_wrapper if is_async else sync_wrapper diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 3278362b6e..49c33f82f1 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -156,14 +156,11 @@ def test_agent_invocation_span_sync( (transaction,) = events spans = transaction["spans"] - agent_workflow_span, invoke_agent_span, ai_client_span = spans - # TODO: why is there a workflow span? + invoke_agent_span, ai_client_span = spans assert transaction["transaction"] == "test_agent workflow" assert transaction["contexts"]["trace"]["origin"] == "auto.ai.openai_agents" - assert agent_workflow_span["description"] == "test_agent workflow" - assert invoke_agent_span["description"] == "invoke_agent test_agent" assert invoke_agent_span["data"]["gen_ai.operation.name"] == "invoke_agent" assert invoke_agent_span["data"]["gen_ai.system"] == "openai" From 99ccbeedd74845f16b5f8aebb9a05d9f08e6c059 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 17 Jun 2025 17:07:56 +0200 Subject: [PATCH 67/89] cleanup --- .../integrations/openai_agents/run_hooks.py | 34 ------------- .../integrations/openai_agents/utils.py | 49 ------------------- 2 files changed, 83 deletions(-) delete mode 100644 sentry_sdk/integrations/openai_agents/run_hooks.py diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py deleted file mode 100644 index 012bec594c..0000000000 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ /dev/null @@ -1,34 +0,0 @@ -from sentry_sdk.integrations import DidNotEnable - -from typing import TYPE_CHECKING - -from .spans import invoke_agent_span, update_invoke_agent_span, handoff_span - -if TYPE_CHECKING: - from typing import Any - -try: - import agents - - -except ImportError: - raise DidNotEnable("OpenAI Agents not installed") - - -class SentryRunHooks(agents.RunHooks): # type: ignore[misc] - async def on_agent_start(self, context, agent): - # type: (agents.RunContextWrapper, agents.Agent) -> None - invoke_agent_span(context, agent) - - async def on_agent_end(self, context, agent, output): - # type: (agents.RunContextWrapper, agents.Agent, Any) -> None - update_invoke_agent_span(context, agent, output) - - async def on_handoff( - self, - context: agents.RunContextWrapper, - from_agent: agents.Agent, - to_agent: agents.Agent, - ) -> None: - handoff_span(context, from_agent, to_agent) - pass diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 31212c0580..13355089dd 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,5 +1,3 @@ -from functools import wraps - import sentry_sdk from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable @@ -15,7 +13,6 @@ try: import agents - from agents import RunHooks except ImportError: raise DidNotEnable("OpenAI Agents not installed") @@ -40,52 +37,6 @@ def _get_start_span_function(): return sentry_sdk.start_span if is_transaction else sentry_sdk.start_transaction -def _create_hook_wrapper(original_hook, sentry_hook): - # type: (Callable[..., Any], Callable[..., Any]) -> Callable[..., Any] - @wraps(original_hook) - async def async_wrapper(*args, **kwargs): - # type: (*Any, **Any) -> Any - await sentry_hook(*args, **kwargs) - return await original_hook(*args, **kwargs) - - return async_wrapper - - -def _wrap_hooks(hooks): - # type: (RunHooks) -> RunHooks - """ - Our integration uses RunHooks to create spans. This function will either - enable our SentryRunHooks or if the users has given custom RunHooks wrap - them so the Sentry hooks and the users hooks are both called - """ - from .run_hooks import SentryRunHooks - - sentry_hooks = SentryRunHooks() - - if hooks is None: - return sentry_hooks - - wrapped_hooks = type("SentryWrappedHooks", (hooks.__class__,), {}) - - # Wrap all methods from RunHooks - for method_name in dir(RunHooks): - if method_name.startswith("on_"): - original_method = getattr(hooks, method_name) - # Only wrap if the method exists in SentryRunHooks - try: - sentry_method = getattr(sentry_hooks, method_name) - setattr( - wrapped_hooks, - method_name, - _create_hook_wrapper(original_method, sentry_method), - ) - except AttributeError: - # If method doesn't exist in SentryRunHooks, just use the original method - setattr(wrapped_hooks, method_name, original_method) - - return wrapped_hooks() - - def _set_agent_data(span, agent): # type: (sentry_sdk.tracing.Span, agents.Agent) -> None span.set_data( From 5c59668cedcc6807a8e6d22172eb06973ca13c4c Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 18 Jun 2025 09:42:40 +0200 Subject: [PATCH 68/89] Bring back handoff span --- .../openai_agents/patches/runner.py | 3 ++ .../openai_agents/spans/handoff.py | 17 +++---- .../integrations/openai_agents/utils.py | 50 +++++++++++++++++++ 3 files changed, 61 insertions(+), 9 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index b4020bff8b..db77d68c84 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -10,6 +10,7 @@ invoke_agent_span, update_invoke_agent_span, ) +from ..utils import _wrap_hooks from typing import TYPE_CHECKING @@ -36,6 +37,7 @@ async def async_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any agent = args[0] with agent_workflow_span(agent) as span: + kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) result = await original_func(*args, **kwargs) update_agent_workflow_span(span, agent, result) @@ -46,6 +48,7 @@ async def async_wrapper(cls, *args, **kwargs): ) def sync_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any + kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) # The sync version (.run_sync()) is just a wrapper around the async version (.run()) return original_func(*args, **kwargs) diff --git a/sentry_sdk/integrations/openai_agents/spans/handoff.py b/sentry_sdk/integrations/openai_agents/spans/handoff.py index b1f4dbda1e..7d9af817e7 100644 --- a/sentry_sdk/integrations/openai_agents/spans/handoff.py +++ b/sentry_sdk/integrations/openai_agents/spans/handoff.py @@ -1,6 +1,8 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA +from ..consts import SPAN_ORIGIN + from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -9,12 +11,9 @@ def handoff_span(context, from_agent, to_agent): # type: (agents.RunContextWrapper, agents.Agent, agents.Agent) -> None - current_span = sentry_sdk.get_current_span() - if current_span: - with current_span.start_child( - op=OP.GEN_AI_HANDOFF, - name=f"handoff from {from_agent.name} to {to_agent.name}", - ) as span: - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "handoff") - - current_span.__exit__(None, None, None) + with sentry_sdk.start_span( + op=OP.GEN_AI_HANDOFF, + name=f"handoff from {from_agent.name} to {to_agent.name}", + origin=SPAN_ORIGIN, + ) as span: + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "handoff") diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 13355089dd..3fd17196ac 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,3 +1,4 @@ +from functools import wraps import sentry_sdk from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable @@ -155,3 +156,52 @@ def _set_output_data(span, result): if len(output_messages["response"]) > 0: span.set_data(SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"]) + + +def _create_hook_wrapper(original_hook, sentry_hook): + # type: (Callable[..., Any], Callable[..., Any]) -> Callable[..., Any] + @wraps(original_hook) + async def async_wrapper(*args, **kwargs): + # type: (*Any, **Any) -> Any + await sentry_hook(*args, **kwargs) + return await original_hook(*args, **kwargs) + + return async_wrapper + + +def _wrap_hooks(hooks): + # type: (agents.RunHooks) -> agents.RunHooks + """ + Our integration uses RunHooks to create spans. This function will either + enable our SentryRunHooks or if the users has given custom RunHooks wrap + them so the Sentry hooks and the users hooks are both called + """ + import ipdb + + ipdb.set_trace() + from .run_hooks import SentryRunHooks + + sentry_hooks = SentryRunHooks() + + if hooks is None: + return sentry_hooks + + wrapped_hooks = type("SentryWrappedHooks", (hooks.__class__,), {}) + + # Wrap all methods from RunHooks + for method_name in dir(agents.RunHooks): + if method_name.startswith("on_"): + original_method = getattr(hooks, method_name) + # Only wrap if the method exists in SentryRunHooks + try: + sentry_method = getattr(sentry_hooks, method_name) + setattr( + wrapped_hooks, + method_name, + _create_hook_wrapper(original_method, sentry_method), + ) + except AttributeError: + # If method doesn't exist in SentryRunHooks, just use the original method + setattr(wrapped_hooks, method_name, original_method) + + return wrapped_hooks() From 269802bb32100daefeb206fcf7b7bbfd3bc2d776 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 18 Jun 2025 09:43:39 +0200 Subject: [PATCH 69/89] Bring back handoff span 2 --- .../integrations/openai_agents/run_hooks.py | 19 +++++++++++++++++++ .../integrations/openai_agents/utils.py | 3 --- 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 sentry_sdk/integrations/openai_agents/run_hooks.py diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py new file mode 100644 index 0000000000..07342047b1 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -0,0 +1,19 @@ +from sentry_sdk.integrations import DidNotEnable + +from .spans import handoff_span + + +try: + import agents +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + + +class SentryRunHooks(agents.RunHooks): # type: ignore[misc] + async def on_handoff( + self, + context: agents.RunContextWrapper, + from_agent: agents.Agent, + to_agent: agents.Agent, + ) -> None: + handoff_span(context, from_agent, to_agent) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 3fd17196ac..ab0b0cc01d 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -176,9 +176,6 @@ def _wrap_hooks(hooks): enable our SentryRunHooks or if the users has given custom RunHooks wrap them so the Sentry hooks and the users hooks are both called """ - import ipdb - - ipdb.set_trace() from .run_hooks import SentryRunHooks sentry_hooks = SentryRunHooks() From 3885fb425cebe4d0735ba1f26eddfe30e6c4ab5a Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 18 Jun 2025 12:45:06 +0200 Subject: [PATCH 70/89] Bring back invoke agent spans based on hooks. --- .../integrations/openai_agents/__init__.py | 6 --- .../openai_agents/patches/__init__.py | 2 +- .../openai_agents/patches/models.py | 1 + .../openai_agents/patches/runner.py | 49 +++++++------------ .../integrations/openai_agents/run_hooks.py | 15 +++++- .../openai_agents/spans/invoke_agent.py | 13 +++-- .../openai_agents/test_openai_agents.py | 11 ++++- 7 files changed, 50 insertions(+), 47 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py index c95e92742b..54ad24e2b8 100644 --- a/sentry_sdk/integrations/openai_agents/__init__.py +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -4,7 +4,6 @@ _create_get_model_wrapper, _create_get_all_tools_wrapper, _create_run_wrapper, - _create_run_single_turn_wrapper, ) try: @@ -25,11 +24,6 @@ def _patch_runner(): ) # TODO-anton: Also patch streaming runner: agents.Runner.run_streamed - # Agent invocation spans - agents.Runner._run_single_turn = classmethod( - _create_run_single_turn_wrapper(agents.Runner._run_single_turn), - ) - def _patch_model(): # type: () -> None diff --git a/sentry_sdk/integrations/openai_agents/patches/__init__.py b/sentry_sdk/integrations/openai_agents/patches/__init__.py index a703cf307d..29e0fd6245 100644 --- a/sentry_sdk/integrations/openai_agents/patches/__init__.py +++ b/sentry_sdk/integrations/openai_agents/patches/__init__.py @@ -1,3 +1,3 @@ from .models import _create_get_model_wrapper # noqa: F401 from .tools import _create_get_all_tools_wrapper # noqa: F401 -from .runner import _create_run_wrapper, _create_run_single_turn_wrapper # noqa: F401 +from .runner import _create_run_wrapper # noqa: F401 diff --git a/sentry_sdk/integrations/openai_agents/patches/models.py b/sentry_sdk/integrations/openai_agents/patches/models.py index 3125fcfa8c..e6f24da6a1 100644 --- a/sentry_sdk/integrations/openai_agents/patches/models.py +++ b/sentry_sdk/integrations/openai_agents/patches/models.py @@ -38,6 +38,7 @@ async def wrapped_get_response(*args, **kwargs): # type: (*Any, **Any) -> Any with ai_client_span(agent, kwargs) as span: result = await original_get_response(*args, **kwargs) + update_ai_client_span(span, agent, kwargs, result) return result diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index db77d68c84..701e56810b 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -2,15 +2,14 @@ from functools import wraps +import sentry_sdk from sentry_sdk.integrations import DidNotEnable from ..spans import ( agent_workflow_span, update_agent_workflow_span, - invoke_agent_span, - update_invoke_agent_span, ) -from ..utils import _wrap_hooks +from ..utils import _capture_exception, _wrap_hooks from typing import TYPE_CHECKING @@ -38,42 +37,28 @@ async def async_wrapper(cls, *args, **kwargs): agent = args[0] with agent_workflow_span(agent) as span: kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) - result = await original_func(*args, **kwargs) - update_agent_workflow_span(span, agent, result) - - return result + result = None + try: + result = await original_func(*args, **kwargs) + return result + except Exception as exc: + _capture_exception(exc) + + # It could be that there is a "invoke agent" span still open (because its create in run_hooks) + current_span = sentry_sdk.get_current_span() + if current_span.timestamp is None: + current_span.__exit__(None, None, None) + + raise exc from None + finally: + update_agent_workflow_span(span, agent, result) @wraps( original_func.__func__ if hasattr(original_func, "__func__") else original_func ) def sync_wrapper(cls, *args, **kwargs): # type: (agents.Runner, *Any, **Any) -> Any - kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) # The sync version (.run_sync()) is just a wrapper around the async version (.run()) return original_func(*args, **kwargs) return async_wrapper if is_async else sync_wrapper - - -def _create_run_single_turn_wrapper(original_func): - # type: (Callable[..., Any]) -> Callable[..., Any] - """ - Wraps the agents.Runner._run_single_turn method to create a span for the agent invocation. - """ - - @wraps( - original_func.__func__ if hasattr(original_func, "__func__") else original_func - ) - async def async_wrapper(cls, *args, **kwargs): - # type: (agents.Runner, *Any, **Any) -> Any - agent = kwargs.get("agent") - if agent is None or not kwargs.get("should_run_agent_start_hooks", False): - return await original_func(*args, **kwargs) - - with invoke_agent_span(agent) as span: - result = await original_func(*args, **kwargs) - update_invoke_agent_span(span, agent, result) - - return result - - return async_wrapper diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py index 07342047b1..b14b9ab5ed 100644 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -1,6 +1,11 @@ from sentry_sdk.integrations import DidNotEnable -from .spans import handoff_span +from .spans import handoff_span, invoke_agent_span, update_invoke_agent_span + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any try: @@ -10,6 +15,14 @@ class SentryRunHooks(agents.RunHooks): # type: ignore[misc] + async def on_agent_start(self, context, agent): + # type: (agents.RunContextWrapper, agents.Agent) -> None + invoke_agent_span(context, agent) + + async def on_agent_end(self, context, agent, output): + # type: (agents.RunContextWrapper, agents.Agent, Any) -> None + update_invoke_agent_span(context, agent, output) + async def on_handoff( self, context: agents.RunContextWrapper, diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index 742cacfee4..549ade1246 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -11,13 +11,14 @@ from typing import Any -def invoke_agent_span(agent): - # type: (agents.Agent) -> sentry_sdk.tracing.Span +def invoke_agent_span(context, agent): + # type: (agents.RunContextWrapper, agents.Agent) -> sentry_sdk.tracing.Span span = sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, name=f"invoke_agent {agent.name}", origin=SPAN_ORIGIN, ) + span.__enter__() span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") @@ -26,6 +27,8 @@ def invoke_agent_span(agent): return span -def update_invoke_agent_span(span, agent, output): - # type: (sentry_sdk.tracing.span, agents.Agent, Any) -> None - pass +def update_invoke_agent_span(context, agent, output): + # type: (agents.RunContextWrapper, agents.Agent, Any) -> None + current_span = sentry_sdk.get_current_span() + if current_span: + current_span.__exit__(None, None, None) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 49c33f82f1..0f95d9aa66 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -426,7 +426,15 @@ async def test_error_handling(sentry_init, capture_events, test_agent): test_agent, "Test input", run_config=test_run_config ) - (transaction,) = events + ( + error_event, + transaction, + ) = events + + assert error_event["exception"]["values"][0]["type"] == "Exception" + assert error_event["exception"]["values"][0]["value"] == "Model Error" + assert error_event["exception"]["values"][0]["mechanism"]["type"] == "openai_agents" + spans = transaction["spans"] (invoke_agent_span, ai_client_span) = spans @@ -435,7 +443,6 @@ async def test_error_handling(sentry_init, capture_events, test_agent): assert invoke_agent_span["description"] == "invoke_agent test_agent" assert invoke_agent_span["origin"] == "auto.ai.openai_agents" - assert invoke_agent_span["tags"]["status"] == "internal_error" assert ai_client_span["description"] == "chat gpt-4" assert ai_client_span["origin"] == "auto.ai.openai_agents" From a2b4d810bee3b4549faef8a08c9d6c93fad31788 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 18 Jun 2025 15:00:30 +0200 Subject: [PATCH 71/89] explanation of defeat --- sentry_sdk/integrations/openai_agents/patches/runner.py | 3 ++- sentry_sdk/integrations/openai_agents/patches/tools.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index 701e56810b..50677402fa 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -44,7 +44,8 @@ async def async_wrapper(cls, *args, **kwargs): except Exception as exc: _capture_exception(exc) - # It could be that there is a "invoke agent" span still open (because its create in run_hooks) + # It could be that there is a "invoke agent" span still open + # (because its create in run_hooks without the context manager) current_span = sentry_sdk.get_current_span() if current_span.timestamp is None: current_span.__exit__(None, None, None) diff --git a/sentry_sdk/integrations/openai_agents/patches/tools.py b/sentry_sdk/integrations/openai_agents/patches/tools.py index c0534dd30d..b359d32678 100644 --- a/sentry_sdk/integrations/openai_agents/patches/tools.py +++ b/sentry_sdk/integrations/openai_agents/patches/tools.py @@ -48,6 +48,13 @@ def create_wrapped_invoke(current_tool, current_on_invoke): async def sentry_wrapped_on_invoke_tool(*args, **kwargs): # type: (*Any, **Any) -> Any with execute_tool_span(current_tool, *args, **kwargs) as span: + # We can not capture exceptions in tool execution here because + # `_on_invoke_tool` is swallowing the exception here: + # https://github.com/openai/openai-agents-python/blob/main/src/agents/tool.py#L409-L422 + # And because function_tool is a decorator with `default_tool_error_function` set as a default parameter + # I was unable to monkey patch it because those are evaluated at module import time + # and the SDK is too late to patch it. I was also unable to patch `_on_invoke_tool_impl` + # because it is nested inside this import time code. As if they made it hard to patch on purpose... result = await current_on_invoke(*args, **kwargs) update_execute_tool_span(span, agent, current_tool, result) From afe1a17cbf131d777f0695509de4f6dddbdce70d Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 18 Jun 2025 15:13:08 +0200 Subject: [PATCH 72/89] updated test matrix --- tox.ini | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index 203499f206..37464a5723 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-06-17T10:29:38.027366+00:00 +# Last generated: 2025-06-18T13:10:50.595385+00:00 [tox] requires = @@ -145,7 +145,7 @@ envlist = {py3.9,py3.11,py3.12}-cohere-v5.11.4 {py3.9,py3.11,py3.12}-cohere-v5.15.0 - {py3.9,py3.11,py3.12}-openai_agents-v0.0.18 + {py3.9,py3.11,py3.12}-openai_agents-v0.0.19 {py3.8,py3.10,py3.11}-huggingface_hub-v0.22.2 {py3.8,py3.11,py3.12}-huggingface_hub-v0.26.5 @@ -182,7 +182,7 @@ envlist = {py3.7,py3.12,py3.13}-statsig-v0.55.3 {py3.7,py3.12,py3.13}-statsig-v0.56.0 {py3.7,py3.12,py3.13}-statsig-v0.57.3 - {py3.7,py3.12,py3.13}-statsig-v0.58.2 + {py3.7,py3.12,py3.13}-statsig-v0.58.3 {py3.8,py3.12,py3.13}-unleash-v6.0.1 {py3.8,py3.12,py3.13}-unleash-v6.1.0 @@ -256,7 +256,7 @@ envlist = {py3.6,py3.9,py3.10}-fastapi-v0.79.1 {py3.7,py3.10,py3.11}-fastapi-v0.91.0 {py3.7,py3.10,py3.11}-fastapi-v0.103.2 - {py3.8,py3.12,py3.13}-fastapi-v0.115.12 + {py3.8,py3.12,py3.13}-fastapi-v0.115.13 # ~~~ Web 2 ~~~ @@ -516,7 +516,7 @@ deps = cohere-v5.11.4: cohere==5.11.4 cohere-v5.15.0: cohere==5.15.0 - openai_agents-v0.0.18: openai-agents==0.0.18 + openai_agents-v0.0.19: openai-agents==0.0.19 openai_agents: pytest-asyncio huggingface_hub-v0.22.2: huggingface_hub==0.22.2 @@ -555,7 +555,7 @@ deps = statsig-v0.55.3: statsig==0.55.3 statsig-v0.56.0: statsig==0.56.0 statsig-v0.57.3: statsig==0.57.3 - statsig-v0.58.2: statsig==0.58.2 + statsig-v0.58.3: statsig==0.58.3 statsig: typing_extensions unleash-v6.0.1: UnleashClient==6.0.1 @@ -685,7 +685,7 @@ deps = fastapi-v0.79.1: fastapi==0.79.1 fastapi-v0.91.0: fastapi==0.91.0 fastapi-v0.103.2: fastapi==0.103.2 - fastapi-v0.115.12: fastapi==0.115.12 + fastapi-v0.115.13: fastapi==0.115.13 fastapi: httpx fastapi: pytest-asyncio fastapi: python-multipart From 43fffe7645d0adc9ed528150b2469785efeb1164 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 18 Jun 2025 15:17:46 +0200 Subject: [PATCH 73/89] linting --- sentry_sdk/integrations/openai_agents/patches/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index 50677402fa..090110a007 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -47,7 +47,7 @@ async def async_wrapper(cls, *args, **kwargs): # It could be that there is a "invoke agent" span still open # (because its create in run_hooks without the context manager) current_span = sentry_sdk.get_current_span() - if current_span.timestamp is None: + if current_span is not None and current_span.timestamp is None: current_span.__exit__(None, None, None) raise exc from None From 145653e44f4242a4d29df885cef248296d30d7c7 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 18 Jun 2025 16:52:30 +0200 Subject: [PATCH 74/89] Updates for version 0.0.19 --- sentry_sdk/integrations/__init__.py | 2 +- .../integrations/openai_agents/__init__.py | 21 +++++++++--------- .../openai_agents/patches/runner.py | 22 +++++-------------- .../integrations/openai_agents/run_hooks.py | 5 +++++ 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/sentry_sdk/integrations/__init__.py b/sentry_sdk/integrations/__init__.py index d63352ac28..e2eadd523d 100644 --- a/sentry_sdk/integrations/__init__.py +++ b/sentry_sdk/integrations/__init__.py @@ -145,7 +145,7 @@ def iter_default_integrations(with_auto_enabling_integrations): "launchdarkly": (9, 8, 0), "loguru": (0, 7, 0), "openai": (1, 0, 0), - "openai_agents": (0, 0, 17), + "openai_agents": (0, 0, 19), "openfeature": (0, 7, 1), "quart": (0, 16, 0), "ray": (2, 7, 0), diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py index 54ad24e2b8..7979995e63 100644 --- a/sentry_sdk/integrations/openai_agents/__init__.py +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -15,27 +15,28 @@ def _patch_runner(): # type: () -> None - # Agent workflow spans - agents.Runner.run = classmethod( - _create_run_wrapper(agents.Runner.run), - ) - agents.Runner.run_sync = classmethod( - _create_run_wrapper(agents.Runner.run_sync), + + # Creating agent workflow spans + # Note agents.run.DEFAULT_AGENT_RUNNER.run_sync is a wrapper around + # agents.run.DEFAULT_AGENT_RUNNER.run. It does not need to be wrapped separately. + agents.run.DEFAULT_AGENT_RUNNER.run = _create_run_wrapper( + agents.run.DEFAULT_AGENT_RUNNER.run ) + # TODO-anton: Also patch streaming runner: agents.Runner.run_streamed def _patch_model(): # type: () -> None - agents.Runner._get_model = classmethod( - _create_get_model_wrapper(agents.Runner._get_model), + agents.run.AgentRunner._get_model = classmethod( + _create_get_model_wrapper(agents.run.AgentRunner._get_model), ) def _patch_tools(): # type: () -> None - agents.Runner._get_all_tools = classmethod( - _create_get_all_tools_wrapper(agents.Runner._get_all_tools), + agents.run.AgentRunner._get_all_tools = classmethod( + _create_get_all_tools_wrapper(agents.run.AgentRunner._get_all_tools), ) diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index 090110a007..34953ad4ac 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -3,7 +3,6 @@ from functools import wraps import sentry_sdk -from sentry_sdk.integrations import DidNotEnable from ..spans import ( agent_workflow_span, @@ -16,11 +15,6 @@ if TYPE_CHECKING: from typing import Any, Callable -try: - import agents -except ImportError: - raise DidNotEnable("OpenAI Agents not installed") - def _create_run_wrapper(original_func): # type: (Callable[..., Any]) -> Callable[..., Any] @@ -29,11 +23,9 @@ def _create_run_wrapper(original_func): """ is_async = asyncio.iscoroutinefunction(original_func) - @wraps( - original_func.__func__ if hasattr(original_func, "__func__") else original_func - ) - async def async_wrapper(cls, *args, **kwargs): - # type: (agents.Runner, *Any, **Any) -> Any + @wraps(original_func) + async def async_wrapper(*args, **kwargs): + # type: (*Any, **Any) -> Any agent = args[0] with agent_workflow_span(agent) as span: kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) @@ -54,11 +46,9 @@ async def async_wrapper(cls, *args, **kwargs): finally: update_agent_workflow_span(span, agent, result) - @wraps( - original_func.__func__ if hasattr(original_func, "__func__") else original_func - ) - def sync_wrapper(cls, *args, **kwargs): - # type: (agents.Runner, *Any, **Any) -> Any + @wraps(original_func) + def sync_wrapper(*args, **kwargs): + # type: (*Any, **Any) -> Any # The sync version (.run_sync()) is just a wrapper around the async version (.run()) return original_func(*args, **kwargs) diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py index b14b9ab5ed..a08b4ca162 100644 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ b/sentry_sdk/integrations/openai_agents/run_hooks.py @@ -17,10 +17,15 @@ class SentryRunHooks(agents.RunHooks): # type: ignore[misc] async def on_agent_start(self, context, agent): # type: (agents.RunContextWrapper, agents.Agent) -> None + print(f"~~~ STARTING AGENT {agent.name}") invoke_agent_span(context, agent) async def on_agent_end(self, context, agent, output): # type: (agents.RunContextWrapper, agents.Agent, Any) -> None + print(f"~~~ STOP AGENT {agent.name}") + # TODO-anton: in my test agent that handsoff to another agent, + # this hook is only called when the second agent finishes but + # never for the first agent. (this used to work that way in 0.0.18) update_invoke_agent_span(context, agent, output) async def on_handoff( From 19c81f22025a3aa21f5520b60ace3824a9a7d8a2 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 23 Jun 2025 10:53:57 +0200 Subject: [PATCH 75/89] Better hooking into agent invokation --- .../integrations/openai_agents/__init__.py | 8 +- .../openai_agents/patches/__init__.py | 1 + .../openai_agents/patches/agent_run.py | 163 ++++++++++++++++++ .../openai_agents/patches/runner.py | 3 +- .../integrations/openai_agents/run_hooks.py | 37 ---- .../integrations/openai_agents/utils.py | 47 ----- 6 files changed, 170 insertions(+), 89 deletions(-) create mode 100644 sentry_sdk/integrations/openai_agents/patches/agent_run.py delete mode 100644 sentry_sdk/integrations/openai_agents/run_hooks.py diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py index 7979995e63..06b6459441 100644 --- a/sentry_sdk/integrations/openai_agents/__init__.py +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -4,6 +4,7 @@ _create_get_model_wrapper, _create_get_all_tools_wrapper, _create_run_wrapper, + _patch_agent_run, ) try: @@ -15,15 +16,16 @@ def _patch_runner(): # type: () -> None - - # Creating agent workflow spans + # Create the root span for one full agent run (including eventual handoffs) # Note agents.run.DEFAULT_AGENT_RUNNER.run_sync is a wrapper around # agents.run.DEFAULT_AGENT_RUNNER.run. It does not need to be wrapped separately. + # TODO-anton: Also patch streaming runner: agents.Runner.run_streamed agents.run.DEFAULT_AGENT_RUNNER.run = _create_run_wrapper( agents.run.DEFAULT_AGENT_RUNNER.run ) - # TODO-anton: Also patch streaming runner: agents.Runner.run_streamed + # Creating the actual spans for each agent run. + _patch_agent_run() def _patch_model(): diff --git a/sentry_sdk/integrations/openai_agents/patches/__init__.py b/sentry_sdk/integrations/openai_agents/patches/__init__.py index 29e0fd6245..06bb1711f8 100644 --- a/sentry_sdk/integrations/openai_agents/patches/__init__.py +++ b/sentry_sdk/integrations/openai_agents/patches/__init__.py @@ -1,3 +1,4 @@ from .models import _create_get_model_wrapper # noqa: F401 from .tools import _create_get_all_tools_wrapper # noqa: F401 from .runner import _create_run_wrapper # noqa: F401 +from .agent_run import _patch_agent_run # noqa: F401 diff --git a/sentry_sdk/integrations/openai_agents/patches/agent_run.py b/sentry_sdk/integrations/openai_agents/patches/agent_run.py new file mode 100644 index 0000000000..cb1bdddbd9 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/patches/agent_run.py @@ -0,0 +1,163 @@ +from functools import wraps + +from sentry_sdk.integrations import DidNotEnable + +from ..spans import invoke_agent_span, update_invoke_agent_span + + +try: + import agents +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + + +def _patch_agent_run(): + """ + Patches AgentRunner methods to create agent invocation spans without using RunHooks. + This directly patches the execution flow to track when agents start and stop. + """ + + # Store original methods + original_run_single_turn = agents.run.AgentRunner._run_single_turn + original_execute_handoffs = agents._run_impl.RunImpl.execute_handoffs + original_execute_final_output = agents._run_impl.RunImpl.execute_final_output + + def _start_invoke_agent_span(context_wrapper, agent): + """Start an agent invocation span""" + # Store the agent on the context wrapper so we can access it later + context_wrapper._sentry_current_agent = agent + invoke_agent_span(context_wrapper, agent) + + def _end_invoke_agent_span(context_wrapper, agent, output): + """End the agent invocation span""" + # Clear the stored agent + if hasattr(context_wrapper, "_sentry_current_agent"): + delattr(context_wrapper, "_sentry_current_agent") + + update_invoke_agent_span(context_wrapper, agent, output) + + def _has_active_agent_span(context_wrapper): + """Check if there's an active agent span for this context""" + return hasattr(context_wrapper, "_sentry_current_agent") + + def _get_current_agent(context_wrapper): + """Get the current agent from context wrapper""" + return getattr(context_wrapper, "_sentry_current_agent", None) + + @wraps(original_run_single_turn) + async def patched_run_single_turn( + cls, + *, + agent, + all_tools, + original_input, + generated_items, + hooks, + context_wrapper, + run_config, + should_run_agent_start_hooks, + tool_use_tracker, + previous_response_id, + ): + """Patched _run_single_turn that creates agent invocation spans""" + + # Start agent span when agent starts (but only once per agent) + if should_run_agent_start_hooks and agent and context_wrapper: + # End any existing span for a different agent + if _has_active_agent_span(context_wrapper): + current_agent = _get_current_agent(context_wrapper) + if current_agent and current_agent != agent: + _end_invoke_agent_span(context_wrapper, current_agent, None) + + _start_invoke_agent_span(context_wrapper, agent) + + # Call original method with all the correct parameters + result = await original_run_single_turn( + agent=agent, + all_tools=all_tools, + original_input=original_input, + generated_items=generated_items, + hooks=hooks, + context_wrapper=context_wrapper, + run_config=run_config, + should_run_agent_start_hooks=should_run_agent_start_hooks, + tool_use_tracker=tool_use_tracker, + previous_response_id=previous_response_id, + ) + + return result + + @wraps(original_execute_handoffs) + async def patched_execute_handoffs( + cls, + *, + agent, + original_input, + pre_step_items, + new_step_items, + new_response, + run_handoffs, + hooks, + context_wrapper, + run_config, + ): + """Patched execute_handoffs that ends agent span for handoffs""" + + # Call original method with all parameters + result = await original_execute_handoffs( + agent=agent, + original_input=original_input, + pre_step_items=pre_step_items, + new_step_items=new_step_items, + new_response=new_response, + run_handoffs=run_handoffs, + hooks=hooks, + context_wrapper=context_wrapper, + run_config=run_config, + ) + + # End span for current agent after handoff processing is complete + if agent and context_wrapper and _has_active_agent_span(context_wrapper): + _end_invoke_agent_span(context_wrapper, agent, None) + + return result + + @wraps(original_execute_final_output) + async def patched_execute_final_output( + cls, + *, + agent, + original_input, + new_response, + pre_step_items, + new_step_items, + final_output, + hooks, + context_wrapper, + ): + """Patched execute_final_output that ends agent span for final outputs""" + + # Call original method with all parameters + result = await original_execute_final_output( + agent=agent, + original_input=original_input, + new_response=new_response, + pre_step_items=pre_step_items, + new_step_items=new_step_items, + final_output=final_output, + hooks=hooks, + context_wrapper=context_wrapper, + ) + + # End span for current agent after final output processing is complete + if agent and context_wrapper and _has_active_agent_span(context_wrapper): + _end_invoke_agent_span(context_wrapper, agent, final_output) + + return result + + # Apply patches + agents.run.AgentRunner._run_single_turn = classmethod(patched_run_single_turn) + agents._run_impl.RunImpl.execute_handoffs = classmethod(patched_execute_handoffs) + agents._run_impl.RunImpl.execute_final_output = classmethod( + patched_execute_final_output + ) diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index 34953ad4ac..de5115202a 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -8,7 +8,7 @@ agent_workflow_span, update_agent_workflow_span, ) -from ..utils import _capture_exception, _wrap_hooks +from ..utils import _capture_exception from typing import TYPE_CHECKING @@ -28,7 +28,6 @@ async def async_wrapper(*args, **kwargs): # type: (*Any, **Any) -> Any agent = args[0] with agent_workflow_span(agent) as span: - kwargs["hooks"] = _wrap_hooks(kwargs.get("hooks")) result = None try: result = await original_func(*args, **kwargs) diff --git a/sentry_sdk/integrations/openai_agents/run_hooks.py b/sentry_sdk/integrations/openai_agents/run_hooks.py deleted file mode 100644 index a08b4ca162..0000000000 --- a/sentry_sdk/integrations/openai_agents/run_hooks.py +++ /dev/null @@ -1,37 +0,0 @@ -from sentry_sdk.integrations import DidNotEnable - -from .spans import handoff_span, invoke_agent_span, update_invoke_agent_span - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from typing import Any - - -try: - import agents -except ImportError: - raise DidNotEnable("OpenAI Agents not installed") - - -class SentryRunHooks(agents.RunHooks): # type: ignore[misc] - async def on_agent_start(self, context, agent): - # type: (agents.RunContextWrapper, agents.Agent) -> None - print(f"~~~ STARTING AGENT {agent.name}") - invoke_agent_span(context, agent) - - async def on_agent_end(self, context, agent, output): - # type: (agents.RunContextWrapper, agents.Agent, Any) -> None - print(f"~~~ STOP AGENT {agent.name}") - # TODO-anton: in my test agent that handsoff to another agent, - # this hook is only called when the second agent finishes but - # never for the first agent. (this used to work that way in 0.0.18) - update_invoke_agent_span(context, agent, output) - - async def on_handoff( - self, - context: agents.RunContextWrapper, - from_agent: agents.Agent, - to_agent: agents.Agent, - ) -> None: - handoff_span(context, from_agent, to_agent) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index ab0b0cc01d..13355089dd 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,4 +1,3 @@ -from functools import wraps import sentry_sdk from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable @@ -156,49 +155,3 @@ def _set_output_data(span, result): if len(output_messages["response"]) > 0: span.set_data(SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"]) - - -def _create_hook_wrapper(original_hook, sentry_hook): - # type: (Callable[..., Any], Callable[..., Any]) -> Callable[..., Any] - @wraps(original_hook) - async def async_wrapper(*args, **kwargs): - # type: (*Any, **Any) -> Any - await sentry_hook(*args, **kwargs) - return await original_hook(*args, **kwargs) - - return async_wrapper - - -def _wrap_hooks(hooks): - # type: (agents.RunHooks) -> agents.RunHooks - """ - Our integration uses RunHooks to create spans. This function will either - enable our SentryRunHooks or if the users has given custom RunHooks wrap - them so the Sentry hooks and the users hooks are both called - """ - from .run_hooks import SentryRunHooks - - sentry_hooks = SentryRunHooks() - - if hooks is None: - return sentry_hooks - - wrapped_hooks = type("SentryWrappedHooks", (hooks.__class__,), {}) - - # Wrap all methods from RunHooks - for method_name in dir(agents.RunHooks): - if method_name.startswith("on_"): - original_method = getattr(hooks, method_name) - # Only wrap if the method exists in SentryRunHooks - try: - sentry_method = getattr(sentry_hooks, method_name) - setattr( - wrapped_hooks, - method_name, - _create_hook_wrapper(original_method, sentry_method), - ) - except AttributeError: - # If method doesn't exist in SentryRunHooks, just use the original method - setattr(wrapped_hooks, method_name, original_method) - - return wrapped_hooks() From 8b1280109610021aca2f842e4ad89687c4c00544 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 23 Jun 2025 12:05:54 +0200 Subject: [PATCH 76/89] Typing --- .../openai_agents/patches/agent_run.py | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/agent_run.py b/sentry_sdk/integrations/openai_agents/patches/agent_run.py index cb1bdddbd9..8608c2280b 100644 --- a/sentry_sdk/integrations/openai_agents/patches/agent_run.py +++ b/sentry_sdk/integrations/openai_agents/patches/agent_run.py @@ -4,6 +4,11 @@ from ..spans import invoke_agent_span, update_invoke_agent_span +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Optional + try: import agents @@ -12,6 +17,7 @@ def _patch_agent_run(): + # type: () -> None """ Patches AgentRunner methods to create agent invocation spans without using RunHooks. This directly patches the execution flow to track when agents start and stop. @@ -23,12 +29,14 @@ def _patch_agent_run(): original_execute_final_output = agents._run_impl.RunImpl.execute_final_output def _start_invoke_agent_span(context_wrapper, agent): + # type: (agents.RunContextWrapper, agents.Agent) -> None """Start an agent invocation span""" # Store the agent on the context wrapper so we can access it later context_wrapper._sentry_current_agent = agent invoke_agent_span(context_wrapper, agent) - def _end_invoke_agent_span(context_wrapper, agent, output): + def _end_invoke_agent_span(context_wrapper, agent, output=None): + # type: (agents.RunContextWrapper, agents.Agent, Optional[Any]) -> None """End the agent invocation span""" # Clear the stored agent if hasattr(context_wrapper, "_sentry_current_agent"): @@ -37,14 +45,20 @@ def _end_invoke_agent_span(context_wrapper, agent, output): update_invoke_agent_span(context_wrapper, agent, output) def _has_active_agent_span(context_wrapper): + # type: (agents.RunContextWrapper) -> bool """Check if there's an active agent span for this context""" return hasattr(context_wrapper, "_sentry_current_agent") def _get_current_agent(context_wrapper): + # type: (agents.RunContextWrapper) -> Optional[agents.Agent] """Get the current agent from context wrapper""" return getattr(context_wrapper, "_sentry_current_agent", None) - @wraps(original_run_single_turn) + @wraps( + original_run_single_turn.__func__ + if hasattr(original_run_single_turn, "__func__") + else original_run_single_turn + ) async def patched_run_single_turn( cls, *, @@ -59,6 +73,7 @@ async def patched_run_single_turn( tool_use_tracker, previous_response_id, ): + # type: (agents.Agent, list[agents.Tool], Any, list[Any], list[Any], agents.RunContextWrapper, agents.RunConfig, bool, Any, Any, Optional[str]) -> Any """Patched _run_single_turn that creates agent invocation spans""" # Start agent span when agent starts (but only once per agent) @@ -67,7 +82,7 @@ async def patched_run_single_turn( if _has_active_agent_span(context_wrapper): current_agent = _get_current_agent(context_wrapper) if current_agent and current_agent != agent: - _end_invoke_agent_span(context_wrapper, current_agent, None) + _end_invoke_agent_span(context_wrapper, current_agent) _start_invoke_agent_span(context_wrapper, agent) @@ -87,7 +102,11 @@ async def patched_run_single_turn( return result - @wraps(original_execute_handoffs) + @wraps( + original_execute_handoffs.__func__ + if hasattr(original_execute_handoffs, "__func__") + else original_execute_handoffs + ) async def patched_execute_handoffs( cls, *, @@ -101,6 +120,7 @@ async def patched_execute_handoffs( context_wrapper, run_config, ): + # type: (Any, agents.Agent, Any, list[Any], list[Any], Any, list[Any], list[Any], agents.RunContextWrapper, agents.RunConfig) -> Any """Patched execute_handoffs that ends agent span for handoffs""" # Call original method with all parameters @@ -118,11 +138,15 @@ async def patched_execute_handoffs( # End span for current agent after handoff processing is complete if agent and context_wrapper and _has_active_agent_span(context_wrapper): - _end_invoke_agent_span(context_wrapper, agent, None) + _end_invoke_agent_span(context_wrapper, agent) return result - @wraps(original_execute_final_output) + @wraps( + original_execute_final_output.__func__ + if hasattr(original_execute_final_output, "__func__") + else original_execute_final_output + ) async def patched_execute_final_output( cls, *, @@ -135,6 +159,7 @@ async def patched_execute_final_output( hooks, context_wrapper, ): + # type: (Any, agents.Agent, Any, Any, list[Any], list[Any], Any, list[Any], agents.RunContextWrapper) -> Any """Patched execute_final_output that ends agent span for final outputs""" # Call original method with all parameters From d7198e2f5bf9332f196b5dffbe3a8c840b814e93 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 23 Jun 2025 12:45:28 +0200 Subject: [PATCH 77/89] Fixed handoff spans --- .../openai_agents/patches/agent_run.py | 10 +- .../openai_agents/spans/handoff.py | 4 +- .../openai_agents/test_openai_agents.py | 223 ++++++++++++++++++ 3 files changed, 233 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/agent_run.py b/sentry_sdk/integrations/openai_agents/patches/agent_run.py index 8608c2280b..7fa217a2dc 100644 --- a/sentry_sdk/integrations/openai_agents/patches/agent_run.py +++ b/sentry_sdk/integrations/openai_agents/patches/agent_run.py @@ -2,7 +2,7 @@ from sentry_sdk.integrations import DidNotEnable -from ..spans import invoke_agent_span, update_invoke_agent_span +from ..spans import invoke_agent_span, update_invoke_agent_span, handoff_span from typing import TYPE_CHECKING @@ -121,7 +121,13 @@ async def patched_execute_handoffs( run_config, ): # type: (Any, agents.Agent, Any, list[Any], list[Any], Any, list[Any], list[Any], agents.RunContextWrapper, agents.RunConfig) -> Any - """Patched execute_handoffs that ends agent span for handoffs""" + """Patched execute_handoffs that creates handoff spans and ends agent span for handoffs""" + + # Create Sentry handoff span for the first handoff (agents library only processes the first one) + if run_handoffs: + first_handoff = run_handoffs[0] + handoff_agent_name = first_handoff.handoff.agent_name + handoff_span(context_wrapper, agent, handoff_agent_name) # Call original method with all parameters result = await original_execute_handoffs( diff --git a/sentry_sdk/integrations/openai_agents/spans/handoff.py b/sentry_sdk/integrations/openai_agents/spans/handoff.py index 7d9af817e7..817b17b2d0 100644 --- a/sentry_sdk/integrations/openai_agents/spans/handoff.py +++ b/sentry_sdk/integrations/openai_agents/spans/handoff.py @@ -9,11 +9,11 @@ import agents -def handoff_span(context, from_agent, to_agent): +def handoff_span(context, from_agent, to_agent_name): # type: (agents.RunContextWrapper, agents.Agent, agents.Agent) -> None with sentry_sdk.start_span( op=OP.GEN_AI_HANDOFF, - name=f"handoff from {from_agent.name} to {to_agent.name}", + name=f"handoff from {from_agent.name} to {to_agent_name}", origin=SPAN_ORIGIN, ) as span: span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "handoff") diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 0f95d9aa66..9d8cc7cea3 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -73,6 +73,78 @@ def test_agent(): ) +@pytest.fixture +def handoff_agents(): + """Create two agents where the first can handoff to the second""" + + secondary_agent = Agent( + name="secondary_agent", + instructions="You are a specialist assistant.", + model="gpt-4o-mini", + model_settings=ModelSettings( + temperature=0.5, + max_tokens=150, + ), + ) + + primary_agent = Agent( + name="primary_agent", + instructions="You are a primary assistant that can handoff to specialists.", + model="gpt-4", + handoffs=[secondary_agent], + model_settings=ModelSettings( + temperature=0.7, + max_tokens=100, + ), + ) + + return primary_agent, secondary_agent + + +@pytest.fixture +def mock_handoff_responses(mock_usage): + """Mock responses that simulate a handoff scenario""" + + # First response: primary agent calls handoff tool + handoff_response = ModelResponse( + output=[ + ResponseFunctionToolCall( + id="call_handoff_123", + call_id="call_handoff_123", + name="transfer_to_secondary_agent", + type="function_call", + arguments="{}", + function=MagicMock(name="transfer_to_secondary_agent", arguments="{}"), + ) + ], + usage=mock_usage, + response_id="resp_handoff_123", + ) + + # Second response: secondary agent produces final output + final_response = ModelResponse( + output=[ + ResponseOutputMessage( + id="msg_final", + type="message", + status="completed", + content=[ + ResponseOutputText( + text="I'm the specialist and I can help with that!", + type="output_text", + annotations=[], + ) + ], + role="assistant", + ) + ], + usage=mock_usage, + response_id="resp_final_123", + ) + + return [handoff_response, final_response] + + @pytest.mark.asyncio async def test_agent_invocation_span( sentry_init, capture_events, test_agent, mock_model_response @@ -180,6 +252,157 @@ def test_agent_invocation_span_sync( assert ai_client_span["data"]["gen_ai.request.top_p"] == 1.0 +@pytest.mark.asyncio +async def test_handoff_span( + sentry_init, capture_events, handoff_agents, mock_handoff_responses +): + """ + Test that the integration creates spans for agent handoffs. + """ + primary_agent, secondary_agent = handoff_agents + + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): + with patch( + "agents.models.openai_responses.OpenAIResponsesModel.get_response" + ) as mock_get_response: + + # First, let's see what handoff tools are actually available + # by mocking get_response to capture the tools + def debug_get_response(*args, **kwargs): + tools = kwargs.get("tools", []) + handoffs = kwargs.get("handoffs", []) + print( + f"Tools available: {[tool.name if hasattr(tool, 'name') else str(tool) for tool in tools]}" + ) + print( + f"Handoffs available: {[h.tool_name if hasattr(h, 'tool_name') else str(h) for h in handoffs]}" + ) + + # Find the correct handoff tool name + handoff_tool_name = None + for handoff in handoffs: + if ( + hasattr(handoff, "tool_name") + and "secondary_agent" in handoff.tool_name + ): + handoff_tool_name = handoff.tool_name + break + + if not handoff_tool_name: + # Fallback - look for any handoff tool + for handoff in handoffs: + if hasattr(handoff, "tool_name"): + handoff_tool_name = handoff.tool_name + break + + print(f"Using handoff tool name: {handoff_tool_name}") + + # Create corrected handoff response with the right tool name + corrected_handoff_response = ModelResponse( + output=[ + ResponseFunctionToolCall( + id="call_handoff_123", + call_id="call_handoff_123", + name=handoff_tool_name, # Use the correct tool name + type="function_call", + arguments="{}", + function=MagicMock(name=handoff_tool_name, arguments="{}"), + ) + ], + usage=Usage( + requests=1, + input_tokens=10, + output_tokens=20, + total_tokens=30, + input_tokens_details=MagicMock(cached_tokens=0), + output_tokens_details=MagicMock(reasoning_tokens=5), + ), + response_id="resp_handoff_123", + ) + + # Return the corrected response for the first call + debug_get_response.call_count += 1 + if debug_get_response.call_count == 1: + return corrected_handoff_response + else: + # Second call - return final response from secondary agent + return mock_handoff_responses[1] + + debug_get_response.call_count = 0 + mock_get_response.side_effect = debug_get_response + + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + ) + + events = capture_events() + + result = await agents.Runner.run( + primary_agent, + "Please help me with this specialist task", + run_config=test_run_config, + ) + + assert result is not None + assert result.final_output == "I'm the specialist and I can help with that!" + + (transaction,) = events + spans = transaction["spans"] + + # Remove the debugger import + # import ipdb; ipdb.set_trace() + print(f"Number of spans: {len(spans)}") + for i, span in enumerate(spans): + print(f"Span {i}: {span['op']} - {span['description']}") + + # Should have: primary invoke, primary ai_client, handoff, secondary invoke, secondary ai_client + assert len(spans) >= 3 + + # Find the spans more flexibly + handoff_span = None + invoke_agent_spans = [] + ai_client_spans = [] + + for span in spans: + if span["op"] == "gen_ai.handoff": + handoff_span = span + elif span["op"] == "gen_ai.invoke_agent": + invoke_agent_spans.append(span) + elif span["op"] == "gen_ai.chat": + ai_client_spans.append(span) + + # Verify transaction + assert transaction["transaction"] == "primary_agent workflow" + assert transaction["contexts"]["trace"]["origin"] == "auto.ai.openai_agents" + + # Verify handoff span exists and has correct properties + assert handoff_span is not None + assert handoff_span["description"] == "handoff primary_agent -> secondary_agent" + assert handoff_span["data"]["gen_ai.operation.name"] == "handoff" + assert handoff_span["data"]["gen_ai.system"] == "openai" + assert handoff_span["data"]["gen_ai.agent.from"] == "primary_agent" + assert handoff_span["data"]["gen_ai.agent.to"] == "secondary_agent" + + # Verify both agent invoke spans exist + assert len(invoke_agent_spans) == 2 + + primary_invoke_span = next( + (s for s in invoke_agent_spans if "primary_agent" in s["description"]), None + ) + secondary_invoke_span = next( + (s for s in invoke_agent_spans if "secondary_agent" in s["description"]), None + ) + + assert primary_invoke_span is not None + assert primary_invoke_span["description"] == "invoke_agent primary_agent" + assert primary_invoke_span["data"]["gen_ai.agent.name"] == "primary_agent" + + assert secondary_invoke_span is not None + assert secondary_invoke_span["description"] == "invoke_agent secondary_agent" + assert secondary_invoke_span["data"]["gen_ai.agent.name"] == "secondary_agent" + + @pytest.mark.asyncio async def test_tool_execution_span(sentry_init, capture_events, test_agent): """ From 3f48a1ccc7b1115d6c706a91c97b20877a8a866e Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 23 Jun 2025 12:57:20 +0200 Subject: [PATCH 78/89] tests for handoff span --- .../openai_agents/spans/handoff.py | 2 +- .../openai_agents/test_openai_agents.py | 251 +++++------------- 2 files changed, 61 insertions(+), 192 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/spans/handoff.py b/sentry_sdk/integrations/openai_agents/spans/handoff.py index 817b17b2d0..78e6788c7d 100644 --- a/sentry_sdk/integrations/openai_agents/spans/handoff.py +++ b/sentry_sdk/integrations/openai_agents/spans/handoff.py @@ -10,7 +10,7 @@ def handoff_span(context, from_agent, to_agent_name): - # type: (agents.RunContextWrapper, agents.Agent, agents.Agent) -> None + # type: (agents.RunContextWrapper, agents.Agent, str) -> None with sentry_sdk.start_span( op=OP.GEN_AI_HANDOFF, name=f"handoff from {from_agent.name} to {to_agent_name}", diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 9d8cc7cea3..1aac912b67 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -73,78 +73,6 @@ def test_agent(): ) -@pytest.fixture -def handoff_agents(): - """Create two agents where the first can handoff to the second""" - - secondary_agent = Agent( - name="secondary_agent", - instructions="You are a specialist assistant.", - model="gpt-4o-mini", - model_settings=ModelSettings( - temperature=0.5, - max_tokens=150, - ), - ) - - primary_agent = Agent( - name="primary_agent", - instructions="You are a primary assistant that can handoff to specialists.", - model="gpt-4", - handoffs=[secondary_agent], - model_settings=ModelSettings( - temperature=0.7, - max_tokens=100, - ), - ) - - return primary_agent, secondary_agent - - -@pytest.fixture -def mock_handoff_responses(mock_usage): - """Mock responses that simulate a handoff scenario""" - - # First response: primary agent calls handoff tool - handoff_response = ModelResponse( - output=[ - ResponseFunctionToolCall( - id="call_handoff_123", - call_id="call_handoff_123", - name="transfer_to_secondary_agent", - type="function_call", - arguments="{}", - function=MagicMock(name="transfer_to_secondary_agent", arguments="{}"), - ) - ], - usage=mock_usage, - response_id="resp_handoff_123", - ) - - # Second response: secondary agent produces final output - final_response = ModelResponse( - output=[ - ResponseOutputMessage( - id="msg_final", - type="message", - status="completed", - content=[ - ResponseOutputText( - text="I'm the specialist and I can help with that!", - type="output_text", - annotations=[], - ) - ], - role="assistant", - ) - ], - usage=mock_usage, - response_id="resp_final_123", - ) - - return [handoff_response, final_response] - - @pytest.mark.asyncio async def test_agent_invocation_span( sentry_init, capture_events, test_agent, mock_model_response @@ -253,83 +181,69 @@ def test_agent_invocation_span_sync( @pytest.mark.asyncio -async def test_handoff_span( - sentry_init, capture_events, handoff_agents, mock_handoff_responses -): +async def test_handoff_span(sentry_init, capture_events, mock_usage): """ - Test that the integration creates spans for agent handoffs. + Test that handoff spans are created when agents hand off to other agents. """ - primary_agent, secondary_agent = handoff_agents + # Create two simple agents with a handoff relationship + secondary_agent = agents.Agent( + name="secondary_agent", + instructions="You are a secondary agent.", + model="gpt-4o-mini", + ) + + primary_agent = agents.Agent( + name="primary_agent", + instructions="You are a primary agent that hands off to secondary agent.", + model="gpt-4o-mini", + handoffs=[secondary_agent], + ) with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): with patch( "agents.models.openai_responses.OpenAIResponsesModel.get_response" ) as mock_get_response: + # Mock two responses: + # 1. Primary agent calls handoff tool + # 2. Secondary agent provides final response + handoff_response = ModelResponse( + output=[ + ResponseFunctionToolCall( + id="call_handoff_123", + call_id="call_handoff_123", + name="transfer_to_secondary_agent", + type="function_call", + arguments="{}", + function=MagicMock( + name="transfer_to_secondary_agent", arguments="{}" + ), + ) + ], + usage=mock_usage, + response_id="resp_handoff_123", + ) - # First, let's see what handoff tools are actually available - # by mocking get_response to capture the tools - def debug_get_response(*args, **kwargs): - tools = kwargs.get("tools", []) - handoffs = kwargs.get("handoffs", []) - print( - f"Tools available: {[tool.name if hasattr(tool, 'name') else str(tool) for tool in tools]}" - ) - print( - f"Handoffs available: {[h.tool_name if hasattr(h, 'tool_name') else str(h) for h in handoffs]}" - ) - - # Find the correct handoff tool name - handoff_tool_name = None - for handoff in handoffs: - if ( - hasattr(handoff, "tool_name") - and "secondary_agent" in handoff.tool_name - ): - handoff_tool_name = handoff.tool_name - break - - if not handoff_tool_name: - # Fallback - look for any handoff tool - for handoff in handoffs: - if hasattr(handoff, "tool_name"): - handoff_tool_name = handoff.tool_name - break - - print(f"Using handoff tool name: {handoff_tool_name}") - - # Create corrected handoff response with the right tool name - corrected_handoff_response = ModelResponse( - output=[ - ResponseFunctionToolCall( - id="call_handoff_123", - call_id="call_handoff_123", - name=handoff_tool_name, # Use the correct tool name - type="function_call", - arguments="{}", - function=MagicMock(name=handoff_tool_name, arguments="{}"), - ) - ], - usage=Usage( - requests=1, - input_tokens=10, - output_tokens=20, - total_tokens=30, - input_tokens_details=MagicMock(cached_tokens=0), - output_tokens_details=MagicMock(reasoning_tokens=5), - ), - response_id="resp_handoff_123", - ) - - # Return the corrected response for the first call - debug_get_response.call_count += 1 - if debug_get_response.call_count == 1: - return corrected_handoff_response - else: - # Second call - return final response from secondary agent - return mock_handoff_responses[1] + final_response = ModelResponse( + output=[ + ResponseOutputMessage( + id="msg_final", + type="message", + status="completed", + content=[ + ResponseOutputText( + text="I'm the specialist and I can help with that!", + type="output_text", + annotations=[], + ) + ], + role="assistant", + ) + ], + usage=mock_usage, + response_id="resp_final_123", + ) - debug_get_response.call_count = 0 - mock_get_response.side_effect = debug_get_response + mock_get_response.side_effect = [handoff_response, final_response] sentry_init( integrations=[OpenAIAgentsIntegration()], @@ -340,67 +254,22 @@ def debug_get_response(*args, **kwargs): result = await agents.Runner.run( primary_agent, - "Please help me with this specialist task", + "Please hand off to secondary agent", run_config=test_run_config, ) assert result is not None - assert result.final_output == "I'm the specialist and I can help with that!" (transaction,) = events spans = transaction["spans"] + handoff_span = spans[2] - # Remove the debugger import - # import ipdb; ipdb.set_trace() - print(f"Number of spans: {len(spans)}") - for i, span in enumerate(spans): - print(f"Span {i}: {span['op']} - {span['description']}") - - # Should have: primary invoke, primary ai_client, handoff, secondary invoke, secondary ai_client - assert len(spans) >= 3 - - # Find the spans more flexibly - handoff_span = None - invoke_agent_spans = [] - ai_client_spans = [] - - for span in spans: - if span["op"] == "gen_ai.handoff": - handoff_span = span - elif span["op"] == "gen_ai.invoke_agent": - invoke_agent_spans.append(span) - elif span["op"] == "gen_ai.chat": - ai_client_spans.append(span) - - # Verify transaction - assert transaction["transaction"] == "primary_agent workflow" - assert transaction["contexts"]["trace"]["origin"] == "auto.ai.openai_agents" - - # Verify handoff span exists and has correct properties + # Verify handoff span was created assert handoff_span is not None - assert handoff_span["description"] == "handoff primary_agent -> secondary_agent" - assert handoff_span["data"]["gen_ai.operation.name"] == "handoff" - assert handoff_span["data"]["gen_ai.system"] == "openai" - assert handoff_span["data"]["gen_ai.agent.from"] == "primary_agent" - assert handoff_span["data"]["gen_ai.agent.to"] == "secondary_agent" - - # Verify both agent invoke spans exist - assert len(invoke_agent_spans) == 2 - - primary_invoke_span = next( - (s for s in invoke_agent_spans if "primary_agent" in s["description"]), None - ) - secondary_invoke_span = next( - (s for s in invoke_agent_spans if "secondary_agent" in s["description"]), None + assert ( + handoff_span["description"] == "handoff from primary_agent to secondary_agent" ) - - assert primary_invoke_span is not None - assert primary_invoke_span["description"] == "invoke_agent primary_agent" - assert primary_invoke_span["data"]["gen_ai.agent.name"] == "primary_agent" - - assert secondary_invoke_span is not None - assert secondary_invoke_span["description"] == "invoke_agent secondary_agent" - assert secondary_invoke_span["data"]["gen_ai.agent.name"] == "secondary_agent" + assert handoff_span["data"]["gen_ai.operation.name"] == "handoff" @pytest.mark.asyncio From cebe0f441c4f6dbbc3b2d6ecee1cf0de223c0ccb Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Mon, 23 Jun 2025 13:55:00 +0200 Subject: [PATCH 79/89] cleanup --- sentry_sdk/integrations/openai_agents/patches/agent_run.py | 2 +- sentry_sdk/integrations/openai_agents/patches/runner.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/agent_run.py b/sentry_sdk/integrations/openai_agents/patches/agent_run.py index 7fa217a2dc..aa21ef48a6 100644 --- a/sentry_sdk/integrations/openai_agents/patches/agent_run.py +++ b/sentry_sdk/integrations/openai_agents/patches/agent_run.py @@ -19,7 +19,7 @@ def _patch_agent_run(): # type: () -> None """ - Patches AgentRunner methods to create agent invocation spans without using RunHooks. + Patches AgentRunner methods to create agent invocation spans. This directly patches the execution flow to track when agents start and stop. """ diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index de5115202a..c39cc8d0f7 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -36,7 +36,6 @@ async def async_wrapper(*args, **kwargs): _capture_exception(exc) # It could be that there is a "invoke agent" span still open - # (because its create in run_hooks without the context manager) current_span = sentry_sdk.get_current_span() if current_span is not None and current_span.timestamp is None: current_span.__exit__(None, None, None) From efe9e39f1e5bba45e4db46c9867a1cb48baa29c2 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 24 Jun 2025 08:37:33 +0200 Subject: [PATCH 80/89] Apply suggestions from code review Co-authored-by: Ivana Kellyer --- sentry_sdk/integrations/openai_agents/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 13355089dd..a77071dd26 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -31,10 +31,10 @@ def _capture_exception(exc): def _get_start_span_function(): # type: () -> Callable[..., Any] current_span = sentry_sdk.get_current_span() - is_transaction = ( + transaction_exists = ( current_span is not None and current_span.containing_transaction == current_span ) - return sentry_sdk.start_span if is_transaction else sentry_sdk.start_transaction + return sentry_sdk.start_span if transaction_exists else sentry_sdk.start_transaction def _set_agent_data(span, agent): From ff760d1665e00dc2073ee3fdabf4681a468f6a1b Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 24 Jun 2025 08:45:44 +0200 Subject: [PATCH 81/89] more resilient monkey patching --- .../openai_agents/patches/agent_run.py | 96 ++++--------------- 1 file changed, 21 insertions(+), 75 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/agent_run.py b/sentry_sdk/integrations/openai_agents/patches/agent_run.py index aa21ef48a6..4da335b984 100644 --- a/sentry_sdk/integrations/openai_agents/patches/agent_run.py +++ b/sentry_sdk/integrations/openai_agents/patches/agent_run.py @@ -59,23 +59,14 @@ def _get_current_agent(context_wrapper): if hasattr(original_run_single_turn, "__func__") else original_run_single_turn ) - async def patched_run_single_turn( - cls, - *, - agent, - all_tools, - original_input, - generated_items, - hooks, - context_wrapper, - run_config, - should_run_agent_start_hooks, - tool_use_tracker, - previous_response_id, - ): - # type: (agents.Agent, list[agents.Tool], Any, list[Any], list[Any], agents.RunContextWrapper, agents.RunConfig, bool, Any, Any, Optional[str]) -> Any + async def patched_run_single_turn(cls, *args, **kwargs): + # type: (agents.Runner, *Any, **Any) -> Any """Patched _run_single_turn that creates agent invocation spans""" + agent = kwargs.get("agent") + context_wrapper = kwargs.get("context_wrapper") + should_run_agent_start_hooks = kwargs.get("should_run_agent_start_hooks") + # Start agent span when agent starts (but only once per agent) if should_run_agent_start_hooks and agent and context_wrapper: # End any existing span for a different agent @@ -87,18 +78,7 @@ async def patched_run_single_turn( _start_invoke_agent_span(context_wrapper, agent) # Call original method with all the correct parameters - result = await original_run_single_turn( - agent=agent, - all_tools=all_tools, - original_input=original_input, - generated_items=generated_items, - hooks=hooks, - context_wrapper=context_wrapper, - run_config=run_config, - should_run_agent_start_hooks=should_run_agent_start_hooks, - tool_use_tracker=tool_use_tracker, - previous_response_id=previous_response_id, - ) + result = await original_run_single_turn(*args, **kwargs) return result @@ -107,22 +87,14 @@ async def patched_run_single_turn( if hasattr(original_execute_handoffs, "__func__") else original_execute_handoffs ) - async def patched_execute_handoffs( - cls, - *, - agent, - original_input, - pre_step_items, - new_step_items, - new_response, - run_handoffs, - hooks, - context_wrapper, - run_config, - ): - # type: (Any, agents.Agent, Any, list[Any], list[Any], Any, list[Any], list[Any], agents.RunContextWrapper, agents.RunConfig) -> Any + async def patched_execute_handoffs(cls, *args, **kwargs): + # type: (agents.Runner, *Any, **Any) -> Any """Patched execute_handoffs that creates handoff spans and ends agent span for handoffs""" + context_wrapper = kwargs.get("context_wrapper") + run_handoffs = kwargs.get("run_handoffs") + agent = kwargs.get("agent") + # Create Sentry handoff span for the first handoff (agents library only processes the first one) if run_handoffs: first_handoff = run_handoffs[0] @@ -130,17 +102,7 @@ async def patched_execute_handoffs( handoff_span(context_wrapper, agent, handoff_agent_name) # Call original method with all parameters - result = await original_execute_handoffs( - agent=agent, - original_input=original_input, - pre_step_items=pre_step_items, - new_step_items=new_step_items, - new_response=new_response, - run_handoffs=run_handoffs, - hooks=hooks, - context_wrapper=context_wrapper, - run_config=run_config, - ) + result = await original_execute_handoffs(*args, **kwargs) # End span for current agent after handoff processing is complete if agent and context_wrapper and _has_active_agent_span(context_wrapper): @@ -153,32 +115,16 @@ async def patched_execute_handoffs( if hasattr(original_execute_final_output, "__func__") else original_execute_final_output ) - async def patched_execute_final_output( - cls, - *, - agent, - original_input, - new_response, - pre_step_items, - new_step_items, - final_output, - hooks, - context_wrapper, - ): - # type: (Any, agents.Agent, Any, Any, list[Any], list[Any], Any, list[Any], agents.RunContextWrapper) -> Any + async def patched_execute_final_output(cls, *args, **kwargs): + # type: (agents.Runner, *Any, **Any) -> Any """Patched execute_final_output that ends agent span for final outputs""" + agent = kwargs.get("agent") + context_wrapper = kwargs.get("context_wrapper") + final_output = kwargs.get("final_output") + # Call original method with all parameters - result = await original_execute_final_output( - agent=agent, - original_input=original_input, - new_response=new_response, - pre_step_items=pre_step_items, - new_step_items=new_step_items, - final_output=final_output, - hooks=hooks, - context_wrapper=context_wrapper, - ) + result = await original_execute_final_output(*args, **kwargs) # End span for current agent after final output processing is complete if agent and context_wrapper and _has_active_agent_span(context_wrapper): From 087bbebd73fc1c62f5b3f4d27dfe9403809c9597 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 24 Jun 2025 08:53:52 +0200 Subject: [PATCH 82/89] removed unused function --- .../integrations/openai_agents/patches/runner.py | 9 ++------- .../integrations/openai_agents/spans/__init__.py | 5 +---- .../integrations/openai_agents/spans/agent_workflow.py | 10 ++-------- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index c39cc8d0f7..d2cb26e217 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -4,10 +4,7 @@ import sentry_sdk -from ..spans import ( - agent_workflow_span, - update_agent_workflow_span, -) +from ..spans import agent_workflow_span from ..utils import _capture_exception from typing import TYPE_CHECKING @@ -27,7 +24,7 @@ def _create_run_wrapper(original_func): async def async_wrapper(*args, **kwargs): # type: (*Any, **Any) -> Any agent = args[0] - with agent_workflow_span(agent) as span: + with agent_workflow_span(agent): result = None try: result = await original_func(*args, **kwargs) @@ -41,8 +38,6 @@ async def async_wrapper(*args, **kwargs): current_span.__exit__(None, None, None) raise exc from None - finally: - update_agent_workflow_span(span, agent, result) @wraps(original_func) def sync_wrapper(*args, **kwargs): diff --git a/sentry_sdk/integrations/openai_agents/spans/__init__.py b/sentry_sdk/integrations/openai_agents/spans/__init__.py index 81e8145cab..3bc453cafa 100644 --- a/sentry_sdk/integrations/openai_agents/spans/__init__.py +++ b/sentry_sdk/integrations/openai_agents/spans/__init__.py @@ -1,7 +1,4 @@ -from .agent_workflow import ( # noqa: F401 - agent_workflow_span, - update_agent_workflow_span, -) +from .agent_workflow import agent_workflow_span # noqa: F401 from .ai_client import ai_client_span, update_ai_client_span # noqa: F401 from .execute_tool import execute_tool_span, update_execute_tool_span # noqa: F401 from .handoff import handoff_span # noqa: F401 diff --git a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py index 05e1a75a1c..de2f28d41e 100644 --- a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py +++ b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py @@ -6,12 +6,11 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from agents import Agent - from typing import Any + import agents def agent_workflow_span(agent): - # type: (Agent) -> sentry_sdk.tracing.Span + # type: (agents.Agent) -> sentry_sdk.tracing.Span # Create a transaction or a span if an transaction is already active span = _get_start_span_function()( @@ -20,8 +19,3 @@ def agent_workflow_span(agent): ) return span - - -def update_agent_workflow_span(span, agent, result): - # type: (sentry_sdk.tracing.Span, Agent, Any) -> None - pass From b55c1ce4d812541c079074e51aa0de173d551945 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 24 Jun 2025 09:02:17 +0200 Subject: [PATCH 83/89] removed unused code --- .../integrations/openai_agents/patches/runner.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index d2cb26e217..e6844685cf 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -1,5 +1,3 @@ -import asyncio - from functools import wraps import sentry_sdk @@ -16,12 +14,11 @@ def _create_run_wrapper(original_func): # type: (Callable[..., Any]) -> Callable[..., Any] """ - Wraps the agents.Runner.run* methods to create a root span for the agent workflow runs. + Wraps the agents.Runner.run methods to create a root span for the agent workflow runs. """ - is_async = asyncio.iscoroutinefunction(original_func) @wraps(original_func) - async def async_wrapper(*args, **kwargs): + async def wrapper(*args, **kwargs): # type: (*Any, **Any) -> Any agent = args[0] with agent_workflow_span(agent): @@ -39,10 +36,4 @@ async def async_wrapper(*args, **kwargs): raise exc from None - @wraps(original_func) - def sync_wrapper(*args, **kwargs): - # type: (*Any, **Any) -> Any - # The sync version (.run_sync()) is just a wrapper around the async version (.run()) - return original_func(*args, **kwargs) - - return async_wrapper if is_async else sync_wrapper + return wrapper From 87adffdb3df47403a1176fb508782d53f0b55af8 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 24 Jun 2025 09:03:45 +0200 Subject: [PATCH 84/89] comment --- sentry_sdk/integrations/openai_agents/patches/runner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index e6844685cf..e1e9a3b50c 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -15,6 +15,9 @@ def _create_run_wrapper(original_func): # type: (Callable[..., Any]) -> Callable[..., Any] """ Wraps the agents.Runner.run methods to create a root span for the agent workflow runs. + + Note agents.Runner.run_sync() is a wrapper around agents.Runner.run(), + so it does not need to be wrapped separately. """ @wraps(original_func) From c6e82303790cf3d86c43169672954621d30af520 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 24 Jun 2025 09:08:07 +0200 Subject: [PATCH 85/89] made finishing spans more resilient --- .../openai_agents/patches/agent_run.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/agent_run.py b/sentry_sdk/integrations/openai_agents/patches/agent_run.py index 4da335b984..c09eea17ce 100644 --- a/sentry_sdk/integrations/openai_agents/patches/agent_run.py +++ b/sentry_sdk/integrations/openai_agents/patches/agent_run.py @@ -102,11 +102,13 @@ async def patched_execute_handoffs(cls, *args, **kwargs): handoff_span(context_wrapper, agent, handoff_agent_name) # Call original method with all parameters - result = await original_execute_handoffs(*args, **kwargs) + try: + result = await original_execute_handoffs(*args, **kwargs) - # End span for current agent after handoff processing is complete - if agent and context_wrapper and _has_active_agent_span(context_wrapper): - _end_invoke_agent_span(context_wrapper, agent) + finally: + # End span for current agent after handoff processing is complete + if agent and context_wrapper and _has_active_agent_span(context_wrapper): + _end_invoke_agent_span(context_wrapper, agent) return result @@ -124,11 +126,12 @@ async def patched_execute_final_output(cls, *args, **kwargs): final_output = kwargs.get("final_output") # Call original method with all parameters - result = await original_execute_final_output(*args, **kwargs) - - # End span for current agent after final output processing is complete - if agent and context_wrapper and _has_active_agent_span(context_wrapper): - _end_invoke_agent_span(context_wrapper, agent, final_output) + try: + result = await original_execute_final_output(*args, **kwargs) + finally: + # End span for current agent after final output processing is complete + if agent and context_wrapper and _has_active_agent_span(context_wrapper): + _end_invoke_agent_span(context_wrapper, agent, final_output) return result From f054a8f4be547602f946f445e602b58a87c1f5a6 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 24 Jun 2025 09:10:08 +0200 Subject: [PATCH 86/89] better check --- sentry_sdk/integrations/openai_agents/patches/agent_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/openai_agents/patches/agent_run.py b/sentry_sdk/integrations/openai_agents/patches/agent_run.py index c09eea17ce..084100878c 100644 --- a/sentry_sdk/integrations/openai_agents/patches/agent_run.py +++ b/sentry_sdk/integrations/openai_agents/patches/agent_run.py @@ -47,7 +47,7 @@ def _end_invoke_agent_span(context_wrapper, agent, output=None): def _has_active_agent_span(context_wrapper): # type: (agents.RunContextWrapper) -> bool """Check if there's an active agent span for this context""" - return hasattr(context_wrapper, "_sentry_current_agent") + return getattr(context_wrapper, "_sentry_current_agent", None) is not None def _get_current_agent(context_wrapper): # type: (agents.RunContextWrapper) -> Optional[agents.Agent] From b4f164e5c52c44418993b07191bb383390e538f2 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 24 Jun 2025 12:00:14 +0200 Subject: [PATCH 87/89] Make sure to never send objects, but rather strings --- .../integrations/openai_agents/utils.py | 59 +++++- .../openai_agents/test_openai_agents.py | 191 +++++++++++------- 2 files changed, 169 insertions(+), 81 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index a77071dd26..1618fede1d 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,3 +1,4 @@ +import json import sentry_sdk from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable @@ -76,7 +77,7 @@ def _set_agent_data(span, agent): if len(agent.tools) > 0: span.set_data( SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, - [vars(tool) for tool in agent.tools], + safe_serialize([vars(tool) for tool in agent.tools]), ) @@ -126,7 +127,7 @@ def _set_input_data(span, get_response_kwargs): if len(messages) > 0: request_messages.append({"role": role, "content": messages}) - span.set_data(SPANDATA.GEN_AI_REQUEST_MESSAGES, request_messages) + span.set_data(SPANDATA.GEN_AI_REQUEST_MESSAGES, safe_serialize(request_messages)) def _set_output_data(span, result): @@ -148,10 +149,58 @@ def _set_output_data(span, result): output_messages["response"].append(output_message.text) except AttributeError: # Unknown output message type, just return the json - output_messages["response"].append(output_message.to_json()) + output_messages["response"].append(output_message.dict()) if len(output_messages["tool"]) > 0: - span.set_data(SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, output_messages["tool"]) + span.set_data( + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, safe_serialize(output_messages["tool"]) + ) if len(output_messages["response"]) > 0: - span.set_data(SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"]) + span.set_data( + SPANDATA.GEN_AI_RESPONSE_TEXT, safe_serialize(output_messages["response"]) + ) + + +def safe_serialize(data): + """Safely serialize to a readable string.""" + + def serialize_item(item): + if callable(item): + try: + module = getattr(item, "__module__", None) + qualname = getattr(item, "__qualname__", None) + name = getattr(item, "__name__", "anonymous") + + if module and qualname: + full_path = f"{module}.{qualname}" + elif module and name: + full_path = f"{module}.{name}" + else: + full_path = name + + return f"" + except Exception: + return f"" + elif isinstance(item, dict): + return {k: serialize_item(v) for k, v in item.items()} + elif isinstance(item, (list, tuple)): + return [serialize_item(x) for x in item] + elif hasattr(item, "__dict__"): + try: + attrs = { + k: serialize_item(v) + for k, v in vars(item).items() + if not k.startswith("_") + } + return f"<{type(item).__name__} {attrs}>" + except Exception: + return repr(item) + else: + return item + + try: + serialized = serialize_item(data) + return json.dumps(serialized, default=str) + except Exception: + return str(data) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 1aac912b67..ec606c8806 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -1,9 +1,10 @@ -from unittest import mock +import re import pytest from unittest.mock import MagicMock, patch import os from sentry_sdk.integrations.openai_agents import OpenAIAgentsIntegration +from sentry_sdk.integrations.openai_agents.utils import safe_serialize import agents from agents import ( @@ -360,22 +361,24 @@ def simple_test_tool(message: str) -> str: ai_client_span2, ) = spans - available_tools = [ - { - "name": "simple_test_tool", - "description": "A simple tool", - "params_json_schema": { - "properties": {"message": {"title": "Message", "type": "string"}}, - "required": ["message"], - "title": "simple_test_tool_args", - "type": "object", - "additionalProperties": False, - }, - "on_invoke_tool": mock.ANY, - "strict_json_schema": True, - "is_enabled": True, - } - ] + available_tools = safe_serialize( + [ + { + "name": "simple_test_tool", + "description": "A simple tool", + "params_json_schema": { + "properties": {"message": {"title": "Message", "type": "string"}}, + "required": ["message"], + "title": "simple_test_tool_args", + "type": "object", + "additionalProperties": False, + }, + "on_invoke_tool": "._create_function_tool.._on_invoke_tool>", + "strict_json_schema": True, + "is_enabled": True, + } + ] + ) assert transaction["transaction"] == "test_agent workflow" assert transaction["contexts"]["trace"]["origin"] == "auto.ai.openai_agents" @@ -397,16 +400,22 @@ def simple_test_tool(message: str) -> str: assert ai_client_span1["data"]["gen_ai.agent.name"] == "test_agent" assert ai_client_span1["data"]["gen_ai.request.available_tools"] == available_tools assert ai_client_span1["data"]["gen_ai.request.max_tokens"] == 100 - assert ai_client_span1["data"]["gen_ai.request.messages"] == [ - { - "role": "system", - "content": [{"type": "text", "text": "You are a helpful test assistant."}], - }, - { - "role": "user", - "content": [{"type": "text", "text": "Please use the simple test tool"}], - }, - ] + assert ai_client_span1["data"]["gen_ai.request.messages"] == safe_serialize( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful test assistant."} + ], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Please use the simple test tool"} + ], + }, + ] + ) assert ai_client_span1["data"]["gen_ai.request.model"] == "gpt-4" assert ai_client_span1["data"]["gen_ai.request.temperature"] == 0.7 assert ai_client_span1["data"]["gen_ai.request.top_p"] == 1.0 @@ -415,22 +424,35 @@ def simple_test_tool(message: str) -> str: assert ai_client_span1["data"]["gen_ai.usage.output_tokens"] == 5 assert ai_client_span1["data"]["gen_ai.usage.output_tokens.reasoning"] == 0 assert ai_client_span1["data"]["gen_ai.usage.total_tokens"] == 15 - assert ai_client_span1["data"]["gen_ai.response.tool_calls"] == [ - { - "arguments": '{"message": "hello"}', - "call_id": "call_123", - "name": "simple_test_tool", - "type": "function_call", - "id": "call_123", - "status": None, - "function": mock.ANY, - } - ] + assert re.sub( + r"SerializationIterator\(.*\)", + "NOT_CHECKED", + ai_client_span1["data"]["gen_ai.response.tool_calls"], + ) == safe_serialize( + [ + { + "arguments": '{"message": "hello"}', + "call_id": "call_123", + "name": "simple_test_tool", + "type": "function_call", + "id": "call_123", + "status": None, + "function": "NOT_CHECKED", + } + ] + ) assert tool_span["description"] == "execute_tool simple_test_tool" assert tool_span["data"]["gen_ai.agent.name"] == "test_agent" assert tool_span["data"]["gen_ai.operation.name"] == "execute_tool" - assert tool_span["data"]["gen_ai.request.available_tools"] == available_tools + assert ( + re.sub( + "<.*>(,)", + r"'NOT_CHECKED'\1", + agent_span["data"]["gen_ai.request.available_tools"], + ) + == available_tools + ) assert tool_span["data"]["gen_ai.request.max_tokens"] == 100 assert tool_span["data"]["gen_ai.request.model"] == "gpt-4" assert tool_span["data"]["gen_ai.request.temperature"] == 0.7 @@ -445,47 +467,64 @@ def simple_test_tool(message: str) -> str: assert ai_client_span2["description"] == "chat gpt-4" assert ai_client_span2["data"]["gen_ai.agent.name"] == "test_agent" assert ai_client_span2["data"]["gen_ai.operation.name"] == "chat" - assert ai_client_span2["data"]["gen_ai.request.available_tools"] == available_tools + assert ( + re.sub( + "<.*>(,)", + r"'NOT_CHECKED'\1", + agent_span["data"]["gen_ai.request.available_tools"], + ) + == available_tools + ) assert ai_client_span2["data"]["gen_ai.request.max_tokens"] == 100 - assert ai_client_span2["data"]["gen_ai.request.messages"] == [ - { - "role": "system", - "content": [{"type": "text", "text": "You are a helpful test assistant."}], - }, - { - "role": "user", - "content": [{"type": "text", "text": "Please use the simple test tool"}], - }, - { - "role": "assistant", - "content": [ - { - "arguments": '{"message": "hello"}', - "call_id": "call_123", - "name": "simple_test_tool", - "type": "function_call", - "id": "call_123", - "function": mock.ANY, - } - ], - }, - { - "role": "tool", - "content": [ - { - "call_id": "call_123", - "output": "Tool executed with: hello", - "type": "function_call_output", - } - ], - }, - ] + assert re.sub( + r"SerializationIterator\(.*\)", + "NOT_CHECKED", + ai_client_span2["data"]["gen_ai.request.messages"], + ) == safe_serialize( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful test assistant."} + ], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Please use the simple test tool"} + ], + }, + { + "role": "assistant", + "content": [ + { + "arguments": '{"message": "hello"}', + "call_id": "call_123", + "name": "simple_test_tool", + "type": "function_call", + "id": "call_123", + "function": "NOT_CHECKED", + } + ], + }, + { + "role": "tool", + "content": [ + { + "call_id": "call_123", + "output": "Tool executed with: hello", + "type": "function_call_output", + } + ], + }, + ] + ) assert ai_client_span2["data"]["gen_ai.request.model"] == "gpt-4" assert ai_client_span2["data"]["gen_ai.request.temperature"] == 0.7 assert ai_client_span2["data"]["gen_ai.request.top_p"] == 1.0 - assert ai_client_span2["data"]["gen_ai.response.text"] == [ - "Task completed using the tool" - ] + assert ai_client_span2["data"]["gen_ai.response.text"] == safe_serialize( + ["Task completed using the tool"] + ) assert ai_client_span2["data"]["gen_ai.system"] == "openai" assert ai_client_span2["data"]["gen_ai.usage.input_tokens.cached"] == 0 assert ai_client_span2["data"]["gen_ai.usage.input_tokens"] == 15 From fbd23c20650bc57b3fa1b0a2cdf83ca18de68ae5 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 24 Jun 2025 12:05:51 +0200 Subject: [PATCH 88/89] typing --- sentry_sdk/integrations/openai_agents/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 1618fede1d..28dbd6bb75 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from typing import Any from typing import Callable + from typing import Union from agents import Usage try: @@ -163,9 +164,11 @@ def _set_output_data(span, result): def safe_serialize(data): + # type: (Any) -> str """Safely serialize to a readable string.""" def serialize_item(item): + # type: (Any) -> Union[str, dict[Any, Any], list[Any], tuple[Any, ...]] if callable(item): try: module = getattr(item, "__module__", None) From 4e4f39cceb164fefbafc3652a2675cc885b08839 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 24 Jun 2025 14:35:47 +0200 Subject: [PATCH 89/89] update test matrix --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f693386396..5c993718d7 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-06-18T13:10:50.595385+00:00 +# Last generated: 2025-06-24T12:35:34.437673+00:00 [tox] requires =