-
Notifications
You must be signed in to change notification settings - Fork 591
Expand file tree
/
Copy pathtest_pydantic_ai.py
More file actions
2941 lines (2278 loc) · 86.9 KB
/
test_pydantic_ai.py
File metadata and controls
2941 lines (2278 loc) · 86.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import asyncio
import json
import pytest
from unittest.mock import MagicMock
from typing import Annotated
from pydantic import Field
import sentry_sdk
from sentry_sdk._types import BLOB_DATA_SUBSTITUTE
from sentry_sdk.consts import SPANDATA
from sentry_sdk.integrations.pydantic_ai import PydanticAIIntegration
from sentry_sdk.integrations.pydantic_ai.spans.ai_client import _set_input_messages
from sentry_sdk.integrations.pydantic_ai.spans.utils import _set_usage_data
from pydantic_ai import Agent
from pydantic_ai.messages import BinaryContent, ImageUrl, UserPromptPart
from pydantic_ai.usage import RequestUsage
from pydantic_ai.exceptions import ModelRetry, UnexpectedModelBehavior
@pytest.fixture
def test_agent():
"""Create a test agent with model settings."""
return Agent(
"test",
name="test_agent",
system_prompt="You are a helpful test assistant.",
)
@pytest.fixture
def test_agent_with_settings():
"""Create a test agent with explicit model settings."""
from pydantic_ai import ModelSettings
return Agent(
"test",
name="test_agent_settings",
system_prompt="You are a test assistant with settings.",
model_settings=ModelSettings(
temperature=0.7,
max_tokens=100,
top_p=0.9,
),
)
@pytest.mark.asyncio
async def test_agent_run_async(sentry_init, capture_events, test_agent):
"""
Test that the integration creates spans for async agent runs.
"""
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
result = await test_agent.run("Test input")
assert result is not None
assert result.output is not None
(transaction,) = events
spans = transaction["spans"]
# Verify transaction (the transaction IS the invoke_agent span)
assert transaction["transaction"] == "invoke_agent test_agent"
assert transaction["contexts"]["trace"]["origin"] == "auto.ai.pydantic_ai"
# The transaction itself should have invoke_agent data
assert transaction["contexts"]["trace"]["op"] == "gen_ai.invoke_agent"
# Find child span types (invoke_agent is the transaction, not a child span)
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
assert len(chat_spans) >= 1
# Check chat span
chat_span = chat_spans[0]
assert "chat" in chat_span["description"]
assert chat_span["data"]["gen_ai.operation.name"] == "chat"
assert chat_span["data"]["gen_ai.response.streaming"] is False
assert "gen_ai.request.messages" in chat_span["data"]
assert "gen_ai.usage.input_tokens" in chat_span["data"]
assert "gen_ai.usage.output_tokens" in chat_span["data"]
@pytest.mark.asyncio
async def test_agent_run_async_usage_data(sentry_init, capture_events, test_agent):
"""
Test that the invoke_agent span includes token usage and model data.
"""
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
result = await test_agent.run("Test input")
assert result is not None
assert result.output is not None
(transaction,) = events
# Verify transaction (the transaction IS the invoke_agent span)
assert transaction["transaction"] == "invoke_agent test_agent"
# The invoke_agent span should have token usage data
trace_data = transaction["contexts"]["trace"].get("data", {})
assert "gen_ai.usage.input_tokens" in trace_data, (
"Missing input_tokens on invoke_agent span"
)
assert "gen_ai.usage.output_tokens" in trace_data, (
"Missing output_tokens on invoke_agent span"
)
assert "gen_ai.usage.total_tokens" in trace_data, (
"Missing total_tokens on invoke_agent span"
)
assert "gen_ai.response.model" in trace_data, (
"Missing response.model on invoke_agent span"
)
# Verify the values are reasonable
assert trace_data["gen_ai.usage.input_tokens"] > 0
assert trace_data["gen_ai.usage.output_tokens"] > 0
assert trace_data["gen_ai.usage.total_tokens"] > 0
assert trace_data["gen_ai.response.model"] == "test" # Test model name
def test_agent_run_sync(sentry_init, capture_events, test_agent):
"""
Test that the integration creates spans for sync agent runs.
"""
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
result = test_agent.run_sync("Test input")
assert result is not None
assert result.output is not None
(transaction,) = events
spans = transaction["spans"]
# Verify transaction
assert transaction["transaction"] == "invoke_agent test_agent"
assert transaction["contexts"]["trace"]["origin"] == "auto.ai.pydantic_ai"
# Find span types
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
assert len(chat_spans) >= 1
# Verify streaming flag is False for sync
for chat_span in chat_spans:
assert chat_span["data"]["gen_ai.response.streaming"] is False
@pytest.mark.asyncio
async def test_agent_run_stream(sentry_init, capture_events, test_agent):
"""
Test that the integration creates spans for streaming agent runs.
"""
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
async with test_agent.run_stream("Test input") as result:
# Consume the stream
async for _ in result.stream_output():
pass
(transaction,) = events
spans = transaction["spans"]
# Verify transaction
assert transaction["transaction"] == "invoke_agent test_agent"
assert transaction["contexts"]["trace"]["origin"] == "auto.ai.pydantic_ai"
# Find chat spans
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
assert len(chat_spans) >= 1
# Verify streaming flag is True for streaming
for chat_span in chat_spans:
assert chat_span["data"]["gen_ai.response.streaming"] is True
assert "gen_ai.request.messages" in chat_span["data"]
assert "gen_ai.usage.input_tokens" in chat_span["data"]
# Streaming responses should still have output data
assert (
"gen_ai.response.text" in chat_span["data"]
or "gen_ai.response.model" in chat_span["data"]
)
@pytest.mark.asyncio
async def test_agent_run_stream_events(sentry_init, capture_events, test_agent):
"""
Test that run_stream_events creates spans (it uses run internally, so non-streaming).
"""
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
# Consume all events
async for _ in test_agent.run_stream_events("Test input"):
pass
(transaction,) = events
# Verify transaction
assert transaction["transaction"] == "invoke_agent test_agent"
# Find chat spans
spans = transaction["spans"]
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
assert len(chat_spans) >= 1
# run_stream_events uses run() internally, so streaming should be False
for chat_span in chat_spans:
assert chat_span["data"]["gen_ai.response.streaming"] is False
@pytest.mark.asyncio
async def test_agent_with_tools(sentry_init, capture_events, test_agent):
"""
Test that tool execution creates execute_tool spans.
"""
@test_agent.tool_plain
def add_numbers(a: int, b: int) -> int:
"""Add two numbers together."""
return a + b
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
result = await test_agent.run("What is 5 + 3?")
assert result is not None
(transaction,) = events
spans = transaction["spans"]
# Find child span types (invoke_agent is the transaction, not a child span)
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
tool_spans = [s for s in spans if s["op"] == "gen_ai.execute_tool"]
# Should have tool spans
assert len(tool_spans) >= 1
# Check tool span
tool_span = tool_spans[0]
assert "execute_tool" in tool_span["description"]
assert tool_span["data"]["gen_ai.operation.name"] == "execute_tool"
assert tool_span["data"]["gen_ai.tool.type"] == "function"
assert tool_span["data"]["gen_ai.tool.name"] == "add_numbers"
assert "gen_ai.tool.input" in tool_span["data"]
assert "gen_ai.tool.output" in tool_span["data"]
# Check chat spans have available_tools
for chat_span in chat_spans:
assert "gen_ai.request.available_tools" in chat_span["data"]
available_tools_str = chat_span["data"]["gen_ai.request.available_tools"]
# Available tools is serialized as a string
assert "add_numbers" in available_tools_str
@pytest.mark.parametrize(
"handled_tool_call_exceptions",
[False, True],
)
@pytest.mark.asyncio
async def test_agent_with_tool_model_retry(
sentry_init, capture_events, test_agent, handled_tool_call_exceptions
):
"""
Test that a handled exception is captured when a tool raises ModelRetry.
"""
retries = 0
@test_agent.tool_plain
def add_numbers(a: int, b: int) -> float:
"""Add two numbers together, but raises an exception on the first attempt."""
nonlocal retries
if retries == 0:
retries += 1
raise ModelRetry(message="Try again with the same arguments.")
return a + b
sentry_init(
integrations=[
PydanticAIIntegration(
handled_tool_call_exceptions=handled_tool_call_exceptions
)
],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
result = await test_agent.run("What is 5 + 3?")
assert result is not None
if handled_tool_call_exceptions:
(error, transaction) = events
else:
(transaction,) = events
spans = transaction["spans"]
if handled_tool_call_exceptions:
assert error["level"] == "error"
assert error["exception"]["values"][0]["mechanism"]["handled"]
# Find child span types (invoke_agent is the transaction, not a child span)
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
tool_spans = [s for s in spans if s["op"] == "gen_ai.execute_tool"]
# Should have tool spans
assert len(tool_spans) >= 1
# Check tool spans
model_retry_tool_span = tool_spans[0]
assert "execute_tool" in model_retry_tool_span["description"]
assert model_retry_tool_span["data"]["gen_ai.operation.name"] == "execute_tool"
assert model_retry_tool_span["data"]["gen_ai.tool.type"] == "function"
assert model_retry_tool_span["data"]["gen_ai.tool.name"] == "add_numbers"
assert "gen_ai.tool.input" in model_retry_tool_span["data"]
tool_span = tool_spans[1]
assert "execute_tool" in tool_span["description"]
assert tool_span["data"]["gen_ai.operation.name"] == "execute_tool"
assert tool_span["data"]["gen_ai.tool.type"] == "function"
assert tool_span["data"]["gen_ai.tool.name"] == "add_numbers"
assert "gen_ai.tool.input" in tool_span["data"]
assert "gen_ai.tool.output" in tool_span["data"]
# Check chat spans have available_tools
for chat_span in chat_spans:
assert "gen_ai.request.available_tools" in chat_span["data"]
available_tools_str = chat_span["data"]["gen_ai.request.available_tools"]
# Available tools is serialized as a string
assert "add_numbers" in available_tools_str
@pytest.mark.parametrize(
"handled_tool_call_exceptions",
[False, True],
)
@pytest.mark.asyncio
async def test_agent_with_tool_validation_error(
sentry_init, capture_events, test_agent, handled_tool_call_exceptions
):
"""
Test that a handled exception is captured when a tool has unsatisfiable constraints.
"""
@test_agent.tool_plain
def add_numbers(a: Annotated[int, Field(gt=0, lt=0)], b: int) -> int:
"""Add two numbers together."""
return a + b
sentry_init(
integrations=[
PydanticAIIntegration(
handled_tool_call_exceptions=handled_tool_call_exceptions
)
],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
result = None
with pytest.raises(UnexpectedModelBehavior):
result = await test_agent.run("What is 5 + 3?")
assert result is None
if handled_tool_call_exceptions:
(error, model_behaviour_error, transaction) = events
else:
(
model_behaviour_error,
transaction,
) = events
spans = transaction["spans"]
if handled_tool_call_exceptions:
assert error["level"] == "error"
assert error["exception"]["values"][0]["mechanism"]["handled"]
# Find child span types (invoke_agent is the transaction, not a child span)
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
tool_spans = [s for s in spans if s["op"] == "gen_ai.execute_tool"]
# Should have tool spans
assert len(tool_spans) >= 1
# Check tool spans
model_retry_tool_span = tool_spans[0]
assert "execute_tool" in model_retry_tool_span["description"]
assert model_retry_tool_span["data"]["gen_ai.operation.name"] == "execute_tool"
assert model_retry_tool_span["data"]["gen_ai.tool.type"] == "function"
assert model_retry_tool_span["data"]["gen_ai.tool.name"] == "add_numbers"
assert "gen_ai.tool.input" in model_retry_tool_span["data"]
# Check chat spans have available_tools
for chat_span in chat_spans:
assert "gen_ai.request.available_tools" in chat_span["data"]
available_tools_str = chat_span["data"]["gen_ai.request.available_tools"]
# Available tools is serialized as a string
assert "add_numbers" in available_tools_str
@pytest.mark.asyncio
async def test_agent_with_tools_streaming(sentry_init, capture_events, test_agent):
"""
Test that tool execution works correctly with streaming.
"""
@test_agent.tool_plain
def multiply(a: int, b: int) -> int:
"""Multiply two numbers."""
return a * b
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
async with test_agent.run_stream("What is 7 times 8?") as result:
async for _ in result.stream_output():
pass
(transaction,) = events
spans = transaction["spans"]
# Find span types
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
tool_spans = [s for s in spans if s["op"] == "gen_ai.execute_tool"]
# Should have tool spans
assert len(tool_spans) >= 1
# Verify streaming flag is True
for chat_span in chat_spans:
assert chat_span["data"]["gen_ai.response.streaming"] is True
# Check tool span
tool_span = tool_spans[0]
assert tool_span["data"]["gen_ai.tool.name"] == "multiply"
assert "gen_ai.tool.input" in tool_span["data"]
assert "gen_ai.tool.output" in tool_span["data"]
@pytest.mark.asyncio
async def test_model_settings(sentry_init, capture_events, test_agent_with_settings):
"""
Test that model settings are captured in spans.
"""
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
)
events = capture_events()
await test_agent_with_settings.run("Test input")
(transaction,) = events
spans = transaction["spans"]
# Find chat span
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
assert len(chat_spans) >= 1
chat_span = chat_spans[0]
# Check that model settings are captured
assert chat_span["data"].get("gen_ai.request.temperature") == 0.7
assert chat_span["data"].get("gen_ai.request.max_tokens") == 100
assert chat_span["data"].get("gen_ai.request.top_p") == 0.9
@pytest.mark.asyncio
@pytest.mark.parametrize(
"send_default_pii, include_prompts",
[
(True, True),
(True, False),
(False, True),
(False, False),
],
)
async def test_system_prompt_attribute(
sentry_init, capture_events, send_default_pii, include_prompts
):
"""
Test that system prompts are included as the first message.
"""
agent = Agent(
"test",
name="test_system",
system_prompt="You are a helpful assistant specialized in testing.",
)
sentry_init(
integrations=[PydanticAIIntegration(include_prompts=include_prompts)],
traces_sample_rate=1.0,
send_default_pii=send_default_pii,
)
events = capture_events()
await agent.run("Hello")
(transaction,) = events
spans = transaction["spans"]
# The transaction IS the invoke_agent span, check for messages in chat spans instead
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
assert len(chat_spans) >= 1
chat_span = chat_spans[0]
if send_default_pii and include_prompts:
system_instructions = chat_span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]
assert json.loads(system_instructions) == [
{
"type": "text",
"content": "You are a helpful assistant specialized in testing.",
}
]
else:
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_span["data"]
@pytest.mark.asyncio
async def test_error_handling(sentry_init, capture_events):
"""
Test error handling in agent execution.
"""
# Use a simpler test that doesn't cause tool failures
# as pydantic-ai has complex error handling for tool errors
agent = Agent(
"test",
name="test_error",
)
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
)
events = capture_events()
# Simple run that should succeed
await agent.run("Hello")
# At minimum, we should have a transaction
assert len(events) >= 1
transaction = [e for e in events if e.get("type") == "transaction"][0]
assert transaction["transaction"] == "invoke_agent test_error"
# Transaction should complete successfully (status key may not exist if no error)
trace_status = transaction["contexts"]["trace"].get("status")
assert trace_status != "error" # Could be None or some other status
@pytest.mark.asyncio
async def test_without_pii(sentry_init, capture_events, test_agent):
"""
Test that PII is not captured when send_default_pii is False.
"""
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
send_default_pii=False,
)
events = capture_events()
await test_agent.run("Sensitive input")
(transaction,) = events
spans = transaction["spans"]
# Find child spans (invoke_agent is the transaction, not a child span)
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
# Verify that messages and response text are not captured
for span in chat_spans:
assert "gen_ai.request.messages" not in span["data"]
assert "gen_ai.response.text" not in span["data"]
@pytest.mark.asyncio
async def test_without_pii_tools(sentry_init, capture_events, test_agent):
"""
Test that tool input/output are not captured when send_default_pii is False.
"""
@test_agent.tool_plain
def sensitive_tool(data: str) -> str:
"""A tool with sensitive data."""
return f"Processed: {data}"
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
send_default_pii=False,
)
events = capture_events()
await test_agent.run("Use sensitive tool with private data")
(transaction,) = events
spans = transaction["spans"]
# Find tool spans
tool_spans = [s for s in spans if s["op"] == "gen_ai.execute_tool"]
# If tool was executed, verify input/output are not captured
for tool_span in tool_spans:
assert "gen_ai.tool.input" not in tool_span["data"]
assert "gen_ai.tool.output" not in tool_span["data"]
@pytest.mark.asyncio
async def test_multiple_agents_concurrent(sentry_init, capture_events, test_agent):
"""
Test that multiple agents can run concurrently without interfering.
"""
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
)
events = capture_events()
async def run_agent(input_text):
return await test_agent.run(input_text)
# Run 3 agents concurrently
results = await asyncio.gather(*[run_agent(f"Input {i}") for i in range(3)])
assert len(results) == 3
assert len(events) == 3
# Verify each transaction is separate
for i, transaction in enumerate(events):
assert transaction["type"] == "transaction"
assert transaction["transaction"] == "invoke_agent test_agent"
# Each should have its own spans
assert len(transaction["spans"]) >= 1
@pytest.mark.asyncio
async def test_message_history(sentry_init, capture_events):
"""
Test that full conversation history is captured in chat spans.
"""
agent = Agent(
"test",
name="test_history",
)
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
# First message
await agent.run("Hello, I'm Alice")
# Second message with history
from pydantic_ai import messages
history = [
messages.ModelRequest(
parts=[messages.UserPromptPart(content="Hello, I'm Alice")]
),
messages.ModelResponse(
parts=[messages.TextPart(content="Hello Alice! How can I help you?")],
model_name="test",
),
]
await agent.run("What is my name?", message_history=history)
# We should have 2 transactions
assert len(events) >= 2
# Check the second transaction has the full history
second_transaction = events[1]
spans = second_transaction["spans"]
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
if chat_spans:
chat_span = chat_spans[0]
if "gen_ai.request.messages" in chat_span["data"]:
messages_data = chat_span["data"]["gen_ai.request.messages"]
# Should have multiple messages including history
assert len(messages_data) > 1
@pytest.mark.asyncio
async def test_gen_ai_system(sentry_init, capture_events, test_agent):
"""
Test that gen_ai.system is set from the model.
"""
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
)
events = capture_events()
await test_agent.run("Test input")
(transaction,) = events
spans = transaction["spans"]
# Find chat span
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
assert len(chat_spans) >= 1
chat_span = chat_spans[0]
# gen_ai.system should be set from the model (TestModel -> 'test')
assert "gen_ai.system" in chat_span["data"]
assert chat_span["data"]["gen_ai.system"] == "test"
@pytest.mark.asyncio
async def test_include_prompts_false(sentry_init, capture_events, test_agent):
"""
Test that prompts are not captured when include_prompts=False.
"""
sentry_init(
integrations=[PydanticAIIntegration(include_prompts=False)],
traces_sample_rate=1.0,
send_default_pii=True, # Even with PII enabled, prompts should not be captured
)
events = capture_events()
await test_agent.run("Sensitive prompt")
(transaction,) = events
spans = transaction["spans"]
# Find child spans (invoke_agent is the transaction, not a child span)
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
# Verify that messages and response text are not captured
for span in chat_spans:
assert "gen_ai.request.messages" not in span["data"]
assert "gen_ai.response.text" not in span["data"]
@pytest.mark.asyncio
async def test_include_prompts_true(sentry_init, capture_events, test_agent):
"""
Test that prompts are captured when include_prompts=True (default).
"""
sentry_init(
integrations=[PydanticAIIntegration(include_prompts=True)],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
await test_agent.run("Test prompt")
(transaction,) = events
spans = transaction["spans"]
# Find child spans (invoke_agent is the transaction, not a child span)
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
# Verify that messages are captured in chat spans
assert len(chat_spans) >= 1
for chat_span in chat_spans:
assert "gen_ai.request.messages" in chat_span["data"]
@pytest.mark.asyncio
async def test_include_prompts_false_with_tools(
sentry_init, capture_events, test_agent
):
"""
Test that tool input/output are not captured when include_prompts=False.
"""
@test_agent.tool_plain
def test_tool(value: int) -> int:
"""A test tool."""
return value * 2
sentry_init(
integrations=[PydanticAIIntegration(include_prompts=False)],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
await test_agent.run("Use the test tool with value 5")
(transaction,) = events
spans = transaction["spans"]
# Find tool spans
tool_spans = [s for s in spans if s["op"] == "gen_ai.execute_tool"]
# If tool was executed, verify input/output are not captured
for tool_span in tool_spans:
assert "gen_ai.tool.input" not in tool_span["data"]
assert "gen_ai.tool.output" not in tool_span["data"]
@pytest.mark.asyncio
async def test_include_prompts_requires_pii(sentry_init, capture_events, test_agent):
"""
Test that include_prompts requires send_default_pii=True.
"""
sentry_init(
integrations=[PydanticAIIntegration(include_prompts=True)],
traces_sample_rate=1.0,
send_default_pii=False, # PII disabled
)
events = capture_events()
await test_agent.run("Test prompt")
(transaction,) = events
spans = transaction["spans"]
# Find child spans (invoke_agent is the transaction, not a child span)
chat_spans = [s for s in spans if s["op"] == "gen_ai.chat"]
# Even with include_prompts=True, if PII is disabled, messages should not be captured
for span in chat_spans:
assert "gen_ai.request.messages" not in span["data"]
assert "gen_ai.response.text" not in span["data"]
@pytest.mark.asyncio
async def test_mcp_tool_execution_spans(sentry_init, capture_events):
"""
Test that MCP (Model Context Protocol) tool calls create execute_tool spans.
Tests MCP tools accessed through CombinedToolset, which is how they're typically
used in practice (when an agent combines regular functions with MCP servers).
"""
pytest.importorskip("mcp")
from unittest.mock import MagicMock
from pydantic_ai.mcp import MCPServerStdio
from pydantic_ai import Agent
from pydantic_ai.toolsets.combined import CombinedToolset
import sentry_sdk
# Create mock MCP server
mock_server = MCPServerStdio(
command="python",
args=["-m", "test_server"],
)
# Mock the server's internal methods
mock_server._client = MagicMock()
mock_server._is_initialized = True
mock_server._server_info = MagicMock()
# Mock tool call response
async def mock_send_request(request, response_type):
from mcp.types import CallToolResult, TextContent
return CallToolResult(
content=[TextContent(type="text", text="MCP tool executed successfully")],
isError=False,
)
mock_server._client.send_request = mock_send_request
# Mock context manager methods
async def mock_aenter():
return mock_server
async def mock_aexit(*args):
pass
mock_server.__aenter__ = mock_aenter
mock_server.__aexit__ = mock_aexit
# Mock _map_tool_result_part
async def mock_map_tool_result_part(part):
return part.text if hasattr(part, "text") else str(part)
mock_server._map_tool_result_part = mock_map_tool_result_part
# Create a CombinedToolset with the MCP server
# This simulates how MCP servers are typically used in practice
from pydantic_ai.toolsets.function import FunctionToolset
function_toolset = FunctionToolset()
combined = CombinedToolset([function_toolset, mock_server])
# Create agent
agent = Agent(
"test",
name="test_mcp_agent",
)
sentry_init(
integrations=[PydanticAIIntegration()],
traces_sample_rate=1.0,
send_default_pii=True,
)
events = capture_events()
# Simulate MCP tool execution within a transaction through CombinedToolset
with sentry_sdk.start_transaction(
op="ai.run", name="invoke_agent test_mcp_agent"
) as transaction:
# Set up the agent context
scope = sentry_sdk.get_current_scope()
scope._contexts["pydantic_ai_agent"] = {
"_agent": agent,
}
# Create a mock tool that simulates an MCP tool from CombinedToolset
from pydantic_ai._run_context import RunContext
from pydantic_ai.result import RunUsage
from pydantic_ai.models.test import TestModel
from pydantic_ai.toolsets.combined import _CombinedToolsetTool
ctx = RunContext(
deps=None,
model=TestModel(),
usage=RunUsage(),
retry=0,
tool_name="test_mcp_tool",
)
tool_name = "test_mcp_tool"
# Create a tool that points to the MCP server
# This simulates how CombinedToolset wraps tools from different sources
tool = _CombinedToolsetTool(
toolset=combined,
tool_def=MagicMock(name=tool_name),
max_retries=0,
args_validator=MagicMock(),
source_toolset=mock_server,
source_tool=MagicMock(),
)
try:
await combined.call_tool(tool_name, {"query": "test"}, ctx, tool)
except Exception:
# MCP tool might raise if not fully mocked, that's okay
pass
events_list = events