diff --git a/.github/workflows/examples-integration-test.yml b/.github/workflows/examples-integration-test.yml new file mode 100644 index 000000000..ccddbd194 --- /dev/null +++ b/.github/workflows/examples-integration-test.yml @@ -0,0 +1,192 @@ +name: Examples Integration Test + +# This workflow runs all example scripts to ensure they work correctly +# and that LLM spans are properly tracked in AgentOps using the +# integrated validation functionality. + +on: + push: + branches: [ main, develop ] + paths: + - 'examples/**/*.py' + - 'agentops/**' + - '.github/workflows/examples-integration-test.yml' + pull_request: + branches: [ main, develop ] + paths: + - 'examples/**/*.py' + - 'agentops/**' + - '.github/workflows/examples-integration-test.yml' + workflow_dispatch: + +env: + PYTHON_VERSION: '3.11' + +jobs: + test-examples: + runs-on: ubuntu-latest + timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + example: + # OpenAI examples + - { path: 'examples/openai/openai_example_sync.py', name: 'OpenAI Sync' } + - { path: 'examples/openai/openai_example_async.py', name: 'OpenAI Async' } + - { path: 'examples/openai/multi_tool_orchestration.py', name: 'OpenAI Multi-Tool' } + - { path: 'examples/openai/web_search.py', name: 'OpenAI Web Search' } + + # Anthropic examples + - { path: 'examples/anthropic/anthropic-example-sync.py', name: 'Anthropic Sync' } + - { path: 'examples/anthropic/anthropic-example-async.py', name: 'Anthropic Async' } + - { path: 'examples/anthropic/agentops-anthropic-understanding-tools.py', name: 'Anthropic Tools' } + + # LangChain examples + - { path: 'examples/langchain/langchain_examples.py', name: 'LangChain' } + + # LiteLLM examples + - { path: 'examples/litellm/litellm_example.py', name: 'LiteLLM' } + + # Google Generative AI examples + - { path: 'examples/google_genai/gemini_example.py', name: 'Google Gemini' } + + # xAI examples + - { path: 'examples/xai/grok_examples.py', name: 'xAI Grok' } + - { path: 'examples/xai/grok_vision_examples.py', name: 'xAI Grok Vision' } + + # CrewAI examples + - { path: 'examples/crewai/job_posting.py', name: 'CrewAI Job Posting' } + - { path: 'examples/crewai/markdown_validator.py', name: 'CrewAI Markdown' } + + # AutoGen examples + - { path: 'examples/autogen/AgentChat.py', name: 'AutoGen Agent Chat' } + - { path: 'examples/autogen/MathAgent.py', name: 'AutoGen Math Agent' } + + # AG2 examples + - { path: 'examples/ag2/async_human_input.py', name: 'AG2 Async Human Input' } + - { path: 'examples/ag2/tools_wikipedia_search.py', name: 'AG2 Wikipedia Search' } + + # Context Manager examples + - { path: 'examples/context_manager/basic_usage.py', name: 'Context Manager Basic' } + - { path: 'examples/context_manager/error_handling.py', name: 'Context Manager Errors' } + - { path: 'examples/context_manager/parallel_traces.py', name: 'Context Manager Parallel' } + - { path: 'examples/context_manager/production_patterns.py', name: 'Context Manager Production' } + + # Agno examples + - { path: 'examples/agno/agno_async_operations.py', name: 'Agno Async Operations' } + - { path: 'examples/agno/agno_basic_agents.py', name: 'Agno Basic Agents' } + - { path: 'examples/agno/agno_research_team.py', name: 'Agno Research Team' } + - { path: 'examples/agno/agno_tool_integrations.py', name: 'Agno Tool Integrations' } + - { path: 'examples/agno/agno_workflow_setup.py', name: 'Agno Workflow Setup' } + + # Google ADK examples + - { path: 'examples/google_adk/human_approval.py', name: 'Google ADK Human Approval' } + + # LlamaIndex examples + - { path: 'examples/llamaindex/llamaindex_example.py', name: 'LlamaIndex' } + + # Mem0 examples + - { path: 'examples/mem0/mem0_memoryclient_example.py', name: 'Mem0 Memory Client' } + + # Watsonx examples + - { path: 'examples/watsonx/watsonx-streaming.py', name: 'Watsonx Streaming' } + - { path: 'examples/watsonx/watsonx-text-chat.py', name: 'Watsonx Text Chat' } + - { path: 'examples/watsonx/watsonx-tokeniation-model.py', name: 'Watsonx Tokenization' } + + # LangGraph examples + - { path: 'examples/langgraph/langgraph_example.py', name: 'LangGraph' } + + # Smolagents examples + - { path: 'examples/smolagents/multi_smolagents_system.py', name: 'Smolagents Multi System' } + - { path: 'examples/smolagents/text_to_sql.py', name: 'Smolagents Text to SQL' } + + # OpenAI Agents examples + - { path: 'examples/openai_agents/agent_guardrails.py', name: 'OpenAI Agents Guardrails' } + - { path: 'examples/openai_agents/agent_patterns.py', name: 'OpenAI Agents Patterns' } + - { path: 'examples/openai_agents/agents_tools.py', name: 'OpenAI Agents Tools' } + - { path: 'examples/openai_agents/customer_service_agent.py', name: 'OpenAI Agents Customer Service' } + + # Add more examples as needed + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install AgentOps + run: | + pip install -e . + + - name: Install example dependencies + run: | + # Install common dependencies + pip install python-dotenv requests + + # Install from requirements.txt in the example's directory + example_dir=$(dirname "${{ matrix.example.path }}") + if [ -f "$example_dir/requirements.txt" ]; then + echo "Installing dependencies from $example_dir/requirements.txt" + pip install -r "$example_dir/requirements.txt" + else + echo "No requirements.txt found in $example_dir" + fi + + - name: Run example - ${{ matrix.example.name }} + env: + AGENTOPS_API_KEY: ${{ secrets.AGENTOPS_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }} + WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }} + WATSONX_URL: ${{ secrets.WATSONX_URL }} + MEM0_API_KEY: ${{ secrets.MEM0_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + AI21_API_KEY: ${{ secrets.AI21_API_KEY }} + TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} + EXA_API_KEY: ${{ secrets.EXA_API_KEY }} + LLAMA_API_KEY: ${{ secrets.LLAMA_API_KEY }} + PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} + REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }} + PYTHONPATH: ${{ github.workspace }} + run: | + echo "Running ${{ matrix.example.name }}..." + python "${{ matrix.example.path }}" || exit 1 + + - name: Check for errors + if: failure() + run: | + echo "Example ${{ matrix.example.name }} failed!" + echo "Path: ${{ matrix.example.path }}" + + # Show last 50 lines of any log files + if [ -f agentops.log ]; then + echo "=== AgentOps Log ===" + tail -n 50 agentops.log + fi + + summary: + needs: test-examples + runs-on: ubuntu-latest + if: always() + + steps: + - name: Summary + run: | + echo "## Examples Integration Test Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ needs.test-examples.result }}" == "success" ]; then + echo "āœ… All examples passed!" >> $GITHUB_STEP_SUMMARY + else + echo "āŒ Some examples failed. Check the logs above." >> $GITHUB_STEP_SUMMARY + fi \ No newline at end of file diff --git a/agentops/__init__.py b/agentops/__init__.py index fc4ceb273..a03089318 100755 --- a/agentops/__init__.py +++ b/agentops/__init__.py @@ -34,6 +34,9 @@ from agentops.helpers.deprecation import deprecated, warn_deprecated_param import threading +# Import validation functions +from agentops.validation import validate_trace_spans, print_validation_summary, ValidationError + # Thread-safe client management _client_lock = threading.Lock() _client = None @@ -442,37 +445,40 @@ def extract_key_from_attr(attr_value: str) -> str: __all__ = [ - "init", - "configure", - "get_client", - "record", - "start_trace", - "end_trace", - "update_trace_metadata", + # Legacy exports "start_session", "end_session", "track_agent", "track_tool", "end_all_sessions", + "Session", "ToolEvent", "ErrorEvent", "ActionEvent", "LLMEvent", - "Session", + # Modern exports + "init", + "start_trace", + "end_trace", + "update_trace_metadata", + "Client", + "get_client", + # Decorators "trace", "session", "agent", "task", "workflow", "operation", - "guardrail", - "tracer", "tool", - # Trace state enums + "guardrail", + # Enums "TraceState", "SUCCESS", "ERROR", "UNSET", - # OpenTelemetry status codes (for advanced users) - "StatusCode", + # Validation + "validate_trace_spans", + "print_validation_summary", + "ValidationError", ] diff --git a/agentops/validation.py b/agentops/validation.py new file mode 100644 index 000000000..7b9cda2e2 --- /dev/null +++ b/agentops/validation.py @@ -0,0 +1,357 @@ +""" +AgentOps Validation Module + +This module provides functions to validate that spans have been sent to AgentOps +using the public API. This is useful for testing and verification purposes. +""" + +import time +import requests +from typing import Optional, Dict, List, Any, Tuple + +from agentops.logging import logger +from agentops.exceptions import ApiServerException + + +class ValidationError(Exception): + """Raised when span validation fails.""" + + pass + + +def get_jwt_token(api_key: Optional[str] = None) -> str: + """ + Exchange API key for JWT token. + + Args: + api_key: Optional API key. If not provided, uses AGENTOPS_API_KEY env var. + + Returns: + JWT bearer token + + Raises: + ApiServerException: If token exchange fails + """ + if api_key is None: + from agentops import get_client + + client = get_client() + if client and client.config.api_key: + api_key = client.config.api_key + else: + import os + + api_key = os.getenv("AGENTOPS_API_KEY") + if not api_key: + raise ValueError("No API key provided and AGENTOPS_API_KEY environment variable not set") + + try: + response = requests.post( + "https://api.agentops.ai/public/v1/auth/access_token", json={"api_key": api_key}, timeout=10 + ) + response.raise_for_status() + return response.json()["bearer"] + except requests.exceptions.RequestException as e: + raise ApiServerException(f"Failed to get JWT token: {e}") + + +def get_trace_details(trace_id: str, jwt_token: str) -> Dict[str, Any]: + """ + Get trace details from AgentOps API. + + Args: + trace_id: The trace ID to query + jwt_token: JWT authentication token + + Returns: + Trace details including spans + + Raises: + ApiServerException: If API request fails + """ + try: + response = requests.get( + f"https://api.agentops.ai/public/v1/traces/{trace_id}", + headers={"Authorization": f"Bearer {jwt_token}"}, + timeout=10, + ) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + raise ApiServerException(f"Failed to get trace details: {e}") + + +def get_trace_metrics(trace_id: str, jwt_token: str) -> Dict[str, Any]: + """ + Get trace metrics from AgentOps API. + + Args: + trace_id: The trace ID to query + jwt_token: JWT authentication token + + Returns: + Trace metrics including token counts and costs + + Raises: + ApiServerException: If API request fails + """ + try: + response = requests.get( + f"https://api.agentops.ai/public/v1/traces/{trace_id}/metrics", + headers={"Authorization": f"Bearer {jwt_token}"}, + timeout=10, + ) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + raise ApiServerException(f"Failed to get trace metrics: {e}") + + +def check_llm_spans(spans: List[Dict[str, Any]]) -> Tuple[bool, List[str]]: + """ + Check if any LLM spans are present in the trace. + + Args: + spans: List of span dictionaries + + Returns: + Tuple of (has_llm_spans, llm_span_names) + """ + llm_spans = [] + + for span in spans: + span_name = span.get("span_name", "unnamed_span") + + # Check span attributes for LLM span kind + span_attributes = span.get("span_attributes", {}) + is_llm_span = False + + # If we have span_attributes, check them + if span_attributes: + # Try different possible structures for span kind + span_kind = None + + # Structure 1: span_attributes.agentops.span.kind + if isinstance(span_attributes, dict): + agentops_attrs = span_attributes.get("agentops", {}) + if isinstance(agentops_attrs, dict): + span_info = agentops_attrs.get("span", {}) + if isinstance(span_info, dict): + span_kind = span_info.get("kind", "") + + # Structure 2: Direct in span_attributes + if not span_kind and isinstance(span_attributes, dict): + # Try looking for agentops.span.kind as a flattened key + span_kind = span_attributes.get("agentops.span.kind", "") + + # Structure 3: Look for SpanAttributes.AGENTOPS_SPAN_KIND + if not span_kind and isinstance(span_attributes, dict): + from agentops.semconv import SpanAttributes + + span_kind = span_attributes.get(SpanAttributes.AGENTOPS_SPAN_KIND, "") + + # Check if this is an LLM span by span kind + is_llm_span = span_kind == "llm" + + # Alternative check: Look for gen_ai.prompt or gen_ai.completion attributes + # These are standard semantic conventions for LLM spans + if not is_llm_span and isinstance(span_attributes, dict): + gen_ai_attrs = span_attributes.get("gen_ai", {}) + if isinstance(gen_ai_attrs, dict): + # If we have prompt or completion data, it's an LLM span + if "prompt" in gen_ai_attrs or "completion" in gen_ai_attrs: + is_llm_span = True + + # Check for LLM_REQUEST_TYPE attribute (used by provider instrumentations) + if not is_llm_span and isinstance(span_attributes, dict): + from agentops.semconv import SpanAttributes, LLMRequestTypeValues + + # Check for LLM request type - try both gen_ai.* and llm.* prefixes + # The instrumentation sets gen_ai.* but the API might return llm.* + llm_request_type = span_attributes.get(SpanAttributes.LLM_REQUEST_TYPE, "") + if not llm_request_type: + # Try the llm.* prefix version + llm_request_type = span_attributes.get("llm.request.type", "") + + # Check if it's a chat or completion request (the main LLM types) + if llm_request_type in [LLMRequestTypeValues.CHAT.value, LLMRequestTypeValues.COMPLETION.value]: + is_llm_span = True + + if is_llm_span: + llm_spans.append(span_name) + + return len(llm_spans) > 0, llm_spans + + +def validate_trace_spans( + trace_id: Optional[str] = None, + trace_context: Optional[Any] = None, + max_retries: int = 10, + retry_delay: float = 1.0, + check_llm: bool = True, + min_spans: int = 1, + api_key: Optional[str] = None, +) -> Dict[str, Any]: + """ + Validate that spans have been sent to AgentOps. + + Args: + trace_id: Direct trace ID to validate + trace_context: TraceContext object from start_trace (alternative to trace_id) + max_retries: Maximum number of retries to wait for spans to appear + retry_delay: Delay between retries in seconds + check_llm: Whether to specifically check for LLM spans + min_spans: Minimum number of spans expected + api_key: Optional API key (uses environment variable if not provided) + + Returns: + Dictionary containing validation results and metrics + + Raises: + ValidationError: If validation fails + ValueError: If neither trace_id nor trace_context is provided + """ + # Extract trace ID + if trace_id is None and trace_context is None: + # Try to get from current span + try: + from opentelemetry.trace import get_current_span + + current_span = get_current_span() + if current_span and hasattr(current_span, "get_span_context"): + span_context = current_span.get_span_context() + if hasattr(span_context, "trace_id") and span_context.trace_id: + if isinstance(span_context.trace_id, int): + trace_id = format(span_context.trace_id, "032x") + else: + trace_id = str(span_context.trace_id) + except ImportError: + pass + + elif trace_context is not None and trace_id is None: + # Extract from TraceContext + if hasattr(trace_context, "span") and trace_context.span: + span_context = trace_context.span.get_span_context() + if hasattr(span_context, "trace_id"): + if isinstance(span_context.trace_id, int): + trace_id = format(span_context.trace_id, "032x") + else: + trace_id = str(span_context.trace_id) + + if trace_id is None: + raise ValueError("No trace ID found. Provide either trace_id or trace_context parameter.") + + # Get JWT token + jwt_token = get_jwt_token(api_key) + + logger.info(f"Validating spans for trace ID: {trace_id}") + + for attempt in range(max_retries): + try: + # Get trace details + trace_details = get_trace_details(trace_id, jwt_token) + spans = trace_details.get("spans", []) + + if len(spans) >= min_spans: + logger.info(f"Found {len(spans)} span(s) in trace") + + # Prepare result + result = { + "trace_id": trace_id, + "span_count": len(spans), + "spans": spans, + "has_llm_spans": False, + "llm_span_names": [], + "metrics": None, + } + + # Get metrics first - if we have token usage, we definitely have LLM spans + try: + metrics = get_trace_metrics(trace_id, jwt_token) + result["metrics"] = metrics + + if metrics: + logger.info( + f"Trace metrics - Total tokens: {metrics.get('total_tokens', 0)}, " + f"Cost: ${metrics.get('total_cost', '0.0000')}" + ) + + # If we have token usage > 0, we definitely have LLM activity + if metrics.get("total_tokens", 0) > 0: + result["has_llm_spans"] = True + logger.info("LLM activity confirmed via token usage metrics") + except Exception as e: + logger.warning(f"Could not retrieve metrics: {e}") + + # Check for LLM spans if requested and not already confirmed via metrics + if check_llm and not result["has_llm_spans"]: + has_llm_spans, llm_span_names = check_llm_spans(spans) + result["has_llm_spans"] = has_llm_spans + result["llm_span_names"] = llm_span_names + + if has_llm_spans: + logger.info(f"Found LLM spans: {', '.join(llm_span_names)}") + else: + logger.warning("No LLM spans found via attribute inspection") + + # Final validation + if check_llm and not result["has_llm_spans"]: + raise ValidationError( + f"No LLM activity detected in trace {trace_id}. " + f"Found spans: {[s.get('span_name', 'unnamed') for s in spans]}, " + f"Token usage: {result.get('metrics', {}).get('total_tokens', 0)}" + ) + + return result + + else: + logger.debug( + f"Only {len(spans)} spans found, expected at least {min_spans}. " + f"Retrying... ({attempt + 1}/{max_retries})" + ) + + except ApiServerException as e: + logger.debug(f"API error during validation: {e}. Retrying... ({attempt + 1}/{max_retries})") + + if attempt < max_retries - 1: + time.sleep(retry_delay) + + raise ValidationError( + f"Validation failed for trace {trace_id} after {max_retries} attempts. " + f"Expected at least {min_spans} spans" + + (", including LLM activity" if check_llm else "") + + ". Please check that tracking is properly configured." + ) + + +def print_validation_summary(result: Dict[str, Any]) -> None: + """ + Print a user-friendly summary of validation results. + + Args: + result: Validation result dictionary from validate_trace_spans + """ + print("\n" + "=" * 50) + print("šŸ” AgentOps Span Validation Results") + print("=" * 50) + + print(f"āœ… Found {result['span_count']} span(s) in trace") + + if result.get("has_llm_spans"): + if result.get("llm_span_names"): + print(f"āœ… Found LLM spans: {', '.join(result['llm_span_names'])}") + else: + # LLM activity confirmed via metrics + print("āœ… LLM activity confirmed via token usage metrics") + elif "has_llm_spans" in result: + print("āš ļø No LLM activity detected") + + if result.get("metrics"): + metrics = result["metrics"] + print("\nšŸ“Š Trace Metrics:") + print(f" - Total tokens: {metrics.get('total_tokens', 0)}") + print(f" - Prompt tokens: {metrics.get('prompt_tokens', 0)}") + print(f" - Completion tokens: {metrics.get('completion_tokens', 0)}") + print(f" - Total cost: ${metrics.get('total_cost', '0.0000')}") + + print("\nāœ… Validation successful!") diff --git a/examples/README_TESTING.md b/examples/README_TESTING.md new file mode 100644 index 000000000..4d9e8105a --- /dev/null +++ b/examples/README_TESTING.md @@ -0,0 +1,103 @@ +# AgentOps Examples Integration Testing + +This directory contains example scripts demonstrating how to use AgentOps with various LLM providers and frameworks. Each example includes automatic validation to ensure that LLM spans are properly tracked by AgentOps. + +## What's Being Tested + +Each example script now includes automated span validation that: + +1. **Runs the example** - Executes the normal example code +2. **Validates span tracking** - Uses the AgentOps integrated validation to verify that: + - Spans were successfully sent to AgentOps + - LLM calls were properly instrumented and tracked + - Token counts and costs were recorded + +## How It Works + +### 1. Integrated Validation + +AgentOps now includes built-in validation functionality (`agentops.validate_trace_spans`) that: +- Exchanges API keys for JWT tokens using the public API +- Queries the AgentOps API for trace and span data +- Validates that expected spans are present +- Retrieves metrics like token usage and costs + +### 2. Example Structure + +Each example follows this pattern: + +```python +import agentops + +# Initialize AgentOps +agentops.init() + +# Start a trace +tracer = agentops.start_trace("example-name") + +# ... perform operations with LLMs ... + +# End the trace +agentops.end_trace(tracer, end_state="Success") + +# Validate spans were tracked +try: + result = agentops.validate_trace_spans(trace_context=tracer) + agentops.print_validation_summary(result) +except agentops.ValidationError as e: + print(f"āŒ Error validating spans: {e}") + raise +``` + +### 3. CI/CD Integration + +The GitHub Actions workflow (`examples-integration-test.yml`) runs all examples automatically on: +- Push to main/develop branches +- Pull requests +- Manual workflow dispatch + +Each example is run in isolation with proper error handling and reporting. + +## Running Tests Locally + +To run a specific example with validation: + +```bash +cd examples/openai +python openai_example_sync.py +``` + +To run all examples: + +```bash +# From the examples directory +for script in $(find . -name "*.py" -type f | grep -v "__pycache__"); do + echo "Running $script..." + python "$script" +done +``` + +## Adding New Examples + +When adding a new example: + +1. Include the standard validation at the end: + ```python + # Validate spans were tracked + try: + result = agentops.validate_trace_spans(trace_context=tracer) + agentops.print_validation_summary(result) + except agentops.ValidationError as e: + print(f"āŒ Error validating spans: {e}") + raise + ``` + +2. Add the example to the GitHub Actions matrix in `.github/workflows/examples-integration-test.yml` + +3. Ensure the example has proper error handling + +## Requirements + +- Valid `AGENTOPS_API_KEY` environment variable +- API keys for the specific LLM provider being tested +- Python 3.12+ with required dependencies \ No newline at end of file diff --git a/examples/ag2/async_human_input.py b/examples/ag2/async_human_input.py index 6484773a4..a534b4758 100644 --- a/examples/ag2/async_human_input.py +++ b/examples/ag2/async_human_input.py @@ -25,7 +25,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="AG2 Async Human Input") tracer = agentops.start_trace( trace_name="AG2 Agent chat with Async Human Inputs", tags=["ag2-chat-async-human-inputs", "agentops-example"] ) @@ -104,3 +104,13 @@ async def main(): # await main() agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/ag2/requirements.txt b/examples/ag2/requirements.txt new file mode 100644 index 000000000..4359d0e30 --- /dev/null +++ b/examples/ag2/requirements.txt @@ -0,0 +1,3 @@ +ag2 +nest-asyncio +wikipedia-api \ No newline at end of file diff --git a/examples/ag2/tools_wikipedia_search.py b/examples/ag2/tools_wikipedia_search.py index 95aff7308..c5a3c5e9f 100644 --- a/examples/ag2/tools_wikipedia_search.py +++ b/examples/ag2/tools_wikipedia_search.py @@ -22,7 +22,7 @@ # ### Agent Configuration # # Configure an assistant agent and user proxy to be used for LLM recommendation and execution respectively. -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="AG2 Wikipedia Search Tools") tracer = agentops.start_trace( trace_name="AG2 Wikipedia Search Tools", tags=["ag2-wikipedia-search-tools", "agentops-example"] ) @@ -70,3 +70,13 @@ ) agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/agno/agno_async_operations.py b/examples/agno/agno_async_operations.py index 5e994e2c1..4ab778b02 100644 --- a/examples/agno/agno_async_operations.py +++ b/examples/agno/agno_async_operations.py @@ -26,7 +26,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_agentops_api_key_here") -agentops.init(auto_start_session=False, tags=["agno-example", "async-operation"]) +agentops.init(auto_start_session=False, trace_name="Agno Async Operations", tags=["agno-example", "async-operation"]) async def demonstrate_async_operations(): @@ -75,6 +75,16 @@ async def task3(): print(f"An error occurred: {e}") agentops.end_trace(tracer, end_state="Error") + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "=" * 50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + if __name__ == "__main__": asyncio.run(demonstrate_async_operations()) diff --git a/examples/agno/agno_basic_agents.py b/examples/agno/agno_basic_agents.py index ef61a642f..25a0cd41f 100644 --- a/examples/agno/agno_basic_agents.py +++ b/examples/agno/agno_basic_agents.py @@ -35,7 +35,9 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_agentops_api_key_here") -agentops.init(auto_start_session=False, tags=["agno-example", "basics", "agents-and-teams"]) +agentops.init( + auto_start_session=False, trace_name="Agno Basic Agents", tags=["agno-example", "basics", "agents-and-teams"] +) def demonstrate_basic_agents(): @@ -88,6 +90,16 @@ def demonstrate_basic_agents(): print(f"An error occurred: {e}") agentops.end_trace(tracer, end_state="Error") + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "=" * 50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + if __name__ == "__main__": demonstrate_basic_agents() diff --git a/examples/agno/agno_research_team.py b/examples/agno/agno_research_team.py index c9ddc5e93..8fbb89a7a 100644 --- a/examples/agno/agno_research_team.py +++ b/examples/agno/agno_research_team.py @@ -67,7 +67,7 @@ load_dotenv() # Initialize AgentOps for monitoring and analytics -agentops.init(auto_start_session=False, tags=["agno-example", "research-team"]) +agentops.init(auto_start_session=False, trace_name="Agno Research Team", tags=["agno-example", "research-team"]) def demonstrate_research_team(): @@ -204,5 +204,15 @@ def demonstrate_research_team(): except Exception: agentops.end_trace(tracer, end_state="Error") + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "=" * 50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + demonstrate_research_team() diff --git a/examples/agno/agno_tool_integrations.py b/examples/agno/agno_tool_integrations.py index 3a28f0e95..859e92f04 100644 --- a/examples/agno/agno_tool_integrations.py +++ b/examples/agno/agno_tool_integrations.py @@ -33,8 +33,7 @@ # Initialize AgentOps agentops.init( - auto_start_session=False, - tags=["agno-tools", "tool-integration", "demo"] + auto_start_session=False, trace_name="Agno Tool Integrations", tags=["agno-tools", "tool-integration", "demo"] ) @@ -56,7 +55,7 @@ def demonstrate_tool_integration(): role="Research information using Google Search", model=OpenAIChat(id="gpt-4o-mini"), tools=[GoogleSearchTools()], - instructions="You are a research assistant. Use Google Search to find accurate, up-to-date information." + instructions="You are a research assistant. Use Google Search to find accurate, up-to-date information.", ) response = search_agent.run("What are the latest developments in AI agents?") @@ -70,15 +69,11 @@ def demonstrate_tool_integration(): name="Research Agent", role="Comprehensive research using multiple tools", model=OpenAIChat(id="gpt-4o-mini"), - tools=[ - GoogleSearchTools(), - ArxivTools(), - DuckDuckGoTools() - ], + tools=[GoogleSearchTools(), ArxivTools(), DuckDuckGoTools()], instructions="""You are a comprehensive research assistant. Use Google Search for general information, Arxiv for academic papers, and DuckDuckGo as an alternative search engine. - Provide well-researched, balanced information from multiple sources.""" + Provide well-researched, balanced information from multiple sources.""", ) response = research_agent.run( @@ -96,12 +91,10 @@ def demonstrate_tool_integration(): role="Find and summarize academic papers", model=OpenAIChat(id="gpt-4o-mini"), tools=[ArxivTools()], - instructions="You are an academic research assistant. Use Arxiv to find relevant papers and provide concise summaries." + instructions="You are an academic research assistant. Use Arxiv to find relevant papers and provide concise summaries.", ) - response = academic_agent.run( - "Find recent papers about tool augmented language models" - ) + response = academic_agent.run("Find recent papers about tool augmented language models") print(f"Academic Agent Response:\n{response.content}") # Example 4: Comparing Search Tools @@ -112,13 +105,10 @@ def demonstrate_tool_integration(): name="Comparison Agent", role="Compare results from different search engines", model=OpenAIChat(id="gpt-4o-mini"), - tools=[ - GoogleSearchTools(), - DuckDuckGoTools() - ], + tools=[GoogleSearchTools(), DuckDuckGoTools()], instructions="""Compare search results from Google and DuckDuckGo. Note any differences in results, ranking, or information quality. - Be objective in your comparison.""" + Be objective in your comparison.""", ) response = comparison_agent.run( @@ -144,6 +134,16 @@ def demonstrate_tool_integration(): agentops.end_trace(tracer, end_state="Error") raise + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "=" * 50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + if __name__ == "__main__": demonstrate_tool_integration() diff --git a/examples/agno/agno_workflow_setup.py b/examples/agno/agno_workflow_setup.py index d3aa0c9ea..42f44d647 100644 --- a/examples/agno/agno_workflow_setup.py +++ b/examples/agno/agno_workflow_setup.py @@ -27,7 +27,7 @@ load_dotenv() -agentops.init(auto_start_session=False, tags=["agno-example", "workflow-setup"]) +agentops.init(auto_start_session=False, trace_name="Agno Workflow Setup", tags=["agno-example", "workflow-setup"]) class CacheWorkflow(Workflow): @@ -113,5 +113,15 @@ def demonstrate_workflows(): except Exception: agentops.end_trace(tracer, end_state="Error") + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "=" * 50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + asyncio.run(demonstrate_workflows()) diff --git a/examples/agno/requirements.txt b/examples/agno/requirements.txt new file mode 100644 index 000000000..96ab8a51e --- /dev/null +++ b/examples/agno/requirements.txt @@ -0,0 +1,2 @@ +agno +aiohttp \ No newline at end of file diff --git a/examples/anthropic/agentops-anthropic-understanding-tools.py b/examples/anthropic/agentops-anthropic-understanding-tools.py index bbce678ff..ed6898250 100644 --- a/examples/anthropic/agentops-anthropic-understanding-tools.py +++ b/examples/anthropic/agentops-anthropic-understanding-tools.py @@ -17,7 +17,7 @@ os.environ["ANTHROPIC_API_KEY"] = os.getenv("ANTHROPIC_API_KEY", "your_anthropic_api_key_here") # # Now let's set the client as Anthropic and make an AgentOps trace -agentops.init(tags=["anthropic-example-tool-tutorials", "agentops-example"]) +agentops.init(trace_name="Anthropic Understanding Tools", tags=["anthropic-example-tool-tutorials", "agentops-example"]) client = Anthropic() # Now to create a simple dummy tool! We are going to make a tool that will tell us about the demon infestation levels for 3 areas. From there, we will have VEGA, our AI determine the best place for the Doom Slayer to attack. locations = [ @@ -371,3 +371,14 @@ def inventoryscan(): message = response.content[0].text print(message) + + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/anthropic/anthropic-example-async.py b/examples/anthropic/anthropic-example-async.py index f1bff79a5..26f6b2b65 100644 --- a/examples/anthropic/anthropic-example-async.py +++ b/examples/anthropic/anthropic-example-async.py @@ -22,7 +22,7 @@ # # Now let's set the client as Anthropic and open an agentops trace! client = Anthropic() -agentops.init(tags=["anthropic-async", "agentops-example"]) +agentops.init(trace_name="Anthropic Async Example", tags=["anthropic-async", "agentops-example"]) # Now we create three personality presets; # # Legion is a relentless and heavy-hitting Titan that embodies brute strength and defensive firepower, Northstar is a precise and agile sniper that excels in long-range combat and flight, while Ronin is a swift and aggressive melee specialist who thrives on close-quarters hit-and-run tactics. @@ -108,3 +108,14 @@ async def main(): # Run the main function asyncio.run(main()) # We can observe the trace in the AgentOps dashboard by going to the trace URL provided above. + + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/anthropic/anthropic-example-sync.py b/examples/anthropic/anthropic-example-sync.py index f97429964..08db9058d 100644 --- a/examples/anthropic/anthropic-example-sync.py +++ b/examples/anthropic/anthropic-example-sync.py @@ -21,7 +21,7 @@ os.environ["ANTHROPIC_API_KEY"] = os.getenv("ANTHROPIC_API_KEY", "your_anthropic_api_key_here") # Now let's set the client as Anthropic and an AgentOps session! client = Anthropic() -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="Anthropic Sync Example") tracer = agentops.start_trace(trace_name="Anthropic Sync Example", tags=["anthropic-example", "agentops-example"]) # Remember that story we made earlier? As of writing, claude-3-5-sonnet-20240620 (the version we will be using) has a 150k word, 680k character length. We also get an 8192 context length. This is great because we can actually set an example for the script! # @@ -110,3 +110,13 @@ # # Now we will end the session with a success message. We can also end the session with a failure or intdeterminate status. By default, the session will be marked as indeterminate. agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/anthropic/requirements.txt b/examples/anthropic/requirements.txt new file mode 100644 index 000000000..d5d39b6df --- /dev/null +++ b/examples/anthropic/requirements.txt @@ -0,0 +1 @@ +anthropic \ No newline at end of file diff --git a/examples/autogen/AgentChat.py b/examples/autogen/AgentChat.py index b0ef8d412..5958e90d8 100644 --- a/examples/autogen/AgentChat.py +++ b/examples/autogen/AgentChat.py @@ -9,9 +9,6 @@ # Then import them import os from dotenv import load_dotenv -from IPython.core.error import ( - StdinNotImplementedError, -) import asyncio import agentops @@ -35,7 +32,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") # When initializing AgentOps, you can pass in optional tags to help filter sessions -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="Autogen Agent Chat Example") tracer = agentops.start_trace( trace_name="Microsoft Agent Chat Example", tags=["autogen-chat", "microsoft-autogen", "agentops-example"] ) @@ -50,7 +47,7 @@ # * Errors # # Simple Chat Example # Define model and API key -model_name = "gpt-4-turbo" # Or "gpt-4o" / "gpt-4o-mini" as per migration guide examples +model_name = "gpt-4o-mini" # Or "gpt-4o" / "gpt-4o-mini" as per migration guide examples api_key = os.getenv("OPENAI_API_KEY") # Create the model client @@ -82,16 +79,22 @@ async def main(): await Console().run(stream) agentops.end_trace(tracer, end_state="Success") - except StdinNotImplementedError: - print("StdinNotImplementedError: This typically happens in non-interactive environments.") - print("Skipping interactive part of chat for automation.") - agentops.end_trace(tracer, end_state="Indeterminate") except Exception as e: print(f"An error occurred: {e}") agentops.end_trace(tracer, end_state="Error") finally: await model_client.close() + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "=" * 50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + if __name__ == "__main__": try: diff --git a/examples/autogen/MathAgent.py b/examples/autogen/MathAgent.py index 64348b65d..c594f05c1 100644 --- a/examples/autogen/MathAgent.py +++ b/examples/autogen/MathAgent.py @@ -11,9 +11,6 @@ import asyncio import os from dotenv import load_dotenv -from IPython.core.error import ( - StdinNotImplementedError, -) import agentops @@ -32,7 +29,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="Autogen Math Agent Example") tracer = agentops.start_trace( trace_name="Microsoft Autogen Tool Example", tags=["autogen-tool", "microsoft-autogen", "agentops-example"] ) @@ -97,15 +94,22 @@ async def main(): agentops.end_trace(tracer, end_state="Success") - except StdinNotImplementedError: - print("StdinNotImplementedError: This typically happens in non-interactive environments.") - agentops.end_trace(tracer, end_state="Indeterminate") except Exception as e: print(f"An error occurred: {e}") agentops.end_trace(tracer, end_state="Error") finally: await model_client.close() + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "=" * 50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + if __name__ == "__main__": try: diff --git a/examples/autogen/requirements.txt b/examples/autogen/requirements.txt new file mode 100644 index 000000000..032705de1 --- /dev/null +++ b/examples/autogen/requirements.txt @@ -0,0 +1 @@ +pyautogen \ No newline at end of file diff --git a/examples/context_manager/basic_usage.py b/examples/context_manager/basic_usage.py index 122991339..94d57c5af 100644 --- a/examples/context_manager/basic_usage.py +++ b/examples/context_manager/basic_usage.py @@ -40,10 +40,14 @@ def basic_context_manager_example(): print("Basic Context Manager Example") # Initialize AgentOps - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Basic Example", + tags=["context-manager", "agentops-example"], + ) # Use native TraceContext context manager - with agentops.start_trace("basic_example", tags=["basic", "demo"]): + with agentops.start_trace("Context Manager Basic Example", tags=["basic", "demo"]): print("Trace started") # Create and use agent @@ -58,10 +62,14 @@ def multiple_parallel_traces(): """Example showing multiple parallel traces.""" print("\nMultiple Parallel Traces") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Multiple Parallel Traces", + tags=["context-manager", "agentops-example"], + ) # First trace - with agentops.start_trace("task_1", tags=["parallel", "task-1"]): + with agentops.start_trace("Context Manager Task 1", tags=["parallel", "task-1"]): print("Task 1 started") agent1 = SimpleAgent("Agent1") result1 = agent1.process_data("task 1 data") @@ -81,7 +89,11 @@ def error_handling_example(): """Example showing error handling with context manager.""" print("\nError Handling Example") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Error Handling", + tags=["context-manager", "agentops-example"], + ) try: with agentops.start_trace("error_example", tags=["error-handling"]): @@ -103,7 +115,11 @@ def nested_traces_example(): """Example showing nested traces (which are parallel, not parent-child).""" print("\nNested Traces Example") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Nested Traces", + tags=["context-manager", "agentops-example"], + ) # Outer trace with agentops.start_trace("main_workflow", tags=["workflow", "main"]): @@ -138,3 +154,13 @@ def nested_traces_example(): nested_traces_example() print("\nAll examples completed!") + + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "=" * 50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/context_manager/error_handling.py b/examples/context_manager/error_handling.py index 9f53c3cb7..f42d0a8ec 100644 --- a/examples/context_manager/error_handling.py +++ b/examples/context_manager/error_handling.py @@ -66,13 +66,17 @@ def basic_exception_handling(): """Basic example of exception handling with context managers.""" print("Basic Exception Handling") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Basic Exception Handling", + tags=["context-manager", "error-handling", "agentops-example"], + ) error_types = ["value_error", "type_error", "runtime_error", "success"] for error_type in error_types: try: - with agentops.start_trace(f"basic_{error_type}", tags=["basic", "error-handling"]): + with agentops.start_trace(f"Context Manager Basic {error_type}", tags=["basic", "error-handling"]): print(f"Started trace for {error_type}") agent = ErrorProneAgent(f"BasicAgent_{error_type}") @@ -93,7 +97,11 @@ def nested_exception_handling(): """Example of exception handling in nested traces.""" print("\nNested Exception Handling") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Nested Exception Handling", + tags=["context-manager", "error-handling", "agentops-example"], + ) try: with agentops.start_trace("outer_operation", tags=["nested", "outer"]): @@ -124,7 +132,11 @@ def retry_pattern(): """Example of retry pattern with context managers.""" print("\nRetry Pattern") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Retry Pattern", + tags=["context-manager", "error-handling", "agentops-example"], + ) max_retries = 3 for attempt in range(max_retries): @@ -160,7 +172,11 @@ def graceful_degradation(): """Example of graceful degradation pattern.""" print("\nGraceful Degradation") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Graceful Degradation", + tags=["context-manager", "error-handling", "agentops-example"], + ) try: with agentops.start_trace("primary_service", tags=["degradation", "primary"]): @@ -188,7 +204,11 @@ def partial_success_handling(): """Example of partial success handling.""" print("\nPartial Success Handling") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Partial Success Handling", + tags=["context-manager", "error-handling", "agentops-example"], + ) steps = ["step1", "step2", "fail", "step4"] @@ -210,7 +230,11 @@ def custom_exception_handling(): """Example of handling custom exceptions.""" print("\nCustom Exception Handling") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Custom Exception Handling", + tags=["context-manager", "error-handling", "agentops-example"], + ) try: with agentops.start_trace("custom_exception", tags=["custom", "exception"]): @@ -232,7 +256,11 @@ def finally_blocks_example(): """Example of exception handling with finally blocks.""" print("\nFinally Blocks Example") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Finally Blocks", + tags=["context-manager", "error-handling", "agentops-example"], + ) cleanup_actions = [] @@ -263,7 +291,11 @@ def exception_chaining_example(): """Example of exception chaining and context preservation.""" print("\nException Chaining Example") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Exception Chaining", + tags=["context-manager", "error-handling", "agentops-example"], + ) try: with agentops.start_trace("exception_chaining", tags=["chaining", "context"]): @@ -315,3 +347,13 @@ def exception_chaining_example(): exception_chaining_example() print("\nAll error handling examples completed!") + + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "=" * 50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/context_manager/parallel_traces.py b/examples/context_manager/parallel_traces.py index 1e05a26af..f8436a44a 100644 --- a/examples/context_manager/parallel_traces.py +++ b/examples/context_manager/parallel_traces.py @@ -51,7 +51,11 @@ def sequential_parallel_traces(): """Example of sequential parallel traces - each trace is independent.""" print("Sequential Parallel Traces") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Sequential Parallel Traces", + tags=["context-manager", "agentops-example"], + ) tasks = ["task_1", "task_2", "task_3"] results = [] @@ -75,7 +79,11 @@ def nested_parallel_traces(): """Example showing that nested context managers create parallel traces.""" print("\nNested Parallel Traces") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Nested Parallel Traces", + tags=["context-manager", "agentops-example"], + ) # Outer trace for the overall workflow with agentops.start_trace("workflow_main", tags=["workflow", "main"]): @@ -110,7 +118,11 @@ def concurrent_traces_with_threads(): """Example of truly concurrent traces using threading.""" print("\nConcurrent Traces with Threading") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Concurrent Threads", + tags=["context-manager", "agentops-example"], + ) def worker_function(worker_id: int, task_data: str): """Function to run in a separate thread with its own trace.""" @@ -149,7 +161,11 @@ def concurrent_traces_with_executor(): """Example using ThreadPoolExecutor for concurrent traces.""" print("\nConcurrent Traces with ThreadPoolExecutor") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Concurrent Executor", + tags=["context-manager", "agentops-example"], + ) def process_with_trace(task_id: int, data: str) -> str: """Process data within its own trace context.""" @@ -195,7 +211,9 @@ def trace_with_different_tag_types(): """Example showing different ways to tag parallel traces.""" print("\nTraces with Different Tag Types") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, trace_name="Context Manager Tag Types", tags=["context-manager", "agentops-example"] + ) # Trace with list tags with agentops.start_trace("list_tags_trace", tags=["list", "example", "demo"]): diff --git a/examples/context_manager/production_patterns.py b/examples/context_manager/production_patterns.py index 8dedc50ec..8cb131f64 100644 --- a/examples/context_manager/production_patterns.py +++ b/examples/context_manager/production_patterns.py @@ -125,7 +125,11 @@ def api_endpoint_pattern(): """Example of using context managers in API endpoint pattern.""" print("API Endpoint Pattern") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager API Endpoint Pattern", + tags=["context-manager", "production", "agentops-example"], + ) # Simulate API requests requests = [ @@ -154,7 +158,11 @@ def batch_processing_pattern(): """Example of batch processing with context managers.""" print("\nBatch Processing Pattern") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Batch Processing Pattern", + tags=["context-manager", "production", "agentops-example"], + ) # Simulate batch data batch_data = [{"id": f"item_{i:03d}", "type": "data_record", "payload": {"value": i * 10}} for i in range(1, 6)] @@ -188,7 +196,11 @@ def microservice_pattern(): """Example of microservice communication pattern.""" print("\nMicroservice Communication Pattern") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Microservice Pattern", + tags=["context-manager", "production", "agentops-example"], + ) def authenticate_user(user_id: str) -> bool: """Simulate authentication service.""" @@ -239,7 +251,11 @@ def monitoring_pattern(): """Example of monitoring with context managers.""" print("\nMonitoring Pattern") - agentops.init(api_key=AGENTOPS_API_KEY) + agentops.init( + api_key=AGENTOPS_API_KEY, + trace_name="Context Manager Monitoring Pattern", + tags=["context-manager", "production", "agentops-example"], + ) class AlertManager: """Simple alert manager for demonstration.""" @@ -338,3 +354,13 @@ def __exit__(self, exc_type, exc_val, exc_tb): monitoring_pattern() print("\nAll production pattern examples completed!") + + # Let's check programmatically that spans were recorded in AgentOps + print("\n" + "=" * 50) + print("Now let's verify that our LLM calls were tracked properly...") + try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") + except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/context_manager/requirements.txt b/examples/context_manager/requirements.txt new file mode 100644 index 000000000..434f10ac2 --- /dev/null +++ b/examples/context_manager/requirements.txt @@ -0,0 +1,2 @@ +# Context manager examples only need agentops +# python-dotenv and requests are installed as common deps \ No newline at end of file diff --git a/examples/crewai/job_posting.py b/examples/crewai/job_posting.py index 3f8d6021c..db0067c2d 100644 --- a/examples/crewai/job_posting.py +++ b/examples/crewai/job_posting.py @@ -20,7 +20,9 @@ os.environ["SERPER_API_KEY"] = os.getenv("SERPER_API_KEY", "your_serper_api_key_here") # Initialize AgentOps client -agentops.init(auto_start_session=False) +agentops.init( + auto_start_session=False, trace_name="CrewAI Job Posting", tags=["crewai", "job-posting", "agentops-example"] +) web_search_tool = WebsiteSearchTool() serper_dev_tool = SerperDevTool() @@ -133,10 +135,10 @@ def industry_analysis_task(self, agent, company_domain, company_description): tracer = agentops.start_trace(trace_name="CrewAI Job Posting", tags=["crew-job-posting-example", "agentops-example"]) tasks = Tasks() agents = Agents() -company_description = input("What is the company description?\n") -company_domain = input("What is the company domain?\n") -hiring_needs = input("What are the hiring needs?\n") -specific_benefits = input("What are specific_benefits you offer?\n") +company_description = "We are a software company that builds AI-powered tools for businesses." +company_domain = "https://www.agentops.ai" +hiring_needs = "We are looking for a software engineer with 3 years of experience in Python and Django." +specific_benefits = "We offer a competitive salary, health insurance, and a 401k plan." # Create Agents researcher_agent = agents.research_agent() @@ -172,3 +174,13 @@ def industry_analysis_task(self, agent, company_domain, company_description): print(result) agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/crewai/markdown_validator.py b/examples/crewai/markdown_validator.py index ea9d36761..1d56cb42e 100644 --- a/examples/crewai/markdown_validator.py +++ b/examples/crewai/markdown_validator.py @@ -24,7 +24,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") # The first step in any AgentOps integration is to call `agentops.init()` -agentops.init(tags=["markdown_validator", "agentops-example"]) +agentops.init(trace_name="CrewAI Markdown Validator", tags=["markdown_validator", "agentops-example"]) # Lets start by creating our markdown validator tool @@ -104,3 +104,14 @@ def markdown_validation_tool(file_path: str) -> str: # Now lets run our task! syntax_review_task.execute_sync() + + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/crewai/requirements.txt b/examples/crewai/requirements.txt new file mode 100644 index 000000000..f2e17dad0 --- /dev/null +++ b/examples/crewai/requirements.txt @@ -0,0 +1,3 @@ +crewai +crewai-tools +pymarkdown \ No newline at end of file diff --git a/examples/google_adk/human_approval.py b/examples/google_adk/human_approval.py index 1cbe55e7f..7bf088449 100644 --- a/examples/google_adk/human_approval.py +++ b/examples/google_adk/human_approval.py @@ -31,7 +31,12 @@ AGENTOPS_API_KEY = os.getenv("AGENTOPS_API_KEY") or "your_agentops_api_key_here" # Initialize AgentOps - Just 2 lines! -agentops.init(AGENTOPS_API_KEY, trace_name="adk-human-approval-notebook", auto_start_session=False) +agentops.init( + AGENTOPS_API_KEY, + trace_name="adk-human-approval-notebook", + auto_start_session=False, + tags=["google-adk", "human-approval", "agentops-example"], +) # Define some constants for our application. APP_NAME = "human_approval_app_notebook" @@ -209,3 +214,13 @@ async def main_notebook(): except Exception as e: print(f"Error: {e}") agentops.end_trace(end_state="Error") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/google_adk/requirements.txt b/examples/google_adk/requirements.txt new file mode 100644 index 000000000..476e7f452 --- /dev/null +++ b/examples/google_adk/requirements.txt @@ -0,0 +1,5 @@ +google-adk +google-genai +nest-asyncio +agentops +python-dotenv \ No newline at end of file diff --git a/examples/google_genai/gemini_example.py b/examples/google_genai/gemini_example.py index 6a8450809..62e4f019c 100644 --- a/examples/google_genai/gemini_example.py +++ b/examples/google_genai/gemini_example.py @@ -15,7 +15,7 @@ os.environ["GEMINI_API_KEY"] = os.getenv("GEMINI_API_KEY", "your_gemini_api_key_here") # Initialize AgentOps and Gemini client -agentops.init(tags=["gemini-example", "agentops-example"]) +agentops.init(trace_name="Google Gemini Example", tags=["gemini-example", "agentops-example"]) client = genai.Client() # Test synchronous generation @@ -46,3 +46,13 @@ model="gemini-1.5-flash", contents="This is a test sentence to count tokens." ) print(f"Token count: {token_response.total_tokens}") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/google_genai/requirements.txt b/examples/google_genai/requirements.txt new file mode 100644 index 000000000..c969438c1 --- /dev/null +++ b/examples/google_genai/requirements.txt @@ -0,0 +1,2 @@ +google-generativeai +google-genai \ No newline at end of file diff --git a/examples/langchain/langchain_examples.py b/examples/langchain/langchain_examples.py index 33455bbd5..1c1307882 100644 --- a/examples/langchain/langchain_examples.py +++ b/examples/langchain/langchain_examples.py @@ -77,3 +77,17 @@ def find_movie(genre: str) -> str: # ## Check your session # Finally, check your run on [AgentOps](https://app.agentops.ai). You will see a session recorded with the LLM calls and tool usage. + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + import agentops + + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except ImportError: + print("\nāŒ Error: agentops library not installed. Please install it to validate spans.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/langchain/requirements.txt b/examples/langchain/requirements.txt new file mode 100644 index 000000000..5497821c0 --- /dev/null +++ b/examples/langchain/requirements.txt @@ -0,0 +1,2 @@ +langchain +langchain-openai \ No newline at end of file diff --git a/examples/langgraph/langgraph_example.py b/examples/langgraph/langgraph_example.py index 2a42b84f6..3a393001a 100644 --- a/examples/langgraph/langgraph_example.py +++ b/examples/langgraph/langgraph_example.py @@ -10,7 +10,11 @@ load_dotenv() -agentops.init(os.getenv("AGENTOPS_API_KEY")) +agentops.init( + os.getenv("AGENTOPS_API_KEY"), + trace_name="LangGraph Tool Usage Example", + tags=["langgraph", "tool-usage", "agentops-example"], +) @tool @@ -116,3 +120,15 @@ def run_example(): if __name__ == "__main__": run_example() print("āœ… Check your AgentOps dashboard for the trace!") + + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that we have enough spans tracked properly...") +try: + # LangGraph doesn't emit LLM spans in the same format, so we just check span count + result = agentops.validate_trace_spans(trace_context=None, check_llm=False, min_spans=5) + print(f"\nāœ… Success! {result['span_count']} spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/langgraph/requirements.txt b/examples/langgraph/requirements.txt new file mode 100644 index 000000000..0450c9884 --- /dev/null +++ b/examples/langgraph/requirements.txt @@ -0,0 +1,2 @@ +langgraph +langchain-openai \ No newline at end of file diff --git a/examples/litellm/litellm_example.py b/examples/litellm/litellm_example.py index 8c9126c83..2e4d51021 100644 --- a/examples/litellm/litellm_example.py +++ b/examples/litellm/litellm_example.py @@ -28,7 +28,7 @@ "OPENAI_API_KEY", "your_openai_api_key_here" ) # or the provider of your choosing -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="LiteLLM Example") tracer = agentops.start_trace(trace_name="LiteLLM Example", tags=["litellm-example", "agentops-example"]) # Note: AgentOps requires that you call LiteLLM completions differently than the LiteLLM's docs mention @@ -46,7 +46,17 @@ # litellm.completion() # ``` messages = [{"role": "user", "content": "Write a 12 word poem about secret agents."}] -response = litellm.completion(model="gpt-4", messages=messages) # or the model of your choosing +response = litellm.completion(model="gpt-4o-mini", messages=messages) # or the model of your choosing print(response.choices[0].message.content) agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/litellm/requirements.txt b/examples/litellm/requirements.txt new file mode 100644 index 000000000..e215fd144 --- /dev/null +++ b/examples/litellm/requirements.txt @@ -0,0 +1 @@ +litellm \ No newline at end of file diff --git a/examples/llamaindex/llamaindex_example.py b/examples/llamaindex/llamaindex_example.py index ed7826a52..d416f1636 100644 --- a/examples/llamaindex/llamaindex_example.py +++ b/examples/llamaindex/llamaindex_example.py @@ -8,7 +8,7 @@ from llama_index.embeddings.huggingface import HuggingFaceEmbedding from llama_index.llms.huggingface import HuggingFaceLLM -handler = AgentOpsHandler() +handler = AgentOpsHandler(tags=["llamaindex", "rag", "agentops-example"]) handler.init() load_dotenv() @@ -64,3 +64,15 @@ print("šŸŽ‰ Example completed successfully!") print("šŸ“Š Check your AgentOps dashboard to see the recorded session with LLM calls and operations.") print("šŸ”— The session link should be printed above by AgentOps.") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + import agentops + + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/llamaindex/requirements.txt b/examples/llamaindex/requirements.txt new file mode 100644 index 000000000..bafc641c4 --- /dev/null +++ b/examples/llamaindex/requirements.txt @@ -0,0 +1,3 @@ +llama-index +llama-index-llms-openai +llama-index-instrumentation-agentops \ No newline at end of file diff --git a/examples/mem0/mem0_memory_example.py b/examples/mem0/mem0_memory_example.py index 4a51e79d9..6f59f07e0 100644 --- a/examples/mem0/mem0_memory_example.py +++ b/examples/mem0/mem0_memory_example.py @@ -26,8 +26,8 @@ os.environ["AGENTOPS_LOG_LEVEL"] = "DEBUG" # Set environment variables before importing -os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY") -os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY") +os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "") +os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "") # Now import mem0 - it will be instrumented by agentops from mem0 import Memory, AsyncMemory # noqa E402 @@ -57,7 +57,7 @@ def demonstrate_sync_memory(local_config, sample_messages, sample_preferences, u must complete before the next one begins. """ - agentops.start_trace("mem0_memory_example", tags=["mem0_memory_example"]) + tracer = agentops.start_trace("Mem0 Memory Example", tags=["mem0_memory_example"]) try: # Initialize sync Memory with local configuration memory = Memory.from_config(local_config) @@ -96,9 +96,10 @@ def demonstrate_sync_memory(local_config, sample_messages, sample_preferences, u delete_all_result = memory.delete_all(user_id=user_id) print(f"Delete all result: {delete_all_result}") - agentops.end_trace(end_state="success") - except Exception: - agentops.end_trace(end_state="error") + agentops.end_trace(tracer, end_state="Success") + except Exception as e: + agentops.end_trace(tracer, end_state="Error") + raise e async def demonstrate_async_memory(local_config, sample_messages, sample_preferences, user_id): @@ -122,7 +123,7 @@ async def demonstrate_async_memory(local_config, sample_messages, sample_prefere by running multiple memory operations in parallel. """ - agentops.start_trace("mem0_memory_async_example") + tracer = agentops.start_trace("Mem0 Memory Async Example", tags=["mem0", "async", "memory-management"]) try: # Initialize async Memory with configuration async_memory = await AsyncMemory.from_config(local_config) @@ -177,12 +178,16 @@ async def search_memory(query): delete_all_result = await async_memory.delete_all(user_id=user_id) print(f"Delete all result: {delete_all_result}") - agentops.end_trace(end_state="success") + agentops.end_trace(tracer, end_state="Success") - except Exception: - agentops.end_trace(end_state="error") + except Exception as e: + agentops.end_trace(tracer, end_state="Error") + raise e +# Initialize AgentOps +agentops.init(trace_name="Mem0 Memory Example", tags=["mem0", "memory-management", "agentops-example"]) + # Configuration for local memory (Memory) # This configuration specifies the LLM provider and model settings local_config = { @@ -222,3 +227,15 @@ async def search_memory(query): # Execute both sync and async demonstrations demonstrate_sync_memory(local_config, sample_messages, sample_preferences, user_id) asyncio.run(demonstrate_async_memory(local_config, sample_messages, sample_preferences, user_id)) + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + # Note: Using trace_id since we ran multiple traces + # In a real application, you would store each tracer and validate individually + result = agentops.validate_trace_spans(check_llm=False) # Don't check for LLM spans as this uses memory operations + agentops.print_validation_summary(result) +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/mem0/mem0_memoryclient_example.py b/examples/mem0/mem0_memoryclient_example.py index cfb060cb2..74d307ac7 100644 --- a/examples/mem0/mem0_memoryclient_example.py +++ b/examples/mem0/mem0_memoryclient_example.py @@ -54,7 +54,7 @@ def demonstrate_sync_memory_client(sample_messages, sample_preferences, user_id) Cloud benefit: All memory operations are handled by Mem0's infrastructure, providing scalability and persistence without local storage management. """ - agentops.start_trace("mem0_memoryclient_example", tags=["mem0_memoryclient_example"]) + agentops.start_trace("Mem0 MemoryClient Example", tags=["mem0_memoryclient_example"]) try: # Initialize sync MemoryClient with API key for cloud access client = MemoryClient(api_key=mem0_api_key) @@ -106,7 +106,7 @@ async def demonstrate_async_memory_client(sample_messages, sample_preferences, u concurrently, significantly reducing total execution time compared to sequential calls. This is especially beneficial when dealing with network I/O to cloud services. """ - agentops.start_trace("mem0_memoryclient_example", tags=["mem0_memoryclient_example"]) + agentops.start_trace("Mem0 MemoryClient Async Example", tags=["mem0_memoryclient_example"]) try: # Initialize async MemoryClient for concurrent cloud operations async_client = AsyncMemoryClient(api_key=mem0_api_key) @@ -183,3 +183,13 @@ async def demonstrate_async_memory_client(sample_messages, sample_preferences, u # Note: The async version typically completes faster due to concurrent operations demonstrate_sync_memory_client(sample_messages, sample_preferences, user_id) asyncio.run(demonstrate_async_memory_client(sample_messages, sample_preferences, user_id)) + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/mem0/requirements.txt b/examples/mem0/requirements.txt new file mode 100644 index 000000000..2713cbc83 --- /dev/null +++ b/examples/mem0/requirements.txt @@ -0,0 +1 @@ +mem0ai \ No newline at end of file diff --git a/examples/openai/multi_tool_orchestration.py b/examples/openai/multi_tool_orchestration.py index 552751b86..f3586a755 100644 --- a/examples/openai/multi_tool_orchestration.py +++ b/examples/openai/multi_tool_orchestration.py @@ -28,9 +28,13 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") os.environ["PINECONE_API_KEY"] = os.getenv("PINECONE_API_KEY", "your_pinecone_api_key_here") -agentops.init(auto_start_session=True) +agentops.init( + auto_start_session=True, + trace_name="OpenAI Multi-Tool Orchestration", + tags=["openai", "multi-tool", "agentops-example"], +) tracer = agentops.start_trace( - trace_name="Multi-Tool Orchestration with RAG", + trace_name="OpenAI Multi-Tool Orchestration with RAG", tags=["multi-tool-orchestration-rag-demo", "openai-responses", "agentops-example"], ) client = OpenAI() @@ -342,4 +346,15 @@ def query_pinecone_index(client, index, model, query_text): print(response_2.output_text) agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # Here, we have seen how to utilize OpenAI's Responses API to implement a Retrieval-Augmented Generation (RAG) approach with multi-tool calling capabilities. It showcases an example where the model selects the appropriate tool based on the input query: general questions may be handled by built-in tools such as web-search, while specific medical inquiries related to internal knowledge are addressed by retrieving context from a vector database (such as Pinecone) via function calls. Additonally, we have showcased how multiple tool calls can be sequentially combined to generate a final response based on our instructions provided to responses API. Happy coding! diff --git a/examples/openai/openai_example_async.py b/examples/openai/openai_example_async.py index 1e32f52b4..0602f72f7 100644 --- a/examples/openai/openai_example_async.py +++ b/examples/openai/openai_example_async.py @@ -21,7 +21,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Next we initialize the AgentOps client. -agentops.init(auto_start_session=True) +agentops.init(auto_start_session=True, trace_name="OpenAI Async Example", tags=["openai", "async", "agentops-example"]) tracer = agentops.start_trace( trace_name="OpenAI Async Example", tags=["openai-async-example", "openai", "agentops-example"] ) @@ -37,7 +37,10 @@ """ user_prompt = [ - {"type": "text", "text": "Write a mystery thriller story based on your understanding of the provided image."}, + { + "type": "text", + "text": "Write a very short mystery thriller story based on your understanding of the provided image.", + }, { "type": "image_url", "image_url": {"url": "https://www.cosy.sbg.ac.at/~pmeerw/Watermarking/lena_color.gif"}, @@ -78,5 +81,16 @@ async def main_stream(): asyncio.run(main_stream()) agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # Note that the response is a generator that yields chunks of the story. We can track this with AgentOps by navigating to the trace url and viewing the run. # All done! diff --git a/examples/openai/openai_example_sync.py b/examples/openai/openai_example_sync.py index 6b535251b..9c7f49cde 100644 --- a/examples/openai/openai_example_sync.py +++ b/examples/openai/openai_example_sync.py @@ -20,7 +20,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Next we initialize the AgentOps client. -agentops.init(auto_start_session=True) +agentops.init(auto_start_session=True, trace_name="OpenAI Sync Example", tags=["openai", "sync", "agentops-example"]) tracer = agentops.start_trace( trace_name="OpenAI Sync Example", tags=["openai-sync-example", "openai", "agentops-example"] ) @@ -35,7 +35,7 @@ You are given a prompt and you need to generate a story based on the prompt. """ -user_prompt = "Write a story about a cyber-warrior trapped in the imperial time period." +user_prompt = "Write a very short story about a cyber-warrior trapped in the imperial time period." messages = [ {"role": "system", "content": system_prompt}, @@ -64,5 +64,15 @@ agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + result = agentops.validate_trace_spans(trace_context=tracer) + agentops.print_validation_summary(result) +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + # Note that the response is a generator that yields chunks of the story. We can track this with AgentOps by navigating to the trace url and viewing the run. # All done! diff --git a/examples/openai/requirements.txt b/examples/openai/requirements.txt new file mode 100644 index 000000000..f066e4c31 --- /dev/null +++ b/examples/openai/requirements.txt @@ -0,0 +1,3 @@ +openai +pandas # for multi_tool_orchestration.py +tavily-python # for web_search.py \ No newline at end of file diff --git a/examples/openai/web_search.py b/examples/openai/web_search.py index d632e0cc4..627d79ced 100644 --- a/examples/openai/web_search.py +++ b/examples/openai/web_search.py @@ -26,7 +26,9 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") -agentops.init(auto_start_session=True) +agentops.init( + auto_start_session=True, trace_name="OpenAI Web Search Example", tags=["openai", "web-search", "agentops-example"] +) tracer = agentops.start_trace( trace_name="OpenAI Responses Example", tags=["openai-responses-example", "openai", "agentops-example"] ) @@ -101,6 +103,17 @@ print(json.dumps(response_multimodal.__dict__, default=lambda o: o.__dict__, indent=4)) agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # In the above example, we were able to use the `web_search` tool to search the web for news related to the image in one API call instead of multiple round trips that would be required if we were using the Chat Completions API. # With the responses API # šŸ”„ a single API call can handle: diff --git a/examples/openai_agents/agent_guardrails.py b/examples/openai_agents/agent_guardrails.py index 2429dab4b..ebfa32ac9 100644 --- a/examples/openai_agents/agent_guardrails.py +++ b/examples/openai_agents/agent_guardrails.py @@ -33,7 +33,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(api_key=os.environ["AGENTOPS_API_KEY"], tags=["agentops-example"]) +agentops.init(api_key=os.environ["AGENTOPS_API_KEY"], trace_name="OpenAI Agents Guardrails", tags=["agentops-example"]) # OpenAI Agents SDK guardrail example with agentops guardrails decorator for observability @@ -81,3 +81,14 @@ async def main(): if __name__ == "__main__": asyncio.run(main()) + + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/openai_agents/agent_patterns.py b/examples/openai_agents/agent_patterns.py index 5482fc7b4..3e6af9026 100644 --- a/examples/openai_agents/agent_patterns.py +++ b/examples/openai_agents/agent_patterns.py @@ -78,7 +78,11 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") # Initialize AgentOps -agentops.init(auto_start_session=False) +agentops.init( + auto_start_session=False, + trace_name="OpenAI Agents Patterns", + tags=["openai-agents", "patterns", "agentops-example"], +) # Note: tracer will be defined in each section's cell for clarity, using the specific tags for that pattern. # ## 1. Agents as Tools Pattern # @@ -88,7 +92,7 @@ # # This pattern demonstrates using agents as callable tools within other agents. The orchestrator agent receives a user message and then picks which specialized agents to call as tools. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Agents as Tools Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace(trace_name="OpenAI Agents as Tools Pattern", tags=["agentops-example", "openai-agents"]) # Define specialized translation agents spanish_agent = Agent( @@ -174,7 +178,9 @@ async def run_agents_as_tools_demo(): # # This pattern demonstrates breaking down a complex task into a series of smaller, sequential steps. Each step is performed by an agent, and the output of one agent is used as input to the next. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Deterministic Flow Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents Deterministic Flow Pattern", tags=["agentops-example", "openai-agents"] +) # Define the story generation workflow story_outline_agent = Agent( @@ -246,7 +252,9 @@ async def run_deterministic_flow_demo(): # # For this demo, we'll allow the user to choose which tool use behavior to test: # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Forcing Tool Use Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents Forcing Tool Use Pattern", tags=["agentops-example", "openai-agents"] +) # Define the weather tool and agent @@ -325,7 +333,9 @@ async def run_forcing_tool_use_demo(tool_use_behavior: str): # # This pattern demonstrates how to use input guardrails to validate user inputs before they reach the main agent. Guardrails can prevent inappropriate or off-topic requests from being processed. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Input Guardrails Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents Input Guardrails Pattern", tags=["agentops-example", "openai-agents"] +) # Define the guardrail @@ -388,7 +398,9 @@ async def run_input_guardrails_demo(): # # This pattern shows how to use one LLM to evaluate and improve the output of another. The first agent generates content, and the second agent judges the quality and provides feedback for improvement. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="LLM as a Judge Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents LLM as a Judge Pattern", tags=["agentops-example", "openai-agents"] +) # Define the story generation and evaluation agents story_outline_generator = Agent( @@ -468,7 +480,9 @@ async def run_llm_as_judge_demo(): # # This pattern demonstrates how to use output guardrails to validate agent outputs after they are generated. This can help prevent sensitive information from being shared or ensure outputs meet quality standards. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Output Guardrails Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents Output Guardrails Pattern", tags=["agentops-example", "openai-agents"] +) # The agent's output type @@ -535,7 +549,9 @@ async def run_output_guardrails_demo(): # # This pattern shows how to run multiple agents in parallel to improve latency or generate multiple options to choose from. In this example, we run translation agents multiple times and pick the best result. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Output Guardrails Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents Parallelization Pattern", tags=["agentops-example", "openai-agents"] +) # Define agents for parallelization spanish_translation_agent = Agent( @@ -592,7 +608,7 @@ async def run_parallelization_demo(): # # This pattern demonstrates handoffs and routing between specialized agents. The triage agent receives the first message and hands off to the appropriate agent based on the language of the request. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Routing Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace(trace_name="OpenAI Agents Routing Pattern", tags=["agentops-example", "openai-agents"]) # Define language-specific agents french_routing_agent = Agent( @@ -645,7 +661,9 @@ async def run_routing_demo(): # # This pattern shows how to use guardrails during streaming to provide real-time validation. Unlike output guardrails that run after completion, streaming guardrails can interrupt the generation process early. # Start the AgentOps trace session -tracer = agentops.start_trace(trace_name="Streaming Guardrails Pattern", tags=["agentops-example", "openai-agents"]) +tracer = agentops.start_trace( + trace_name="OpenAI Agents Streaming Guardrails Pattern", tags=["agentops-example", "openai-agents"] +) # Define streaming guardrail agent streaming_agent = Agent( @@ -723,6 +741,17 @@ async def run_streaming_guardrails_demo(): # End the AgentOps trace session agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + # ## Conclusion # # This notebook has demonstrated 9 key agent patterns that are commonly used in production AI applications. Each pattern showcases how agents can be orchestrated to perform complex tasks, validate inputs and outputs, and improve overall application performance. diff --git a/examples/openai_agents/agents_tools.py b/examples/openai_agents/agents_tools.py index 71c8644ea..ed01e7520 100644 --- a/examples/openai_agents/agents_tools.py +++ b/examples/openai_agents/agents_tools.py @@ -48,7 +48,11 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(auto_start_session=False, tags=["agentops-example"]) +agentops.init( + auto_start_session=False, + trace_name="OpenAI Agents Tools Examples", + tags=["openai-agents", "tools", "agentops-example"], +) # ## 1. Code Interpreter Tool # @@ -61,7 +65,7 @@ # - Handle data processing tasks # Start the AgentOps trace session tracer = agentops.start_trace( - trace_name="Code Interpreter Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] + trace_name="OpenAI Agents Code Interpreter Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] ) @@ -111,7 +115,7 @@ async def run_code_interpreter_demo(): # **Note:** This example requires a pre-configured vector store ID. # Start the AgentOps trace session tracer = agentops.start_trace( - trace_name="File Search Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] + trace_name="OpenAI Agents File Search Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] ) @@ -157,7 +161,7 @@ async def run_file_search_demo(): # - Automatic image saving and display # Start the AgentOps trace session tracer = agentops.start_trace( - trace_name="Image Generation Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] + trace_name="OpenAI Agents Image Generation Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] ) @@ -222,7 +226,7 @@ async def run_image_generation_demo(): # - Configurable search parameters # Start the AgentOps trace session tracer = agentops.start_trace( - trace_name="Web Search Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] + trace_name="OpenAI Agents Web Search Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"] ) @@ -248,6 +252,17 @@ async def run_web_search_demo(): # End the AgentOps trace session agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # ## Conclusion # # Each tool extends agent capabilities and enables sophisticated automation. **AgentOps makes tool observability effortless** - simply import the library and all your tool interactions are automatically tracked, visualized, and analyzed. This enables you to: diff --git a/examples/openai_agents/customer_service_agent.py b/examples/openai_agents/customer_service_agent.py index 92fa34bb1..dffad335f 100644 --- a/examples/openai_agents/customer_service_agent.py +++ b/examples/openai_agents/customer_service_agent.py @@ -53,8 +53,12 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(tags=["customer-service-agent", "openai-agents", "agentops-example"], auto_start_session=False) -tracer = agentops.start_trace(trace_name="Customer Service Agent") +agentops.init( + trace_name="OpenAI Agents Customer Service", + tags=["customer-service-agent", "openai-agents", "agentops-example"], + auto_start_session=False, +) +tracer = agentops.start_trace(trace_name="OpenAI Agents Customer Service Agent") # Context model for the airline agent @@ -102,13 +106,13 @@ async def update_seat(context: RunContextWrapper[AirlineAgentContext], confirmat return f"Updated seat to {new_seat} for confirmation number {confirmation_number}" -### HOOKS +# HOOKS async def on_seat_booking_handoff(context: RunContextWrapper[AirlineAgentContext]) -> None: flight_number = f"FLT-{random.randint(100, 999)}" context.context.flight_number = flight_number -### AGENTS +# AGENTS faq_agent = Agent[AirlineAgentContext]( name="FAQ Agent", handoff_description="A helpful agent that can answer questions about the airline.", @@ -187,6 +191,17 @@ async def main(): # await main() agentops.end_trace(tracer, status="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # ## Conclusion # # **AgentOps makes observability effortless** - simply import the library and all your interactions are automatically tracked, visualized, and analyzed. This enables you to: diff --git a/examples/openai_agents/requirements.txt b/examples/openai_agents/requirements.txt new file mode 100644 index 000000000..11a0a416d --- /dev/null +++ b/examples/openai_agents/requirements.txt @@ -0,0 +1,2 @@ +openai +openai-agents \ No newline at end of file diff --git a/examples/smolagents/multi_smolagents_system.py b/examples/smolagents/multi_smolagents_system.py index c14740b30..acbb6965a 100644 --- a/examples/smolagents/multi_smolagents_system.py +++ b/examples/smolagents/multi_smolagents_system.py @@ -47,9 +47,10 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="Smolagents Multi-Agent System") tracer = agentops.start_trace( - trace_name="Orchestrate a Multi-Agent System", tags=["smolagents", "example", "multi-agent", "agentops-example"] + trace_name="Smolagents Multi-Agent System Orchestration", + tags=["smolagents", "example", "multi-agent", "agentops-example"], ) model = LiteLLMModel("openai/gpt-4o-mini") # ## Create a Web Search Tool @@ -82,7 +83,7 @@ def visit_webpage(url: str) -> str: except RequestException as e: return f"Error fetching the webpage: {str(e)}" - except Exception as e: + except agentops.ValidationError as e: return f"An unexpected error occurred: {str(e)}" @@ -111,4 +112,15 @@ def visit_webpage(url: str) -> str: print(answer) # Awesome! We've successfully run a multi-agent system. Let's end the agentops session with a "Success" state. You can also end the session with a "Failure" or "Indeterminate" state, which is set as default. agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + # You can view the session in the [AgentOps dashboard](https://app.agentops.ai/sessions) by clicking the link provided after ending the session. diff --git a/examples/smolagents/requirements.txt b/examples/smolagents/requirements.txt new file mode 100644 index 000000000..9a3d792c5 --- /dev/null +++ b/examples/smolagents/requirements.txt @@ -0,0 +1,4 @@ +smolagents +pandas +duckduckgo-search +sqlalchemy \ No newline at end of file diff --git a/examples/smolagents/text_to_sql.py b/examples/smolagents/text_to_sql.py index 10eeefa66..32282b6f8 100644 --- a/examples/smolagents/text_to_sql.py +++ b/examples/smolagents/text_to_sql.py @@ -98,9 +98,9 @@ def sql_engine(query: str) -> str: os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") -agentops.init(auto_start_session=False) +agentops.init(auto_start_session=False, trace_name="Smolagents Text-to-SQL") tracer = agentops.start_trace( - trace_name="Text-to-SQL", tags=["smolagents", "example", "text-to-sql", "agentops-example"] + trace_name="Smolagents Text-to-SQL", tags=["smolagents", "example", "text-to-sql", "agentops-example"] ) model = LiteLLMModel("openai/gpt-4o-mini") agent = CodeAgent( @@ -156,4 +156,15 @@ def sql_engine(query: str) -> str: agent.run("Which waiter got more total money from tips?") # All done! Now we can end the agentops session with a "Success" state. You can also end the session with a "Failure" or "Indeterminate" state, where the "Indeterminate" state is used by default. agentops.end_trace(tracer, end_state="Success") + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + # You can view the session in the [AgentOps dashboard](https://app.agentops.ai/sessions) by clicking the link provided after ending the session. diff --git a/examples/watsonx/requirements.txt b/examples/watsonx/requirements.txt new file mode 100644 index 000000000..b3e3ac418 --- /dev/null +++ b/examples/watsonx/requirements.txt @@ -0,0 +1 @@ +ibm-watsonx-ai \ No newline at end of file diff --git a/examples/watsonx/watsonx-streaming.py b/examples/watsonx/watsonx-streaming.py index a78214f4f..c62d27327 100644 --- a/examples/watsonx/watsonx-streaming.py +++ b/examples/watsonx/watsonx-streaming.py @@ -15,7 +15,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Initialize AgentOps -agentops.init(tags=["watsonx-streaming", "agentops-example"]) +agentops.init(trace_name="WatsonX Streaming Example", tags=["watsonx-streaming", "agentops-example"]) # ## Initialize IBM Watsonx AI Credentials # # To use IBM Watsonx AI, you need to set up your credentials and project ID. @@ -117,3 +117,13 @@ # Close connections gen_model.close_persistent_connection() chat_model.close_persistent_connection() + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/watsonx/watsonx-text-chat.py b/examples/watsonx/watsonx-text-chat.py index c21391584..af49599a6 100644 --- a/examples/watsonx/watsonx-text-chat.py +++ b/examples/watsonx/watsonx-text-chat.py @@ -15,7 +15,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Initialize AgentOps -agentops.init(tags=["watsonx-text-chat", "agentops-example"]) +agentops.init(trace_name="WatsonX Text Chat Example", tags=["watsonx-text-chat", "agentops-example"]) # ## Initialize IBM Watsonx AI Credentials # # To use IBM Watsonx AI, you need to set up your credentials and project ID. @@ -76,3 +76,13 @@ # Close connections gen_model.close_persistent_connection() chat_model.close_persistent_connection() + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/watsonx/watsonx-tokeniation-model.py b/examples/watsonx/watsonx-tokeniation-model.py index 90d8210d9..4d039e528 100644 --- a/examples/watsonx/watsonx-tokeniation-model.py +++ b/examples/watsonx/watsonx-tokeniation-model.py @@ -15,7 +15,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Initialize AgentOps -agentops.init(tags=["watsonx-tokenization", "agentops-example"]) +agentops.init(trace_name="WatsonX Tokenization Model Example", tags=["watsonx-tokenization", "agentops-example"]) # ## Initialize IBM Watsonx AI Credentials # # To use IBM Watsonx AI, you need to set up your credentials and project ID. @@ -95,3 +95,13 @@ # Close connections model.close_persistent_connection() llama_model.close_persistent_connection() + +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=None) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise diff --git a/examples/xai/grok_examples.py b/examples/xai/grok_examples.py index 43718a2ca..860f93b05 100644 --- a/examples/xai/grok_examples.py +++ b/examples/xai/grok_examples.py @@ -17,8 +17,8 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here") # Next we initialize the AgentOps client. -agentops.init(auto_start_session=False) -tracer = agentops.start_trace(trace_name="XAI Example", tags=["xai-example", "grok", "agentops-example"]) +agentops.init(auto_start_session=False, trace_name="XAI Grok Example", tags=["xai", "grok", "agentops-example"]) +tracer = agentops.start_trace(trace_name="XAI Grok Example", tags=["xai-example", "grok", "agentops-example"]) # And we are all set! Note the seesion url above. We will use it to track the chatbot. # @@ -77,4 +77,15 @@ # Awesome! We can now transliterate from English to any language! And all of this can be tracked with AgentOps by going to the session url above. agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # We end the session with a success state and a success reason. This is useful if you want to track the success or failure of the chatbot. In that case you can set the end state to failure and provide a reason. By default the session will have an indeterminate end state. diff --git a/examples/xai/grok_vision_examples.py b/examples/xai/grok_vision_examples.py index 3f620d637..151c3bdf1 100644 --- a/examples/xai/grok_vision_examples.py +++ b/examples/xai/grok_vision_examples.py @@ -18,8 +18,12 @@ os.environ["XAI_API_KEY"] = os.getenv("XAI_API_KEY", "your_xai_api_key_here") # Next we initialize the AgentOps client. -agentops.init(auto_start_session=False) -tracer = agentops.start_trace(trace_name="XAI Vision Example", tags=["xai-example", "grok-vision", "agentops-example"]) +agentops.init( + auto_start_session=False, trace_name="XAI Grok Vision Example", tags=["xai", "grok-vision", "agentops-example"] +) +tracer = agentops.start_trace( + trace_name="XAI Grok Vision Example", tags=["xai-example", "grok-vision", "agentops-example"] +) # And we are all set! Note the seesion url above. We will use it to track the program's performance. # @@ -60,4 +64,15 @@ # Awesome! It returns a fascinating response explaining the image and also deciphering the text content. All of this can be tracked with AgentOps by going to the session url above. agentops.end_trace(tracer, end_state="Success") +# Let's check programmatically that spans were recorded in AgentOps +print("\n" + "=" * 50) +print("Now let's verify that our LLM calls were tracked properly...") +try: + agentops.validate_trace_spans(trace_context=tracer) + print("\nāœ… Success! All LLM spans were properly recorded in AgentOps.") +except agentops.ValidationError as e: + print(f"\nāŒ Error validating spans: {e}") + raise + + # We end the session with a success state and a success reason. This is useful if you want to track the success or failure of the chatbot. In that case you can set the end state to failure and provide a reason. By default the session will have an indeterminate end state. diff --git a/examples/xai/requirements.txt b/examples/xai/requirements.txt new file mode 100644 index 000000000..85a3cb06b --- /dev/null +++ b/examples/xai/requirements.txt @@ -0,0 +1 @@ +openai # xAI uses OpenAI client \ No newline at end of file diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py new file mode 100644 index 000000000..ac717358f --- /dev/null +++ b/tests/unit/test_validation.py @@ -0,0 +1,405 @@ +""" +Unit tests for the AgentOps validation module. +""" + +import pytest +from unittest.mock import patch, Mock +import requests + +from agentops.validation import ( + get_jwt_token, + get_trace_details, + check_llm_spans, + validate_trace_spans, + ValidationError, + print_validation_summary, +) +from agentops.exceptions import ApiServerException + + +class TestGetJwtToken: + """Test JWT token exchange functionality.""" + + @patch("agentops.validation.requests.post") + def test_get_jwt_token_success(self, mock_post): + """Test successful JWT token retrieval.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"bearer": "test-token"} + mock_post.return_value = mock_response + + token = get_jwt_token("test-api-key") + assert token == "test-token" + + mock_post.assert_called_once_with( + "https://api.agentops.ai/public/v1/auth/access_token", json={"api_key": "test-api-key"}, timeout=10 + ) + + @patch("agentops.validation.requests.post") + def test_get_jwt_token_failure(self, mock_post): + """Test JWT token retrieval failure.""" + mock_response = Mock() + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("401 Unauthorized") + mock_post.return_value = mock_response + + with pytest.raises(ApiServerException, match="Failed to get JWT token"): + get_jwt_token("invalid-api-key") + + @patch("os.getenv") + @patch("agentops.get_client") + @patch("agentops.validation.requests.post") + def test_get_jwt_token_from_env(self, mock_post, mock_get_client, mock_getenv): + """Test JWT token retrieval using environment variable.""" + mock_get_client.return_value = None + mock_getenv.return_value = "env-api-key" + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"bearer": "env-token"} + mock_post.return_value = mock_response + + token = get_jwt_token() + assert token == "env-token" + + mock_getenv.assert_called_once_with("AGENTOPS_API_KEY") + + +class TestGetTraceDetails: + """Test trace details retrieval.""" + + @patch("agentops.validation.requests.get") + def test_get_trace_details_success(self, mock_get): + """Test successful trace details retrieval.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"trace_id": "test-trace", "spans": [{"span_name": "test-span"}]} + mock_get.return_value = mock_response + + details = get_trace_details("test-trace", "test-token") + assert details["trace_id"] == "test-trace" + assert len(details["spans"]) == 1 + + mock_get.assert_called_once_with( + "https://api.agentops.ai/public/v1/traces/test-trace", + headers={"Authorization": "Bearer test-token"}, + timeout=10, + ) + + @patch("agentops.validation.requests.get") + def test_get_trace_details_failure(self, mock_get): + """Test trace details retrieval failure.""" + mock_response = Mock() + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("404 Not Found") + mock_get.return_value = mock_response + + with pytest.raises(ApiServerException, match="Failed to get trace details"): + get_trace_details("invalid-trace", "test-token") + + +class TestCheckLlmSpans: + """Test LLM span checking.""" + + def test_check_llm_spans_found(self): + """Test when LLM spans are found.""" + spans = [ + {"span_name": "OpenAI Chat Completion", "span_attributes": {"agentops.span.kind": "llm"}}, + {"span_name": "Some other span"}, + {"span_name": "anthropic.messages.create", "span_attributes": {"agentops": {"span": {"kind": "llm"}}}}, + ] + + has_llm, llm_names = check_llm_spans(spans) + assert has_llm is True + assert len(llm_names) == 2 + assert "OpenAI Chat Completion" in llm_names + assert "anthropic.messages.create" in llm_names + + def test_check_llm_spans_not_found(self): + """Test when no LLM spans are found.""" + spans = [{"span_name": "database.query"}, {"span_name": "http.request"}] + + has_llm, llm_names = check_llm_spans(spans) + assert has_llm is False + assert len(llm_names) == 0 + + def test_check_llm_spans_empty(self): + """Test with empty spans list.""" + has_llm, llm_names = check_llm_spans([]) + assert has_llm is False + assert len(llm_names) == 0 + + def test_check_llm_spans_with_request_type(self): + """Test when LLM spans are identified by LLM_REQUEST_TYPE attribute.""" + from agentops.semconv import SpanAttributes, LLMRequestTypeValues + + spans = [ + { + "span_name": "openai.chat.completion", + "span_attributes": {SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value}, + }, + { + "span_name": "anthropic.messages.create", + "span_attributes": {SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value}, + }, + { + "span_name": "llm.completion", + "span_attributes": {SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.COMPLETION.value}, + }, + { + "span_name": "embedding.create", + "span_attributes": {SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.EMBEDDING.value}, + }, + {"span_name": "database.query"}, + ] + + has_llm, llm_names = check_llm_spans(spans) + assert has_llm is True + assert len(llm_names) == 3 # Only chat and completion types count as LLM + assert "openai.chat.completion" in llm_names + assert "anthropic.messages.create" in llm_names + assert "llm.completion" in llm_names + assert "embedding.create" not in llm_names # Embeddings are not LLM spans + + def test_check_llm_spans_real_world(self): + """Test with real-world span structures from OpenAI and Anthropic.""" + from agentops.semconv import SpanAttributes, LLMRequestTypeValues + + # This simulates what we actually get from the OpenAI and Anthropic instrumentations + spans = [ + { + "span_name": "openai.chat.completion", + "span_attributes": { + SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value, + SpanAttributes.LLM_SYSTEM: "OpenAI", + SpanAttributes.LLM_REQUEST_MODEL: "gpt-4", + }, + }, + { + "span_name": "anthropic.messages.create", + "span_attributes": { + SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value, + SpanAttributes.LLM_SYSTEM: "Anthropic", + SpanAttributes.LLM_REQUEST_MODEL: "claude-3-opus-20240229", + }, + }, + ] + + has_llm, llm_names = check_llm_spans(spans) + assert has_llm is True + assert len(llm_names) == 2 + assert "openai.chat.completion" in llm_names + assert "anthropic.messages.create" in llm_names + + +class TestValidateTraceSpans: + """Test the main validation function.""" + + @patch("agentops.validation.get_jwt_token") + @patch("agentops.validation.get_trace_details") + @patch("agentops.validation.get_trace_metrics") + def test_validate_trace_spans_success(self, mock_metrics, mock_details, mock_token): + """Test successful validation.""" + mock_token.return_value = "test-token" + mock_details.return_value = { + "spans": [ + {"span_name": "OpenAI Chat Completion", "span_attributes": {"agentops.span.kind": "llm"}}, + {"span_name": "Other span"}, + ] + } + mock_metrics.return_value = {"total_tokens": 100, "total_cost": "0.0025"} + + result = validate_trace_spans(trace_id="test-trace") + + assert result["trace_id"] == "test-trace" + assert result["span_count"] == 2 + assert result["has_llm_spans"] is True + # LLM activity can be confirmed via metrics or span inspection + assert result["metrics"]["total_tokens"] == 100 + + @patch("agentops.validation.get_jwt_token") + @patch("agentops.validation.get_trace_details") + @patch("agentops.validation.get_trace_metrics") + def test_validate_trace_spans_success_via_metrics(self, mock_metrics, mock_details, mock_token): + """Test successful validation when LLM activity is confirmed via metrics.""" + mock_token.return_value = "test-token" + mock_details.return_value = { + "spans": [ + { + "span_name": "openai.chat.completion", + "span_attributes": {}, # No specific LLM attributes + }, + {"span_name": "Other span"}, + ] + } + # But we have token usage, proving LLM activity + mock_metrics.return_value = {"total_tokens": 1066, "total_cost": "0.0006077"} + + result = validate_trace_spans(trace_id="test-trace") + + assert result["trace_id"] == "test-trace" + assert result["span_count"] == 2 + assert result["has_llm_spans"] is True # Confirmed via metrics + assert result["metrics"]["total_tokens"] == 1066 + + @patch("agentops.validation.get_jwt_token") + @patch("agentops.validation.get_trace_details") + @patch("agentops.validation.get_trace_metrics") + def test_validate_trace_spans_no_llm(self, mock_metrics, mock_details, mock_token): + """Test validation failure when no LLM spans found and no token usage.""" + mock_token.return_value = "test-token" + mock_details.return_value = {"spans": [{"span_name": "database.query"}]} + # No token usage either + mock_metrics.return_value = {"total_tokens": 0, "total_cost": "0.0000"} + + with pytest.raises(ValidationError, match="No LLM activity detected"): + validate_trace_spans(trace_id="test-trace", check_llm=True) + + @patch("agentops.validation.get_jwt_token") + @patch("agentops.validation.get_trace_details") + @patch("agentops.validation.get_trace_metrics") + def test_validate_trace_spans_retry(self, mock_metrics, mock_details, mock_token): + """Test validation with retries.""" + mock_token.return_value = "test-token" + + # First two calls return empty, third returns spans + mock_details.side_effect = [ + {"spans": []}, + {"spans": []}, + {"spans": [{"span_name": "OpenAI Chat Completion", "span_attributes": {"agentops.span.kind": "llm"}}]}, + ] + + # Mock metrics for the successful attempt + mock_metrics.return_value = {"total_tokens": 100, "total_cost": "0.0025"} + + result = validate_trace_spans(trace_id="test-trace", max_retries=3, retry_delay=0.01) + + assert result["span_count"] == 1 + assert mock_details.call_count == 3 + + @patch("opentelemetry.trace.get_current_span") + def test_validate_trace_spans_no_trace_id(self, mock_get_current_span): + """Test validation without trace ID.""" + # Mock get_current_span to return None + mock_get_current_span.return_value = None + + with pytest.raises(ValueError, match="No trace ID found"): + validate_trace_spans() + + @patch("opentelemetry.trace.get_current_span") + @patch("agentops.validation.get_jwt_token") + @patch("agentops.validation.get_trace_details") + @patch("agentops.validation.get_trace_metrics") + def test_validate_trace_spans_from_current_span(self, mock_metrics, mock_details, mock_token, mock_get_span): + """Test extracting trace ID from current span.""" + # Mock the current span + mock_span_context = Mock() + mock_span_context.trace_id = 12345678901234567890 + + mock_span = Mock() + mock_span.get_span_context.return_value = mock_span_context + + mock_get_span.return_value = mock_span + + mock_token.return_value = "test-token" + mock_details.return_value = { + "spans": [{"span_name": "OpenAI Chat Completion", "span_attributes": {"agentops.span.kind": "llm"}}] + } + mock_metrics.return_value = {"total_tokens": 100, "total_cost": "0.0025"} + + result = validate_trace_spans() + assert result["trace_id"] == "0000000000000000ab54a98ceb1f0ad2" # hex format of trace ID + + +class TestPrintValidationSummary: + """Test validation summary printing.""" + + def test_print_validation_summary(self, capsys): + """Test printing validation summary.""" + result = { + "span_count": 3, + "has_llm_spans": True, + "llm_span_names": ["OpenAI Chat", "Claude Message"], + "metrics": {"total_tokens": 150, "prompt_tokens": 100, "completion_tokens": 50, "total_cost": "0.0030"}, + } + + print_validation_summary(result) + + captured = capsys.readouterr() + assert "Found 3 span(s)" in captured.out + assert "OpenAI Chat" in captured.out + assert "Total tokens: 150" in captured.out + assert "Total cost: $0.0030" in captured.out + assert "āœ… Validation successful!" in captured.out + + def test_print_validation_summary_metrics_only(self, capsys): + """Test printing validation summary when LLM activity confirmed via metrics only.""" + result = { + "span_count": 2, + "has_llm_spans": True, + "llm_span_names": [], # No specific LLM span names found + "metrics": { + "total_tokens": 1066, + "prompt_tokens": 800, + "completion_tokens": 266, + "total_cost": "0.0006077", + }, + } + + print_validation_summary(result) + + captured = capsys.readouterr() + assert "Found 2 span(s)" in captured.out + assert "LLM activity confirmed via token usage metrics" in captured.out + assert "Total tokens: 1066" in captured.out + assert "Total cost: $0.0006077" in captured.out + assert "āœ… Validation successful!" in captured.out + + def test_print_validation_summary_llm_prefix(self, capsys): + """Test with spans using llm.* prefix (as returned by API).""" + result = { + "span_count": 1, + "has_llm_spans": True, + "llm_span_names": ["openai.chat.completion"], + "metrics": {"total_tokens": 150, "prompt_tokens": 100, "completion_tokens": 50, "total_cost": "0.0030"}, + } + + print_validation_summary(result) + + captured = capsys.readouterr() + assert "Found 1 span(s)" in captured.out + assert "openai.chat.completion" in captured.out + assert "āœ… Validation successful!" in captured.out + + +class TestCheckLlmSpansWithLlmPrefix: + """Test LLM span checking with llm.* prefix attributes.""" + + def test_check_llm_spans_with_llm_prefix(self): + """Test when spans use llm.request.type instead of gen_ai.request.type.""" + spans = [ + { + "span_name": "openai.chat.completion", + "span_attributes": { + "llm.request.type": "chat", + "llm.system": "OpenAI", + "llm.request.model": "gpt-4", + "llm.usage.total_tokens": 150, + }, + }, + { + "span_name": "anthropic.messages.create", + "span_attributes": { + "llm.request.type": "chat", + "llm.system": "Anthropic", + "llm.request.model": "claude-3-opus", + "llm.usage.total_tokens": 300, + }, + }, + {"span_name": "embedding.create", "span_attributes": {"llm.request.type": "embedding"}}, + {"span_name": "database.query"}, + ] + + has_llm, llm_names = check_llm_spans(spans) + assert has_llm is True + assert len(llm_names) == 2 # Only chat types + assert "openai.chat.completion" in llm_names + assert "anthropic.messages.create" in llm_names