diff --git a/develop-docs/sdk/telemetry/traces/modules/llm-monitoring.mdx b/develop-docs/sdk/telemetry/traces/modules/llm-monitoring.mdx deleted file mode 100644 index b58dfd9df46f2..0000000000000 --- a/develop-docs/sdk/telemetry/traces/modules/llm-monitoring.mdx +++ /dev/null @@ -1,75 +0,0 @@ ---- -title: LLM Monitoring -sidebar_hidden: true ---- - - - -This documentation is deprecated. Please use the [AI Agents Module](/sdk/telemetry/traces/modules/ai-agents) instead for AI/LLM monitoring and instrumentation. - - - -Sentry auto-generates LLM Monitoring data for common providers in Python, but you may need to manually annotate spans for other frameworks. - -## Span conventions - -### Span Operations - -| Span OP | Description | -| :---------------------- | :----------------------------------------------------------------------------------- | -| `ai.pipeline.*` | The top-level span which corresponds to one or more AI operations & helper functions | -| `ai.run.*` | A unit of work - a tool call, LLM execution, or helper method. | -| `ai.chat_completions.*` | A LLM chat operation | -| `ai.embeddings.*` | An LLM embedding creation operation | - -### Span Data - -| Attribute | Type | Description | Examples | Notes | -| --------------------------- | ------- | ----------------------------------------------------- | ---------------------------------------- | ---------------------------------------- | -| `ai.input_messages` | string | The input messages sent to the model | `[{"role": "user", "message": "hello"}]` | | -| `ai.completion_tоkens.used` | int | The number of tokens used to respond to the message | `10` | required for cost calculation | -| `ai.prompt_tоkens.used` | int | The number of tokens used to process just the prompt | `20` | required for cost calculation | -| `ai.total_tоkens.used` | int | The total number of tokens used to process the prompt | `30` | required for charts and cost calculation | -| `ai.model_id` | list | The vendor-specific ID of the model used | `"gpt-4"` | required for cost calculation | -| `ai.streaming` | boolean | Whether the request was streamed back | `true` | | -| `ai.responses` | list | The response messages sent back by the AI model | `["hello", "world"]` | | -| `ai.pipeline.name` | string | The description of the parent ai.pipeline span | `My AI pipeline` | required for charts | - -## Instrumentation - -When a user creates a new AI pipeline, the SDK automatically creates spans that instrument both the pipeline and its AI operations. - -**Example** - -```python -from sentry_sdk.ai.monitoring import ai_track -from openai import OpenAI - -sentry.init(...) - -openai = OpenAI() - -@ai_track("My AI pipeline") -def invoke_pipeline(): - result = openai.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ).choices[0].message.content - - return openai.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": result}] - ).choices[0].message.content - - - -``` - -This should result in the following spans. - -``` - - - - -``` - -Notice that the ai.pipeline.name span of the children spans is the description of the `ai.pipeline.*` span parent. diff --git a/docs/platforms/python/integrations/anthropic/index.mdx b/docs/platforms/python/integrations/anthropic/index.mdx deleted file mode 100644 index 9836159a70d73..0000000000000 --- a/docs/platforms/python/integrations/anthropic/index.mdx +++ /dev/null @@ -1,101 +0,0 @@ ---- -title: Anthropic -description: "Learn about using Sentry for Anthropic." -sidebar_hidden: true ---- - - - -This integration is designed for legacy [LLM Monitoring](/product/insights/ai/llm-monitoring/) and is not currently compatible with [AI Agents Insights](/product/insights/agents/). - -We are working on this, and it will soon be compatible with [AI Agents Insights](/product/insights/agents/). - - - - -This integration connects Sentry with the [Anthropic Python SDK](https://github.com/anthropics/anthropic-sdk-python) and works with Anthropic versions `0.16.0` and above. - -Once you've installed this SDK, you can use Sentry LLM Monitoring, a Sentry dashboard that helps you understand what's going on with your AI pipelines. - -Sentry LLM Monitoring will automatically collect information about prompts, tokens, and models from providers like OpenAI. Learn more about it [here](/product/insights/llm-monitoring). - -## Install - -Install `sentry-sdk` from PyPI with the `anthropic` extra: - -```bash {tabTitle:pip} -pip install "sentry-sdk[anthropic]" -``` -```bash {tabTitle:uv} -uv add "sentry-sdk[anthropic]" -``` - -## Configure - -If you have the `anthropic` package in your dependencies, the Anthropic integration will be enabled automatically when you initialize the Sentry SDK. - - - -## Verify - -Verify that the integration works by creating an AI pipeline. The resulting data should show up in your LLM monitoring dashboard. - -```python -import sentry_sdk -from sentry_sdk.ai.monitoring import ai_track -from anthropic import Anthropic - - -sentry_sdk.init(...) # same as above - -client = Anthropic(api_key="(your Anthropic API key)") - -@ai_track("My AI pipeline") -def my_pipeline(): - with sentry_sdk.start_transaction(op="ai-inference", name="The result of the AI inference"): - print( - client.messages.create( - max_tokens=42, - model="some-model", - messages=[{"role": "system", "content": "Hello, Anthropic!"}] - ) - ) -``` - -After running this script, a pipeline will be created in the [LLM Monitoring](/product/insights/llm-monitoring) section of the Sentry dashboard. - -The pipeline will have an associated Anthropic span for the `messages.create` operation. - -It may take a couple of moments for the data to appear in [sentry.io](https://sentry.io). - -## Behavior - -- The supported modules are currently `chat.messages.create` with `stream=True` and `stream=False`. - -- All exceptions leading to an `AnthropicError` are reported. - -- Sentry considers LLM and tokenizer inputs/outputs as PII and doesn't include PII data by default. If you want to include the data, set `send_default_pii=True` in the `sentry_sdk.init()` call. To explicitly exclude prompts and outputs despite `send_default_pii=True`, configure the integration with `include_prompts=False` as shown in the [Options section](#options) below. - -## Options - -The `AnthropicIntegration` takes an optional `include_prompts` parameter. If set to `False`, prompts are excluded from being sent to Sentry, despite `send_default_pii=True`. - -```python -import sentry_sdk -from sentry_sdk.integrations.anthropic import AnthropicIntegration - -sentry_sdk.init( - # same options as above - send_default_pii=True, - integrations=[ - AnthropicIntegration( - include_prompts=False, # Exclude prompts from being sent to Sentry, despite send_default_pii=True - ), - ], -) -``` - -## Supported Versions - -- Anthropic: 0.16.0+ -- Python: 3.7+ diff --git a/docs/platforms/python/integrations/cohere/index.mdx b/docs/platforms/python/integrations/cohere/index.mdx deleted file mode 100644 index 1070fe31245cf..0000000000000 --- a/docs/platforms/python/integrations/cohere/index.mdx +++ /dev/null @@ -1,100 +0,0 @@ ---- -title: Cohere -description: "Learn about using Sentry for Cohere." -sidebar_hidden: true ---- - - - -This integration is designed for legacy [LLM Monitoring](/product/insights/ai/llm-monitoring/) and is not currently compatible with [AI Agents Insights](/product/insights/agents/). - -We are working on this, and it will soon be compatible with [AI Agents Insights](/product/insights/agents/). - - - - -This integration connects Sentry with the [Cohere Python SDK](https://github.com/cohere-ai/cohere-python). -The integration has been confirmed to work with Cohere 5.3.3. - -Once you've installed this SDK, you can use Sentry LLM Monitoring, a Sentry dashboard that helps you understand what's going on with your AI pipelines. - -Sentry LLM Monitoring will automatically collect information about prompts, tokens, and models from providers like Cohere. Learn more about it [here](/product/insights/llm-monitoring). - -## Install - -Install `sentry-sdk` and `cohere` from PyPI: - -```bash {tabTitle:pip} -pip install "sentry-sdk" "cohere" -``` -```bash {tabTitle:uv} -uv add "sentry-sdk" "cohere" -``` - -## Configure - -If you have the `cohere` package in your dependencies, the integration will be enabled automatically when you initialize the Sentry SDK. - - - -## Verify - -Verify that the integration works by creating an AI pipeline. The resulting data should show up in your LLM monitoring dashboard. - - -```python -import sentry_sdk -from sentry_sdk.ai.monitoring import ai_track -from cohere import Client - -sentry_sdk.init(...) # same as above - -client = Client(api_key="(your Cohere API key)") - -@ai_track("My AI pipeline") -def my_pipeline(): - with sentry_sdk.start_transaction(op="ai-inference", name="The result of the AI inference"): - print( - client.chat( - model="command", message="say hello" - ) - ) -``` - -After running this script, a pipeline will be created in the [LLM Monitoring](/product/insights/llm-monitoring) section of the Sentry dashboard. The pipeline will have an associated Cohere span for the `chat.completions.create` operation. - -It may take a couple of moments for the data to appear in [sentry.io](https://sentry.io). - -## Behavior - -- The Cohere integration will connect Sentry with all supported Cohere methods automatically. - -- All exceptions caused by Cohere methods are reported to Sentry. - -- The supported modules are currently `chat` and `embed`. - -- Sentry considers LLM and tokenizer inputs/outputs as PII and doesn't include PII data by default. If you want to include that data, set `send_default_pii=True` in the `sentry_sdk.init()` call. To explicitly exclude prompts and outputs despite `send_default_pii=True`, configure the integration with `include_prompts=False` as shown in the [Options section](#options) below. - -## Options - -By adding `CohereIntegration` to your `sentry_sdk.init()` call explicitly, you can set options for `CohereIntegration` to change its behavior: - -```python -import sentry_sdk -from sentry_sdk.integrations.cohere import CohereIntegration - -sentry_sdk.init( - # ... - send_default_pii=True, - integrations = [ - CohereIntegration( - include_prompts=False, # LLM/tokenizer inputs/outputs will be not sent to Sentry, despite send_default_pii=True - ), - ], -) -``` - -## Supported Versions - -- Cohere: 5.0+ -- Python: 3.9+ diff --git a/docs/platforms/python/integrations/huggingface_hub/index.mdx b/docs/platforms/python/integrations/huggingface_hub/index.mdx deleted file mode 100644 index 7d57134b97cfc..0000000000000 --- a/docs/platforms/python/integrations/huggingface_hub/index.mdx +++ /dev/null @@ -1,97 +0,0 @@ ---- -title: Huggingface Hub -description: "Learn about using Sentry for Huggingface Hub." -sidebar_hidden: true ---- - - - -This integration is designed for legacy [LLM Monitoring](/product/insights/ai/llm-monitoring/) and is not currently compatible with [AI Agents Insights](/product/insights/agents/). - -We are working on this, and it will soon be compatible with [AI Agents Insights](/product/insights/agents/). - - - -This integration connects Sentry with the [Huggingface Hub Python SDK](https://github.com/huggingface/huggingface_hub) and has been confirmed to work with Huggingface Hub version 0.21.4. - -Once you've installed this SDK, you can use Sentry LLM Monitoring, a Sentry dashboard that helps you understand what's going on with your AI pipelines. - -Sentry LLM Monitoring will automatically collect information about prompts, tokens, and models from providers like OpenAI. Learn more about it [here](/product/insights/llm-monitoring). - -## Install - -Install `sentry-sdk` from PyPI with the `huggingface_hub` extra: - -```bash {tabTitle:pip} -pip install "sentry-sdk[huggingface_hub]" -``` - -```bash {tabTitle:uv} -uv add "sentry-sdk[huggingface_hub]" -``` - -## Configure - -If you have the `huggingface_hub` package in your dependencies, the Huggingface Hub integration will be enabled automatically when you initialize the Sentry SDK. - - - -## Verify - -Verify that the integration works by creating an AI pipeline. The resulting data should show up in your LLM monitoring dashboard. - -```python -import sentry_sdk -from sentry_sdk.ai.monitoring import ai_track -from huggingface_hub import InferenceClient - -sentry_sdk.init(...) # same as above - -client = InferenceClient(token="(your Huggingface Hub API token)", model="HuggingFaceH4/zephyr-7b-beta") - -@ai_track("My AI pipeline") -def my_pipeline(): - with sentry_sdk.start_transaction(op="ai-inference", name="The result of the AI inference"): - print(client.text_generation(prompt="say hello", details=True)) - -``` - -After running this script, a pipeline will be created in the [LLM Monitoring](/product/insights/llm-monitoring) section of the Sentry dashboard. The pipeline will have an associated Huggingface Hub span for the `text_generation` operation. - -It may take a couple of moments for the data to appear in [sentry.io](https://sentry.io). - -## Behavior - -- The Huggingface Hub integration will connect Sentry with all supported Huggingface Hub methods automatically. - -- All exceptions in supported SDK methods are reported to Sentry automatically. - -- Currently, the only supported module is `InferenceClient.text_generation`. - -- Sentry considers LLM and tokenizer inputs/outputs as PII and doesn't include PII data by default. If you want to include the data, set `send_default_pii=True` in the `sentry_sdk.init()` call. To explicitly exclude prompts and outputs despite `send_default_pii=True`, configure the integration with `include_prompts=False` as shown in the [Options section](#options) below. - -## Options - -After adding `HuggingfaceHubIntegration` to your `sentry_sdk.init()` call explicitly, you'll be able to set options to change its behavior: - -```python -import sentry_sdk -from sentry_sdk.integrations.huggingface_hub import HuggingfaceHubIntegration - -sentry_sdk.init( - # ... - # Add data like request headers and IP for users, if applicable; - # see https://docs.sentry.io/platforms/python/data-management/data-collected/ for more info - send_default_pii=True, - integrations=[ - HuggingfaceHubIntegration( - include_prompts=False, # LLM/tokenizer inputs/outputs will be not sent to Sentry, despite send_default_pii=True - ), - ], -) -``` - -## Supported Versions - -- huggingface_hub: 0.21.4+ -- Python: 3.9+ diff --git a/docs/platforms/python/integrations/langchain/index.mdx b/docs/platforms/python/integrations/langchain/index.mdx deleted file mode 100644 index d952fc7943e3b..0000000000000 --- a/docs/platforms/python/integrations/langchain/index.mdx +++ /dev/null @@ -1,146 +0,0 @@ ---- -title: Langchain -description: "Learn about using Sentry for Langchain." -sidebar_hidden: true ---- - - - -This integration is designed for legacy [LLM Monitoring](/product/insights/ai/llm-monitoring/) and is not currently compatible with [AI Agents Insights](/product/insights/agents/). - -We are working on this, and it will soon be compatible with [AI Agents Insights](/product/insights/agents/). - - - - -This integration connects Sentry with [Langchain](https://github.com/langchain-ai/langchain). -The integration has been confirmed to work with Langchain 0.1.11. - -## Install - -Install `sentry-sdk` from PyPI and the appropriate langchain packages: - -```bash {tabTitle:pip} -pip install "sentry-sdk" "langchain-openai" "langchain-core" -``` -```bash {tabTitle:uv} -uv add "sentry-sdk" "langchain-openai" "langchain-core" -``` - -## Configure - -If you have the `langchain` package in your dependencies, the Langchain integration will be enabled automatically when you initialize the Sentry SDK. - -An additional dependency, `tiktoken`, is required to be installed if you want to calculate token usage for streaming chat responses. - - -In addition to capturing errors, you can monitor interactions between multiple services or applications by [enabling tracing](/concepts/key-terms/tracing/). You can also collect and analyze performance profiles from real users with [profiling](/product/explore/profiling/). - -Select which Sentry features you'd like to install in addition to Error Monitoring to get the corresponding installation and configuration instructions below. - - - -```python -import sentry_sdk - -sentry_sdk.init( - dsn="___PUBLIC_DSN___", - # Send personally-identifiable information like LLM responses to Sentry; - # see https://docs.sentry.io/platforms/python/data-management/data-collected/ for more info - send_default_pii=True, - # ___PRODUCT_OPTION_START___ performance - # Set traces_sample_rate to 1.0 to capture 100% - # of transactions for performance monitoring. - traces_sample_rate=1.0, - # ___PRODUCT_OPTION_END___ performance - # ___PRODUCT_OPTION_START___ profiling - # To collect profiles for all profile sessions, - # set `profile_session_sample_rate` to 1.0. - profile_session_sample_rate=1.0, - # Profiles will be automatically collected while - # there is an active span. - profile_lifecycle="trace", - # ___PRODUCT_OPTION_END___ profiling -) -``` - -## Verify - -Verify that the integration works by inducing an error: - -```python -from langchain_openai import ChatOpenAI -import sentry_sdk - -sentry_sdk.init(...) # same as above - -llm = ChatOpenAI(model="gpt-3.5-turbo-0125", temperature=0, api_key="bad API key") -with sentry_sdk.start_transaction(op="ai-inference", name="The result of the AI inference"): - response = llm.invoke([("system", "What is the capital of paris?")]) - print(response) -``` - -After running this script, a transaction will be created in the Performance section of [sentry.io](https://sentry.io). Additionally, an error event (about the bad API key) will be sent to [sentry.io](https://sentry.io) and will be connected to the transaction. - -It may take a couple of moments for the data to appear in [sentry.io](https://sentry.io). - -## Behavior - -- The Langchain integration will connect Sentry with Langchain and automatically monitor all LLM, tool, and function calls. - -- All exceptions in the execution of the chain are reported. - -- Sentry considers LLM and tokenizer inputs/outputs as PII and, by default, does not include PII data. If you want to include the data, set `send_default_pii=True` in the `sentry_sdk.init()` call. To explicitly exclude prompts and outputs despite `send_default_pii=True`, configure the integration with `include_prompts=False` as shown in the [Options section](#options) below. - -## Options - -By adding `LangchainIntegration` to your `sentry_sdk.init()` call explicitly, you can set options for `LangchainIntegration` to change its behavior: - -```python -import sentry_sdk -from sentry_sdk.integrations.langchain import LangchainIntegration - -sentry_sdk.init( - # ... - integrations=[ - LangchainIntegration( - include_prompts=False, # LLM/tokenizer inputs/outputs will be not sent to Sentry, despite send_default_pii=True - max_spans=500, - tiktoken_encoding_name="cl100k_base", - ), - ], -) -``` - -You can pass the following keyword arguments to `LangchainIntegration()`: - -- `include_prompts`: - - Whether LLM and tokenizer inputs and outputs should be sent to Sentry. Sentry considers this data personal identifiable data (PII) by default. If you want to include the data, set `send_default_pii=True` in the `sentry_sdk.init()` call. To explicitly exclude prompts and outputs despite `send_default_pii=True`, configure the integration with `include_prompts=False`. - - The default is `True`. - -- `max_spans`: - - The most number of spans (e.g., LLM calls) that can be processed at the same time. - - The default is `1024`. - -- `tiktoken_encoding_name`: - - If you want to calculate token usage for streaming chat responses you need to have an additional dependency, [tiktoken](https://pypi.org/project/tiktoken/) installed and specify the `tiktoken_encoding_name` that you use for tokenization. See the [OpenAI Cookbook](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for possible values. - - The default is `None`. - - -## Supported Versions - -- Langchain: 0.1.11+ -- tiktoken: 0.6.0+ -- Python: 3.9+ diff --git a/docs/platforms/python/integrations/openai/index.mdx b/docs/platforms/python/integrations/openai/index.mdx deleted file mode 100644 index 907fee87e15ab..0000000000000 --- a/docs/platforms/python/integrations/openai/index.mdx +++ /dev/null @@ -1,121 +0,0 @@ ---- -title: OpenAI -description: "Learn about using Sentry for OpenAI." -sidebar_hidden: true ---- - - - -This integration is designed for legacy [LLM Monitoring](/product/insights/ai/llm-monitoring/) and is not currently compatible with [AI Agents Insights](/product/insights/agents/). - -We are working on this, and it will soon be compatible with [AI Agents Insights](/product/insights/agents/). - - - -This integration connects Sentry with the [OpenAI Python SDK](https://github.com/openai/openai-python). -The integration has been confirmed to work with OpenAI 1.13.3. - -Once you've installed this SDK, you can use Sentry LLM Monitoring, a Sentry dashboard that helps you understand what's going on with your AI pipelines. - -Sentry LLM Monitoring will automatically collect information about prompts, tokens, and models from providers like OpenAI. Learn more about it [here](/product/insights/llm-monitoring). - -## Install - -Install `sentry-sdk` from PyPI with the `openai` extra: - -```bash {tabTitle:pip} -pip install "sentry-sdk[openai]" -``` - -```bash {tabTitle:uv} -uv add "sentry-sdk[openai]" -``` - -## Configure - -If you have the `openai` package in your dependencies, the OpenAI integration will be enabled automatically when you initialize the Sentry SDK. - -An additional dependency, `tiktoken`, is required if you want to calculate token usage for streaming chat responses. - - - -## Verify - -Verify that the integration works by creating an AI pipeline. The resulting data should show up in your LLM monitoring dashboard. - -```python -import sentry_sdk -from sentry_sdk.ai.monitoring import ai_track -from openai import OpenAI - -sentry_sdk.init(...) # same as above - -client = OpenAI(api_key="(your OpenAI key)") - -@ai_track("My AI pipeline") -def my_pipeline(): - with sentry_sdk.start_transaction(op="ai-inference", name="The result of the AI inference"): - print( - client.chat.completions.create( - model="gpt-3.5", messages=[{"role": "system", "content": "say hello"}] - ) - .choices[0] - .message.content - ) -``` - -After running this script, a pipeline will be created in the [LLM Monitoring](/product/insights/llm-monitoring) section of the Sentry dashboard. The pipeline will have an associated OpenAI span for the `chat.completions.create` operation. - -It may take a couple of moments for the data to appear in [sentry.io](https://sentry.io). - -## Behavior - -- The OpenAI integration will connect Sentry with all supported OpenAI methods automatically. - -- All exceptions leading to an OpenAIException are reported. - -- The supported modules are currently `chat.completions.create` and `embeddings.create`. - -- Sentry considers LLM and tokenizer inputs/outputs as PII and doesn't include PII data by default. If you want to include the data, set `send_default_pii=True` in the `sentry_sdk.init()` call. To explicitly exclude prompts and outputs despite `send_default_pii=True`, configure the integration with `include_prompts=False` as shown in the [Options section](#options) below. - -## Options - -By adding `OpenAIIntegration` to your `sentry_sdk.init()` call explicitly, you can set options for `OpenAIIntegration` to change its behavior: - -```python -import sentry_sdk -from sentry_sdk.integrations.openai import OpenAIIntegration - -sentry_sdk.init( - # ... - # Add data like inputs and responses; - # see https://docs.sentry.io/platforms/python/data-management/data-collected/ for more info - send_default_pii=True, - integrations=[ - OpenAIIntegration( - include_prompts=False, # LLM/tokenizer inputs/outputs will be not sent to Sentry, despite send_default_pii=True - tiktoken_encoding_name="cl100k_base", - ), - ], -) -``` - -You can pass the following keyword arguments to `OpenAIIntegration()`: - -- `include_prompts`: - - Whether LLM and tokenizer inputs and outputs should be sent to Sentry. Sentry considers this data personal identifiable data (PII) by default. If you want to include the data, set `send_default_pii=True` in the `sentry_sdk.init()` call. To explicitly exclude prompts and outputs despite `send_default_pii=True`, configure the integration with `include_prompts=False`. - - The default is `True`. - -- `tiktoken_encoding_name`: - - If you want to calculate token usage for streaming chat responses you need to have an additional dependency, [tiktoken](https://pypi.org/project/tiktoken/) installed and specify the `tiktoken_encoding_name` that you use for tokenization. See the [OpenAI Cookbook](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for possible values. - - The default is `None`. - -## Supported Versions - -- OpenAI: 1.0+ -- tiktoken: 0.6.0+ -- Python: 3.9+ diff --git a/docs/product/index.mdx b/docs/product/index.mdx index e5e358dcaba33..a130534c1a0bb 100644 --- a/docs/product/index.mdx +++ b/docs/product/index.mdx @@ -44,9 +44,9 @@ Giving Sentry visibility into your [**releases**](/product/releases/) makes it p Releases are integrated with the rest of Sentry so you can directly see how an error or performance issue was affected by a release, in addition to being able to resolve release-specific issues. -### LLM Monitoring +### AI Agents Monitoring -Our [**LLM Monitoring**](/product/insights/llm-monitoring/) feature gives you insights into your AI pipelines within the broader context of your app. When you `pip install sentry` into a project that's also using an AI provider like OpenAI, Sentry will automatically pick up useful metrics like token usage, prompts, and model IDs, and send them to our LLM Monitoring dashboard. +Our [**AI Agents Monitoring**](/product/insights/agents/) feature gives you insights into your AI agent workflows within the broader context of your app. When you `pip install sentry` into a project using AI agents, Sentry will automatically pick up useful metrics like agent invocations, tool executions, handoffs between agents, and token usage, sending them to our AI Agents Insights dashboard. ### Uptime Monitoring diff --git a/docs/product/insights/agents/dashboard.mdx b/docs/product/insights/agents/dashboard.mdx index 1d6ec9635c227..b50178f83bec9 100644 --- a/docs/product/insights/agents/dashboard.mdx +++ b/docs/product/insights/agents/dashboard.mdx @@ -61,7 +61,3 @@ This detailed view reveals: - **Error Context**: Detailed information about any failures or issues When your AI agents are part of larger applications (like web servers or APIs), the trace view will include context from other Sentry integrations, giving you a complete picture of how your agents fit into your overall application architecture. - - - - diff --git a/docs/product/insights/agents/index.mdx b/docs/product/insights/agents/index.mdx index ce4d3f46154b7..0fd5de912960f 100644 --- a/docs/product/insights/agents/index.mdx +++ b/docs/product/insights/agents/index.mdx @@ -4,7 +4,7 @@ sidebar_order: 40 description: "Learn how to use Sentry's AI Agent monitoring tools to trace and debug your AI agent workflows, including agent runs, tool calls, and model interactions." --- - + Sentry's AI Agent monitoring tools help you understand what's going on with your AI agent workflows. They automatically collect information about agent runs, tool calls, model interactions, and errors across your entire AI pipeline—from user interaction to final response. diff --git a/docs/product/insights/ai/llm-monitoring/getting-started/index.mdx b/docs/product/insights/ai/llm-monitoring/getting-started/index.mdx deleted file mode 100644 index 745a1ed05f0a3..0000000000000 --- a/docs/product/insights/ai/llm-monitoring/getting-started/index.mdx +++ /dev/null @@ -1,213 +0,0 @@ ---- -title: Set Up -sidebar_order: 0 -description: "Learn how to set up Sentry LLM Monitoring" ---- - -Sentry LLM Monitoring helps you track and debug AI-powered applications using our supported SDKs and integrations. - -![LLM Monitoring User Interface](../img/pipelines-view.png) - -To start sending LLM data to Sentry, make sure you've created a Sentry project for your AI-enabled repository and follow one of the guides below: - -## Supported SDKs - -### Python - -The Sentry Python SDK supports LLM monitoring with integrations for OpenAI, Langchain, Anthropic, Huggingface, and Cohere - -#### Official AI Integrations - -- -- -- -- -- - -### JavaScript - -The JavaScript SDK supports LLM monitoring through the Vercel AI integration for Node.js and Bun runtimes. - -#### Supported Platforms - -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- - - - -We'll be adding AI integrations continuously. You can also instrument AI manually with the Sentry Python SDK. - - - -## Pipelines and LLMs - -The Sentry LLM Monitoring feature relies on the fact that you have an orchestrator (like LangChain) creating pipelines of one or more LLMs (such as gpt-4). In the LLM Monitoring dashboard, we show you a table of the AI pipelines and pull the token usage from your LLMs. - -If you're using a provider like OpenAI without an orchestrator like LangChain, you'll need to manually create pipelines with the `@ai_track` annotation. If you're using a non-supported LLM provider and want to record token usage, use the `record_token_usage()` helper function. Both manual helpers are documented below. - -### Python SDK Decorators - -The [Python SDK](/platforms/python) includes an `@ai_track` decorator which will mark functions as AI-related and -cause them to show up in the LLM Monitoring dashboard. - -```python - -import time -from sentry_sdk.ai.monitoring import ai_track, record_token_usage -import sentry_sdk -import requests - -@ai_track("AI tool") -def some_workload_function(**kwargs): - """ - This function is an example of calling arbitrary code with @ai_track so that it shows up in the Sentry trace - """ - time.sleep(5) - -@ai_track("LLM") -def some_llm_call(): - """ - This function is an example of calling an LLM provider that isn't officially supported by Sentry. - """ - with sentry_sdk.start_span(op="ai.chat_completions.create.examplecom", name="Example.com LLM") as span: - result = requests.get('https://example.com/api/llm-chat?question=say+hello').json() - # this annotates the tokens used by the LLM so that they show up in the graphs in the dashboard - record_token_usage(span, total_tokens=result["usage"]["total_tokens"]) - return result["text"] - -@ai_track("My AI pipeline") -def some_pipeline(): - """ - The topmost level function with @ai_track gets the operation "ai.pipeline", which makes it show up - in the table of AI pipelines in the Sentry LLM Monitoring dashboard. - """ - client = OpenAI() - # Data can be passed to the @ai_track annotation to include metadata - some_workload_function(sentry_tags={"username": "my_user"}, sentry_data={"data": "some longer data that provides context"}) - some_llm_call() - response = ( - client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "say hello"}] - ) - .choices[0] - .message.content - ) - print(response) - -with sentry_sdk.start_transaction(op="ai-inference", name="The result of the AI inference"): - some_pipeline() - -``` diff --git a/docs/product/insights/ai/llm-monitoring/getting-started/the-dashboard.mdx b/docs/product/insights/ai/llm-monitoring/getting-started/the-dashboard.mdx deleted file mode 100644 index 292314a8c2680..0000000000000 --- a/docs/product/insights/ai/llm-monitoring/getting-started/the-dashboard.mdx +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: LLM Monitoring Dashboard -sidebar_order: 10 -description: "Learn how to use Sentry's LLM Monitoring Dashboard." ---- - - - - -Once you've [configured the Sentry SDK](/product/insights/llm-monitoring/getting-started/) for your AI project, you'll start -receiving data in the Sentry LLM Monitoring dashboard. - -![LLM Monitoring Dashboard](../img/pipelines-view.png) - - -## The Per-pipeline Dashboard -In the example below, there are two LangChain pipelines (whose `.name` is the name that shows up in the table). -One has used 58,000 tokens in the past hour, and the other has used 44,900 tokens. When you click one of the pipelines in the table, you can see details about that particular pipeline. - -![LLM Monitoring for a specific pipeline](../img/details-view.png) - -As you can see in the example above, the "Ask Sentry" pipeline has used 59 thousand tokens and taken 3.2 seconds on average. - - -Creating an AI pipeline is different than calling an LLM. If you're creating AI pipelines by calling LLMs directly (without using a tool like LangChain), consider using [manual AI instrumentation](/product/insights/llm-monitoring/getting-started/#manually-instrumenting-ai-workloads). - - -## Where AI Data Shows Up In the Trace View - -If configured to include PII, the Sentry SDK will add prompts and responses to LLMs and other AI models to spans in the trace view. - -![LLM Monitoring trace example](../img/trace-view.png) - -In the example above, you can see input messages and LLM responses related to the `ai.chat_completions.create.langchain` span. Other spans like `ai.chat_completions.create.openai` show the number of tokens used for that particular chat completion. - -This view can show other data as well. For example, if you call your LLM from a webserver, the trace will include details about the webserver through other integrations, and you'll get a holistic view of all the related parts. diff --git a/docs/product/insights/ai/llm-monitoring/img/details-view.png b/docs/product/insights/ai/llm-monitoring/img/details-view.png deleted file mode 100644 index e79f47179dbee..0000000000000 Binary files a/docs/product/insights/ai/llm-monitoring/img/details-view.png and /dev/null differ diff --git a/docs/product/insights/ai/llm-monitoring/img/pipelines-view.png b/docs/product/insights/ai/llm-monitoring/img/pipelines-view.png deleted file mode 100644 index 8f3132dfa978f..0000000000000 Binary files a/docs/product/insights/ai/llm-monitoring/img/pipelines-view.png and /dev/null differ diff --git a/docs/product/insights/ai/llm-monitoring/img/trace-view.png b/docs/product/insights/ai/llm-monitoring/img/trace-view.png deleted file mode 100644 index 0b0f82366c982..0000000000000 Binary files a/docs/product/insights/ai/llm-monitoring/img/trace-view.png and /dev/null differ diff --git a/docs/product/insights/ai/llm-monitoring/index.mdx b/docs/product/insights/ai/llm-monitoring/index.mdx deleted file mode 100644 index c52b4c39c9703..0000000000000 --- a/docs/product/insights/ai/llm-monitoring/index.mdx +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: "LLM Monitoring" -sidebar_order: 0 -description: "Sentry LLM monitoring helps you understand your LLM calls." -sidebar_hidden: true ---- - - - -This Insights module is now considered deprecated and will be faded out in the future. Please use [AI Agents Insights](/product/insights/agents/) instead. - - - - - -Sentry's LLM Monitoring tools help you understand what's going on with your AI pipelines. They automatically collect information about prompts, tokens, and models from providers like OpenAI and Anthropic. - -## Example LLM Monitoring Use Cases - -- Users are reporting issues with an LLM workflow, and you want to investigate responses from the relevant large language models. -- You'd like to receive an alert if a specific pipeline costs more than $100 in a single day. -- Users report that LLM workflows are taking longer than usual, and you want to understand what steps in a workflow are slowest. - -## Get Started - -To use LLM Monitoring, you must have an existing Sentry account and project set up. If you don't have one, [create an account here](https://sentry.io/signup/). - -![LLM Monitoring User Interface](./img/details-view.png) - -Learn how to [set up Sentry's LLM Monitoring](/product/insights/llm-monitoring/getting-started/). diff --git a/includes/feature-stage-beta-ai-agents.mdx b/includes/feature-stage-beta-ai-agents.mdx deleted file mode 100644 index beac15c44728a..0000000000000 --- a/includes/feature-stage-beta-ai-agents.mdx +++ /dev/null @@ -1,5 +0,0 @@ - - -This feature is currently in Beta. If you are looking for old LLM Monitoring insights, see [LLM Monitoring](/product/insights/ai/). - - diff --git a/redirects.js b/redirects.js index 9146ef9302708..64d12308c77ab 100644 --- a/redirects.js +++ b/redirects.js @@ -967,16 +967,8 @@ const userDocsRedirects = [ destination: '/product/insights/backend/queue-monitoring/queues-page/', }, { - source: '/product/insights/llm-monitoring/', - destination: '/product/insights/ai/llm-monitoring/', - }, - { - source: '/product/insights/llm-monitoring/getting-started/', - destination: '/product/insights/ai/llm-monitoring/getting-started/', - }, - { - source: '/product/insights/llm-monitoring/getting-started/the-dashboard/', - destination: '/product/insights/ai/llm-monitoring/getting-started/the-dashboard/', + source: '/product/insights/llm-monitoring/:path*', + destination: '/product/insights/ai/:path*', }, { source: '/product/insights/retention-priorities/', diff --git a/src/middleware.ts b/src/middleware.ts index bb859df45429f..7d6eec70b247b 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -3124,27 +3124,19 @@ const USER_DOCS_REDIRECTS: Redirect[] = [ }, { from: '/product/ai-monitoring/', - to: '/product/insights/llm-monitoring/', + to: '/product/insights/agents/', }, { - from: '/product/ai-monitoring/getting-started/', - to: '/product/insights/llm-monitoring/getting-started/', + from: '/product/insights/llm-monitoring/', + to: '/product/insights/agents/', }, { - from: '/product/ai-monitoring/getting-started/the-dashboard/', - to: '/product/insights/llm-monitoring/getting-started/the-dashboard/', + from: '/product/insights/llm-monitoring/getting-started/', + to: '/product/insights/agents/getting-started/', }, { - from: '/product/llm-monitoring/', - to: '/product/insights/llm-monitoring/', - }, - { - from: '/product/llm-monitoring/getting-started/', - to: '/product/insights/llm-monitoring/getting-started/', - }, - { - from: '/product/llm-monitoring/getting-started/the-dashboard/', - to: '/product/insights/llm-monitoring/getting-started/the-dashboard/', + from: '/product/insights/llm-monitoring/getting-started/the-dashboard/', + to: '/product/insights/agents/getting-started/the-dashboard/', }, { from: '/product/metrics/',