ref(llm-detection): Send enhanced span data to support improved Seer analysis (#103871)

Extends span data sent to Seer with additional fields needed for more
accurate LLM issue detection.

This provides the telemetry improvements requested through
`missing_telemetry` feedback and supports stricter detection controls in
getsentry/seer#4096.

Changes:
- Add `EvidenceSpan` model (matches the model defined in Seer)
- create new method for getting the trace for a transaction
- Adds additional columns to `selected_columns` when searching for
traces

---------

Co-authored-by: Richard Roggenkemper <[email protected]>

Author: nora-shap

src/sentry/seer/explorer/index_data.py

@@ -195,7 +195,7 @@ def get_trace_for_transaction(transaction_name: str, project_id: int) -> TraceDa
         sampling_mode="NORMAL",
     )
 
-    # Step 4: Build span objects
+    # Step 3: Build span objects
     spans = []
     for row in spans_result.get("data", []):
         span_id = row.get("span_id")

src/sentry/seer/sentry_data_models.py

@@ -22,6 +22,16 @@ class Span(BaseModel):
     span_description: str | None
 
 
+class EvidenceSpan(BaseModel):
+    span_id: str | None = None
+    parent_span_id: str | None = None
+    timestamp: float | None = None
+    op: str | None = None
+    description: str | None = None
+    exclusive_time: float | None = None  # duration in milliseconds
+    data: dict[str, Any] | None = None
+
+
 class TraceData(BaseModel):
     trace_id: str
     project_id: int
@@ -30,6 +40,14 @@ class TraceData(BaseModel):
     spans: list[Span]
 
 
+class EvidenceTraceData(BaseModel):
+    trace_id: str
+    project_id: int
+    transaction_name: str
+    total_spans: int
+    spans: list[EvidenceSpan]
+
+
 class EAPTrace(BaseModel):
     """
     Based on the Seer model. `trace` can contain both span and error events (see `SerializedEvent`).

src/sentry/tasks/llm_issue_detection/__init__.py

@@ -0,0 +1,13 @@
+from sentry.tasks.llm_issue_detection.detection import (
+    DetectedIssue,
+    create_issue_occurrence_from_detection,
+    detect_llm_issues_for_project,
+    run_llm_issue_detection,
+)
+
+__all__ = [
+    "DetectedIssue",
+    "create_issue_occurrence_from_detection",
+    "detect_llm_issues_for_project",
+    "run_llm_issue_detection",
+]

src/sentry/tasks/llm_issue_detection.py renamed to src/sentry/tasks/llm_issue_detection/detection.py

@@ -15,11 +15,12 @@
 from sentry.issues.producer import PayloadType, produce_occurrence_to_kafka
 from sentry.models.project import Project
 from sentry.net.http import connection_from_url
-from sentry.seer.explorer.index_data import get_trace_for_transaction, get_transactions_for_project
+from sentry.seer.explorer.index_data import get_transactions_for_project
 from sentry.seer.models import SeerApiError
-from sentry.seer.sentry_data_models import TraceData
+from sentry.seer.sentry_data_models import EvidenceTraceData
 from sentry.seer.signed_seer_api import make_signed_seer_api_request
 from sentry.tasks.base import instrumented_task
+from sentry.tasks.llm_issue_detection.trace_data import get_evidence_trace_for_llm_detection
 from sentry.taskworker.namespaces import issues_tasks
 from sentry.utils import json
 
@@ -73,7 +74,7 @@ def __init__(
 
 def create_issue_occurrence_from_detection(
     detected_issue: DetectedIssue,
-    trace: TraceData,
+    trace: EvidenceTraceData,
     project_id: int,
     transaction_name: str,
 ) -> None:
@@ -181,6 +182,7 @@ def detect_llm_issues_for_project(project_id: int) -> None:
     """
     project = Project.objects.get_from_cache(id=project_id)
     organization = project.organization
+    organization_id = organization.id
 
     has_access = features.has("organizations:gen-ai-features", organization) and not bool(
         organization.get_option("sentry:hide_ai_features")
@@ -203,9 +205,8 @@ def detect_llm_issues_for_project(project_id: int) -> None:
             break
 
         try:
-            trace: TraceData | None = get_trace_for_transaction(
-                transaction.name, transaction.project_id
-            )
+            trace = get_evidence_trace_for_llm_detection(transaction.name, transaction.project_id)
+
             if (
                 not trace
                 or trace.total_spans < LOWER_SPAN_LIMIT
@@ -226,7 +227,7 @@ def detect_llm_issues_for_project(project_id: int) -> None:
 
             seer_request = {
                 "telemetry": [{**trace.dict(), "kind": "trace"}],
-                "organization_id": organization.id,
+                "organization_id": organization_id,
                 "project_id": project_id,
             }
             response = make_signed_seer_api_request(

src/sentry/tasks/llm_issue_detection/trace_data.py

@@ -0,0 +1,142 @@
+"""
+Functions for fetching trace data optimized for LLM issue detection.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from datetime import UTC, datetime, timedelta
+
+from sentry.models.project import Project
+from sentry.search.eap.types import SearchResolverConfig
+from sentry.search.events.types import SnubaParams
+from sentry.seer.sentry_data_models import EvidenceSpan, EvidenceTraceData
+from sentry.snuba.referrer import Referrer
+from sentry.snuba.spans_rpc import Spans
+
+logger = logging.getLogger(__name__)
+
+# Regex to match unescaped quotes (not preceded by backslash)
+UNESCAPED_QUOTE_RE = re.compile('(?<!\\\\)"')
+
+
+def get_evidence_trace_for_llm_detection(
+    transaction_name: str, project_id: int
+) -> EvidenceTraceData | None:
+    """
+    Get trace data with performance metrics for LLM issue detection.
+
+    Args:
+        transaction_name: The name of the transaction to find traces for
+        project_id: The ID of the project
+
+    Returns:
+        EvidenceTraceData with spans including performance metrics, or None if no traces found
+    """
+    try:
+        project = Project.objects.get(id=project_id)
+    except Project.DoesNotExist:
+        logger.exception(
+            "Project does not exist; cannot fetch traces for LLM detection",
+            extra={"project_id": project_id, "transaction_name": transaction_name},
+        )
+        return None
+
+    end_time = datetime.now(UTC)
+    start_time = end_time - timedelta(hours=24)
+
+    snuba_params = SnubaParams(
+        start=start_time,
+        end=end_time,
+        projects=[project],
+        organization=project.organization,
+    )
+    config = SearchResolverConfig(
+        auto_fields=True,
+    )
+
+    escaped_transaction_name = UNESCAPED_QUOTE_RE.sub('\\"', transaction_name)
+    traces_result = Spans.run_table_query(
+        params=snuba_params,
+        query_string=f'transaction:"{escaped_transaction_name}" project.id:{project_id}',
+        selected_columns=[
+            "trace",
+            "precise.start_ts",
+        ],
+        orderby=["precise.start_ts"],
+        offset=0,
+        limit=1,
+        referrer=Referrer.SEER_RPC,
+        config=config,
+        sampling_mode="NORMAL",
+    )
+
+    trace_id = None
+    for row in traces_result.get("data", []):
+        trace_id = row.get("trace")
+        if trace_id:
+            break
+
+    if not trace_id:
+        logger.info(
+            "No traces found for transaction (LLM detection)",
+            extra={"transaction_name": transaction_name, "project_id": project_id},
+        )
+        return None
+
+    spans_result = Spans.run_table_query(
+        params=snuba_params,
+        query_string=f"trace:{trace_id}",
+        selected_columns=[
+            "span_id",
+            "parent_span",
+            "span.op",
+            "span.description",
+            "precise.start_ts",
+            "span.self_time",
+            "span.duration",
+            "span.status",
+        ],
+        orderby=["precise.start_ts"],
+        offset=0,
+        limit=1000,
+        referrer=Referrer.SEER_RPC,
+        config=config,
+        sampling_mode="NORMAL",
+    )
+
+    evidence_spans: list[EvidenceSpan] = []
+    for row in spans_result.get("data", []):
+        span_id = row.get("span_id")
+        parent_span_id = row.get("parent_span")
+        span_op = row.get("span.op")
+        span_description = row.get("span.description")
+        span_exclusive_time = row.get("span.self_time")
+        span_duration = row.get("span.duration")
+        span_status = row.get("span.status")
+        span_timestamp = row.get("precise.start_ts")
+
+        if span_id:
+            evidence_spans.append(
+                EvidenceSpan(
+                    span_id=span_id,
+                    parent_span_id=parent_span_id,
+                    op=span_op,
+                    description=span_description or "",
+                    exclusive_time=span_exclusive_time,
+                    timestamp=span_timestamp,
+                    data={
+                        "duration": span_duration,
+                        "status": span_status,
+                    },
+                )
+            )
+
+    return EvidenceTraceData(
+        trace_id=trace_id,
+        project_id=project_id,
+        transaction_name=transaction_name,
+        total_spans=len(evidence_spans),
+        spans=evidence_spans,
+    )