feat(feedback): label generation at ingest, stored as tags (#96390)

vishnupsatish · web-flow · commit d26758e4cff0 · 2025-07-29T10:08:32.000-07:00
diff --git a/src/sentry/feedback/usecases/ingest/create_feedback.py b/src/sentry/feedback/usecases/ingest/create_feedback.py
@@ -8,9 +8,14 @@
 
 import jsonschema
 
-from sentry import options
+from sentry import features, options
 from sentry.constants import DataCategory
 from sentry.feedback.lib.utils import UNREAL_FEEDBACK_UNATTENDED_MESSAGE, FeedbackCreationSource
+from sentry.feedback.usecases.label_generation import (
+    AI_LABEL_TAG_PREFIX,
+    MAX_AI_LABELS,
+    generate_labels,
+)
 from sentry.feedback.usecases.spam_detection import is_spam, spam_detection_enabled
 from sentry.issues.grouptype import FeedbackGroup
 from sentry.issues.issue_occurrence import IssueEvidence, IssueOccurrence
@@ -346,6 +351,30 @@ def create_feedback_issue(
         }
     )
 
+    # Generating labels using Seer, which will later be used to categorize feedbacks
+    if (
+        not is_message_spam
+        and features.has("organizations:user-feedback-ai-categorization", project.organization)
+        and features.has("organizations:gen-ai-features", project.organization)
+    ):
+        try:
+            labels = generate_labels(feedback_message, project.organization_id)
+            if len(labels) > MAX_AI_LABELS:
+                logger.info(
+                    "Feedback message has more than the maximum allowed labels.",
+                    extra={
+                        "project_id": project.id,
+                        "entrypoint": "create_feedback_issue",
+                        "feedback_message": feedback_message[:100],
+                    },
+                )
+                labels = labels[:MAX_AI_LABELS]
+
+            for idx, label in enumerate(labels):
+                event_fixed["tags"][f"{AI_LABEL_TAG_PREFIX}.{idx}"] = label
+        except Exception:
+            logger.exception("Error generating labels", extra={"project_id": project.id})
+
     # Set the user.email tag since we want to be able to display user.email on the feedback UI as a tag
     # as well as be able to write alert conditions on it
     user_email = get_path(event_fixed, "user", "email")
diff --git a/src/sentry/feedback/usecases/label_generation.py b/src/sentry/feedback/usecases/label_generation.py
@@ -0,0 +1,70 @@
+import logging
+from typing import TypedDict
+
+import requests
+from django.conf import settings
+
+from sentry.seer.signed_seer_api import sign_with_seer_secret
+from sentry.utils import json, metrics
+
+logger = logging.getLogger(__name__)
+
+
+class LabelRequest(TypedDict):
+    """Corresponds to GenerateFeedbackLabelsRequest in Seer."""
+
+    organization_id: int
+    feedback_message: str
+
+
+AI_LABEL_TAG_PREFIX = "ai_categorization.label"
+# If Seer generates more labels, we truncate it to this many labels
+MAX_AI_LABELS = 15
+
+SEER_GENERATE_LABELS_URL = f"{settings.SEER_AUTOFIX_URL}/v1/automation/summarize/feedback/labels"
+
+
+@metrics.wraps("feedback.generate_labels", sample_rate=1.0)
+def generate_labels(feedback_message: str, organization_id: int) -> list[str]:
+    """
+    Generate labels for a feedback message.
+
+    The possible errors this can throw are:
+    - request.exceptions.Timeout, request.exceptions.ConnectionError, etc. while making the request
+    - request.exceptions.HTTPError (for raise_for_status)
+    - requests.exceptions.JSONDecodeError or another decode error if the response is not valid JSON
+    - KeyError / ValueError if the response JSON doesn't have the expected structure
+    """
+    request = LabelRequest(
+        organization_id=organization_id,
+        feedback_message=feedback_message,
+    )
+
+    serialized_request = json.dumps(request)
+
+    response = requests.post(
+        SEER_GENERATE_LABELS_URL,
+        data=serialized_request,
+        headers={
+            "content-type": "application/json;charset=utf-8",
+            **sign_with_seer_secret(serialized_request.encode()),
+        },
+        timeout=10,
+    )
+
+    if response.status_code != 200:
+        logger.error(
+            "Failed to generate labels",
+            extra={
+                "status_code": response.status_code,
+                "response": response.text,
+                "content": response.content,
+            },
+        )
+
+    response.raise_for_status()
+
+    labels = response.json()["data"]["labels"]
+
+    # Guaranteed to be a list of strings (validated in Seer)
+    return labels
diff --git a/tests/sentry/feedback/usecases/ingest/test_create_feedback.py b/tests/sentry/feedback/usecases/ingest/test_create_feedback.py
@@ -13,6 +13,7 @@
     get_feedback_title,
     validate_issue_platform_event_schema,
 )
+from sentry.feedback.usecases.label_generation import AI_LABEL_TAG_PREFIX, MAX_AI_LABELS
 from sentry.models.group import Group, GroupStatus
 from sentry.signals import first_feedback_received, first_new_feedback_received
 from sentry.testutils.helpers import Feature
@@ -935,3 +936,118 @@ def test_create_feedback_issue_title(default_project, mock_produce_occurrence_to
         "User Feedback: This is a very long feedback message that describes multiple..."
     )
     assert occurrence.issue_title == expected_title
+
+
+@django_db_all
+def test_create_feedback_adds_ai_labels(
+    default_project, mock_produce_occurrence_to_kafka, monkeypatch
+):
+    """Test that create_feedback_issue adds AI labels to tags when label generation succeeds."""
+    with Feature(
+        {
+            "organizations:user-feedback-ai-categorization": True,
+            "organizations:gen-ai-features": True,
+        }
+    ):
+        event = mock_feedback_event(default_project.id)
+        event["contexts"]["feedback"]["message"] = "The login button is broken and the UI is slow"
+
+        # This assumes that the maximum number of labels allowed is greater than 3
+        def mock_generate_labels(*args, **kwargs):
+            return ["User Interface", "Authentication", "Performance"]
+
+        monkeypatch.setattr(
+            "sentry.feedback.usecases.ingest.create_feedback.generate_labels",
+            mock_generate_labels,
+        )
+
+        create_feedback_issue(event, default_project, FeedbackCreationSource.NEW_FEEDBACK_ENVELOPE)
+
+        assert mock_produce_occurrence_to_kafka.call_count == 1
+        produced_event = mock_produce_occurrence_to_kafka.call_args.kwargs["event_data"]
+        tags = produced_event["tags"]
+
+        ai_labels = [value for key, value in tags.items() if key.startswith(AI_LABEL_TAG_PREFIX)]
+        assert len(ai_labels) == 3
+        assert set(ai_labels) == {"User Interface", "Authentication", "Performance"}
+
+
+@django_db_all
+def test_create_feedback_handles_label_generation_errors(
+    default_project, mock_produce_occurrence_to_kafka, monkeypatch
+):
+    """Test that create_feedback_issue continues to work even when generate_labels raises an error."""
+    with Feature(
+        {
+            "organizations:user-feedback-ai-categorization": True,
+            "organizations:gen-ai-features": True,
+        }
+    ):
+        event = mock_feedback_event(default_project.id)
+        event["contexts"]["feedback"]["message"] = "This is a valid feedback message"
+
+        # Mock generate_labels to raise an exception
+        def mock_generate_labels(*args, **kwargs):
+            raise Exception("Label generation failed")
+
+        monkeypatch.setattr(
+            "sentry.feedback.usecases.ingest.create_feedback.generate_labels",
+            mock_generate_labels,
+        )
+
+        # This should not raise an exception and should still create the feedback
+        create_feedback_issue(event, default_project, FeedbackCreationSource.NEW_FEEDBACK_ENVELOPE)
+
+        # Verify that the feedback was still created successfully
+        assert mock_produce_occurrence_to_kafka.call_count == 1
+
+        produced_event = mock_produce_occurrence_to_kafka.call_args.kwargs["event_data"]
+        tags = produced_event["tags"]
+
+        ai_labels = [tag for tag in tags.keys() if tag.startswith(AI_LABEL_TAG_PREFIX)]
+        assert (
+            len(ai_labels) == 0
+        ), "No AI categorization labels should be present when label generation fails"
+
+
+@django_db_all
+def test_create_feedback_truncates_ai_labels(
+    default_project, mock_produce_occurrence_to_kafka, monkeypatch
+):
+    """Test that create_feedback_issue truncates AI labels when more than MAX_AI_LABELS are returned."""
+    with Feature(
+        {
+            "organizations:user-feedback-ai-categorization": True,
+            "organizations:gen-ai-features": True,
+        }
+    ):
+        event = mock_feedback_event(default_project.id)
+        event["contexts"]["feedback"][
+            "message"
+        ] = "This is a very complex feedback with many issues"
+
+        # Mock generate_labels to return more than MAX_AI_LABELS labels
+        def mock_generate_labels(*args, **kwargs):
+            return [f"Label {i}" for i in range(MAX_AI_LABELS + 5)]
+
+        monkeypatch.setattr(
+            "sentry.feedback.usecases.ingest.create_feedback.generate_labels",
+            mock_generate_labels,
+        )
+
+        create_feedback_issue(event, default_project, FeedbackCreationSource.NEW_FEEDBACK_ENVELOPE)
+
+        assert mock_produce_occurrence_to_kafka.call_count == 1
+
+        produced_event = mock_produce_occurrence_to_kafka.call_args.kwargs["event_data"]
+        tags = produced_event["tags"]
+
+        ai_labels = [value for key, value in tags.items() if key.startswith(AI_LABEL_TAG_PREFIX)]
+        assert len(ai_labels) == MAX_AI_LABELS, "Should be truncated to exactly MAX_AI_LABELS"
+
+        for i in range(MAX_AI_LABELS):
+            assert tags[f"{AI_LABEL_TAG_PREFIX}.{i}"] == f"Label {i}"
+
+        # Verify that labels beyond MAX_AI_LABELS are not present
+        for i in range(MAX_AI_LABELS, MAX_AI_LABELS + 5):
+            assert f"{AI_LABEL_TAG_PREFIX}.{i}" not in tags
diff --git a/tests/sentry/feedback/usecases/test_label_generation.py b/tests/sentry/feedback/usecases/test_label_generation.py
@@ -0,0 +1,77 @@
+import pytest
+import requests
+import responses
+
+from sentry.feedback.usecases.label_generation import SEER_GENERATE_LABELS_URL, generate_labels
+from sentry.testutils.cases import TestCase
+from sentry.utils import json
+
+
+def mock_seer_response(**kwargs) -> None:
+    """Use with @responses.activate to cleanup after tests. Not compatible with store_replay."""
+    responses.add(
+        responses.POST,
+        SEER_GENERATE_LABELS_URL,
+        **kwargs,
+    )
+
+
+class TestGenerateLabels(TestCase):
+    @responses.activate
+    def test_generate_labels_success_response(self):
+        mock_seer_response(
+            status=200,
+            json={"data": {"labels": ["User Interface", "Navigation", "Right Sidebar"]}},
+        )
+
+        labels = generate_labels(
+            "I don't like the new right sidebar, it makes navigating everywhere hard!", 1
+        )
+
+        test_request = responses.calls[0].request
+        test_response = responses.calls[0].response
+
+        assert labels == ["User Interface", "Navigation", "Right Sidebar"]
+        assert json.loads(test_request.body) == {
+            "feedback_message": "I don't like the new right sidebar, it makes navigating everywhere hard!",
+            "organization_id": 1,
+        }
+        assert test_response.status_code == 200
+
+    @responses.activate
+    def test_generate_labels_failed_response(self):
+        mock_seer_response(
+            status=500,
+            json={"error": "Internal Server Error"},
+        )
+
+        with pytest.raises(requests.exceptions.HTTPError):
+            generate_labels(
+                "I don't like the new right sidebar, it makes navigating everywhere hard!", 1
+            )
+
+        test_request = responses.calls[0].request
+        test_response = responses.calls[0].response
+
+        assert test_response.status_code == 500
+        assert json.loads(test_request.body) == {
+            "feedback_message": "I don't like the new right sidebar, it makes navigating everywhere hard!",
+            "organization_id": 1,
+        }
+
+    @responses.activate
+    def test_generate_labels_network_error(self):
+        mock_seer_response(body=requests.exceptions.Timeout("Request timed out"))
+
+        with pytest.raises(requests.exceptions.Timeout):
+            generate_labels(
+                "I don't like the new right sidebar, it makes navigating everywhere hard!", 1
+            )
+
+        test_request = responses.calls[0].request
+
+        assert len(responses.calls) == 1
+        assert json.loads(test_request.body) == {
+            "feedback_message": "I don't like the new right sidebar, it makes navigating everywhere hard!",
+            "organization_id": 1,
+        }
