feat(feedback): reduce labels passed to LLM, cap associated label count (#97702)

Author: vishnupsatish

src/sentry/feedback/endpoints/organization_feedback_categories.py

@@ -35,6 +35,11 @@
 
 MAX_RETURN_CATEGORIES = 4
 
+MAX_ASSOCIATED_LABELS = 12
+
+# Number of top labels to pass to Seer to ask for similar labels
+NUM_TOP_LABELS = 6
+
 # Two days because the largest granularity we cache at is the day
 CATEGORIES_CACHE_TIMEOUT = 172800
 
@@ -169,21 +174,21 @@ def get(self, request: Request, organization: Organization) -> Response:
                 LabelGroupFeedbacksContext(feedback=feedback["feedback"], labels=feedback["labels"])
             )
 
-        # Gets the top 10 labels by feedbacks to augment the context that the LLM has, instead of just asking it to generate categories without knowing the most common labels
-        top_10_labels_result = query_top_ai_labels_by_feedback_count(
+        # Gets the top labels by feedbacks to augment the context that the LLM has, instead of just asking it to generate categories without knowing the most common labels
+        top_labels_result = query_top_ai_labels_by_feedback_count(
             organization_id=organization.id,
             project_ids=numeric_project_ids,
             start=start,
             end=end,
-            limit=10,
+            limit=NUM_TOP_LABELS,
         )
 
         # Guaranteed to be non-empty since recent_feedbacks is non-empty
-        top_10_labels = [result["label"] for result in top_10_labels_result]
+        top_labels = [result["label"] for result in top_labels_result]
 
         seer_request = LabelGroupsRequest(
             organization_id=organization.id,
-            labels=top_10_labels,
+            labels=top_labels,
             feedbacks_context=context_feedbacks,
         )
 
@@ -192,31 +197,31 @@ def get(self, request: Request, organization: Organization) -> Response:
         )["data"]
 
         # If the LLM just forgets or adds extra primary labels, log it but still generate categories
-        if len(label_groups) != len(top_10_labels):
+        if len(label_groups) != len(top_labels):
             logger.warning(
                 "Number of label groups does not match number of primary labels passed in Seer",
                 extra={
                     "label_groups": label_groups,
-                    "top_10_labels": top_10_labels,
+                    "top_labels": top_labels,
                 },
             )
 
         # If the LLM hallucinates primary label(s), log it but still generate categories
         for label_group in label_groups:
-            if label_group["primaryLabel"] not in top_10_labels:
+            if label_group["primaryLabel"] not in top_labels:
                 logger.warning(
                     "LLM hallucinated primary label",
                     extra={"label_group": label_group},
                 )
 
         # Converts label_groups (which maps primary label to associated labels) to a list of lists, where the first element is the primary label and the rest are the associated labels
         label_groups_lists: list[list[str]] = [
-            [label_group["primaryLabel"]] + label_group["associatedLabels"]
+            [label_group["primaryLabel"]] + label_group["associatedLabels"][:MAX_ASSOCIATED_LABELS]
             for label_group in label_groups
         ]
 
         # label_groups_lists might be empty if the LLM just decides not to give us any primary labels (leading to ValueError, then 500)
-        # This will be logged since top_10_labels is guaranteed to be non-empty, but label_groups_lists will be empty
+        # This will be logged since top_labels is guaranteed to be non-empty, but label_groups_lists will be empty
         label_feedback_counts = query_label_group_counts(
             organization_id=organization.id,
             project_ids=numeric_project_ids,

tests/sentry/feedback/endpoints/test_organization_feedback_categories.py

@@ -253,6 +253,79 @@ def test_get_feedback_categories_with_project_filter(self) -> None:
             elif category["primaryLabel"] == "Authentication":
                 assert category["feedbackCount"] == 3
 
+    @django_db_all
+    @responses.activate
+    def test_max_associated_labels_limit(self) -> None:
+        """Test that MAX_ASSOCIATED_LABELS constant is respected when processing label groups."""
+        # Mock Seer to return a label group with more than MAX_ASSOCIATED_LABELS associated labels
+        mock_seer_category_response(
+            status=200,
+            json={
+                "data": [
+                    {
+                        "primaryLabel": "User Interface",
+                        "associatedLabels": [
+                            "Usability",
+                            "Design",
+                            "Layout",
+                            "Colors",
+                            "Typography",
+                            "Spacing",
+                            "Alignment",
+                            "Responsiveness",
+                            "Accessibility",
+                            "Navigation",
+                            "Buttons",
+                            "Forms",
+                            "Extra Label 1",  # This should be truncated
+                            "Extra Label 2",  # This should be truncated
+                            "Extra Label 3",  # This should be truncated
+                        ],
+                    }
+                ]
+            },
+        )
+
+        with self.feature(self.features):
+            response = self.get_success_response(self.org.slug)
+
+        assert response.data["success"] is True
+        assert "categories" in response.data
+
+        categories = response.data["categories"]
+        assert len(categories) == 1
+
+        user_interface_category = categories[0]
+        assert user_interface_category["primaryLabel"] == "User Interface"
+
+        # Verify that associatedLabels is truncated to MAX_ASSOCIATED_LABELS (12)
+        associated_labels = user_interface_category["associatedLabels"]
+        assert (
+            len(associated_labels) == 12
+        ), f"Expected 12 associated labels, got {len(associated_labels)}"
+
+        # Verify the first 12 labels are preserved
+        expected_labels = [
+            "Usability",
+            "Design",
+            "Layout",
+            "Colors",
+            "Typography",
+            "Spacing",
+            "Alignment",
+            "Responsiveness",
+            "Accessibility",
+            "Navigation",
+            "Buttons",
+            "Forms",
+        ]
+        assert associated_labels == expected_labels
+
+        # Verify that the extra labels beyond MAX_ASSOCIATED_LABELS are not included
+        assert "Extra Label 1" not in associated_labels
+        assert "Extra Label 2" not in associated_labels
+        assert "Extra Label 3" not in associated_labels
+
     @django_db_all
     @responses.activate
     def test_seer_timeout(self) -> None: