ref(replay-feedback): check Seer consent in Seer callsites (#97654)

Create and use a shared util for checking Seer org consent in Sentry,
along with other required FFs and options, using
[autofix](https://github.com/getsentry/sentry/blob/d1241729972adf1cb2601691c759827c758280e8/src/sentry/seer/autofix/autofix.py#L406-L415)
as example. In tests, mocking this util decouples the specific Seer
requirements from our dependencies.
- ~~Now requires seer access in replay delete endpoints (before we only
checked the summaries FF, but once that's rolled out we'll need the
access check)~~
- Refactors title generation a bit so we check access in create_feedback
instead of a util.

**Followups:**
Once this is deployed we can remove the org consent RPC calls for
replay/feedback endpoints in Seer. Then a 2nd followup PR in Sentry can
remove the org_id param from requests.

Fixes REPLAY-619

---------

Co-authored-by: getsantry[bot] <66042841+getsantry[bot]@users.noreply.github.com>

Author: aliu39

src/sentry/feedback/endpoints/organization_feedback_categories.py

@@ -22,6 +22,7 @@
 )
 from sentry.grouping.utils import hash_from_values
 from sentry.models.organization import Organization
+from sentry.seer.seer_setup import has_seer_access
 from sentry.seer.signed_seer_api import sign_with_seer_secret
 from sentry.utils import json
 from sentry.utils.cache import cache
@@ -114,8 +115,10 @@ def get(self, request: Request, organization: Organization) -> Response:
             "organizations:user-feedback-ai-categorization-features",
             organization,
             actor=request.user,
-        ) or not features.has("organizations:gen-ai-features", organization, actor=request.user):
-            return Response(status=404)
+        ) or not has_seer_access(organization, actor=request.user):
+            return Response(
+                {"detail": "AI categorization is not available for this organization."}, status=403
+            )
 
         try:
             start, end = get_date_range_from_stats_period(

src/sentry/feedback/endpoints/organization_feedback_summary.py

@@ -19,6 +19,7 @@
 from sentry.issues.grouptype import FeedbackGroup
 from sentry.models.group import Group, GroupStatus
 from sentry.models.organization import Organization
+from sentry.seer.seer_setup import has_seer_access
 from sentry.seer.signed_seer_api import sign_with_seer_secret
 from sentry.utils import json
 from sentry.utils.cache import cache
@@ -67,8 +68,10 @@ def get(self, request: Request, organization: Organization) -> Response:
 
         if not features.has(
             "organizations:user-feedback-ai-summaries", organization, actor=request.user
-        ) or not features.has("organizations:gen-ai-features", organization, actor=request.user):
-            return Response(status=403)
+        ) or not has_seer_access(organization, actor=request.user):
+            return Response(
+                {"detail": "AI summaries are not available for this organization."}, status=403
+            )
 
         try:
             start, end = get_date_range_from_stats_period(

src/sentry/feedback/usecases/ingest/create_feedback.py

@@ -18,18 +18,15 @@
     generate_labels,
 )
 from sentry.feedback.usecases.spam_detection import is_spam, spam_detection_enabled
-from sentry.feedback.usecases.title_generation import (
-    format_feedback_title,
-    get_feedback_title_from_seer,
-    should_get_ai_title,
-)
+from sentry.feedback.usecases.title_generation import get_feedback_title
 from sentry.issues.grouptype import FeedbackGroup
 from sentry.issues.issue_occurrence import IssueEvidence, IssueOccurrence
 from sentry.issues.json_schemas import EVENT_PAYLOAD_SCHEMA
 from sentry.issues.producer import PayloadType, produce_occurrence_to_kafka
 from sentry.issues.status_change_message import StatusChangeMessage
 from sentry.models.group import GroupStatus
 from sentry.models.project import Project
+from sentry.seer.seer_setup import has_seer_access
 from sentry.signals import first_feedback_received, first_new_feedback_received
 from sentry.types.group import GroupSubStatus
 from sentry.utils import json, metrics
@@ -280,6 +277,8 @@ def create_feedback_issue(
             },
         )
 
+    should_query_seer = not is_message_spam and has_seer_access(project.organization)
+
     # Prepare the data for issue platform processing and attach useful tags.
 
     # Note that some of the fields below like title and subtitle
@@ -291,17 +290,34 @@ def create_feedback_issue(
     )
     issue_fingerprint = [uuid4().hex]
 
-    ai_title = None
-    if not is_message_spam and should_get_ai_title(project.organization):
-        ai_title = get_feedback_title_from_seer(feedback_message, project.organization_id)
-    formatted_title = format_feedback_title(ai_title or feedback_message)
+    # TODO: clean up these metrics after the feature is rolled out.
+    if is_message_spam:
+        metrics.incr(
+            "feedback.ai_title_generation.skipped",
+            tags={"reason": "is_spam"},
+        )
+    elif not should_query_seer:
+        metrics.incr(
+            "feedback.ai_title_generation.skipped",
+            tags={"reason": "gen_ai_disabled"},
+        )
+    elif not features.has("organizations:user-feedback-ai-titles", project.organization):
+        metrics.incr(
+            "feedback.ai_title_generation.skipped",
+            tags={"reason": "feedback_ai_titles_disabled"},
+        )
+
+    use_ai_title = should_query_seer and features.has(
+        "organizations:user-feedback-ai-titles", project.organization
+    )
+    title = get_feedback_title(feedback_message, project.organization_id, use_ai_title)
 
     occurrence = IssueOccurrence(
         id=uuid4().hex,
         event_id=event["event_id"],
         project_id=project.id,
         fingerprint=issue_fingerprint,  # random UUID for fingerprint so feedbacks are grouped individually
-        issue_title=formatted_title,
+        issue_title=title,
         subtitle=feedback_message,
         resource_id=None,
         evidence_data=evidence_data,
@@ -323,10 +339,8 @@ def create_feedback_issue(
     )
 
     # Generating labels using Seer, which will later be used to categorize feedbacks
-    if (
-        not is_message_spam
-        and features.has("organizations:user-feedback-ai-categorization", project.organization)
-        and features.has("organizations:gen-ai-features", project.organization)
+    if should_query_seer and features.has(
+        "organizations:user-feedback-ai-categorization", project.organization
     ):
         try:
             labels = generate_labels(feedback_message, project.organization_id)

src/sentry/feedback/usecases/title_generation.py

@@ -6,8 +6,6 @@
 import requests
 from django.conf import settings
 
-from sentry import features
-from sentry.models.organization import Organization
 from sentry.seer.signed_seer_api import sign_with_seer_secret
 from sentry.utils import json, metrics
 
@@ -23,17 +21,6 @@ class GenerateFeedbackTitleRequest(TypedDict):
     feedback_message: str
 
 
-def should_get_ai_title(organization: Organization) -> bool:
-    """Check if AI title generation should be used for the given organization."""
-    if not features.has("organizations:gen-ai-features", organization):
-        return False
-
-    if not features.has("organizations:user-feedback-ai-titles", organization):
-        return False
-
-    return True
-
-
 def format_feedback_title(title: str, max_words: int = 10) -> str:
     """
     Clean and format a title for user feedback issues.
@@ -133,3 +120,15 @@ def make_seer_request(request: GenerateFeedbackTitleRequest) -> bytes:
         response.raise_for_status()
 
     return response.content
+
+
+def get_feedback_title(feedback_message: str, organization_id: int, use_seer: bool) -> str:
+    if use_seer:
+        # Message is fallback if Seer fails.
+        raw_title = (
+            get_feedback_title_from_seer(feedback_message, organization_id) or feedback_message
+        )
+    else:
+        raw_title = feedback_message
+
+    return format_feedback_title(raw_title)

src/sentry/replays/endpoints/project_replay_details.py

@@ -96,10 +96,13 @@ def delete(self, request: Request, project: Project, replay_id: str) -> Response
         if has_archived_segment(project.id, replay_id):
             return Response(status=404)
 
+        # We don't check Seer features because an org may have previously had them on, then turned them off.
+        has_seer_data = features.has("organizations:replay-ai-summaries", project.organization)
+
         delete_replay.delay(
             project_id=project.id,
             replay_id=replay_id,
-            has_seer_data=features.has("organizations:replay-ai-summaries", project.organization),
+            has_seer_data=has_seer_data,
         )
 
         return Response(status=204)

src/sentry/replays/endpoints/project_replay_jobs_delete.py

@@ -102,6 +102,7 @@ def post(self, request: Request, project) -> Response:
             status="pending",
         )
 
+        # We don't check Seer features because an org may have previously had them on, then turned them off.
         has_seer_data = features.has("organizations:replay-ai-summaries", project.organization)
 
         # We always start with an offset of 0 (obviously) but future work doesn't need to obey

src/sentry/replays/endpoints/project_replay_summary.py

@@ -22,6 +22,7 @@
 from sentry.replays.post_process import process_raw_response
 from sentry.replays.query import query_replay_instance
 from sentry.replays.usecases.reader import fetch_segments_metadata, iter_segment_data
+from sentry.seer.seer_setup import has_seer_access
 from sentry.seer.signed_seer_api import sign_with_seer_secret
 from sentry.utils import json
 
@@ -69,11 +70,6 @@ class ProjectReplaySummaryEndpoint(ProjectEndpoint):
     def __init__(self, **options) -> None:
         storage.initialize_client()
         super().__init__(**options)
-        self.features = [
-            "organizations:session-replay",
-            "organizations:replay-ai-summaries",
-            "organizations:gen-ai-features",
-        ]
 
     def make_seer_request(self, url: str, post_body: dict[str, Any]) -> Response:
         """Make a POST request to a Seer endpoint. Raises HTTPError and logs non-200 status codes."""
@@ -127,13 +123,23 @@ def make_seer_request(self, url: str, post_body: dict[str, Any]) -> Response:
         # Note any headers in the Seer response aren't returned.
         return Response(data=response.json(), status=response.status_code)
 
+    def has_replay_summary_access(self, project: Project, request: Request) -> bool:
+        return (
+            features.has("organizations:session-replay", project.organization, actor=request.user)
+            and features.has(
+                "organizations:replay-ai-summaries", project.organization, actor=request.user
+            )
+            and has_seer_access(project.organization, actor=request.user)
+        )
+
     def get(self, request: Request, project: Project, replay_id: str) -> Response:
         """Poll for the status of a replay summary task in Seer."""
-        if not all(
-            features.has(feature, project.organization, actor=request.user)
-            for feature in self.features
-        ):
-            return self.respond(status=404)
+        if not self.has_replay_summary_access(project, request):
+            return self.respond(
+                {"detail": "Replay summaries are not available for this organization."}, status=403
+            )
+
+        # We skip checking Seer permissions here for performance, and because summaries can't be created without them anyway.
 
         # Request Seer for the state of the summary task.
         return self.make_seer_request(
@@ -145,11 +151,10 @@ def get(self, request: Request, project: Project, replay_id: str) -> Response:
 
     def post(self, request: Request, project: Project, replay_id: str) -> Response:
         """Download replay segment data and parse it into logs. Then post to Seer to start a summary task."""
-        if not all(
-            features.has(feature, project.organization, actor=request.user)
-            for feature in self.features
-        ):
-            return self.respond(status=404)
+        if not self.has_replay_summary_access(project, request):
+            return self.respond(
+                {"detail": "Replay summaries are not available for this organization."}, status=403
+            )
 
         filter_params = self.get_filter_params(request, project)
 

src/sentry/seer/seer_setup.py

@@ -1,4 +1,10 @@
+from django.contrib.auth.models import AnonymousUser
+
+from sentry import features
+from sentry.models.organization import Organization
 from sentry.models.promptsactivity import PromptsActivity
+from sentry.users.models.user import User
+from sentry.users.services.user.model import RpcUser
 
 feature_name = "seer_autofix_setup_acknowledged"
 
@@ -18,3 +24,13 @@ def get_seer_org_acknowledgement(org_id: int) -> bool:
         organization_id=org_id,
         project_id=0,
     ).exists()
+
+
+def has_seer_access(
+    organization: Organization, actor: User | AnonymousUser | RpcUser | None = None
+) -> bool:
+    return (
+        features.has("organizations:gen-ai-features", organization, actor=actor)
+        and not bool(organization.get_option("sentry:hide_ai_features"))
+        and get_seer_org_acknowledgement(org_id=organization.id)
+    )

tests/sentry/feedback/endpoints/test_organization_feedback_categories.py

@@ -11,7 +11,6 @@
 from sentry.issues.grouptype import FeedbackGroup
 from sentry.testutils.cases import APITestCase, SnubaTestCase
 from sentry.testutils.helpers.datetime import before_now
-from sentry.testutils.pytest.fixtures import django_db_all
 from sentry.testutils.silo import region_silo_test
 from tests.sentry.issues.test_utils import SearchIssueTestMixin
 
@@ -46,14 +45,23 @@ def setUp(self) -> None:
         self.project2 = self.create_project(teams=[self.team])
         self.features = {
             "organizations:user-feedback-ai-categorization-features": True,
-            "organizations:gen-ai-features": True,
         }
         self.url = reverse(
             self.endpoint,
             kwargs={"organization_id_or_slug": self.org.slug},
         )
+        self.mock_has_seer_access_patcher = patch(
+            "sentry.feedback.endpoints.organization_feedback_categories.has_seer_access",
+            return_value=True,
+        )
+        self.mock_has_seer_access = self.mock_has_seer_access_patcher.start()
+
         self._create_standard_feedbacks(self.project1.id)
 
+    def tearDown(self) -> None:
+        self.mock_has_seer_access_patcher.stop()
+        super().tearDown()
+
     def _create_standard_feedbacks(self, project_id: int) -> None:
         """Create a standard set of feedbacks for testing."""
         insert_time = before_now(hours=12)
@@ -156,12 +164,16 @@ def _create_standard_feedbacks(self, project_id: int) -> None:
                 insert_time=insert_time,
             )
 
-    @django_db_all
     def test_get_feedback_categories_without_feature_flag(self) -> None:
         response = self.get_error_response(self.org.slug)
-        assert response.status_code == 404
+        assert response.status_code == 403
+
+    def test_get_feedback_categories_without_seer_access(self) -> None:
+        self.mock_has_seer_access.return_value = False
+        with self.feature(self.features):
+            response = self.get_error_response(self.org.slug)
+            assert response.status_code == 403
 
-    @django_db_all
     @patch(
         "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS",
         1,
@@ -213,7 +225,6 @@ def test_get_feedback_categories_basic(self) -> None:
             elif category["primaryLabel"] == "Authentication":
                 assert category["feedbackCount"] == 3
 
-    @django_db_all
     @patch(
         "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS",
         1,
@@ -264,7 +275,6 @@ def test_get_feedback_categories_with_project_filter(self) -> None:
             elif category["primaryLabel"] == "Authentication":
                 assert category["feedbackCount"] == 3
 
-    @django_db_all
     @patch(
         "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS",
         1,
@@ -333,7 +343,6 @@ def test_max_group_labels_limit(self) -> None:
         assert "Extra Label 2" not in associated_labels
         assert "Extra Label 3" not in associated_labels
 
-    @django_db_all
     @patch(
         "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS",
         1,
@@ -421,7 +430,6 @@ def test_filter_invalid_associated_labels_by_count_ratio(self) -> None:
         assert len(associated_labels) == 1
         assert associated_labels == ["Design"]
 
-    @django_db_all
     @patch(
         "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS",
         1,
@@ -435,7 +443,6 @@ def test_seer_timeout(self) -> None:
 
         assert response.status_code == 500
 
-    @django_db_all
     @patch(
         "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS",
         1,
@@ -449,7 +456,6 @@ def test_seer_connection_error(self) -> None:
 
         assert response.status_code == 500
 
-    @django_db_all
     @patch(
         "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS",
         1,
@@ -465,7 +471,6 @@ def test_seer_request_error(self) -> None:
 
         assert response.status_code == 500
 
-    @django_db_all
     @patch(
         "sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS",
         1,
@@ -480,7 +485,6 @@ def test_seer_http_errors(self) -> None:
 
             assert response.status_code == 500
 
-    @django_db_all
     @responses.activate
     def test_fallback_to_primary_labels_when_below_threshold(self) -> None:
         """Test that when feedback count is below THRESHOLD_TO_GET_ASSOCIATED_LABELS, we fall back to primary labels only."""