chore(flags): add metrics for events with flags and flag audit log (#83315)

closes getsentry/team-replay#531

adds logging for events w/ feature flags (& number of flags being sent)
and audit log posts

Author: michellewzhang

src/sentry/flags/endpoints/hooks.py

@@ -10,6 +10,7 @@
 from sentry.api.exceptions import ResourceDoesNotExist
 from sentry.flags.providers import DeserializationError, get_provider, write
 from sentry.models.organization import Organization
+from sentry.utils import metrics
 
 
 @region_silo_endpoint
@@ -33,6 +34,7 @@ def post(self, request: Request, organization: Organization, provider: str) -> R
                 return Response("Not authorized.", status=401)
             else:
                 write(provider_cls.handle(request.data))
+                metrics.incr("feature_flags.audit_log_event_posted", tags={"provider": provider})
                 return Response(status=200)
         except DeserializationError as exc:
             sentry_sdk.capture_exception()

src/sentry/tasks/post_process.py

@@ -1497,8 +1497,12 @@ def check_if_flags_sent(job: PostProcessJob) -> None:
     event = job["event"]
     project = event.project
     flag_context = get_path(event.data, "contexts", "flags")
-    if flag_context and not project.flags.has_flags:
-        first_flag_received.send_robust(project=project, sender=Project)
+
+    if flag_context:
+        metrics.incr("feature_flags.event_has_flags_context")
+        metrics.distribution("feature_flags.num_flags_sent", len(flag_context))
+        if not project.flags.has_flags:
+            first_flag_received.send_robust(project=project, sender=Project)
 
 
 GROUP_CATEGORY_POST_PROCESS_PIPELINE = {

tests/sentry/flags/endpoints/test_hooks.py

@@ -1,3 +1,5 @@
+from unittest.mock import call, patch
+
 from django.urls import reverse
 
 from sentry.flags.models import (
@@ -11,6 +13,7 @@
 from sentry.utils import json
 
 
+@patch("sentry.utils.metrics.incr")
 class OrganizationFlagsHooksEndpointTestCase(APITestCase):
     endpoint = "sentry-api-0-organization-flag-hooks"
 
@@ -22,7 +25,7 @@ def setUp(self):
     def features(self):
         return {"organizations:feature-flag-audit-log": True}
 
-    def test_generic_post_create(self):
+    def test_generic_post_create(self, mock_incr):
         request_data = {
             "data": [
                 {
@@ -47,9 +50,12 @@ def test_generic_post_create(self):
                 headers={"X-Sentry-Signature": signature},
             )
             assert response.status_code == 200, response.content
+            mock_incr.assert_any_call(
+                "feature_flags.audit_log_event_posted", tags={"provider": "generic"}
+            )
             assert FlagAuditLogModel.objects.count() == 1
 
-    def test_unleash_post_create(self):
+    def test_unleash_post_create(self, mock_incr):
         request_data = {
             "id": 28,
             "tags": [{"type": "simple", "value": "testvalue"}],
@@ -75,9 +81,12 @@ def test_unleash_post_create(self):
                 headers={"Authorization": signature},
             )
             assert response.status_code == 200, response.content
+            mock_incr.assert_any_call(
+                "feature_flags.audit_log_event_posted", tags={"provider": "unleash"}
+            )
             assert FlagAuditLogModel.objects.count() == 1
 
-    def test_launchdarkly_post_create(self):
+    def test_launchdarkly_post_create(self, mock_incr):
         request_data = LD_REQUEST
         signature = hmac_sha256_hex_digest(key="456", message=json.dumps(request_data).encode())
 
@@ -95,6 +104,9 @@ def test_launchdarkly_post_create(self):
             )
 
         assert response.status_code == 200
+        mock_incr.assert_any_call(
+            "feature_flags.audit_log_event_posted", tags={"provider": "launchdarkly"}
+        )
         assert FlagAuditLogModel.objects.count() == 1
         flag = FlagAuditLogModel.objects.first()
         assert flag is not None
@@ -106,13 +118,14 @@ def test_launchdarkly_post_create(self):
         assert flag.tags is not None
         assert flag.tags["description"] == "flag was created"
 
-    def test_launchdarkly_post_create_invalid_signature(self):
+    def test_launchdarkly_post_create_invalid_signature(self, mock_incr):
         with self.feature(self.features):
             sig = hmac_sha256_hex_digest(key="123", message=b"456")
             response = self.client.post(self.url, LD_REQUEST, headers={"X-LD-Signature": sig})
             assert response.status_code == 401
+            assert call("feature_flags.audit_log_event_posted") not in mock_incr.call_args_list
 
-    def test_post_launchdarkly_deserialization_failed(self):
+    def test_post_launchdarkly_deserialization_failed(self, mock_incr):
         signature = hmac_sha256_hex_digest(key="123", message=json.dumps({}).encode())
         FlagWebHookSigningSecretModel.objects.create(
             organization=self.organization, provider="launchdarkly", secret="123"
@@ -122,22 +135,26 @@ def test_post_launchdarkly_deserialization_failed(self):
             response = self.client.post(self.url, {}, headers={"X-LD-Signature": signature})
             assert response.status_code == 200
             assert FlagAuditLogModel.objects.count() == 0
+            assert call("feature_flags.audit_log_event_posted") not in mock_incr.call_args_list
 
-    def test_post_invalid_provider(self):
+    def test_post_invalid_provider(self, mock_incr):
         url = reverse(self.endpoint, args=(self.organization.slug, "test"))
         with self.feature(self.features):
             response = self.client.post(url, {})
             assert response.status_code == 404
+            assert call("feature_flags.audit_log_event_posted") not in mock_incr.call_args_list
 
-    def test_post_disabled(self):
+    def test_post_disabled(self, mock_incr):
         response = self.client.post(self.url, data={})
         assert response.status_code == 404
         assert response.content == b'"Not enabled."'
+        assert call("feature_flags.audit_log_event_posted") not in mock_incr.call_args_list
 
-    def test_post_missing_signature(self):
+    def test_post_missing_signature(self, mock_incr):
         with self.feature(self.features):
             response = self.client.post(self.url, {})
             assert response.status_code == 401, response.content
+            assert call("feature_flags.audit_log_event_posted") not in mock_incr.call_args_list
 
 
 LD_REQUEST = {

tests/sentry/tasks/test_post_process.py

@@ -2199,14 +2199,33 @@ def test_uptime_detection_no_feature(self):
 
 
 @patch("sentry.analytics.record")
+@patch("sentry.utils.metrics.incr")
+@patch("sentry.utils.metrics.distribution")
 class CheckIfFlagsSentTestMixin(BasePostProgressGroupMixin):
-    def test_set_has_flags(self, mock_record):
+    def test_set_has_flags(self, mock_dist, mock_incr, mock_record):
         project = self.create_project()
         event_id = "a" * 32
         event = self.create_event(
-            data={"event_id": event_id, "contexts": {"flags": {"values": []}}},
+            data={
+                "event_id": event_id,
+                "contexts": {
+                    "flags": {
+                        "values": [
+                            {
+                                "flag": "test-flag-1",
+                                "result": False,
+                            },
+                            {
+                                "flag": "test-flag-2",
+                                "result": True,
+                            },
+                        ]
+                    }
+                },
+            },
             project_id=project.id,
         )
+
         self.call_post_process_group(
             is_new=True,
             is_regression=False,
@@ -2217,6 +2236,8 @@ def test_set_has_flags(self, mock_record):
         project.refresh_from_db()
         assert project.flags.has_flags
 
+        mock_incr.assert_any_call("feature_flags.event_has_flags_context")
+        mock_dist.assert_any_call("feature_flags.num_flags_sent", 2)
         mock_record.assert_called_with(
             "first_flag.sent",
             organization_id=self.organization.id,