ref(sdk): Add more checks to send_envelope (#26182)

Almost there. Worker.submit rate and the send_envelope rate match the captured envelopes, but again 'sent_request' drops. There are only a few lines of code between these functions, check_disabled doesn't appear to be getting called, and there are a couple file operations which might be throwing exceptions, so wrapped those in metrics as well.

Author: k-fish

src/sentry/utils/sdk.py

@@ -229,7 +229,15 @@ def patched_worker_submit(*args, **kwargs):
 
         def patched_send_envelope(*args, **kwargs):
             metrics.incr(f"internal.send_envelope.{transport_name}.events")
-            return _send_envelope(*args, **kwargs)
+            try:
+                result = _send_envelope(*args, **kwargs)
+                metrics.incr(f"internal.send_envelope.{transport_name}.sent.events")
+                return result
+            except Exception as error:
+                error_name = type(error).__name__
+                metrics.incr(
+                    f"internal.send_envelope.{transport_name}.error", tags={"error": error_name}
+                )
 
         transport._send_envelope = patched_send_envelope
 
@@ -264,17 +272,20 @@ def patched_update_rate_limits(*args, **kwargs):
     _check_disabled = transport._check_disabled
     if _check_disabled:
 
+        def check_disabled_bucket(bucket):
+            ts = transport._disabled_until.get(bucket)
+
+            # Confirm the transaction bucket is disabled
+            if ts is not None and ts > datetime.utcnow():
+                metrics.incr(f"internal.check_disabled.{transport_name}.bucket.{bucket}.disabled")
+
         def patched_check_disabled(*args, **kwargs):
             result = _check_disabled(*args, **kwargs)
+            metrics.incr(f"internal.check_disabled.{transport_name}.events.count")
             if result:
-                if getattr(transport, "_disabled_until", None):
-                    ts = transport._disabled_until.get("transaction")
 
-                    # Confirm the transaction bucket is disabled
-                    if ts is not None and ts > datetime.utcnow():
-                        metrics.incr(
-                            f"internal.check_disabled.{transport_name}.events.transactions_disabled"
-                        )
+                if getattr(transport, "_disabled_until", None):
+                    check_disabled_bucket("transaction")
 
                 metrics.incr(f"internal.check_disabled.{transport_name}.events.is_disabled")
             return result