Tighten API latency budget to 300ms (#532)

Author: shayancoin

docs/release-checklist.md

@@ -15,7 +15,7 @@ This checklist ties together continuous integration signal, Grafana alerting, an
   - `SUPABASE_URL`
   - `SUPABASE_SERVICE_ROLE_KEY`
   - Optional: `SHOPIFY_MODULE_CONFIG_TABLE` and `SHOPIFY_ORDER_TABLE` when the Supabase schema uses custom table names.
-- Confirm that the **observability-budgets** job has passed. It queries Prometheus and Tempo spanmetrics using `observability-budgets.yml` and fails when P95 latency or error-rate thresholds are exceeded compared to the previous day.
+- Confirm that the **observability-budgets** job has passed. It queries Prometheus and Tempo spanmetrics using `observability-budgets.yml` and fails when P95 latency or error-rate thresholds are exceeded compared to the previous day. The API latency SLO is currently **≤ 300 ms at P95**, so any regression beyond that should block the release until resolved.
 - Export any new failure signatures into the on-call runbook.
 
 ## 2. Review Grafana Dashboards

observability-budgets.yml

@@ -10,7 +10,7 @@ prometheus:
       description: "P95 API latency across all HTTP handlers"
       query: histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{service=\"paform-api\",route!~\"/healthcheck\"}[5m])) by (le))
       unit: ms
-      threshold: 3000
+      threshold: 300
       scale: 1000
     - id: api_error_rate
       description: "5xx error rate for the API"

ops/grafana/provisioning/dashboards/backend-golden-signals.json

@@ -99,7 +99,24 @@
       },
       "fieldConfig": {
         "defaults": {
-          "unit": "s"
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 250
+              },
+              {
+                "color": "red",
+                "value": 300
+              }
+            ]
+          },
+          "unit": "ms"
         },
         "overrides": []
       },
@@ -122,7 +139,7 @@
       },
       "targets": [
         {
-          "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"backend\"}[5m])) by (le))",
+          "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"backend\"}[5m])) by (le)) * 1000",
           "legendFormat": "P95 latency",
           "refId": "A"
         }

ops/grafana/provisioning/dashboards/backend_golden_signals.json

@@ -103,13 +103,17 @@
                 "color": "green",
                 "value": null
               },
+              {
+                "color": "orange",
+                "value": 250
+              },
               {
                 "color": "red",
-                "value": 0.3
+                "value": 300
               }
             ]
           },
-          "unit": "s"
+          "unit": "ms"
         },
         "overrides": []
       },
@@ -134,7 +138,7 @@
       },
       "targets": [
         {
-          "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{service=\"backend\"}[5m])) by (le))",
+          "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{service=\"backend\"}[5m])) by (le)) * 1000",
           "legendFormat": "p95 latency",
           "refId": "A"
         }

ops/prometheus/alerts.yml

@@ -20,14 +20,14 @@ groups:
           histogram_quantile(
             0.95,
             sum(rate(http_request_duration_seconds_bucket{job="backend"}[5m])) by (le)
-          ) > 0.75
+          ) > 0.3
         for: 10m
         labels:
           severity: critical
         annotations:
           summary: "Backend p95 latency is elevated"
           description: |
-            The 95th percentile API latency has been above 750ms for more than 10 minutes.
+            The 95th percentile API latency has been above 300ms for more than 10 minutes.
 
   - name: frontend-alerts
     rules: