Fix: Many false positives for waf_blocked alarm in idle deployments (#7642)

deployments/prod/environment.py

@@ -1927,5 +1927,8 @@ def env() -> Mapping[str, str | None]:
 
         'AZUL_MIRROR_BUCKET': 'humancellatlas',
 
-        'AZUL_MIRRORING_CONCURRENCY': '128'
+        'AZUL_MIRRORING_CONCURRENCY': '128',
+
+        # This deployment is busy so we can afford a more sensitive threshold.
+        'azul_waf_blocked_alarm_threshold': '25'
     }

environment.py

@@ -919,5 +919,11 @@ def env() -> Mapping[str, str | None]:
         # $1 per one million requests above ten million requests. The blocking
         # only applies to URLs disallowed via robots.txt.
         #
-        'azul_waf_bot_control': '0'
+        'azul_waf_bot_control': '0',
+
+        # The maximum allowed percentage of blocked requests (number of blocked
+        # requests, divided by the number of all requests, times 100) for a
+        # configured period before a metric alarm is tripped.
+        #
+        'azul_waf_blocked_alarm_threshold': '50'
     }

src/azul/__init__.py

@@ -1870,6 +1870,10 @@ def __attrs_post_init__(self):
     def waf_bot_control(self) -> bool:
         return self._boolean(self.environ['azul_waf_bot_control'])
 
+    @property
+    def waf_blocked_alarm_threshold(self) -> int:
+        return int(self.environ['azul_waf_blocked_alarm_threshold'])
+
     @property
     def vpc_cidr(self) -> str:
         return self.environ['azul_vpc_cidr']

terraform/api_gateway.tf.json.template.py

@@ -181,8 +181,9 @@ def waf_match_path(path_regex: str) -> JSON:
 def add_waf_blocked_alarm(resources: JSON) -> JSON:
     """
     Add a metric alarm that trips if the ratio between blocked and overall
-    requests goes above 25%. Note that requests blocked by rules listed in
-    :py:attr:`Config.waf_rules_not_logged` are not considered.
+    requests goes above a deployment-specific threshold. Note that requests
+    blocked by rules listed in :py:attr:`Config.waf_rules_not_logged` are not
+    considered for the alarm.
     """
     if not config.enable_monitoring:
         return resources
@@ -240,7 +241,7 @@ def add_waf_blocked_alarm(resources: JSON) -> JSON:
                         }
                     ],
                     'comparison_operator': 'GreaterThanThreshold',
-                    'threshold': 25,  # percent blocked of total requests in a period
+                    'threshold': config.waf_blocked_alarm_threshold,
                     'evaluation_periods': 1,
                     'datapoints_to_alarm': 1,
                     'alarm_actions': ['${data.aws_sns_topic.monitoring.arn}'],