From a766ed185500dd4b309e755a250abc1fca5021b2 Mon Sep 17 00:00:00 2001
From: Thanh Ha <thanh.ha@linuxfoundation.org>
Date: Wed, 30 Jul 2025 15:05:23 -0400
Subject: [PATCH] Add monitor for ALI ValidationException Detected

This adds a monitor for the ALI ValidationException Detected
CloudWatch alarm.

Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>
---
 monitors.tf | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/monitors.tf b/monitors.tf
index 9e4436a..89840c6 100644
--- a/monitors.tf
+++ b/monitors.tf
@@ -58,3 +58,33 @@ The number of visible messages in `{{queuename.name}}` is outside of the typical
 @webhook-lf-incident-io
 EOT
 }
+
+resource "datadog_monitor" "ALI_ValidationException_Detected" {
+  include_tags        = false
+  require_full_window = false
+  monitor_thresholds {
+    critical = 0
+  }
+  name    = "ALI ValidationException Detected"
+  type    = "event-v2 alert"
+  query   = <<EOT
+events("source:amazon_sns @title:\"ALI ValidationException Detected\"").rollup("count").last("5m") > 0
+EOT
+  message = <<EOT
+# ValidationException
+
+We've detected that a ValidationException has happened in the ALI. This could
+mean the ALI is having issues scaling up runners. Perhaps test-infra release
+was recently updated which may affect this.
+
+## Action
+
+Review scale-up lambda logs in CloudWatch to triage issue and take any
+necessary action. Revert test-infra release to last known working version if
+necessary.
+
+@slack-PyTorch-pytorch-infra-alerts
+@slack-Linux_Foundation-pytorch-alerts
+@webhook-lf-incident-io
+EOT
+}