Don't fire on alerts at sev info

Author: dgoodwin

pkg/monitortests/testframework/legacytestframeworkmonitortests/alerts.go

@@ -244,9 +244,10 @@ func isSkippedAlert(alertName string) bool {
 	return false
 }
 
-// runNoNewAlertsFiringTest checks all alerts to see if we:
-//  1. have no historical data for that alert in that namespace for this release
-//  2. have historical data but it was first observed less than 2 weeks ago
+// runNoNewAlertsFiringTest checks all firing non-info alerts to see if we:
+//
+//   - have no historical data for that alert in that namespace for this release, or
+//   - have historical data but it was first observed less than 2 weeks ago
 //
 // If either is true, this test will fail. We do not want new product alerts being added to the product that
 // will trigger routinely and affect the fleet when they ship.
@@ -272,6 +273,13 @@ func runNoNewAlertsFiringTest(historicalData *historicaldata.AlertBestMatcher,
 			continue
 		}
 
+		// Skip alerts with severity info, I don't totally understand the semantics here but it appears some components
+		// use this for informational alerts. (saw two examples from Insights)
+		// We're only interested in warning+critical for the purposes of this test.
+		if interval.StructuredMessage.Annotations[monitorapi.AnnotationSeverity] == "info" {
+			continue
+		}
+
 		// Scan historical data to see if this alert appears new. Ignore release, namespace, level, everything but
 		// AlertName, just see if it's in the data file and when we first observed it in either of the two releases we expect in there.
 		var firstObserved *time.Time

pkg/monitortests/testframework/legacytestframeworkmonitortests/alerts_test.go

@@ -82,6 +82,35 @@ func TestNoNewAlertsFiringBackstop(t *testing.T) {
 			firingIntervals: monitorapi.Intervals{interval},
 			expectedStatus:  []string{"pass", "fail"},
 		},
+		{
+			name: "firing severity info alert never seen before",
+			historicalData: historicaldata.NewAlertMatcherWithHistoricalData(
+				map[historicaldata.AlertDataKey]historicaldata.AlertStatisticalData{}),
+			firingIntervals: monitorapi.Intervals{monitorapi.Interval{
+				Condition: monitorapi.Condition{
+					Level: monitorapi.Warning,
+					StructuredLocator: monitorapi.Locator{
+						Type: monitorapi.LocatorTypeAlert,
+						Keys: map[monitorapi.LocatorKey]string{
+							monitorapi.LocatorAlertKey:     "FakeAlert",
+							monitorapi.LocatorNamespaceKey: "fakens",
+						},
+					},
+					StructuredMessage: monitorapi.Message{
+						HumanMessage: "jibberish",
+						Annotations: map[monitorapi.AnnotationKey]string{
+							monitorapi.AnnotationAlertState: "firing",
+							monitorapi.AnnotationSeverity:   "info",
+						},
+					},
+				},
+				Source:  monitorapi.SourceAlert,
+				Display: false,
+				From:    time.Now().Add(-5 * time.Hour),
+				To:      time.Now().Add(-6 * time.Hour),
+			}},
+			expectedStatus: []string{"pass"}, // info severity alerts should not fail this test
+		},
 		{
 			name: "firing alert observed more than two weeks ago",
 			historicalData: historicaldata.NewAlertMatcherWithHistoricalData(