Support pinning alerts that we want to fire on any occurrence

New initiative to allow pinning alerts that are simply not allowed to
fire, regardless of historical data.

Author: dgoodwin

pkg/alerts/check.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 	"time"
 
-	allowedalerts2 "github.com/openshift/origin/pkg/monitortestlibrary/allowedalerts"
+	allowedalerts "github.com/openshift/origin/pkg/monitortestlibrary/allowedalerts"
 	"github.com/openshift/origin/pkg/monitortestlibrary/platformidentification"
 
 	o "github.com/onsi/gomega"
@@ -18,20 +18,14 @@ import (
 	prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/client-go/rest"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
 type AllowedAlertsFunc func(featureSet configv1.FeatureSet) (allowedFiringWithBugs, allowedFiring, allowedPendingWithBugs, allowedPending helper.MetricConditions)
 
 // CheckAlerts will query prometheus and ensure no-unexpected alerts were pending or firing.
 // Used by both upgrade and conformance suites, with different allowances for each.
-func CheckAlerts(allowancesFunc AllowedAlertsFunc,
-	restConfig *rest.Config,
-	prometheusClient prometheusv1.API, // TODO: remove
-	configClient configclient.Interface, // TODO: remove
-	testDuration time.Duration,
-	f *framework.Framework) {
+func CheckAlerts(allowancesFunc AllowedAlertsFunc, prometheusClient prometheusv1.API, configClient configclient.Interface, testDuration time.Duration, f *framework.Framework) {
 
 	featureSet := configv1.Default
 	featureGate, err := configClient.ConfigV1().FeatureGates().Get(context.TODO(), "cluster", metav1.GetOptions{})
@@ -46,11 +40,11 @@ func CheckAlerts(allowancesFunc AllowedAlertsFunc,
 	// In addition to the alert allowances passed in (which can differ for upgrades vs conformance),
 	// we also exclude alerts that have their own separate tests codified. This is a backstop test for
 	// everything else.
-	for _, alertTest := range allowedalerts2.AllAlertTests(&platformidentification.JobType{},
-		allowedalerts2.DefaultAllowances) {
+	for _, alertTest := range allowedalerts.AllAlertTests(&platformidentification.JobType{},
+		allowedalerts.DefaultAllowances) {
 
 		switch alertTest.AlertState() {
-		case allowedalerts2.AlertPending:
+		case allowedalerts.AlertPending:
 			// a pending test covers pending and everything above (firing)
 			allowedPendingAlerts = append(allowedPendingAlerts,
 				helper.MetricCondition{
@@ -64,7 +58,7 @@ func CheckAlerts(allowancesFunc AllowedAlertsFunc,
 					Text:     "has a separate e2e test",
 				},
 			)
-		case allowedalerts2.AlertInfo:
+		case allowedalerts.AlertInfo:
 			// an info test covers all firing
 			allowedFiringAlerts = append(allowedFiringAlerts,
 				helper.MetricCondition{

pkg/monitortestlibrary/allowedalerts/all.go

@@ -15,6 +15,15 @@ func AllAlertTests(jobType *platformidentification.JobType, etcdAllowance AlertT
 	ret = append(ret, newNamespacedAlert("KubePodNotReady", jobType).pending().neverFail().toTests()...)
 	ret = append(ret, newNamespacedAlert("KubePodNotReady", jobType).firing().toTests()...)
 
+	// ClusterOperatorDown and ClusterOperatorDegraded should not occur during a normal upgrade.
+	// In CI firing is rare, but does happen a few times a month. Pending however occurs all the time, which implies operators are routinely in
+	// a state they're not supposed to be during upgrade.
+	// https://github.com/openshift/enhancements/blob/cb81452fddf86c1099acd87610b88369cd6192db/dev-guide/cluster-version-operator/dev/clusteroperator.md#there-are-a-set-of-guarantees-components-are-expected-to-honor-in-return
+	ret = append(ret, newAlert("Cluster Version Operator", "ClusterOperatorDown", jobType).pending().alwaysFail().toTests()...)
+	ret = append(ret, newAlert("Cluster Version Operator", "ClusterOperatorDegraded", jobType).pending().alwaysFail().toTests()...)
+	ret = append(ret, newAlert("Cluster Version Operator", "ClusterOperatorDown", jobType).firing().alwaysFail().toTests()...)
+	ret = append(ret, newAlert("Cluster Version Operator", "ClusterOperatorDegraded", jobType).firing().alwaysFail().toTests()...)
+
 	ret = append(ret, newAlert("etcd", "etcdMembersDown", jobType).pending().neverFail().toTests()...)
 	ret = append(ret, newAlert("etcd", "etcdMembersDown", jobType).firing().toTests()...)
 	ret = append(ret, newAlert("etcd", "etcdGRPCRequestsSlow", jobType).pending().neverFail().toTests()...)

pkg/monitortestlibrary/allowedalerts/basic_alert.go

@@ -111,6 +111,11 @@ func (a *alertBuilder) neverFail() *alertBuilder {
 	return a
 }
 
+func (a *alertBuilder) alwaysFail() *alertBuilder {
+	a.allowanceCalculator = alwaysFail()
+	return a
+}
+
 func (a *alertBuilder) toTests() []AlertTest {
 	if !a.divideByNamespaces {
 		return []AlertTest{

pkg/monitortestlibrary/allowedalerts/matches.go

@@ -54,3 +54,21 @@ func (d *percentileAllowances) FlakeAfter(key historicaldata2.AlertDataKey) time
 func getClosestPercentilesValues(key historicaldata2.AlertDataKey) (historicaldata2.StatisticalDuration, string, error) {
 	return getCurrentResults().BestMatchDuration(key)
 }
+
+func alwaysFail() AlertTestAllowanceCalculator {
+	return &alwaysFailAllowance{}
+}
+
+// alwaysFailAllowance is for alerts we want to fail a test if they occur at all.
+type alwaysFailAllowance struct {
+}
+
+func (d *alwaysFailAllowance) FailAfter(key historicaldata2.AlertDataKey) (time.Duration, error) {
+	// TODO: right now we're just flaking until we're certain this doesn't happen too often. Once we're sure,
+	// change to 1 second.
+	return 24 * time.Hour, nil
+}
+
+func (d *alwaysFailAllowance) FlakeAfter(key historicaldata2.AlertDataKey) time.Duration {
+	return 1 * time.Second
+}

test/e2e/upgrade/alert/alert.go

@@ -48,8 +48,7 @@ func (t *UpgradeTest) Test(ctx context.Context, f *framework.Framework, done <-c
 
 	testDuration := time.Now().Sub(startTime).Round(time.Second)
 
-	alerts.CheckAlerts(alerts.AllowedAlertsDuringUpgrade, t.oc.AdminConfig(),
-		t.oc.NewPrometheusClient(context.TODO()), t.oc.AdminConfigClient(), testDuration, f)
+	alerts.CheckAlerts(alerts.AllowedAlertsDuringUpgrade, t.oc.NewPrometheusClient(context.TODO()), t.oc.AdminConfigClient(), testDuration, f)
 }
 
 // Teardown cleans up any remaining resources.

test/extended/prometheus/prometheus.go

@@ -255,7 +255,7 @@ var _ = g.Describe("[sig-instrumentation][Late] Alerts", func() {
 	g.It("shouldn't report any unexpected alerts in firing or pending state", func() {
 		// we only consider samples since the beginning of the test
 		testDuration := exutil.DurationSinceStartInSeconds()
-		alerts.CheckAlerts(alerts.AllowedAlertsDuringConformance, oc.AdminConfig(), oc.NewPrometheusClient(context.TODO()), oc.AdminConfigClient(), testDuration, nil)
+		alerts.CheckAlerts(alerts.AllowedAlertsDuringConformance, oc.NewPrometheusClient(context.TODO()), oc.AdminConfigClient(), testDuration, nil)
 	})
 
 	g.It("shouldn't exceed the series limit of total series sent via telemetry from each cluster", func() {