Clarify that we're flaking on any occurrence of alert, not failing

Author: dgoodwin

pkg/monitortestlibrary/allowedalerts/all.go

@@ -19,10 +19,10 @@ func AllAlertTests(jobType *platformidentification.JobType, etcdAllowance AlertT
 	// In CI firing is rare, but does happen a few times a month. Pending however occurs all the time, which implies operators are routinely in
 	// a state they're not supposed to be during upgrade.
 	// https://github.com/openshift/enhancements/blob/cb81452fddf86c1099acd87610b88369cd6192db/dev-guide/cluster-version-operator/dev/clusteroperator.md#there-are-a-set-of-guarantees-components-are-expected-to-honor-in-return
-	ret = append(ret, newAlert("Cluster Version Operator", "ClusterOperatorDown", jobType).pending().alwaysFail().toTests()...)
-	ret = append(ret, newAlert("Cluster Version Operator", "ClusterOperatorDegraded", jobType).pending().alwaysFail().toTests()...)
-	ret = append(ret, newAlert("Cluster Version Operator", "ClusterOperatorDown", jobType).firing().alwaysFail().toTests()...)
-	ret = append(ret, newAlert("Cluster Version Operator", "ClusterOperatorDegraded", jobType).firing().alwaysFail().toTests()...)
+	ret = append(ret, newAlert("Cluster Version Operator", "ClusterOperatorDown", jobType).pending().alwaysFlake().toTests()...)
+	ret = append(ret, newAlert("Cluster Version Operator", "ClusterOperatorDegraded", jobType).pending().alwaysFlake().toTests()...)
+	ret = append(ret, newAlert("Cluster Version Operator", "ClusterOperatorDown", jobType).firing().alwaysFlake().toTests()...)
+	ret = append(ret, newAlert("Cluster Version Operator", "ClusterOperatorDegraded", jobType).firing().alwaysFlake().toTests()...)
 
 	ret = append(ret, newAlert("etcd", "etcdMembersDown", jobType).pending().neverFail().toTests()...)
 	ret = append(ret, newAlert("etcd", "etcdMembersDown", jobType).firing().toTests()...)

pkg/monitortestlibrary/allowedalerts/basic_alert.go

@@ -111,8 +111,8 @@ func (a *alertBuilder) neverFail() *alertBuilder {
 	return a
 }
 
-func (a *alertBuilder) alwaysFail() *alertBuilder {
-	a.allowanceCalculator = alwaysFail()
+func (a *alertBuilder) alwaysFlake() *alertBuilder {
+	a.allowanceCalculator = alwaysFlake()
 	return a
 }
 

pkg/monitortestlibrary/allowedalerts/matches.go

@@ -55,20 +55,19 @@ func getClosestPercentilesValues(key historicaldata2.AlertDataKey) (historicalda
 	return getCurrentResults().BestMatchDuration(key)
 }
 
-func alwaysFail() AlertTestAllowanceCalculator {
-	return &alwaysFailAllowance{}
+func alwaysFlake() AlertTestAllowanceCalculator {
+	return &alwaysFlakeAllowance{}
 }
 
-// alwaysFailAllowance is for alerts we want to fail a test if they occur at all.
-type alwaysFailAllowance struct {
+// alwaysFlakeAllowance is for alerts we want to flake a test if they occur at all.
+type alwaysFlakeAllowance struct {
 }
 
-func (d *alwaysFailAllowance) FailAfter(key historicaldata2.AlertDataKey) (time.Duration, error) {
-	// TODO: right now we're just flaking until we're certain this doesn't happen too often. Once we're sure,
-	// change to 1 second.
+func (d *alwaysFlakeAllowance) FailAfter(key historicaldata2.AlertDataKey) (time.Duration, error) {
+	// make it effectively impossible for a test failure here, we only want flakes
 	return 24 * time.Hour, nil
 }
 
-func (d *alwaysFailAllowance) FlakeAfter(key historicaldata2.AlertDataKey) time.Duration {
+func (d *alwaysFlakeAllowance) FlakeAfter(key historicaldata2.AlertDataKey) time.Duration {
 	return 1 * time.Second
 }