Extract the cluster operator name from Down/Degraded Alerts

This will now appear in the locator and help with analysis about which
ClusterOperator is down/degraded. It may also prevent overlap problems
we could be experiencing when more than one is down.

Author: dgoodwin

pkg/monitor/monitorapi/construction.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/prometheus/common/model"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/kube-openapi/pkg/util/sets"
 )
@@ -92,23 +93,39 @@ func (b *LocatorBuilder) NodeFromName(nodeName string) Locator {
 		Build()
 }
 
-func (b *LocatorBuilder) AlertFromNames(alertName, node, namespace, pod, container string) Locator {
+func (b *LocatorBuilder) AlertFromPromSampleStream(alert *model.SampleStream) Locator {
 	b.targetType = LocatorTypeAlert
+
+	alertName := string(alert.Metric[model.AlertNameLabel])
 	if len(alertName) > 0 {
 		b.annotations[LocatorAlertKey] = alertName
 	}
+	node := string(alert.Metric["instance"])
 	if len(node) > 0 {
 		b.annotations[LocatorNodeKey] = node
 	}
+	namespace := string(alert.Metric["namespace"])
 	if len(namespace) > 0 {
 		b.annotations[LocatorNamespaceKey] = namespace
 	}
+	pod := string(alert.Metric["pod"])
 	if len(pod) > 0 {
 		b.annotations[LocatorPodKey] = pod
 	}
+	container := string(alert.Metric["container"])
 	if len(container) > 0 {
 		b.annotations[LocatorContainerKey] = container
 	}
+
+	// Some alerts include a very useful name field, ClusterOperator[Down|Degraded] for example,
+	// always comes from the namespace openshift-cluster-version, but this field is the actual
+	// name of the operator that was detected to be down. This is very useful for locators and
+	// analysis.
+	additionalName := string(alert.Metric["name"])
+	if len(additionalName) > 0 {
+		b.annotations[LocatorNameKey] = additionalName
+	}
+
 	return b.Build()
 }
 

pkg/monitortests/testframework/alertanalyzer/alerts.go

@@ -13,6 +13,7 @@ import (
 	"github.com/openshift/origin/pkg/monitor/monitorapi"
 	prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	prometheustypes "github.com/prometheus/common/model"
+	"github.com/sirupsen/logrus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -296,15 +297,8 @@ func createEventIntervalsForAlerts(ctx context.Context, alerts prometheustypes.V
 	case alerts.Type() == prometheustypes.ValMatrix:
 		matrixAlert := alerts.(prometheustypes.Matrix)
 		for _, alert := range matrixAlert {
-			alertName := alert.Metric[prometheustypes.AlertNameLabel]
 
-			lb := monitorapi.NewLocator().AlertFromNames(
-				string(alertName),
-				string(alert.Metric["instance"]),
-				string(alert.Metric["namespace"]),
-				string(alert.Metric["pod"]),
-				string(alert.Metric["container"]),
-			)
+			lb := monitorapi.NewLocator().AlertFromPromSampleStream(alert)
 
 			var level monitorapi.IntervalLevel
 			switch {
@@ -364,10 +358,7 @@ func createEventIntervalsForAlerts(ctx context.Context, alerts prometheustypes.V
 		}
 
 	default:
-		ret = append(ret, monitorapi.NewInterval(monitorapi.SourceAlert, monitorapi.Error).
-			Locator(monitorapi.NewLocator().AlertFromNames("all", "", "", "", "")).
-			Message(monitorapi.NewMessage().HumanMessagef("unhandled type: %v", alerts.Type())).
-			Build(startTime, time.Now()))
+		logrus.WithField("type", alerts.Type()).Warning("unhandled prometheus alert type received in alert monitor")
 	}
 
 	return ret, nil