Show muted alerts in Alert Groups API (prometheus#3797)

This commit updates /api/v2/alerts/groups to show if an alert is
suppressed from one or more active or mute time intervals. While
the muted by field can be found in /api/v2/alerts, it is not
used here because /api/v2/alerts does not take aggregation
or routing into consideration.

It also updates the UI to support filtering muted alerts via the
Muted checkbox.

Signed-off-by: George Robinson <[email protected]>

Author: grobinson-grafana

api/api.go

@@ -45,16 +45,20 @@ type API struct {
 	inFlightSem              chan struct{}
 }
 
-// Options for the creation of an API object. Alerts, Silences, and StatusFunc
-// are mandatory to set. The zero value for everything else is a safe default.
+// Options for the creation of an API object. Alerts, Silences, AlertStatusFunc
+// and GroupMutedFunc are mandatory. The zero value for everything else is a safe
+// default.
 type Options struct {
 	// Alerts to be used by the API. Mandatory.
 	Alerts provider.Alerts
 	// Silences to be used by the API. Mandatory.
 	Silences *silence.Silences
-	// StatusFunc is used be the API to retrieve the AlertStatus of an
+	// AlertStatusFunc is used be the API to retrieve the AlertStatus of an
 	// alert. Mandatory.
-	StatusFunc func(model.Fingerprint) types.AlertStatus
+	AlertStatusFunc func(model.Fingerprint) types.AlertStatus
+	// GroupMutedFunc is used be the API to know if an alert is muted.
+	// Mandatory.
+	GroupMutedFunc func(routeID, groupKey string) ([]string, bool)
 	// Peer from the gossip cluster. If nil, no clustering will be used.
 	Peer cluster.ClusterPeer
 	// Timeout for all HTTP connections. The zero value (and negative
@@ -83,8 +87,11 @@ func (o Options) validate() error {
 	if o.Silences == nil {
 		return errors.New("mandatory field Silences not set")
 	}
-	if o.StatusFunc == nil {
-		return errors.New("mandatory field StatusFunc not set")
+	if o.AlertStatusFunc == nil {
+		return errors.New("mandatory field AlertStatusFunc not set")
+	}
+	if o.GroupMutedFunc == nil {
+		return errors.New("mandatory field GroupMutedFunc not set")
 	}
 	if o.GroupFunc == nil {
 		return errors.New("mandatory field GroupFunc not set")
@@ -113,7 +120,8 @@ func New(opts Options) (*API, error) {
 	v2, err := apiv2.NewAPI(
 		opts.Alerts,
 		opts.GroupFunc,
-		opts.StatusFunc,
+		opts.AlertStatusFunc,
+		opts.GroupMutedFunc,
 		opts.Silences,
 		opts.Peer,
 		log.With(l, "version", "v2"),

api/v2/api.go

@@ -60,6 +60,7 @@ type API struct {
 	alerts         provider.Alerts
 	alertGroups    groupsFn
 	getAlertStatus getAlertStatusFn
+	groupMutedFunc groupMutedFunc
 	uptime         time.Time
 
 	// mtx protects alertmanagerConfig, setAlertStatus and route.
@@ -78,6 +79,7 @@ type API struct {
 
 type (
 	groupsFn         func(func(*dispatch.Route) bool, func(*types.Alert, time.Time) bool) (dispatch.AlertGroups, map[prometheus_model.Fingerprint][]string)
+	groupMutedFunc   func(routeID, groupKey string) ([]string, bool)
 	getAlertStatusFn func(prometheus_model.Fingerprint) types.AlertStatus
 	setAlertStatusFn func(prometheus_model.LabelSet)
 )
@@ -86,16 +88,18 @@ type (
 func NewAPI(
 	alerts provider.Alerts,
 	gf groupsFn,
-	sf getAlertStatusFn,
+	asf getAlertStatusFn,
+	gmf groupMutedFunc,
 	silences *silence.Silences,
 	peer cluster.ClusterPeer,
 	l log.Logger,
 	r prometheus.Registerer,
 ) (*API, error) {
 	api := API{
 		alerts:         alerts,
-		getAlertStatus: sf,
+		getAlertStatus: asf,
 		alertGroups:    gf,
+		groupMutedFunc: gmf,
 		peer:           peer,
 		silences:       silences,
 		logger:         l,
@@ -290,7 +294,7 @@ func (api *API) getAlertsHandler(params alert_ops.GetAlertsParams) middleware.Re
 			continue
 		}
 
-		alert := AlertToOpenAPIAlert(a, api.getAlertStatus(a.Fingerprint()), receivers)
+		alert := AlertToOpenAPIAlert(a, api.getAlertStatus(a.Fingerprint()), receivers, nil)
 
 		res = append(res, alert)
 	}
@@ -407,6 +411,11 @@ func (api *API) getAlertGroupsHandler(params alertgroup_ops.GetAlertGroupsParams
 	res := make(open_api_models.AlertGroups, 0, len(alertGroups))
 
 	for _, alertGroup := range alertGroups {
+		mutedBy, isMuted := api.groupMutedFunc(alertGroup.RouteID, alertGroup.GroupKey)
+		if !*params.Muted && isMuted {
+			continue
+		}
+
 		ag := &open_api_models.AlertGroup{
 			Receiver: &open_api_models.Receiver{Name: &alertGroup.Receiver},
 			Labels:   ModelLabelSetToAPILabelSet(alertGroup.Labels),
@@ -417,7 +426,7 @@ func (api *API) getAlertGroupsHandler(params alertgroup_ops.GetAlertGroupsParams
 			fp := alert.Fingerprint()
 			receivers := allReceivers[fp]
 			status := api.getAlertStatus(fp)
-			apiAlert := AlertToOpenAPIAlert(alert, status, receivers)
+			apiAlert := AlertToOpenAPIAlert(alert, status, receivers, mutedBy)
 			ag.Alerts = append(ag.Alerts, apiAlert)
 		}
 		res = append(res, ag)

api/v2/api_test.go

@@ -484,17 +484,12 @@ func TestAlertToOpenAPIAlert(t *testing.T) {
 			UpdatedAt: updated,
 		}
 	)
-	openAPIAlert := AlertToOpenAPIAlert(alert, types.AlertStatus{State: types.AlertStateActive}, receivers)
+	openAPIAlert := AlertToOpenAPIAlert(alert, types.AlertStatus{State: types.AlertStateActive}, receivers, nil)
 	require.Equal(t, &open_api_models.GettableAlert{
 		Annotations: open_api_models.LabelSet{},
 		Alert: open_api_models.Alert{
 			Labels: open_api_models.LabelSet{"severity": "critical", "alertname": "alert1"},
 		},
-		Status: &open_api_models.AlertStatus{
-			State:       &active,
-			InhibitedBy: []string{},
-			SilencedBy:  []string{},
-		},
 		StartsAt:    convertDateTime(start),
 		EndsAt:      convertDateTime(time.Time{}),
 		UpdatedAt:   convertDateTime(updated),
@@ -503,6 +498,12 @@ func TestAlertToOpenAPIAlert(t *testing.T) {
 			{Name: &receivers[0]},
 			{Name: &receivers[1]},
 		},
+		Status: &open_api_models.AlertStatus{
+			State:       &active,
+			InhibitedBy: []string{},
+			SilencedBy:  []string{},
+			MutedBy:     []string{},
+		},
 	}, openAPIAlert)
 }
 

api/v2/client/alertgroup/get_alert_groups_parameters.go

@@ -117,7 +117,7 @@ func PostableSilenceToProto(s *open_api_models.PostableSilence) (*silencepb.Sile
 }
 
 // AlertToOpenAPIAlert converts internal alerts, alert types, and receivers to *open_api_models.GettableAlert.
-func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers []string) *open_api_models.GettableAlert {
+func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers, mutedBy []string) *open_api_models.GettableAlert {
 	startsAt := strfmt.DateTime(alert.StartsAt)
 	updatedAt := strfmt.DateTime(alert.UpdatedAt)
 	endsAt := strfmt.DateTime(alert.EndsAt)
@@ -128,7 +128,13 @@ func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers
 	}
 
 	fp := alert.Fingerprint().String()
+
 	state := string(status.State)
+	if len(mutedBy) > 0 {
+		// If the alert is muted, change the state to suppressed.
+		state = open_api_models.AlertStatusStateSuppressed
+	}
+
 	aa := &open_api_models.GettableAlert{
 		Alert: open_api_models.Alert{
 			GeneratorURL: strfmt.URI(alert.GeneratorURL),
@@ -144,6 +150,7 @@ func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers
 			State:       &state,
 			SilencedBy:  status.SilencedBy,
 			InhibitedBy: status.InhibitedBy,
+			MutedBy:     mutedBy,
 		},
 	}
 
@@ -155,6 +162,10 @@ func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers
 		aa.Status.InhibitedBy = []string{}
 	}
 
+	if aa.Status.MutedBy == nil {
+		aa.Status.MutedBy = []string{}
+	}
+
 	return aa
 }
 

api/v2/compat.go

@@ -223,6 +223,11 @@ paths:
           type: boolean
           description: Show inhibited alerts
           default: true
+        - in: query
+          name: muted
+          type: boolean
+          description: Show muted alerts
+          default: true
         - name: filter
           in: query
           description: A list of matchers to filter alerts by
@@ -501,10 +506,15 @@ definitions:
         type: array
         items:
           type: string
+      mutedBy:
+        type: array
+        items:
+          type: string
     required:
       - state
       - silencedBy
       - inhibitedBy
+      - mutedBy
   receiver:
     type: object
     properties: