metrics: add route validation gauge metrics (zalando#3550)

**Prometheus metrics:**
- `skipper_route_invalid_routes_gauge{reason}` (gauge) - Current number
of invalid routes by reason

**CodaHale metrics:**
- `route.invalid.gauge.{reason}` (gauge) - Current invalid routes count
by reason

**Failure Reasons Tracked:**
- `failed_backend_split` - Backend URL parsing errors
- `unknown_filter` - Unknown or disabled filters used
- `unknown_predicate` - Unknown predicates used
- `invalid_filter_params` - Invalid filter parameters
- `invalid_predicate_params` - Invalid predicate parameters
- Any other validation errors that may occur

## How error rate will be calculated

- **Graph Representation**: Displays a graph with points over specific
time intervals.

- **Calculation**: `(routes.total - sum(route.invalid.gauge.%s)) /
routes.total`, where `routes.total` is an existed gauge metric

Signed-off-by: Veronika Volokitina <[email protected]>

Author: vlktna

docs/operation/operation.md

@@ -789,6 +789,72 @@ Metrics explanation:
 
 If you want to read more about RouteSRV see [deploy RouteSRV](../kubernetes/ingress-controller.md#routesrv).
 
+### Route validation metrics
+
+Skipper provides metrics to track the success and failure rates of route processing during configuration updates. These
+metrics help monitor the health of route definitions and identify common configuration issues.
+
+#### Gauge metrics
+
+The following gauge metrics show the current count of invalid routes by failure reason:
+
+- `skipper_route_invalid{reason="<reason>"}`: Current number of invalid routes by reason
+- `routes.total`: Total number of valid routes currently loaded (available as
+  `routesrv_custom_gauges{key="routes.total"}` in RouteSRV)
+
+The `routes.total` gauge metric represents the current number of successfully loaded and valid routes. This metric can
+be used in combination with the invalid route metrics to calculate error rates during route processing.
+
+#### Failure reasons
+
+The metrics track different types of route validation failures:
+
+- `unknown_filter`: Route uses a filter that is not registered or available
+- `invalid_filter_params`: Route has a filter with invalid parameters
+- `unknown_predicate`: Route uses a predicate that is not registered or available
+- `invalid_predicate_params`: Route has a predicate with invalid parameters
+- `failed_backend_split`: Route has an invalid backend URL or configuration
+- `other`: Route has other unclassified validation errors
+
+#### Prometheus example
+
+```
+# HELP skipper_route_invalid Number of invalid routes by reason.
+# TYPE skipper_route_invalid gauge
+skipper_route_invalid{reason="unknown_filter"} 3
+skipper_route_invalid{reason="invalid_filter_params"} 1
+skipper_route_invalid{reason="failed_backend_split"} 2
+
+# HELP skipper_custom_gauges Gauges number of custom metrics.
+# TYPE skipper_custom_gauges gauge
+skipper_custom_gauges{key="routes.total"} 1250
+```
+
+#### Codahale example
+
+```json
+{
+  "gauges": {
+    "route.invalid.unknown_filter": {
+      "value": 3
+    },
+    "route.invalid.invalid_filter_params": {
+      "value": 1
+    },
+    "route.invalid.failed_backend_split": {
+      "value": 2
+    }
+  }
+}
+```
+
+These metrics are particularly useful for:
+
+- Monitoring configuration deployment health
+- Identifying common route definition errors
+- Alerting on configuration issues
+- Tracking the success rate of route updates
+  
 ## OpenTracing
 
 Skipper has support for different [OpenTracing API](http://opentracing.io/) vendors, including

metrics/all_kind.go

@@ -131,14 +131,9 @@ func (a *All) IncErrorsStreaming(routeId string) {
 
 }
 
-func (a *All) IncValidRoutes() {
-	a.prometheus.IncValidRoutes()
-	a.codaHale.IncValidRoutes()
-}
-
-func (a *All) IncInvalidRoutes(reason string) {
-	a.prometheus.IncInvalidRoutes(reason)
-	a.codaHale.IncInvalidRoutes(reason)
+func (a *All) UpdateInvalidRoute(reasonCounts map[string]int) {
+	a.prometheus.UpdateInvalidRoute(reasonCounts)
+	a.codaHale.UpdateInvalidRoute(reasonCounts)
 }
 
 func (a *All) Close() {

metrics/codahale.go

@@ -33,7 +33,6 @@ const (
 
 	KeyErrorsBackend   = "errors.backend.%s"
 	KeyErrorsStreaming = "errors.streaming.%s"
-	KeyValidRoutes     = "route.valid"
 	KeyInvalidRoutes   = "route.invalid.%s"
 
 	statsRefreshDuration = time.Duration(5 * time.Second)
@@ -266,12 +265,10 @@ func (c *CodaHale) IncErrorsStreaming(routeId string) {
 	}
 }
 
-func (c *CodaHale) IncValidRoutes() {
-	c.incCounter(KeyValidRoutes, 1)
-}
-
-func (c *CodaHale) IncInvalidRoutes(reason string) {
-	c.incCounter(fmt.Sprintf(KeyInvalidRoutes, reason), 1)
+func (c *CodaHale) UpdateInvalidRoute(reasonCounts map[string]int) {
+	for reason, count := range reasonCounts {
+		c.UpdateGauge(fmt.Sprintf(KeyInvalidRoutes, reason), float64(count))
+	}
 }
 
 func (c *CodaHale) Close() {

metrics/metrics.go

@@ -80,8 +80,7 @@ type Metrics interface {
 	IncErrorsStreaming(routeId string)
 	RegisterHandler(path string, handler *http.ServeMux)
 	UpdateGauge(key string, value float64)
-	IncValidRoutes()
-	IncInvalidRoutes(reason string)
+	UpdateInvalidRoute(reasonCounts map[string]int)
 	Close()
 }
 

metrics/metricstest/metricsmock.go

@@ -225,12 +225,10 @@ func (m *MockMetrics) Gauge(key string) (v float64, ok bool) {
 	return
 }
 
-func (m *MockMetrics) IncValidRoutes() {
-	m.IncCounter("route.valid")
-}
-
-func (m *MockMetrics) IncInvalidRoutes(reason string) {
-	m.IncCounter("route.invalid." + reason)
+func (m *MockMetrics) UpdateInvalidRoute(reasonCounts map[string]int) {
+	for reason, count := range reasonCounts {
+		m.UpdateGauge("route.invalid."+reason, float64(count))
+	}
 }
 
 func (m *MockMetrics) Close() {}

metrics/prometheus.go

@@ -71,8 +71,7 @@ type Prometheus struct {
 	customHistogramM           *prometheus.HistogramVec
 	customCounterM             *prometheus.CounterVec
 	customGaugeM               *prometheus.GaugeVec
-	validRoutesM               *prometheus.CounterVec
-	invalidRoutesM             *prometheus.CounterVec
+	invalidRouteM              *prometheus.GaugeVec
 
 	opts     Options
 	registry *prometheus.Registry
@@ -308,18 +307,11 @@ func NewPrometheus(opts Options) *Prometheus {
 		Buckets:   opts.HistogramBuckets,
 	}, []string{"key"}))
 
-	p.validRoutesM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{
+	p.invalidRouteM = register(p, prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: namespace,
 		Subsystem: promRouteSubsystem,
-		Name:      "valid_routes",
-		Help:      "Total number of successfully processed routes.",
-	}, []string{}))
-
-	p.invalidRoutesM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: namespace,
-		Subsystem: promRouteSubsystem,
-		Name:      "invalid_routes",
-		Help:      "Total number of invalid routes with failure reasons.",
+		Name:      "invalid",
+		Help:      "Number of invalid routes by reason.",
 	}, []string{"reason"}))
 
 	// Register prometheus runtime collectors if required.
@@ -531,14 +523,11 @@ func (p *Prometheus) IncErrorsStreaming(routeID string) {
 	p.proxyStreamingErrorsM.WithLabelValues(routeID).Inc()
 }
 
-// IncValidRoutes satisfies Metrics interface.
-func (p *Prometheus) IncValidRoutes() {
-	p.validRoutesM.WithLabelValues().Inc()
-}
-
-// IncInvalidRoutes satisfies Metrics interface.
-func (p *Prometheus) IncInvalidRoutes(reason string) {
-	p.invalidRoutesM.WithLabelValues(reason).Inc()
+// UpdateInvalidRoute satisfies Metrics interface.
+func (p *Prometheus) UpdateInvalidRoute(reasonCounts map[string]int) {
+	for reason, count := range reasonCounts {
+		p.invalidRouteM.WithLabelValues(reason).Set(float64(count))
+	}
 }
 
 func (p *Prometheus) Close() {}

routing/datasource.go

@@ -34,6 +34,7 @@ var (
 	errUnknownPredicate       = invalidDefinitionError("unknown_predicate")
 	errInvalidPredicateParams = invalidDefinitionError("invalid_predicate_params")
 	errFailedBackendSplit     = invalidDefinitionError("failed_backend_split")
+	errInvalidMatcher         = invalidDefinitionError("invalid_matcher")
 )
 
 func (it incomingType) String() string {
@@ -509,10 +510,6 @@ func processRouteDef(o *Options, cpm map[string]PredicateSpec, def *eskip.Route)
 		return nil, err
 	}
 
-	if o.Metrics != nil {
-		o.Metrics.IncValidRoutes()
-	}
-
 	return r, nil
 }
 
@@ -529,6 +526,8 @@ func mapPredicates(cps []PredicateSpec) map[string]PredicateSpec {
 // processes a set of route definitions for the routing table
 func processRouteDefs(o *Options, defs []*eskip.Route) (routes []*Route, invalidDefs []*eskip.Route) {
 	cpm := mapPredicates(o.Predicates)
+	reasonCounts := make(map[string]int)
+
 	for _, def := range defs {
 		route, err := processRouteDef(o, cpm, def)
 		if err == nil {
@@ -538,13 +537,19 @@ func processRouteDefs(o *Options, defs []*eskip.Route) (routes []*Route, invalid
 			o.Log.Errorf("failed to process route %s: %v", def.Id, err)
 
 			var defErr invalidDefinitionError
-			if errors.As(err, &defErr) && o.Metrics != nil {
-				o.Metrics.IncInvalidRoutes(defErr.Code())
-			} else if o.Metrics != nil {
-				o.Metrics.IncInvalidRoutes("other")
+			reason := "other"
+			if errors.As(err, &defErr) {
+				reason = defErr.Code()
 			}
+
+			reasonCounts[reason]++
 		}
 	}
+
+	if o.Metrics != nil {
+		o.Metrics.UpdateInvalidRoute(reasonCounts)
+	}
+
 	return
 }
 
@@ -613,6 +618,10 @@ func receiveRouteMatcher(o Options, out chan<- *routeTable, quit <-chan struct{}
 				invalidRouteIds[err.ID] = struct{}{}
 			}
 
+			if o.Metrics != nil {
+				o.Metrics.UpdateInvalidRoute(map[string]int{errInvalidMatcher.Code(): len(errs)})
+			}
+
 			for i := range routes {
 				r := routes[i]
 				if _, found := invalidRouteIds[r.Id]; found {