Merge pull request kubernetes#73847 from wojtek-t/watch_metric

Add metric exposing amount of processed init events in watchcache

Author: k8s-ci-robot

staging/src/k8s.io/apiserver/pkg/storage/cacher/BUILD

@@ -27,6 +27,7 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/storage:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
         "//vendor/k8s.io/utils/trace:go_default_library",
     ],

staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go

@@ -41,8 +41,24 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/cache"
 	utiltrace "k8s.io/utils/trace"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	initCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "apiserver_init_events_total",
+			Help: "Counter of init events processed in watchcache broken by resource type",
+		},
+		[]string{"resource"},
+	)
 )
 
+func init() {
+	prometheus.MustRegister(initCounter)
+}
+
 // Config contains the configuration for a given Cache.
 type Config struct {
 	// Maximum size of the history cached in memory.
@@ -951,6 +967,10 @@ func (c *cacheWatcher) process(initEvents []*watchCacheEvent, resourceVersion ui
 	for _, event := range initEvents {
 		c.sendWatchCacheEvent(event)
 	}
+	if len(initEvents) > 0 {
+		objType := reflect.TypeOf(initEvents[0].Object).String()
+		initCounter.WithLabelValues(objType).Add(float64(len(initEvents)))
+	}
 	processingTime := time.Since(startTime)
 	if processingTime > initProcessThreshold {
 		objType := "<null>"

test/e2e/framework/metrics_util.go

@@ -140,12 +140,7 @@ var InterestingApiServerMetrics = []string{
 	// TODO(krzysied): apiserver_request_latencies_summary is a deprecated metric.
 	// It should be replaced with new metric.
 	"apiserver_request_latencies_summary",
-	"etcd_helper_cache_entry_total",
-	"etcd_helper_cache_hit_total",
-	"etcd_helper_cache_miss_total",
-	"etcd_request_cache_add_latency_seconds",
-	"etcd_request_cache_get_latency_seconds",
-	"etcd_request_latency_seconds",
+	"apiserver_init_events_total",
 }
 
 var InterestingControllerManagerMetrics = []string{