Collect some of scheduling metrics and scheduling throughput

In addition to getting overall performance measurements from golang benchmark,
collect metrics that provides information about insides of the scheduler itself.
This is a first step towards improving what we collect about the scheduler.

Metrics in question:
- scheduler_scheduling_algorithm_predicate_evaluation_seconds
- scheduler_scheduling_algorithm_priority_evaluation_seconds
- scheduler_binding_duration_seconds
- scheduler_e2e_scheduling_duration_seconds

Scheduling throughput is computed on the fly inside perfScheduling.

Author: ingvagabund

staging/src/k8s.io/component-base/metrics/testutil/BUILD

@@ -13,6 +13,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library",
         "//staging/src/k8s.io/component-base/metrics:go_default_library",
         "//vendor/github.com/prometheus/client_golang/prometheus/testutil:go_default_library",
+        "//vendor/github.com/prometheus/client_model/go:go_default_library",
         "//vendor/github.com/prometheus/common/expfmt:go_default_library",
         "//vendor/github.com/prometheus/common/model:go_default_library",
     ],
@@ -34,7 +35,14 @@ filegroup(
 
 go_test(
     name = "go_default_test",
-    srcs = ["testutil_test.go"],
+    srcs = [
+        "metrics_test.go",
+        "testutil_test.go",
+    ],
     embed = [":go_default_library"],
-    deps = ["//staging/src/k8s.io/component-base/metrics:go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/component-base/metrics:go_default_library",
+        "//vendor/github.com/prometheus/client_model/go:go_default_library",
+        "//vendor/k8s.io/utils/pointer:go_default_library",
+    ],
 )

staging/src/k8s.io/component-base/metrics/testutil/metrics.go

@@ -19,11 +19,16 @@ package testutil
 import (
 	"fmt"
 	"io"
+	"math"
 	"reflect"
+	"sort"
 	"strings"
 
+	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
 	"github.com/prometheus/common/model"
+
+	"k8s.io/component-base/metrics"
 )
 
 var (
@@ -178,3 +183,140 @@ func ValidateMetrics(metrics Metrics, metricName string, expectedLabels ...strin
 	}
 	return nil
 }
+
+// Histogram wraps prometheus histogram DTO (data transfer object)
+type Histogram struct {
+	*dto.Histogram
+}
+
+// GetHistogramFromGatherer collects a metric from a gatherer implementing k8s.io/component-base/metrics.Gatherer interface.
+// Used only for testing purposes where we need to gather metrics directly from a running binary (without metrics endpoint).
+func GetHistogramFromGatherer(gatherer metrics.Gatherer, metricName string) (Histogram, error) {
+	var metricFamily *dto.MetricFamily
+	m, err := gatherer.Gather()
+	if err != nil {
+		return Histogram{}, err
+	}
+	for _, mFamily := range m {
+		if mFamily.Name != nil && *mFamily.Name == metricName {
+			metricFamily = mFamily
+			break
+		}
+	}
+
+	if metricFamily == nil {
+		return Histogram{}, fmt.Errorf("Metric %q not found", metricName)
+	}
+
+	if metricFamily.GetMetric() == nil {
+		return Histogram{}, fmt.Errorf("Metric %q is empty", metricName)
+	}
+
+	if len(metricFamily.GetMetric()) == 0 {
+		return Histogram{}, fmt.Errorf("Metric %q is empty", metricName)
+	}
+
+	return Histogram{
+		// Histograms are stored under the first index (based on observation).
+		// Given there's only one histogram registered per each metric name, accessing
+		// the first index is sufficient.
+		metricFamily.GetMetric()[0].GetHistogram(),
+	}, nil
+}
+
+func uint64Ptr(u uint64) *uint64 {
+	return &u
+}
+
+// Bucket of a histogram
+type bucket struct {
+	upperBound float64
+	count      float64
+}
+
+func bucketQuantile(q float64, buckets []bucket) float64 {
+	if q < 0 {
+		return math.Inf(-1)
+	}
+	if q > 1 {
+		return math.Inf(+1)
+	}
+
+	if len(buckets) < 2 {
+		return math.NaN()
+	}
+
+	rank := q * buckets[len(buckets)-1].count
+	b := sort.Search(len(buckets)-1, func(i int) bool { return buckets[i].count >= rank })
+
+	if b == 0 {
+		return buckets[0].upperBound * (rank / buckets[0].count)
+	}
+
+	// linear approximation of b-th bucket
+	brank := rank - buckets[b-1].count
+	bSize := buckets[b].upperBound - buckets[b-1].upperBound
+	bCount := buckets[b].count - buckets[b-1].count
+
+	return buckets[b-1].upperBound + bSize*(brank/bCount)
+}
+
+// Quantile computes q-th quantile of a cumulative histogram.
+// It's expected the histogram is valid (by calling Validate)
+func (hist *Histogram) Quantile(q float64) float64 {
+	buckets := []bucket{}
+
+	for _, bckt := range hist.Bucket {
+		buckets = append(buckets, bucket{
+			count:      float64(*bckt.CumulativeCount),
+			upperBound: *bckt.UpperBound,
+		})
+	}
+
+	// bucketQuantile expects the upper bound of the last bucket to be +inf
+	// buckets[len(buckets)-1].upperBound = math.Inf(+1)
+
+	return bucketQuantile(q, buckets)
+}
+
+// Average computes histogram's average value
+func (hist *Histogram) Average() float64 {
+	return *hist.SampleSum / float64(*hist.SampleCount)
+}
+
+// Clear clears all fields of the wrapped histogram
+func (hist *Histogram) Clear() {
+	if hist.SampleCount != nil {
+		*hist.SampleCount = 0
+	}
+	if hist.SampleSum != nil {
+		*hist.SampleSum = 0
+	}
+	for _, b := range hist.Bucket {
+		if b.CumulativeCount != nil {
+			*b.CumulativeCount = 0
+		}
+	}
+}
+
+// Validate makes sure the wrapped histogram has all necessary fields set and with valid values.
+func (hist *Histogram) Validate() error {
+	if hist.SampleCount == nil || *hist.SampleCount == 0 {
+		return fmt.Errorf("nil or empty histogram SampleCount")
+	}
+
+	if hist.SampleSum == nil || *hist.SampleSum == 0 {
+		return fmt.Errorf("nil or empty histogram SampleSum")
+	}
+
+	for _, bckt := range hist.Bucket {
+		if bckt == nil {
+			return fmt.Errorf("empty histogram bucket")
+		}
+		if bckt.UpperBound == nil || *bckt.UpperBound < 0 {
+			return fmt.Errorf("nil or negative histogram bucket UpperBound")
+		}
+	}
+
+	return nil
+}