Update bucket for scheduler framework latency histograms.

Author: liu-cong

pkg/scheduler/metrics/metrics.go

@@ -159,7 +159,7 @@ var (
 			StabilityLevel: metrics.ALPHA,
 		},
 	)
-	SchedulingAlgorithmPremptionEvaluationDuration = metrics.NewHistogram(
+	SchedulingAlgorithmPreemptionEvaluationDuration = metrics.NewHistogram(
 		&metrics.HistogramOpts{
 			Subsystem:      SchedulerSubsystem,
 			Name:           "scheduling_algorithm_preemption_evaluation_seconds",
@@ -168,7 +168,7 @@ var (
 			StabilityLevel: metrics.ALPHA,
 		},
 	)
-	DeprecatedSchedulingAlgorithmPremptionEvaluationDuration = metrics.NewHistogram(
+	DeprecatedSchedulingAlgorithmPreemptionEvaluationDuration = metrics.NewHistogram(
 		&metrics.HistogramOpts{
 			Subsystem:      SchedulerSubsystem,
 			Name:           "scheduling_algorithm_preemption_evaluation",
@@ -228,9 +228,10 @@ var (
 
 	PodSchedulingDuration = metrics.NewHistogram(
 		&metrics.HistogramOpts{
-			Subsystem:      SchedulerSubsystem,
-			Name:           "pod_scheduling_duration_seconds",
-			Help:           "E2e latency for a pod being scheduled which may include multiple scheduling attempts.",
+			Subsystem: SchedulerSubsystem,
+			Name:      "pod_scheduling_duration_seconds",
+			Help:      "E2e latency for a pod being scheduled which may include multiple scheduling attempts.",
+			// Start with 1ms with the last bucket being [~16s, Inf)
 			Buckets:        metrics.ExponentialBuckets(0.001, 2, 15),
 			StabilityLevel: metrics.ALPHA,
 		})
@@ -246,20 +247,23 @@ var (
 
 	FrameworkExtensionPointDuration = metrics.NewHistogramVec(
 		&metrics.HistogramOpts{
-			Subsystem:      SchedulerSubsystem,
-			Name:           "framework_extension_point_duration_seconds",
-			Help:           "Latency for running all plugins of a specific extension point.",
-			Buckets:        nil,
+			Subsystem: SchedulerSubsystem,
+			Name:      "framework_extension_point_duration_seconds",
+			Help:      "Latency for running all plugins of a specific extension point.",
+			// Start with 0.1ms with the last bucket being [~200ms, Inf)
+			Buckets:        metrics.ExponentialBuckets(0.0001, 2, 12),
 			StabilityLevel: metrics.ALPHA,
 		},
 		[]string{"extension_point", "status"})
 
 	PluginExecutionDuration = metrics.NewHistogramVec(
 		&metrics.HistogramOpts{
-			Subsystem:      SchedulerSubsystem,
-			Name:           "plugin_execution_duration_seconds",
-			Help:           "Duration for running a plugin at a specific extension point.",
-			Buckets:        nil,
+			Subsystem: SchedulerSubsystem,
+			Name:      "plugin_execution_duration_seconds",
+			Help:      "Duration for running a plugin at a specific extension point.",
+			// Start with 0.01ms with the last bucket being [~22ms, Inf). We use a small factor (1.5)
+			// so that we have better granularity since plugin latency is very sensitive.
+			Buckets:        metrics.ExponentialBuckets(0.00001, 1.5, 20),
 			StabilityLevel: metrics.ALPHA,
 		},
 		[]string{"plugin", "extension_point", "status"})
@@ -304,8 +308,8 @@ var (
 		DeprecatedSchedulingAlgorithmPredicateEvaluationDuration,
 		SchedulingAlgorithmPriorityEvaluationDuration,
 		DeprecatedSchedulingAlgorithmPriorityEvaluationDuration,
-		SchedulingAlgorithmPremptionEvaluationDuration,
-		DeprecatedSchedulingAlgorithmPremptionEvaluationDuration,
+		SchedulingAlgorithmPreemptionEvaluationDuration,
+		DeprecatedSchedulingAlgorithmPreemptionEvaluationDuration,
 		PreemptionVictims,
 		PreemptionAttempts,
 		pendingPods,

pkg/scheduler/scheduler.go

@@ -614,8 +614,8 @@ func (sched *Scheduler) scheduleOne(ctx context.Context) {
 				preemptionStartTime := time.Now()
 				sched.preempt(schedulingCycleCtx, state, fwk, pod, fitError)
 				metrics.PreemptionAttempts.Inc()
-				metrics.SchedulingAlgorithmPremptionEvaluationDuration.Observe(metrics.SinceInSeconds(preemptionStartTime))
-				metrics.DeprecatedSchedulingAlgorithmPremptionEvaluationDuration.Observe(metrics.SinceInMicroseconds(preemptionStartTime))
+				metrics.SchedulingAlgorithmPreemptionEvaluationDuration.Observe(metrics.SinceInSeconds(preemptionStartTime))
+				metrics.DeprecatedSchedulingAlgorithmPreemptionEvaluationDuration.Observe(metrics.SinceInMicroseconds(preemptionStartTime))
 				metrics.SchedulingLatency.WithLabelValues(metrics.PreemptionEvaluation).Observe(metrics.SinceInSeconds(preemptionStartTime))
 				metrics.DeprecatedSchedulingLatency.WithLabelValues(metrics.PreemptionEvaluation).Observe(metrics.SinceInSeconds(preemptionStartTime))
 			}

test/integration/scheduler_perf/scheduler_bench_test.go

@@ -48,7 +48,7 @@ var (
 
 // BenchmarkScheduling benchmarks the scheduling rate when the cluster has
 // various quantities of nodes and scheduled pods.
-func BenchmarkSchedulingV(b *testing.B) {
+func BenchmarkScheduling(b *testing.B) {
 	tests := []struct{ nodes, existingPods, minPods int }{
 		{nodes: 100, existingPods: 0, minPods: 100},
 		{nodes: 100, existingPods: 1000, minPods: 100},