fix: async metric recording

Author: sanposhiho

pkg/scheduler/internal/queue/scheduling_queue.go

@@ -510,7 +510,7 @@ func (p *PriorityQueue) isPodWorthRequeuing(logger klog.Logger, pInfo *framework
 				}
 				hint = framework.Queue
 			}
-			metrics.QueueingHintExecutionDuration.WithLabelValues(hintfn.PluginName, event.Label, queueingHintToLabel(hint, err)).Observe(metrics.SinceInSeconds(start))
+			p.metricsRecorder.ObserveQueueingHintDurationAsync(hintfn.PluginName, event.Label, queueingHintToLabel(hint, err), metrics.SinceInSeconds(start))
 
 			if hint == framework.QueueSkip {
 				continue

pkg/scheduler/metrics/metric_recorder.go

@@ -119,9 +119,19 @@ func NewMetricsAsyncRecorder(bufferSize int, interval time.Duration, stopCh <-ch
 // ObservePluginDurationAsync observes the plugin_execution_duration_seconds metric.
 // The metric will be flushed to Prometheus asynchronously.
 func (r *MetricAsyncRecorder) ObservePluginDurationAsync(extensionPoint, pluginName, status string, value float64) {
+	r.observeMetricAsync(PluginExecutionDuration, value, pluginName, extensionPoint, status)
+}
+
+// ObserveQueueingHintDurationAsync observes the queueing_hint_execution_duration_seconds metric.
+// The metric will be flushed to Prometheus asynchronously.
+func (r *MetricAsyncRecorder) ObserveQueueingHintDurationAsync(pluginName, event, hint string, value float64) {
+	r.observeMetricAsync(queueingHintExecutionDuration, value, pluginName, event, hint)
+}
+
+func (r *MetricAsyncRecorder) observeMetricAsync(m *metrics.HistogramVec, value float64, labelsValues ...string) {
 	newMetric := &metric{
-		metric:      PluginExecutionDuration,
-		labelValues: []string{pluginName, extensionPoint, status},
+		metric:      m,
+		labelValues: labelsValues,
 		value:       value,
 	}
 	select {

pkg/scheduler/metrics/metrics.go

@@ -207,11 +207,11 @@ var (
 		[]string{"plugin", "extension_point", "status"})
 
 	// This is only available when the QHint feature gate is enabled.
-	QueueingHintExecutionDuration = metrics.NewHistogramVec(
+	queueingHintExecutionDuration = metrics.NewHistogramVec(
 		&metrics.HistogramOpts{
 			Subsystem: SchedulerSubsystem,
 			Name:      "queueing_hint_execution_duration_seconds",
-			Help:      "Duration for running a queueing hint from a plugin.",
+			Help:      "Duration for running a queueing hint function of a plugin.",
 			// Start with 0.01ms with the last bucket being [~22ms, Inf). We use a small factor (1.5)
 			// so that we have better granularity since plugin latency is very sensitive.
 			Buckets:        metrics.ExponentialBuckets(0.00001, 1.5, 20),
@@ -291,7 +291,7 @@ func Register() {
 	registerMetrics.Do(func() {
 		RegisterMetrics(metricsList...)
 		if utilfeature.DefaultFeatureGate.Enabled(features.SchedulerQueueingHints) {
-			RegisterMetrics(QueueingHintExecutionDuration)
+			RegisterMetrics(queueingHintExecutionDuration)
 		}
 		volumebindingmetrics.RegisterVolumeSchedulingMetrics()
 	})