Merge pull request #1926 from serathius/fix-metrics

Fix measuring remaining timestamp/duration metrics

Author: piosz

events/manager/manager.go

@@ -90,7 +90,9 @@ func (rm *realManager) Housekeep() {
 }
 
 func (rm *realManager) housekeep() {
-	defer lastHousekeepTimestamp.Set(float64(time.Now().Unix()))
+	defer func() {
+		lastHousekeepTimestamp.Set(float64(time.Now().Unix()))
+	}()
 
 	LatestScrapeTime = time.Now()
 

events/sinks/manager.go

@@ -29,13 +29,13 @@ const (
 )
 
 var (
-	// Time spent exporting events to sink in microseconds.
+	// Time spent exporting events to sink in milliseconds.
 	exporterDuration = prometheus.NewSummaryVec(
 		prometheus.SummaryOpts{
 			Namespace: "eventer",
 			Subsystem: "exporter",
-			Name:      "duration_microseconds",
-			Help:      "Time spent exporting events to sink in microseconds.",
+			Name:      "duration_milliseconds",
+			Help:      "Time spent exporting events to sink in milliseconds.",
 		},
 		[]string{"exporter"},
 	)
@@ -137,8 +137,10 @@ func (this *sinkManager) Stop() {
 
 func export(s core.EventSink, data *core.EventBatch) {
 	startTime := time.Now()
-	defer exporterDuration.
-		WithLabelValues(s.Name()).
-		Observe(float64(time.Since(startTime)) / float64(time.Microsecond))
+	defer func() {
+		exporterDuration.
+			WithLabelValues(s.Name()).
+			Observe(float64(time.Since(startTime)) / float64(time.Millisecond))
+	}()
 	s.ExportEvents(data)
 }

events/sources/kubernetes/kubernetes_source.go

@@ -55,8 +55,8 @@ var (
 		prometheus.SummaryOpts{
 			Namespace: "eventer",
 			Subsystem: "scraper",
-			Name:      "duration_microseconds",
-			Help:      "Time spent scraping events in microseconds.",
+			Name:      "duration_milliseconds",
+			Help:      "Time spent scraping events in milliseconds.",
 		})
 )
 
@@ -78,8 +78,10 @@ type KubernetesEventSource struct {
 
 func (this *KubernetesEventSource) GetNewEvents() *core.EventBatch {
 	startTime := time.Now()
-	defer lastEventTimestamp.Set(float64(time.Now().Unix()))
-	defer scrapEventsDuration.Observe(float64(time.Since(startTime)) / float64(time.Microsecond))
+	defer func() {
+		lastEventTimestamp.Set(float64(time.Now().Unix()))
+		scrapEventsDuration.Observe(float64(time.Since(startTime)) / float64(time.Millisecond))
+	}()
 	result := core.EventBatch{
 		Timestamp: time.Now(),
 		Events:    []*kubeapi.Event{},

metrics/manager/manager.go

@@ -29,13 +29,13 @@ const (
 )
 
 var (
-	// The Time spent in a processor in microseconds.
+	// The Time spent in a processor in milliseconds.
 	processorDuration = prometheus.NewSummaryVec(
 		prometheus.SummaryOpts{
 			Namespace: "heapster",
 			Subsystem: "processor",
-			Name:      "duration_microseconds",
-			Help:      "The Time spent in a processor in microseconds.",
+			Name:      "duration_milliseconds",
+			Help:      "The Time spent in a processor in milliseconds.",
 		},
 		[]string{"processor"},
 	)
@@ -149,9 +149,11 @@ func (rm *realManager) housekeep(start, end time.Time) {
 
 func process(p core.DataProcessor, data *core.DataBatch) (*core.DataBatch, error) {
 	startTime := time.Now()
-	defer processorDuration.
-		WithLabelValues(p.Name()).
-		Observe(float64(time.Since(startTime)) / float64(time.Microsecond))
+	defer func() {
+		processorDuration.
+			WithLabelValues(p.Name()).
+			Observe(float64(time.Since(startTime)) / float64(time.Millisecond))
+	}()
 
 	return p.Process(data)
 }

metrics/sinks/manager.go

@@ -152,8 +152,6 @@ func export(s core.DataSink, data *core.DataBatch) {
 		lastExportTimestamp.
 			WithLabelValues(s.Name()).
 			Set(float64(time.Now().Unix()))
-	}()
-	defer func() {
 		exporterDuration.
 			WithLabelValues(s.Name()).
 			Observe(float64(time.Since(startTime)) / float64(time.Millisecond))

metrics/sources/kubelet/kubelet.go

@@ -50,8 +50,8 @@ var (
 		prometheus.SummaryOpts{
 			Namespace: "heapster",
 			Subsystem: "kubelet",
-			Name:      "request_duration_microseconds",
-			Help:      "The Kubelet request latencies in microseconds.",
+			Name:      "request_duration_milliseconds",
+			Help:      "The Kubelet request latencies in milliseconds.",
 		},
 		[]string{"node"},
 	)
@@ -260,7 +260,9 @@ func (this *kubeletMetricsSource) ScrapeMetrics(start, end time.Time) (*DataBatc
 
 func (this *kubeletMetricsSource) scrapeKubelet(client *KubeletClient, host Host, start, end time.Time) ([]cadvisor.ContainerInfo, error) {
 	startTime := time.Now()
-	defer kubeletRequestLatency.WithLabelValues(this.hostname).Observe(float64(time.Since(startTime)))
+	defer func() {
+		kubeletRequestLatency.WithLabelValues(this.hostname).Observe(float64(time.Since(startTime)) / float64(time.Millisecond))
+	}()
 	return client.GetAllRawContainers(host, start, end)
 }
 

metrics/sources/manager.go

@@ -42,13 +42,13 @@ var (
 		[]string{"source"},
 	)
 
-	// Time spent exporting scraping sources in microseconds..
+	// Time spent exporting scraping sources in milliseconds.
 	scraperDuration = prometheus.NewSummaryVec(
 		prometheus.SummaryOpts{
 			Namespace: "heapster",
 			Subsystem: "scraper",
-			Name:      "duration_microseconds",
-			Help:      "Time spent scraping sources in microseconds.",
+			Name:      "duration_milliseconds",
+			Help:      "Time spent scraping sources in milliseconds.",
 		},
 		[]string{"source"},
 	)
@@ -164,12 +164,14 @@ responseloop:
 func scrape(s MetricsSource, start, end time.Time) (*DataBatch, error) {
 	sourceName := s.Name()
 	startTime := time.Now()
-	defer lastScrapeTimestamp.
-		WithLabelValues(sourceName).
-		Set(float64(time.Now().Unix()))
-	defer scraperDuration.
-		WithLabelValues(sourceName).
-		Observe(float64(time.Since(startTime)) / float64(time.Microsecond))
+	defer func() {
+		lastScrapeTimestamp.
+			WithLabelValues(sourceName).
+			Set(float64(time.Now().Unix()))
+		scraperDuration.
+			WithLabelValues(sourceName).
+			Observe(float64(time.Since(startTime)) / float64(time.Millisecond))
+	}()
 
 	return s.ScrapeMetrics(start, end)
 }

metrics/sources/summary/summary.go

@@ -38,8 +38,8 @@ var (
 		prometheus.SummaryOpts{
 			Namespace: "heapster",
 			Subsystem: "kubelet_summary",
-			Name:      "request_duration_microseconds",
-			Help:      "The Kubelet summary request latencies in microseconds.",
+			Name:      "request_duration_milliseconds",
+			Help:      "The Kubelet summary request latencies in milliseconds.",
 		},
 		[]string{"node"},
 	)
@@ -89,7 +89,9 @@ func (this *summaryMetricsSource) ScrapeMetrics(start, end time.Time) (*DataBatc
 
 	summary, err := func() (*stats.Summary, error) {
 		startTime := time.Now()
-		defer summaryRequestLatency.WithLabelValues(this.node.HostName).Observe(float64(time.Since(startTime)))
+		defer func() {
+			summaryRequestLatency.WithLabelValues(this.node.HostName).Observe(float64(time.Since(startTime)) / float64(time.Millisecond))
+		}()
 		return this.kubeletClient.GetSummary(this.node.Host)
 	}()