Correctly determine CPU usage since last call of Set()

Author: MMulthaupt

cmd/exporter/handler.go

@@ -2,25 +2,28 @@ package main
 
 import (
 	"net/http"
+	"sync"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/setlog/process_exporter/cmd/exporter/metrics"
 )
 
-type httpHandler struct {
+type httpMetricsRequestHandler struct {
 	metricsHandler http.Handler
-	metricsSet     *metrics.ProcessMetricsSet
+	metricsSet     *metrics.PrometheusProcessMetricsSet
+	metricsMutex   *sync.Mutex
 }
 
-func newHttpHandler(set *metrics.ProcessMetricsSet) *httpHandler {
-	return &httpHandler{
+func newHttpMetricsRequestHandler(set *metrics.PrometheusProcessMetricsSet, metricsMutex *sync.Mutex) *httpMetricsRequestHandler {
+	return &httpMetricsRequestHandler{
 		metricsHandler: promhttp.Handler(),
 		metricsSet:     set,
+		metricsMutex:   metricsMutex,
 	}
 }
 
-func (h *httpHandler) ServeHTTP(response http.ResponseWriter, request *http.Request) {
-	h.metricsSet.UpdateMonitoredSet()
-	h.metricsSet.UpdateMetrics()
+func (h *httpMetricsRequestHandler) ServeHTTP(response http.ResponseWriter, request *http.Request) {
+	h.metricsMutex.Lock()
+	defer h.metricsMutex.Unlock()
 	h.metricsHandler.ServeHTTP(response, request)
 }

cmd/exporter/main.go

@@ -2,16 +2,48 @@ package main
 
 import (
 	"fmt"
-	"log"
 	"net/http"
 	"os"
+	"sync"
+	"time"
 
 	"github.com/setlog/process_exporter/cmd/exporter/flags"
 	"github.com/setlog/process_exporter/cmd/exporter/metrics"
+	log "github.com/sirupsen/logrus"
 )
 
 func main() {
 	namespace, procBinaryName, nameFlag, port := flags.Parse(os.Args[1:])
-	http.Handle("/metrics", newHttpHandler(metrics.NewProcessMetricsSet(namespace, procBinaryName, nameFlag)))
+	metricsSet := metrics.NewPrometheusProcessMetricsSet(namespace, procBinaryName, nameFlag)
+	mu := &sync.Mutex{}
+	updateMetricsSet(metricsSet, mu)
+	// ctx, cancelFunc := context.WithCancel(context.Background())
+	go keepMetricsUpToDate(metricsSet, mu)
+	http.Handle("/metrics", newHttpMetricsRequestHandler(metricsSet, mu))
 	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil))
 }
+
+func keepMetricsUpToDate(set *metrics.PrometheusProcessMetricsSet, mu *sync.Mutex) {
+	defer func() {
+		if r := recover(); r != nil {
+			defer func() { go func() { time.Sleep(time.Second); log.Exit(1) }() }()
+			log.Panicf("Panic in keepMetricsUpToDate(): %v", r)
+		}
+	}()
+	ticker := time.NewTicker(time.Second * 10)
+	for {
+		select {
+		case <-ticker.C:
+			{
+				updateMetricsSet(set, mu)
+			}
+		}
+	}
+}
+
+func updateMetricsSet(set *metrics.PrometheusProcessMetricsSet, mu *sync.Mutex) {
+	mu.Lock()
+	defer mu.Unlock()
+	set.UpdateMonitoredSet()
+	set.UpdateMetrics()
+}

cmd/exporter/metrics/proc_metrics.go

@@ -2,12 +2,14 @@ package metrics
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/shirou/gopsutil/process"
 )
 
 type ProcessMetrics struct {
-	cpu             float64
+	cpuDuration     float64
+	cpuSampleTime   time.Time
 	ram             uint64
 	swap            uint64
 	diskReadBytes   uint64
@@ -24,10 +26,11 @@ func getProcMetrics(pid int) (processMetrics *ProcessMetrics, err error) {
 		return nil, fmt.Errorf("could not open PID %d: %w", pid, err)
 	}
 	m := &ProcessMetrics{}
-	m.cpu, err = proc.CPUPercent()
+	timeStat, err := proc.Times()
 	if err != nil {
-		return nil, fmt.Errorf("could not read CPU usage of PID %d: %w", pid, err)
+		return nil, fmt.Errorf("could not read CPU times of PID %d: %w", pid, err)
 	}
+	m.cpuDuration, m.cpuSampleTime = timeStat.Total(), time.Now()
 	mem, err := proc.MemoryInfo()
 	if err != nil {
 		return nil, fmt.Errorf("could not read memory info of PID %d: %w", pid, err)

cmd/exporter/metrics/prom_proc_metrics.go

@@ -6,9 +6,11 @@ import (
 
 	"github.com/mitchellh/go-ps"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
 )
 
 type PrometheusProcessMetrics struct {
+	previousMetrics      *ProcessMetrics
 	cpuGauge             prometheus.Gauge
 	ramGauge             prometheus.Gauge
 	swapGauge            prometheus.Gauge
@@ -20,7 +22,7 @@ type PrometheusProcessMetrics struct {
 	networkOutBytesGauge prometheus.Gauge
 }
 
-func newProcessMetrics(proc ps.Process, descriptiveName, metricNamespace string) (processMetrics *PrometheusProcessMetrics) {
+func newPrometheusProcessMetrics(proc ps.Process, descriptiveName, metricNamespace string) (processMetrics *PrometheusProcessMetrics) {
 	processMetrics = &PrometheusProcessMetrics{}
 	binaryName := filepath.Base(proc.Executable())
 	pid := fmt.Sprintf("%d", proc.Pid())
@@ -130,7 +132,14 @@ func (pm *PrometheusProcessMetrics) Update() {
 }
 
 func (pm *PrometheusProcessMetrics) Set(processMetrics *ProcessMetrics) {
-	pm.cpuGauge.Set(processMetrics.cpu)
+	if pm.previousMetrics != nil {
+		deltaTime := processMetrics.cpuSampleTime.Sub(pm.previousMetrics.cpuSampleTime).Seconds()
+		if deltaTime > 0 {
+			pm.cpuGauge.Set((processMetrics.cpuDuration - pm.previousMetrics.cpuDuration) / deltaTime)
+		} else {
+			log.Warn("deltaTime <= 0")
+		}
+	}
 	pm.ramGauge.Set(float64(processMetrics.ram))
 	pm.swapGauge.Set(float64(processMetrics.swap))
 	pm.diskReadBytesGauge.Set(float64(processMetrics.diskReadBytes))
@@ -139,4 +148,5 @@ func (pm *PrometheusProcessMetrics) Set(processMetrics *ProcessMetrics) {
 	pm.diskWriteCountGauge.Set(float64(processMetrics.diskWriteCount))
 	pm.networkInBytesGauge.Set(float64(processMetrics.networkInBytes))
 	pm.networkOutBytesGauge.Set(float64(processMetrics.networkOutBytes))
+	pm.previousMetrics = processMetrics
 }

cmd/exporter/metrics/prom_proc_metrics_set.go

@@ -9,23 +9,23 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-type ProcessMetricsSet struct {
+type PrometheusProcessMetricsSet struct {
 	processMetrics map[int]*PrometheusProcessMetrics
 	namespace      string
 	procBinaryName string
 	nameFlag       string
 }
 
-func NewProcessMetricsSet(namespace, procBinaryName, nameFlag string) *ProcessMetricsSet {
-	return &ProcessMetricsSet{
+func NewPrometheusProcessMetricsSet(namespace, procBinaryName, nameFlag string) *PrometheusProcessMetricsSet {
+	return &PrometheusProcessMetricsSet{
 		processMetrics: make(map[int]*PrometheusProcessMetrics),
 		namespace:      namespace,
 		procBinaryName: procBinaryName,
 		nameFlag:       nameFlag,
 	}
 }
 
-func (set *ProcessMetricsSet) UpdateMonitoredSet() {
+func (set *PrometheusProcessMetricsSet) UpdateMonitoredSet() {
 	if set.processMetrics == nil {
 		panic("called update on disposed ProcessMetricsSet")
 	}
@@ -36,7 +36,7 @@ func (set *ProcessMetricsSet) UpdateMonitoredSet() {
 	}
 }
 
-func (set *ProcessMetricsSet) Dispose() {
+func (set *PrometheusProcessMetricsSet) Dispose() {
 	for _, metrics := range set.processMetrics {
 		metrics.Unregister()
 	}
@@ -67,7 +67,7 @@ func AdjustMetricsMap(metricMap map[int]*PrometheusProcessMetrics, pids map[int]
 			errs = append(errs, fmt.Errorf("failed to get descriptive process name for PID %d: %w", pid, err))
 			continue
 		}
-		m := newProcessMetrics(pids[pid], name, metricNamespace)
+		m := newPrometheusProcessMetrics(pids[pid], name, metricNamespace)
 		err = m.Register()
 		if err != nil {
 			errs = append(errs, fmt.Errorf("failed to register process metrics for PID %d: %w", pid, err))
@@ -99,7 +99,7 @@ func FindPidDifferences(pidMap map[int]*PrometheusProcessMetrics, wantedPids map
 	return removePids, newPids
 }
 
-func (set *ProcessMetricsSet) UpdateMetrics() {
+func (set *PrometheusProcessMetricsSet) UpdateMetrics() {
 	for pid, processMetrics := range set.processMetrics {
 		updateMetrics(processMetrics, pid)
 	}