Expose more stats via Prometheus (#29)

* Expose more stats

Author: localhots

rpc/decoder.go

@@ -241,3 +241,49 @@ func Decode(b []byte) (*EnvelopeDecoder, error) {
 	}
 	return &env, nil
 }
+
+// nolint:gocyclo
+func (env EnvelopeDecoder) Method() string {
+	switch {
+	case env.Body.GetRPCMethods != nil:
+		return "GetRPCMethods"
+	case env.Body.SetParameterValues != nil:
+		return "SetParameterValues"
+	case env.Body.GetParameterValues != nil:
+		return "GetParameterValues"
+	case env.Body.GetParameterNames != nil:
+		return "GetParameterNames"
+	case env.Body.SetParameterAttributes != nil:
+		return "SetParameterAttributes"
+	case env.Body.GetParameterAttributes != nil:
+		return "GetParameterAttributes"
+	case env.Body.AddObject != nil:
+		return "AddObject"
+	case env.Body.DeleteObject != nil:
+		return "DeleteObject"
+	case env.Body.Reboot != nil:
+		return "Reboot"
+	case env.Body.Download != nil:
+		return "Download"
+	case env.Body.Upload != nil:
+		return "Upload"
+	case env.Body.FactoryReset != nil:
+		return "FactoryReset"
+	case env.Body.GetQueuedTransfers != nil:
+		return "GetQueuedTransfers"
+	case env.Body.GetAllQueuedTransfers != nil:
+		return "GetAllQueuedTransfers"
+	case env.Body.ScheduleInform != nil:
+		return "ScheduleInform"
+	case env.Body.SetVouchers != nil:
+		return "SetVouchers"
+	case env.Body.GetOptions != nil:
+		return "GetOptions"
+	case env.Body.Fault != nil:
+		return "Fault"
+	case env.Body.TransferCompleteResponse != nil:
+		return "TransferCompleteResponse"
+	default:
+		return "Unknown"
+	}
+}

simulator/get_parameter_values.go

@@ -32,6 +32,7 @@ func (s *Simulator) handleGetParameterValues(envID string, r *rpc.GetParameterVa
 		}
 	}
 
+	s.metrics.ParametersRead.Add(float64(len(params)))
 	resp.Body.GetParameterValuesResponse = &rpc.GetParameterValuesResponseEncoder{
 		ParameterList: rpc.ParameterListEncoder{
 			ArrayType:       rpc.ArrayType("cwmp:ParameterValue", len(params)),

simulator/inform.go

@@ -10,7 +10,6 @@ import (
 	"net"
 	"net/http"
 	"net/url"
-	"strconv"
 	"time"
 
 	"github.com/icholy/digest"
@@ -83,20 +82,15 @@ func (s *Simulator) startSession(ctx context.Context, handler sessionHandler) {
 	}
 	defer s.sessionMux.Unlock()
 
+	s.metrics.SessionsAttempted.Inc()
 	u, err := url.Parse(Config.ACSURL)
 	if err != nil {
 		log.Error().Err(err).Msg("Failed to parse ACS URL")
 		return
 	}
 
-	connectionStartTime := time.Now()
-	defer func() {
-		s.metrics.ConcurrentInforms.Dec()
-		s.metrics.InformDuration.Observe(float64(time.Since(connectionStartTime).Milliseconds()))
-	}()
-	s.metrics.ConcurrentInforms.Inc()
-
 	log.Info().Str("acs_url", Config.ACSURL).Msg("Connecting to ACS")
+	connectionStartTime := time.Now()
 	client, closeFn, err := newClient(u.Hostname(), tcpPort(u))
 	s.metrics.ConnectionLatency.Observe(float64(time.Since(connectionStartTime).Milliseconds()))
 	if err != nil {
@@ -107,13 +101,26 @@ func (s *Simulator) startSession(ctx context.Context, handler sessionHandler) {
 	}
 	defer func() { _ = closeFn() }()
 
+	s.metrics.SessionsEstablished.Inc()
 	handler(ctx, &client)
 }
 
 // nolint:gocyclo
 func (s *Simulator) informHandler(ctx context.Context, client *http.Client) {
 	log.Info().Msg("Starting inform")
 	informEnv := s.makeInformEnvelope()
+
+	evt := informEnv.Body.Inform.Event.Events[0]
+	startedAt := time.Now()
+	s.metrics.ConcurrentSessions.Inc()
+	s.metrics.InformEvents.With(prometheus.Labels{"event": evt.EventCode}).Inc()
+	defer func() {
+		s.metrics.ConcurrentSessions.Dec()
+		s.metrics.SessionDuration.With(prometheus.Labels{
+			"event": evt.EventCode,
+		}).Observe(float64(time.Since(startedAt).Milliseconds()))
+	}()
+
 	resp, err := s.request(ctx, client, informEnv)
 	if err != nil {
 		log.Error().Err(err).Msg("Failed to make request")
@@ -174,9 +181,11 @@ pendingRequests:
 		}
 	}
 
+	s.metrics.SessionsCompleted.Inc()
 	for _, evt := range informEnv.Body.Inform.Event.Events {
 		if evt.EventCode == rpc.EventBootstrap {
 			s.dm.SetBootstrapped(true)
+			s.metrics.Bootstrapped.Inc()
 			break
 		}
 	}
@@ -282,9 +291,7 @@ func (s *Simulator) request(ctx context.Context, client *http.Client, env *rpc.E
 		return nil, fmt.Errorf("execute request: %w", err)
 	}
 	s.cookies.SetCookies(req.URL, resp.Cookies())
-	s.metrics.ResponseStatus.With(prometheus.Labels{
-		"status": strconv.Itoa(resp.StatusCode),
-	}).Inc()
+	s.metrics.ResponseStatus.With(prometheus.Labels{"status": resp.Status}).Inc()
 
 	return resp, nil
 }

simulator/metrics/metrics.go

@@ -7,59 +7,121 @@ import (
 )
 
 type Metrics struct {
-	Registrer         prometheus.Registerer
-	RequestFailures   prometheus.Counter
-	ResponseStatus    prometheus_CounterVec
-	ConnectionLatency prometheus.Histogram
-	InformDuration    prometheus.Histogram
-	ConcurrentInforms prometheus.Gauge
+	Bootstrapped        prometheus.Counter
+	ConnectionLatency   prometheus.Histogram
+	MethodCalls         prometheus_CounterVec
+	RequestFailures     prometheus.Counter
+	ResponseStatus      prometheus_CounterVec
+	SessionsAttempted   prometheus.Counter
+	SessionsEstablished prometheus.Counter
+	SessionsCompleted   prometheus.Counter
+	SessionDuration     prometheus_HistogramVec
+	ConcurrentSessions  prometheus.Gauge
+	InformEvents        prometheus_CounterVec
+	ParametersRead      prometheus.Counter
+	ParametersWritten   prometheus.Counter
 }
 
-// Odd name, I know.
+// prometheus.CounterVec is a struct, not an interface. We can't reimplement it
+// so instead a custom interface is defined.
 // nolint:revive,stylecheck
 type prometheus_CounterVec interface {
 	prometheus.Collector
 	With(prometheus.Labels) prometheus.Counter
 }
 
+// prometheus.HistogramVec is a struct, not an interface. We can't reimplement
+// it so instead a custom interface is defined.
+// nolint:revive,stylecheck
+type prometheus_HistogramVec interface {
+	prometheus.Collector
+	With(prometheus.Labels) prometheus.Observer
+}
+
 func New(reg prometheus.Registerer) *Metrics {
 	m := &Metrics{
+		Bootstrapped: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "bootstrapped",
+			Help: "Number of successful bootstraps",
+		}),
+		ConnectionLatency: prometheus.NewHistogram(prometheus.HistogramOpts{
+			Name:                        "acs_connection_latency",
+			Help:                        "Time it takes to establish a connection to ACS in milliseconds",
+			Buckets:                     prometheus.LinearBuckets(0.5, 0.5, 10),
+			NativeHistogramMaxExemplars: 100,
+			NativeHistogramExemplarTTL:  15 * time.Second,
+		}),
+		MethodCalls: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "rpc_method_calls",
+				Help: "Number of RPC method calls",
+			},
+			[]string{"method"},
+		),
 		RequestFailures: prometheus.NewCounter(prometheus.CounterOpts{
 			Name: "acs_request_failures",
-			Help: "Number of times requests to ACS failed.",
+			Help: "Number of times requests to ACS failed",
 		}),
 		ResponseStatus: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
 				Name: "acs_response_status",
-				Help: "ACS response status code.",
+				Help: "ACS response status code",
 			},
 			[]string{"status"},
 		),
-		ConcurrentInforms: prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: "concurrent_informs",
-			Help: "The number of inform sessions in progress.",
+		SessionsAttempted: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "sessions_attempted",
+			Help: "Number of attempted sessions",
 		}),
-		ConnectionLatency: prometheus.NewHistogram(prometheus.HistogramOpts{
-			Name:                        "acs_connection_latency",
-			Help:                        "Time it takes to establish a connection to ACS in milliseconds.",
-			Buckets:                     prometheus.LinearBuckets(0.5, 0.5, 10),
-			NativeHistogramMaxExemplars: 100,
-			NativeHistogramExemplarTTL:  15 * time.Second,
+		SessionsEstablished: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "sessions_established",
+			Help: "Number of established sessions",
+		}),
+		SessionsCompleted: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "sessions_completed",
+			Help: "Number of completed sessions",
 		}),
-		InformDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
-			Name:                        "inform_duration",
-			Help:                        "Inform length in milliseconds.",
+		SessionDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:                        "session_duration_per_event",
+			Help:                        "Session length in milliseconds",
 			Buckets:                     prometheus.ExponentialBuckets(1, 10, 10),
 			NativeHistogramMaxExemplars: 100,
 			NativeHistogramExemplarTTL:  15 * time.Minute,
+		}, []string{"event"}),
+		ConcurrentSessions: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "concurrent_sessions",
+			Help: "The number of concurrent in-progress sessions",
+		}),
+		InformEvents: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "inform_events",
+				Help: "Inform event count by event",
+			},
+			[]string{"event"},
+		),
+		ParametersRead: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "parameters_read",
+			Help: "Number of parameters accessed via GetParameterValues",
+		}),
+		ParametersWritten: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "parameters_written",
+			Help: "Number of parameters changed via SetParameterValues",
 		}),
 	}
 	reg.MustRegister(
+		m.Bootstrapped,
+		m.ConnectionLatency,
+		m.MethodCalls,
 		m.RequestFailures,
 		m.ResponseStatus,
-		m.ConcurrentInforms,
-		m.ConnectionLatency,
-		m.InformDuration,
+		m.SessionsAttempted,
+		m.SessionsEstablished,
+		m.SessionsCompleted,
+		m.SessionDuration,
+		m.ConcurrentSessions,
+		m.InformEvents,
+		m.ParametersRead,
+		m.ParametersWritten,
 	)
 	return m
 }

simulator/metrics/noop.go

@@ -14,22 +14,31 @@ var (
 )
 
 type (
-	noopRegisterer struct{}
-	noopCollector  struct{}
-	noopCounter    struct{ noopCollector }
-	noopCounterVec struct{ noopCollector }
-	noopHistogram  struct{ noopCollector }
-	noopGauge      struct{ noopCollector }
+	noopRegisterer   struct{}
+	noopCollector    struct{}
+	noopCounter      struct{ noopCollector }
+	noopCounterVec   struct{ noopCollector }
+	noopHistogram    struct{ noopCollector }
+	noopHistogramVec struct{ noopCollector }
+	noopGauge        struct{ noopCollector }
+	noopObserver     struct{}
 )
 
 func NewNoop() *Metrics {
 	return &Metrics{
-		Registrer:         noopRegisterer{},
-		RequestFailures:   noopCounter{},
-		ResponseStatus:    noopCounterVec{},
-		ConnectionLatency: noopHistogram{},
-		InformDuration:    noopHistogram{},
-		ConcurrentInforms: noopGauge{},
+		Bootstrapped:        noopCounter{},
+		ConnectionLatency:   noopHistogram{},
+		MethodCalls:         noopCounterVec{},
+		RequestFailures:     noopCounter{},
+		ResponseStatus:      noopCounterVec{},
+		SessionsAttempted:   noopCounter{},
+		SessionsEstablished: noopCounter{},
+		SessionsCompleted:   noopCounter{},
+		SessionDuration:     noopHistogramVec{},
+		ConcurrentSessions:  noopGauge{},
+		InformEvents:        noopCounterVec{},
+		ParametersRead:      noopCounter{},
+		ParametersWritten:   noopCounter{},
 	}
 }
 
@@ -43,9 +52,11 @@ func (noopCollector) Describe(chan<- *prometheus.Desc)         {}
 func (noopCollector) Inc()                                     {}
 func (noopCollector) Write(*io_prometheus_client.Metric) error { return nil }
 
-func (noopCounterVec) With(prometheus.Labels) prometheus.Counter { return noopCounter{} }
+func (noopCounterVec) With(prometheus.Labels) prometheus.Counter    { return noopCounter{} }
+func (noopHistogramVec) With(prometheus.Labels) prometheus.Observer { return noopObserver{} }
 
-func (noopCounter) Add(float64) {}
+func (noopCounter) Add(float64)      {}
+func (noopObserver) Observe(float64) {}
 
 func (noopHistogram) Observe(float64) {}
 

simulator/set_parameter_values.go

@@ -47,6 +47,7 @@ func (s *Simulator) handleSetParameterValues(envID string, r *rpc.SetParameterVa
 		defer s.resetInformTimer()
 	}
 
+	s.metrics.ParametersWritten.Add(float64(len(params)))
 	s.dm.SetValues(params)
 	s.dm.SetParameterKey(r.ParameterKey)
 	resp := rpc.NewEnvelope(envID)

simulator/simulator.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/go-xmlfmt/xmlfmt"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/rs/zerolog/log"
 
 	"github.com/localhots/SimulaTR69/datamodel"
@@ -118,6 +119,7 @@ func (s *Simulator) handleConnectionRequest(_ context.Context) error {
 
 // nolint:gocyclo
 func (s *Simulator) handleEnvelope(env *rpc.EnvelopeDecoder) *rpc.EnvelopeEncoder {
+	s.metrics.MethodCalls.With(prometheus.Labels{"method": env.Method()}).Inc()
 	envID := env.Header.ID.Value
 	switch {
 	case env.Body.GetRPCMethods != nil: