Update + document Prometheus metrics.

Author: brocaar

docs/content/backends/basic-station.md

@@ -34,3 +34,27 @@ Basic Station. The flow for this is:
 
 * The Basic Station does not send RX / TX stats
 
+## Prometheus metrics
+
+The Semtech Basic Station packet-forwarder backend exposes several [Prometheus](https://prometheus.io/)
+metrics for monitoring.
+
+### backend_basicstation_websocket_ping_pong_count
+
+The number of WebSocket Ping/Pong requests sent and received (per event type).
+
+### backend_basicstation_websocket_received_count
+
+The number of WebSocket messages received by the backend (per msgtype).
+
+### backend_basicstation_websocket_sent_count
+
+The number of WebSocket messages sent by the backend (per msgtype).
+
+### backend_basicstation_gateway_connect_count
+
+The number of gateway connections received by the backend.
+
+### backend_basicstation_gateway_disconnect_count
+
+The number of gateways that disconnected from the backend.

docs/content/backends/semtech-udp.md

@@ -65,3 +65,24 @@ This works as follow:
 For the above to work, you must configure the _Mandaged packet-forwarder configuration_
 section in the `lora-gateway-bridge.toml`.
 
+## Prometheus metrics
+
+The Semtech UDP packet-forwarder backend exposes several [Prometheus](https://prometheus.io/)
+metrics for monitoring.
+
+### backend_semtechudp_udp_sent_count
+
+The number of UDP packets sent by the backend (per packet_type).
+
+
+### backend_semtechudp_udp_received_count
+
+The number of UDP packets received by the backend (per packet_type).
+
+### backend_semtechudp_gateway_connect_count
+
+The number of gateway connections received by the backend.
+
+### backend_semtechudp_gateway_disconnect_count
+
+The number of gateways that disconnected from the backend.

docs/content/integrate/metrics.md

@@ -0,0 +1,34 @@
+---
+title: Prometheus metrics
+description: Prometheus metrics exposed by the MQTT backend.
+menu:
+  main:
+    parent: integrate
+    weight: 4
+---
+
+# Prometheus metrics
+
+Independent from the chosen MQTT authentication type, the MQTT integration
+expses the following [Prometheus](https://prometheus.io/) metrics for monitoring.
+
+### integration_mqtt_event_count
+
+The number of gateway events published by the MQTT integration (per event).
+
+### integration_mqtt_command_count
+
+The number of commands received by the MQTT integration (per command).
+
+### integration_mqtt_connect_count
+
+The number of times the integration connected to the MQTT broker.
+
+### integration_mqtt_disconnect_count
+
+The number of times the integration disconnected from the MQTT broker.
+
+### integration_mqtt_reconnect_count
+
+The number of times the integration reconnected to the MQTT broker (this also increments the disconnect and connect counters).
+

go.mod

@@ -7,14 +7,23 @@ require (
 	github.com/brocaar/lorawan v0.0.0-20190402092148-5bca41b178e9
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/eclipse/paho.mqtt.golang v1.2.0
-	github.com/golang/protobuf v1.3.1
+	github.com/go-kit/kit v0.9.0 // indirect
+	github.com/golang/protobuf v1.3.2
 	github.com/goreleaser/goreleaser v0.106.0
 	github.com/gorilla/websocket v1.4.0
 	github.com/pkg/errors v0.8.1
-	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
-	github.com/sirupsen/logrus v1.4.1
+	github.com/prometheus/client_golang v1.0.0
+	github.com/prometheus/common v0.6.0 // indirect
+	github.com/prometheus/procfs v0.0.3 // indirect
+	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/viper v1.3.2
+	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/stretchr/testify v1.3.0
+	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect
 	golang.org/x/lint v0.0.0-20190409202823-959b441ac422
+	golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect
+	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect
+	golang.org/x/text v0.3.2 // indirect
+	golang.org/x/tools v0.0.0-20190709211700-7b25e351ac0e // indirect
 )

go.sum

@@ -112,9 +112,9 @@ func NewBackend(conf config.Config) (*Backend, error) {
 		b.websocketWrap(b.handleRouterInfo, w, r)
 	})
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		bsEventCounter("connect")
+		connectCounter().Inc()
 		b.websocketWrap(b.handleGateway, w, r)
-		bsEventCounter("disconnect")
+		disconnectCounter().Inc()
 	})
 
 	// using net.Listen makes it easier to test as we can bind to ":0" and
@@ -199,7 +199,7 @@ func (b *Backend) SendDownlinkFrame(df gw.DownlinkFrame) error {
 	var gatewayID lorawan.EUI64
 	copy(gatewayID[:], df.TxInfo.GatewayId)
 
-	bsWebsocketSendCounter("dnmsg")
+	websocketSendCounter("dnmsg").Inc()
 	if err := b.sendToGateway(gatewayID, pl); err != nil {
 		return errors.Wrap(err, "send to gateway error")
 	}
@@ -218,7 +218,7 @@ func (b *Backend) ApplyConfiguration(gwConfig gw.GatewayConfiguration) error {
 	var gatewayID lorawan.EUI64
 	copy(gatewayID[:], gwConfig.GatewayId)
 
-	bsWebsocketSendCounter("router_config")
+	websocketSendCounter("router_config").Inc()
 	if err := b.sendToGateway(gatewayID, rc); err != nil {
 		return errors.Wrap(err, "send router config to gateway error")
 	}
@@ -235,7 +235,7 @@ func (b *Backend) Close() error {
 }
 
 func (b *Backend) handleRouterInfo(r *http.Request, c *websocket.Conn) {
-	bsWebsocketReceiveCounter("router_info")
+	websocketReceiveCounter("router_info").Inc()
 	var req structs.RouterInfoRequest
 
 	if err := c.ReadJSON(&req); err != nil {
@@ -331,7 +331,7 @@ func (b *Backend) handleGateway(r *http.Request, c *websocket.Conn) {
 			continue
 		}
 
-		bsWebsocketReceiveCounter(string(msgType))
+		websocketReceiveCounter(string(msgType)).Inc()
 
 		// handle message-type
 		switch msgType {
@@ -524,7 +524,7 @@ func (b *Backend) websocketWrap(handler func(*http.Request, *websocket.Conn), w
 
 	conn.SetReadDeadline(time.Now().Add(b.readTimeout))
 	conn.SetPongHandler(func(string) error {
-		bsWebsocketPingPongCounter("pong")
+		websocketPingPongCounter("pong").Inc()
 		conn.SetReadDeadline(time.Now().Add(b.readTimeout))
 		return nil
 	})
@@ -536,7 +536,7 @@ func (b *Backend) websocketWrap(handler func(*http.Request, *websocket.Conn), w
 		for {
 			select {
 			case <-ticker.C:
-				bsWebsocketPingPongCounter("ping")
+				websocketPingPongCounter("ping").Inc()
 				conn.SetWriteDeadline(time.Now().Add(b.writeTimeout))
 				if err := conn.WriteMessage(websocket.PingMessage, nil); err != nil {
 					log.WithError(err).Error("backend/basicstation: send ping message error")

internal/backend/basicstation/backend.go

@@ -2,55 +2,52 @@ package basicstation
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/brocaar/lora-gateway-bridge/internal/metrics"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 var (
-	bsEventCounter             func(string)
-	bsWebsocketSendCounter     func(string)
-	bsWebsocketReceiveCounter  func(string)
-	bsWebsocketPingPongCounter func(string)
+	ppc = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "backend_basicstation_websocket_ping_pong_count",
+		Help: "The number of WebSocket Ping/Pong requests sent and received (per event type).",
+	}, []string{"type"})
+
+	wsr = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "backend_basicstation_websocket_received_count",
+		Help: "The number of WebSocket messages received by the backend (per msgtype).",
+	}, []string{"msgtype"})
+
+	wss = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "backend_basicstation_websocket_sent_count",
+		Help: "The number of WebSocket messages sent by the backend (per msgtype).",
+	}, []string{"msgtype"})
+
+	gwc = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "backend_basicstation_gateway_connect_count",
+		Help: "The number of gateway connections received by the backend.",
+	})
+
+	gwd = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "backend_basicstation_gateway_disconnect_count",
+		Help: "The number of gateways that disconnected from the backend.",
+	})
 )
 
-func init() {
-	ec := metrics.MustRegisterNewCounter(
-		"backend_basicstation_event",
-		"Per gateway event type counter.",
-		[]string{"event"},
-	)
-
-	wsc := metrics.MustRegisterNewCounter(
-		"backend_basicstation_websocket_send",
-		"Per message-type websocket write counter.",
-		[]string{"msgtype"},
-	)
-
-	wrc := metrics.MustRegisterNewCounter(
-		"backend_basicstation_websocket_receive",
-		"Per message-type websocket receive counter.",
-		[]string{"msgtype"},
-	)
-
-	ppc := metrics.MustRegisterNewCounter(
-		"backend_basicstation_websocket_ping_pong",
-		"Websocket Ping/Pong counter.",
-		[]string{"type"},
-	)
-
-	bsEventCounter = func(event string) {
-		ec(prometheus.Labels{"event": event})
-	}
-
-	bsWebsocketReceiveCounter = func(msgtype string) {
-		wsc(prometheus.Labels{"msgtype": msgtype})
-	}
-
-	bsWebsocketSendCounter = func(msgtype string) {
-		wrc(prometheus.Labels{"msgtype": msgtype})
-	}
-
-	bsWebsocketPingPongCounter = func(typ string) {
-		ppc(prometheus.Labels{"type": typ})
-	}
+func websocketPingPongCounter(typ string) prometheus.Counter {
+	return ppc.With(prometheus.Labels{"type": typ})
+}
+
+func websocketReceiveCounter(msgtype string) prometheus.Counter {
+	return wsr.With(prometheus.Labels{"msgtype": msgtype})
+}
+
+func websocketSendCounter(msgtype string) prometheus.Counter {
+	return wss.With(prometheus.Labels{"msgtype": msgtype})
+}
+
+func connectCounter() prometheus.Counter {
+	return gwc
+}
+
+func disconnectCounter() prometheus.Counter {
+	return gwd
 }

internal/backend/basicstation/metrics.go

@@ -194,8 +194,6 @@ func (b *Backend) SendDownlinkFrame(frame gw.DownlinkFrame) error {
 // ApplyConfiguration applies the given configuration to the gateway
 // (packet-forwarder).
 func (b *Backend) ApplyConfiguration(config gw.GatewayConfiguration) error {
-	eventCounter("configuration")
-
 	var gatewayID lorawan.EUI64
 	copy(gatewayID[:], config.GatewayId)
 
@@ -317,7 +315,6 @@ func (b *Backend) sendPackets() error {
 			"protocol_version": p.data[0],
 		}).Debug("backend/semtechudp: sending udp packet to gateway")
 
-		udpWriteCounter(pt.String())
 		_, err = b.conn.WriteToUDP(p.data, p.addr)
 		if err != nil {
 			log.WithFields(log.Fields{
@@ -326,6 +323,8 @@ func (b *Backend) sendPackets() error {
 				"protocol_version": p.data[0],
 			}).WithError(err).Error("backend/semtechudp: write to udp error")
 		}
+
+		udpWriteCounter(pt.String()).Inc()
 	}
 	return nil
 }
@@ -348,7 +347,7 @@ func (b *Backend) handlePacket(up udpPacket) error {
 		"protocol_version": up.data[0],
 	}).Debug("backend/semtechudp: received udp packet from gateway")
 
-	udpReadCounter(pt.String())
+	udpReadCounter(pt.String()).Inc()
 
 	switch pt {
 	case packets.PushData:

internal/backend/semtechudp/backend.go

@@ -2,44 +2,43 @@ package semtechudp
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/brocaar/lora-gateway-bridge/internal/metrics"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 var (
-	eventCounter    func(string)
-	udpWriteCounter func(string)
-	udpReadCounter  func(string)
+	uwc = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "backend_semtechudp_udp_sent_count",
+		Help: "The number of UDP packets sent by the backend (per packet_type).",
+	}, []string{"packet_type"})
+
+	urc = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "backend_semtechudp_udp_received_count",
+		Help: "The number of UDP packets received by the backend (per packet_type).",
+	}, []string{"packet_type"})
+
+	gwc = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "backend_semtechudp_gateway_connect_count",
+		Help: "The number of gateway connections received by the backend.",
+	})
+
+	gwd = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "backend_semtechudp_gateway_diconnect_count",
+		Help: "The number of gateways that disconnected from the backend.",
+	})
 )
 
-func init() {
-	ec := metrics.MustRegisterNewCounter(
-		"backend_semtechudp_event",
-		"Per gateway event type counter.",
-		[]string{"event"},
-	)
-
-	uwc := metrics.MustRegisterNewCounter(
-		"backend_semtechudp_udp_write",
-		"UDP packets written by packet type.",
-		[]string{"packet_type"},
-	)
-
-	urc := metrics.MustRegisterNewCounter(
-		"backend_semtechudp_udp_read",
-		"UDP packets read by packet type.",
-		[]string{"packet_type"},
-	)
-
-	eventCounter = func(event string) {
-		ec(prometheus.Labels{"event": event})
-	}
-
-	udpWriteCounter = func(pt string) {
-		uwc(prometheus.Labels{"packet_type": pt})
-	}
-
-	udpReadCounter = func(pt string) {
-		urc(prometheus.Labels{"packet_type": pt})
-	}
+func udpWriteCounter(pt string) prometheus.Counter {
+	return uwc.With(prometheus.Labels{"packet_type": pt})
+}
+
+func udpReadCounter(pt string) prometheus.Counter {
+	return urc.With(prometheus.Labels{"packet_type": pt})
+}
+
+func connectCounter() prometheus.Counter {
+	return gwc
+}
+
+func disconnectCounter() prometheus.Counter {
+	return gwd
 }