gossip: add messages sent/received metric

This commit adds two metrics: (1) Messages sent, and (2) Messages
received. These metrics will be handy when adding message batch sending
on both the server and the client as it will help verify that the
batching works.

References: #119420

Release note: None

Author: iskettaneh

pkg/gossip/client.go

@@ -219,8 +219,10 @@ func (c *client) sendGossip(g *Gossip, stream Gossip_GossipClient, firstReq bool
 		infosSent := int64(len(delta))
 		c.clientMetrics.BytesSent.Inc(bytesSent)
 		c.clientMetrics.InfosSent.Inc(infosSent)
+		c.clientMetrics.MessagesSent.Inc(1)
 		c.nodeMetrics.BytesSent.Inc(bytesSent)
 		c.nodeMetrics.InfosSent.Inc(infosSent)
+		c.nodeMetrics.MessagesSent.Inc(1)
 
 		if log.V(1) {
 			ctx := c.AnnotateCtx(stream.Context())
@@ -247,8 +249,10 @@ func (c *client) handleResponse(ctx context.Context, g *Gossip, reply *Response)
 	infosReceived := int64(len(reply.Delta))
 	c.clientMetrics.BytesReceived.Inc(bytesReceived)
 	c.clientMetrics.InfosReceived.Inc(infosReceived)
+	c.clientMetrics.MessagesReceived.Inc(1)
 	c.nodeMetrics.BytesReceived.Inc(bytesReceived)
 	c.nodeMetrics.InfosReceived.Inc(infosReceived)
+	c.nodeMetrics.MessagesReceived.Inc(1)
 
 	// Combine remote node's infostore delta with ours.
 	if reply.Delta != nil {

pkg/gossip/gossip.go

@@ -141,6 +141,18 @@ var (
 		Measurement: "Connections",
 		Unit:        metric.Unit_COUNT,
 	}
+	MetaMessagesSent = metric.Metadata{
+		Name:        "gossip.messages.sent",
+		Help:        "Number of sent gossip messages",
+		Measurement: "Messages",
+		Unit:        metric.Unit_COUNT,
+	}
+	MetaMessagesReceived = metric.Metadata{
+		Name:        "gossip.messages.received",
+		Help:        "Number of received gossip messages",
+		Measurement: "Messages",
+		Unit:        metric.Unit_COUNT,
+	}
 	MetaInfosSent = metric.Metadata{
 		Name:        "gossip.infos.sent",
 		Help:        "Number of sent gossip Info objects",

pkg/gossip/gossip.proto

@@ -76,6 +76,8 @@ message MetricSnap {
   int64 bytes_sent = 3;
   int64 infos_received = 4;
   int64 infos_sent = 5;
+  int64 messages_received = 7;
+  int64 messages_sent = 8;
   int64 conns_refused = 6;
 }
 

pkg/gossip/server.go

@@ -125,8 +125,10 @@ func (s *server) Gossip(stream Gossip_GossipServer) error {
 			infoCount := int64(len(reply.Delta))
 			s.nodeMetrics.BytesSent.Inc(bytesSent)
 			s.nodeMetrics.InfosSent.Inc(infoCount)
+			s.nodeMetrics.MessagesSent.Inc(1)
 			s.serverMetrics.BytesSent.Inc(bytesSent)
 			s.serverMetrics.InfosSent.Inc(infoCount)
+			s.serverMetrics.MessagesSent.Inc(1)
 
 			return stream.Send(reply)
 		}
@@ -307,8 +309,10 @@ func (s *server) gossipReceiver(
 		infosReceived := int64(len(args.Delta))
 		s.nodeMetrics.BytesReceived.Inc(bytesReceived)
 		s.nodeMetrics.InfosReceived.Inc(infosReceived)
+		s.nodeMetrics.MessagesReceived.Inc(1)
 		s.serverMetrics.BytesReceived.Inc(bytesReceived)
 		s.serverMetrics.InfosReceived.Inc(infosReceived)
+		s.serverMetrics.MessagesReceived.Inc(1)
 
 		freshCount, err := s.mu.is.combine(args.Delta, args.NodeID)
 		if err != nil {

pkg/gossip/status.go

@@ -26,6 +26,8 @@ type Metrics struct {
 	BytesSent                   *metric.Counter
 	InfosReceived               *metric.Counter
 	InfosSent                   *metric.Counter
+	MessagesSent                *metric.Counter
+	MessagesReceived            *metric.Counter
 	CallbacksProcessed          *metric.Counter
 	CallbacksPending            *metric.Gauge
 	CallbacksProcessingDuration metric.IHistogram
@@ -39,6 +41,8 @@ func makeMetrics() Metrics {
 		BytesSent:          metric.NewCounter(MetaBytesSent),
 		InfosReceived:      metric.NewCounter(MetaInfosReceived),
 		InfosSent:          metric.NewCounter(MetaInfosSent),
+		MessagesReceived:   metric.NewCounter(MetaMessagesReceived),
+		MessagesSent:       metric.NewCounter(MetaMessagesSent),
 		CallbacksProcessed: metric.NewCounter(MetaCallbacksProcessed),
 		CallbacksPending:   metric.NewGauge(MetaCallbacksPending),
 		CallbacksProcessingDuration: metric.NewHistogram(metric.HistogramOptions{
@@ -63,11 +67,13 @@ func (m Metrics) String() string {
 // Snapshot returns a snapshot of the metrics.
 func (m Metrics) Snapshot() MetricSnap {
 	return MetricSnap{
-		ConnsRefused:  m.ConnectionsRefused.Count(),
-		BytesReceived: m.BytesReceived.Count(),
-		BytesSent:     m.BytesSent.Count(),
-		InfosReceived: m.InfosReceived.Count(),
-		InfosSent:     m.InfosSent.Count(),
+		ConnsRefused:     m.ConnectionsRefused.Count(),
+		BytesReceived:    m.BytesReceived.Count(),
+		BytesSent:        m.BytesSent.Count(),
+		InfosReceived:    m.InfosReceived.Count(),
+		InfosSent:        m.InfosSent.Count(),
+		MessagesReceived: m.MessagesReceived.Count(),
+		MessagesSent:     m.MessagesSent.Count(),
 	}
 }
 
@@ -77,7 +83,8 @@ func (m MetricSnap) String() string {
 
 // SafeFormat implements the redact.SafeFormatter interface.
 func (m MetricSnap) SafeFormat(w redact.SafePrinter, _ rune) {
-	w.Printf("infos %d/%d sent/received, bytes %dB/%dB sent/received",
+	w.Printf("messages %d/%d sent/received, infos %d/%d sent/received, bytes %dB/%dB sent/received",
+		m.MessagesSent, m.MessagesReceived,
 		m.InfosSent, m.InfosReceived,
 		m.BytesSent, m.BytesReceived)
 	if m.ConnsRefused > 0 {

pkg/gossip/status_test.go

@@ -21,14 +21,16 @@ func TestGossipStatus(t *testing.T) {
 		},
 		MaxConns: 3,
 		MetricSnap: MetricSnap{
-			BytesReceived: 1000,
-			BytesSent:     2000,
-			InfosReceived: 10,
-			InfosSent:     20,
-			ConnsRefused:  17,
+			BytesReceived:    1000,
+			BytesSent:        2000,
+			InfosReceived:    10,
+			InfosSent:        20,
+			MessagesSent:     2,
+			MessagesReceived: 1,
+			ConnsRefused:     17,
 		},
 	}
-	if exp, act := `gossip server (2/3 cur/max conns, infos 20/10 sent/received, bytes 2000B/1000B sent/received, refused 17 conns)
+	if exp, act := `gossip server (2/3 cur/max conns, messages 2/1 sent/received, infos 20/10 sent/received, bytes 2000B/1000B sent/received, refused 17 conns)
   1: localhost:1234 (17s)
   4: localhost:4567 (18s)
 `, ss.String(); exp != act {
@@ -39,13 +41,13 @@ func TestGossipStatus(t *testing.T) {
 		ConnStatus: []OutgoingConnStatus{
 			{
 				ConnStatus: ss.ConnStatus[0],
-				MetricSnap: MetricSnap{BytesReceived: 77, BytesSent: 88, InfosReceived: 11, InfosSent: 22},
+				MetricSnap: MetricSnap{BytesReceived: 77, BytesSent: 88, InfosReceived: 11, InfosSent: 22, MessagesReceived: 3, MessagesSent: 4},
 			},
 		},
 		MaxConns: 3,
 	}
 	if exp, act := `gossip client (1/3 cur/max conns)
-  1: localhost:1234 (17s: infos 22/11 sent/received, bytes 88B/77B sent/received)
+  1: localhost:1234 (17s: messages 4/3 sent/received, infos 22/11 sent/received, bytes 88B/77B sent/received)
 `, cs.String(); exp != act {
 		t.Errorf("expected:\n%q\ngot:\n%q", exp, act)
 	}