Export metrics for assigned and requested capacity, and client count.

I wrote a custom Prometheus collector that exports the server status
at collection time. This is necessary because updating the metrics as
the values change would be quite expensive.

Author: ryszard

go/cmd/doorman/doorman_server.go

@@ -227,6 +227,10 @@ func main() {
 
 	http.Handle("/metrics", prometheus.Handler())
 
+	if err := prometheus.Register(doorman.NewCollector(dm)); err != nil {
+		log.Exitf("prometheus.Register: %v", err)
+	}
+
 	go http.ListenAndServe(fmt.Sprintf(":%v", *debugPort), nil)
 
 	// Waits for the server to get its initial configuration. This guarantees that

go/server/doorman/collector.go

@@ -0,0 +1,71 @@
+package doorman
+
+import (
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type collector struct {
+	server *Server
+	mu     sync.Mutex
+	has    *prometheus.GaugeVec
+	wants  *prometheus.GaugeVec
+	count  *prometheus.GaugeVec
+}
+
+// NewCollector returns a custom Prometheus collector that creates
+// metrics for how much capacity has been assigned
+// (doorman_server_sum_has), requested (doorman_server_sum_wants), and
+// the total number of clients (doorman_server_client_count), with the
+// resource id as the label. It has to be registered using
+// prometheus.Register.
+func NewCollector(server *Server) prometheus.Collector {
+	labels := []string{"resource"}
+	return &collector{
+		server: server,
+		has: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "doorman",
+			Subsystem: "server",
+			Name:      "sum_has",
+			Help:      "All capacity assigned to clients for a resource.",
+		}, labels),
+		wants: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "doorman",
+			Subsystem: "server",
+			Name:      "sum_wants",
+			Help:      "All capacity requested by clients for a resource.",
+		}, labels),
+		count: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "doorman",
+			Subsystem: "server",
+			Name:      "client_count",
+			Help:      "Number of clients requesting this resource.",
+		}, labels),
+	}
+}
+
+func (c *collector) Describe(ch chan<- *prometheus.Desc) {
+	c.has.Describe(ch)
+	c.wants.Describe(ch)
+	c.count.Describe(ch)
+}
+
+func (c *collector) Collect(ch chan<- prometheus.Metric) {
+	status := c.server.Status()
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	for id, res := range status.Resources {
+		c.has.WithLabelValues(id).Set(res.SumHas)
+		c.wants.WithLabelValues(id).Set(res.SumWants)
+		c.count.WithLabelValues(id).Set(float64(res.Count))
+	}
+	c.has.Collect(ch)
+	c.wants.Collect(ch)
+	c.count.Collect(ch)
+
+	c.has.Reset()
+	c.wants.Reset()
+	c.count.Reset()
+}

go/server/doorman/collector_test.go

@@ -0,0 +1,21 @@
+package doorman
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	pb "github.com/youtube/doorman/proto/doorman"
+)
+
+func TestCollector(t *testing.T) {
+	s, err := MakeTestServer(makeResourceTemplate("*", pb.Algorithm_FAIR_SHARE))
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := NewCollector(s)
+	prometheus.EnableCollectChecks(true)
+	if err := prometheus.Register(c); err != nil {
+		t.Fatal(err)
+	}
+}

go/server/doorman/server.go

@@ -524,10 +524,7 @@ func NewIntermediate(ctx context.Context, id string, addr string, leader electio
 	return server, nil
 }
 
-// New returns a new unconfigured server. It will use id and
-// masterLock (an etcd path) in a master election. masterDelay is the
-// amount of time in which a deposed master can try to reacquire
-// mastership.
+// New returns a new unconfigured server.
 func New(ctx context.Context, id string, leader election.Election, opts ...connection.Option) (*Server, error) {
 	return NewIntermediate(ctx, id, "", leader, opts...)
 }