WFE: Track in-flight HTTP requests by endpoint using a gauge (#7758)

Author: beautifulentropy

metrics/measured_http/http.go

@@ -45,6 +45,9 @@ type MeasuredHandler struct {
 	clk clock.Clock
 	// Normally this is always responseTime, but we override it for testing.
 	stat *prometheus.HistogramVec
+	// inFlightRequestsGauge is a gauge that tracks the number of requests
+	// currently in flight, labeled by endpoint.
+	inFlightRequestsGauge *prometheus.GaugeVec
 }
 
 func New(m serveMux, clk clock.Clock, stats prometheus.Registerer, opts ...otelhttp.Option) http.Handler {
@@ -55,17 +58,34 @@ func New(m serveMux, clk clock.Clock, stats prometheus.Registerer, opts ...otelh
 		},
 		[]string{"endpoint", "method", "code"})
 	stats.MustRegister(responseTime)
+
+	inFlightRequestsGauge := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "in_flight_requests",
+			Help: "Tracks the number of WFE requests currently in flight, labeled by endpoint.",
+		},
+		[]string{"endpoint"},
+	)
+	stats.MustRegister(inFlightRequestsGauge)
+
 	return otelhttp.NewHandler(&MeasuredHandler{
-		serveMux: m,
-		clk:      clk,
-		stat:     responseTime,
+		serveMux:              m,
+		clk:                   clk,
+		stat:                  responseTime,
+		inFlightRequestsGauge: inFlightRequestsGauge,
 	}, "server", opts...)
 }
 
 func (h *MeasuredHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	begin := h.clk.Now()
 	rwws := &responseWriterWithStatus{w, 0}
 
+	subHandler, pattern := h.Handler(r)
+	if pattern != "/" {
+		h.inFlightRequestsGauge.WithLabelValues(pattern).Inc()
+		defer h.inFlightRequestsGauge.WithLabelValues(pattern).Dec()
+	}
+
 	// Use the method string only if it's a recognized HTTP method. This avoids
 	// ballooning timeseries with invalid methods from public input.
 	var method string
@@ -78,7 +98,6 @@ func (h *MeasuredHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		method = "unknown"
 	}
 
-	subHandler, pattern := h.Handler(r)
 	defer func() {
 		h.stat.With(prometheus.Labels{
 			"endpoint": pattern,

metrics/measured_http/http_test.go

@@ -42,12 +42,21 @@ func TestMeasuring(t *testing.T) {
 		},
 		[]string{"endpoint", "method", "code"})
 
+	inFlightRequestsGauge := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "in_flight_requests",
+			Help: "Tracks the number of WFE requests currently in flight, labeled by endpoint.",
+		},
+		[]string{"endpoint"},
+	)
+
 	mux := http.NewServeMux()
 	mux.Handle("/foo", sleepyHandler{clk})
 	mh := MeasuredHandler{
-		serveMux: mux,
-		clk:      clk,
-		stat:     stat,
+		serveMux:              mux,
+		clk:                   clk,
+		stat:                  stat,
+		inFlightRequestsGauge: inFlightRequestsGauge,
 	}
 	mh.ServeHTTP(httptest.NewRecorder(), &http.Request{
 		URL:    &url.URL{Path: "/foo"},
@@ -95,13 +104,21 @@ func TestUnknownMethod(t *testing.T) {
 			Help: "fake",
 		},
 		[]string{"endpoint", "method", "code"})
+	inFlightRequestsGauge := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "in_flight_requests",
+			Help: "Tracks the number of WFE requests currently in flight, labeled by endpoint.",
+		},
+		[]string{"endpoint"},
+	)
 
 	mux := http.NewServeMux()
 	mux.Handle("/foo", sleepyHandler{clk})
 	mh := MeasuredHandler{
-		serveMux: mux,
-		clk:      clk,
-		stat:     stat,
+		serveMux:              mux,
+		clk:                   clk,
+		stat:                  stat,
+		inFlightRequestsGauge: inFlightRequestsGauge,
 	}
 	mh.ServeHTTP(httptest.NewRecorder(), &http.Request{
 		URL:    &url.URL{Path: "/foo"},
@@ -140,14 +157,22 @@ func TestWrite(t *testing.T) {
 		},
 		[]string{"endpoint", "method", "code"})
 
+	inFlightRequestsGauge := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "in_flight_requests",
+			Help: "Tracks the number of WFE requests currently in flight, labeled by endpoint.",
+		},
+		[]string{"endpoint"})
+
 	mux := http.NewServeMux()
 	mux.HandleFunc("/foo", func(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte{})
 	})
 	mh := MeasuredHandler{
-		serveMux: mux,
-		clk:      clk,
-		stat:     stat,
+		serveMux:              mux,
+		clk:                   clk,
+		stat:                  stat,
+		inFlightRequestsGauge: inFlightRequestsGauge,
 	}
 	mh.ServeHTTP(httptest.NewRecorder(), &http.Request{
 		URL:    &url.URL{Path: "/foo"},
@@ -162,6 +187,7 @@ func TestWrite(t *testing.T) {
 		},
 		[]string{"endpoint", "method", "code"})
 	mh.stat = stat
+	mh.inFlightRequestsGauge = inFlightRequestsGauge
 	expectedLabels := map[string]string{
 		"endpoint": "/foo",
 		"method":   "GET",