Merge pull request #15 from slok/inflight

Add requests inflights metric

Author: slok

CHANGELOG

@@ -1,3 +1,7 @@
+## 0.x.x / 2019-xx-xx
+
+* [FEATURE] Add inflight requests metric per handler.
+
 ## 0.2.0 / 2019-03-22
 
 * [FEATURE] Add metrics of HTTP response size in bytes.

Readme.md

@@ -16,6 +16,7 @@ If you are using a framework that isn't directly compatible with go's `http.Hand
     - [Recorder](#recorder)
     - [GroupedStatus](#groupedstatus)
     - [DisableMeasureSize](#disablemeasuresize)
+    - [DisableMeasureInflight](#disablemeasureinflight)
     - [Custom handler ID](#custom-handler-id)
   - [Prometheus recorder options](#prometheus-recorder-options)
     - [Prefix](#prefix)
@@ -31,6 +32,7 @@ The metrics obtained with this middleware are the [most important ones][red] for
 - Records the duration of the requests(with: code, handler, method).
 - Records the count of the requests(with: code, handler, method).
 - Records the size of the responses(with: code, handler, method).
+- Records the number requests being handled concurrently at a given time a.k.a inflight requests (with: handler).
 
 ## Metrics recorder implementations
 
@@ -141,6 +143,10 @@ Storing all the status codes could increase the cardinality of the metrics, usua
 
 This setting will disable measuring the size of the responses. By default measuring the size is enabled.
 
+#### DisableMeasureInflight
+
+This settings will disable measuring the number of requests being handled concurrently by the handlers.
+
 #### Custom handler ID
 
 One of the options that you need to pass when wrapping the handler with the middleware is `handlerID`, this has 2 working ways.
@@ -174,10 +180,11 @@ The label names of the Prometheus metrics can be configured using `HandlerIDLabe
 ```text
 pkg: github.com/slok/go-http-metrics/middleware
 
-BenchmarkMiddlewareHandler/benchmark_with_default_settings.-4            1000000              1062 ns/op             256 B/op          6 allocs/op
-BenchmarkMiddlewareHandler/benchmark_disabling_measuring_size.-4         1000000              1101 ns/op             256 B/op          6 allocs/op
-BenchmarkMiddlewareHandler/benchmark_with_grouped_status_code.-4         1000000              1324 ns/op             256 B/op          7 allocs/op
-BenchmarkMiddlewareHandler/benchmark_with_predefined_handler_ID-4        1000000              1155 ns/op             256 B/op          6 allocs/op
+BenchmarkMiddlewareHandler/benchmark_with_default_settings.-4         	 1000000	      1206 ns/op	     256 B/op	       6 allocs/op
+BenchmarkMiddlewareHandler/benchmark_disabling_measuring_size.-4      	 1000000	      1198 ns/op	     256 B/op	       6 allocs/op
+BenchmarkMiddlewareHandler/benchmark_disabling_inflights.-4           	 1000000	      1139 ns/op	     256 B/op	       6 allocs/op
+BenchmarkMiddlewareHandler/benchmark_with_grouped_status_code.-4      	 1000000	      1534 ns/op	     256 B/op	       7 allocs/op
+BenchmarkMiddlewareHandler/benchmark_with_predefined_handler_ID-4     	 1000000	      1258 ns/op	     256 B/op	       6 allocs/op
 ```
 
 [travis-image]: https://travis-ci.org/slok/go-http-metrics.svg?branch=master

examples/default/main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	metrics "github.com/slok/go-http-metrics/metrics/prometheus"
@@ -32,7 +33,10 @@ func main() {
 
 	// Create our server.
 	mux := http.NewServeMux()
-	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(200 * time.Millisecond)
+		w.WriteHeader(http.StatusOK)
+	})
 	mux.HandleFunc("/test1", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) })
 	mux.HandleFunc("/test1/test2", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) })
 	mux.HandleFunc("/test1/test4", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNonAuthoritativeInfo) })

internal/mocks/metrics/Recorder.go

@@ -12,6 +12,9 @@ type Recorder interface {
 	ObserveHTTPRequestDuration(id string, duration time.Duration, method, code string)
 	// ObserveHTTPResponseSize measures the size of an HTTP response in bytes.
 	ObserveHTTPResponseSize(id string, sizeBytes int64, method, code string)
+	// AddInflightRequests increments and decrements the number of inflight request being
+	// processed.
+	AddInflightRequests(id string, quantity int)
 }
 
 // Dummy is a dummy recorder.
@@ -21,3 +24,4 @@ type dummy struct{}
 
 func (dummy) ObserveHTTPRequestDuration(id string, duration time.Duration, method, code string) {}
 func (dummy) ObserveHTTPResponseSize(id string, sizeBytes int64, method, code string)           {}
+func (dummy) AddInflightRequests(id string, quantity int)                                       {}

metrics/metrics.go

@@ -58,6 +58,7 @@ func (c *Config) defaults() {
 type recorder struct {
 	httpRequestDurHistogram   *prometheus.HistogramVec
 	httpResponseSizeHistogram *prometheus.HistogramVec
+	httpRequestsInflight      *prometheus.GaugeVec
 
 	cfg Config
 }
@@ -82,6 +83,12 @@ func NewRecorder(cfg Config) metrics.Recorder {
 			Help:      "The size of the HTTP responses.",
 			Buckets:   cfg.SizeBuckets,
 		}, []string{cfg.HandlerIDLabel, cfg.MethodLabel, cfg.StatusCodeLabel}),
+		httpRequestsInflight: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: cfg.Prefix,
+			Subsystem: "http",
+			Name:      "requests_inflight",
+			Help:      "The number of inflight requests being handled at the same time.",
+		}, []string{cfg.HandlerIDLabel}),
 
 		cfg: cfg,
 	}
@@ -95,6 +102,7 @@ func (r recorder) registerMetrics() {
 	r.cfg.Registry.MustRegister(
 		r.httpRequestDurHistogram,
 		r.httpResponseSizeHistogram,
+		r.httpRequestsInflight,
 	)
 }
 
@@ -105,3 +113,7 @@ func (r recorder) ObserveHTTPRequestDuration(id string, duration time.Duration,
 func (r recorder) ObserveHTTPResponseSize(id string, sizeBytes int64, method, code string) {
 	r.httpResponseSizeHistogram.WithLabelValues(id, method, code).Observe(float64(sizeBytes))
 }
+
+func (r recorder) AddInflightRequests(id string, quantity int) {
+	r.httpRequestsInflight.WithLabelValues(id).Add(float64(quantity))
+}

metrics/prometheus/prometheus.go

@@ -33,6 +33,9 @@ func TestPrometheusRecorder(t *testing.T) {
 				r.ObserveHTTPResponseSize("test4", 529930, http.MethodPost, "500")
 				r.ObserveHTTPResponseSize("test4", 231, http.MethodPost, "500")
 				r.ObserveHTTPResponseSize("test4", 99999999, http.MethodPatch, "429")
+				r.AddInflightRequests("test1", 5)
+				r.AddInflightRequests("test1", -3)
+				r.AddInflightRequests("test2", 9)
 			},
 			expMetrics: []string{
 				`http_request_duration_seconds_bucket{code="200",handler="test1",method="GET",le="0.005"} 0`,
@@ -98,6 +101,9 @@ func TestPrometheusRecorder(t *testing.T) {
 				`http_response_size_bytes_bucket{code="500",handler="test4",method="POST",le="1e+09"} 2`,
 				`http_response_size_bytes_bucket{code="500",handler="test4",method="POST",le="+Inf"} 2`,
 				`http_response_size_bytes_count{code="500",handler="test4",method="POST"} 2`,
+
+				`http_requests_inflight{handler="test1"} 2`,
+				`http_requests_inflight{handler="test2"} 9`,
 			},
 		},
 		{

metrics/prometheus/prometheus_test.go

@@ -47,8 +47,10 @@ func TestMiddlewareIntegration(t *testing.T) {
 
 			// Mocks.
 			mr := &mmetrics.Recorder{}
-			mr.On("ObserveHTTPRequestDuration", test.expHandlerID, mock.Anything, test.expMethod, test.expStatusCode)
-			mr.On("ObserveHTTPResponseSize", test.expHandlerID, mock.Anything, test.expMethod, test.expStatusCode)
+			mr.On("ObserveHTTPRequestDuration", test.expHandlerID, mock.Anything, test.expMethod, test.expStatusCode).Once()
+			mr.On("ObserveHTTPResponseSize", test.expHandlerID, mock.Anything, test.expMethod, test.expStatusCode).Once()
+			mr.On("AddInflightRequests", test.expHandlerID, 1).Once()
+			mr.On("AddInflightRequests", test.expHandlerID, -1).Once()
 
 			// Create our instance with the middleware.
 			mdlw := middleware.New(middleware.Config{Recorder: mr})

middleware/gorestful/gorestful_test.go

@@ -47,8 +47,10 @@ func TestMiddlewareIntegration(t *testing.T) {
 
 			// Mocks.
 			mr := &mmetrics.Recorder{}
-			mr.On("ObserveHTTPRequestDuration", test.expHandlerID, mock.Anything, test.expMethod, test.expStatusCode)
-			mr.On("ObserveHTTPResponseSize", test.expHandlerID, mock.Anything, test.expMethod, test.expStatusCode)
+			mr.On("ObserveHTTPRequestDuration", test.expHandlerID, mock.Anything, test.expMethod, test.expStatusCode).Once()
+			mr.On("ObserveHTTPResponseSize", test.expHandlerID, mock.Anything, test.expMethod, test.expStatusCode).Once()
+			mr.On("AddInflightRequests", test.expHandlerID, 1).Once()
+			mr.On("AddInflightRequests", test.expHandlerID, -1).Once()
 
 			// Create our instance with the middleware.
 			mdlw := middleware.New(middleware.Config{Recorder: mr})

middleware/httprouter/httprouter_test.go

@@ -26,6 +26,10 @@ type Config struct {
 	// DisableMeasureSize will disable the recording metrics about the response size,
 	// by default measuring size is enabled (`DisableMeasureSize` is false).
 	DisableMeasureSize bool
+
+	// DisableMeasureInflight will disable the recording metrics about the inflight requests number,
+	// by default measuring inflights is enabled (`DisableMeasureInflight` is false).
+	DisableMeasureInflight bool
 }
 
 func (c *Config) validate() {
@@ -81,6 +85,12 @@ func (m *middleware) Handler(handlerID string, h http.Handler) http.Handler {
 			hid = r.URL.Path
 		}
 
+		// Measure inflights if required.
+		if !m.cfg.DisableMeasureInflight {
+			m.cfg.Recorder.AddInflightRequests(hid, 1)
+			defer m.cfg.Recorder.AddInflightRequests(hid, -1)
+		}
+
 		// Start the timer and when finishing measure the duration.
 		start := time.Now()
 		defer func() {