feat: add api http metrics

- measure api http request duration

Signed-off-by: Siavash Safi <[email protected]>

Author: siavashs

api/api.go

@@ -19,9 +19,11 @@ import (
 	"log/slog"
 	"net/http"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
 	"github.com/prometheus/common/route"
@@ -40,6 +42,7 @@ type API struct {
 	v2                *apiv2.API
 	deprecationRouter *V1DeprecationRouter
 
+	requestDuration          *prometheus.HistogramVec
 	requestsInFlight         prometheus.Gauge
 	concurrencyLimitExceeded prometheus.Counter
 	timeout                  time.Duration
@@ -75,6 +78,8 @@ type Options struct {
 	// Registry is used to register Prometheus metrics. If nil, no metrics
 	// registration will happen.
 	Registry prometheus.Registerer
+	// RequestDuration is used to measure the duration of HTTP requests.
+	RequestDuration *prometheus.HistogramVec
 	// GroupFunc returns a list of alert groups. The alerts are grouped
 	// according to the current active configuration. Alerts returned are
 	// filtered by the arguments provided to the function.
@@ -132,8 +137,6 @@ func New(opts Options) (*API, error) {
 		return nil, err
 	}
 
-	// TODO(beorn7): For now, this hardcodes the method="get" label. Other
-	// methods should get the same instrumentation.
 	requestsInFlight := prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:        "alertmanager_http_requests_in_flight",
 		Help:        "Current number of HTTP requests being processed.",
@@ -156,6 +159,7 @@ func New(opts Options) (*API, error) {
 	return &API{
 		deprecationRouter:        NewV1DeprecationRouter(l.With("version", "v1")),
 		v2:                       v2,
+		requestDuration:          opts.RequestDuration,
 		requestsInFlight:         requestsInFlight,
 		concurrencyLimitExceeded: concurrencyLimitExceeded,
 		timeout:                  opts.Timeout,
@@ -181,13 +185,17 @@ func (api *API) Register(r *route.Router, routePrefix string) *http.ServeMux {
 	if routePrefix != "/" {
 		apiPrefix = routePrefix
 	}
-	// TODO(beorn7): HTTP instrumentation is only in place for Router. Since
-	// /api/v2 works on the Handler level, it is currently not instrumented
-	// at all (with the exception of requestsInFlight, which is handled in
-	// limitHandler below).
 	mux.Handle(
 		apiPrefix+"/api/v2/",
-		api.limitHandler(http.StripPrefix(apiPrefix, api.v2.Handler)),
+		api.instrumentHandler(
+			apiPrefix,
+			api.limitHandler(
+				http.StripPrefix(
+					apiPrefix,
+					api.v2.Handler,
+				),
+			),
+		),
 	)
 
 	return mux
@@ -226,3 +234,17 @@ func (api *API) limitHandler(h http.Handler) http.Handler {
 		"Exceeded configured timeout of %v.\n", api.timeout,
 	))
 }
+
+func (api *API) instrumentHandler(prefix string, h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		path, _ := strings.CutPrefix(r.URL.Path, prefix)
+		// avoid high cardinality label values by replacing the actual silence IDs with a placeholder
+		if strings.HasPrefix(path, "/api/v2/silence/") {
+			path = "/api/v2/silence/{silenceID}"
+		}
+		promhttp.InstrumentHandlerDuration(
+			api.requestDuration.MustCurryWith(prometheus.Labels{"handler": path}),
+			h,
+		).ServeHTTP(w, r)
+	})
+}

cmd/alertmanager/main.go

@@ -68,12 +68,12 @@ var (
 		prometheus.HistogramOpts{
 			Name:                            "alertmanager_http_request_duration_seconds",
 			Help:                            "Histogram of latencies for HTTP requests.",
-			Buckets:                         []float64{.05, 0.1, .25, .5, .75, 1, 2, 5, 20, 60},
+			Buckets:                         prometheus.DefBuckets,
 			NativeHistogramBucketFactor:     1.1,
 			NativeHistogramMaxBucketNumber:  100,
 			NativeHistogramMinResetDuration: 1 * time.Hour,
 		},
-		[]string{"handler", "method"},
+		[]string{"handler", "method", "code"},
 	)
 	responseSize = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
@@ -377,6 +377,7 @@ func run() int {
 		Concurrency:     *getConcurrency,
 		Logger:          logger.With("component", "api"),
 		Registry:        prometheus.DefaultRegisterer,
+		RequestDuration: requestDuration,
 		GroupFunc:       groupFn,
 	})
 	if err != nil {