✨ add metrics for webhook

Author: Mengqi Yu

example/main.go

@@ -108,8 +108,8 @@ func main() {
 
 	entryLog.Info("setting up webhook server")
 	as, err := webhook.NewServer("foo-admission-server", mgr, webhook.ServerOptions{
-		Port:    9876,
-		CertDir: "/tmp/cert",
+		Port:                          9876,
+		CertDir:                       "/tmp/cert",
 		DisableWebhookConfigInstaller: &disableWebhookConfigInstaller,
 		BootstrapOptions: &webhook.BootstrapOptions{
 			Secret: &apitypes.NamespacedName{

pkg/webhook/admission/http.go

@@ -24,13 +24,15 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"time"
 
 	"k8s.io/api/admission/v1beta1"
 	admissionv1beta1 "k8s.io/api/admission/v1beta1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission/types"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics"
 )
 
 var admissionv1beta1scheme = runtime.NewScheme()
@@ -47,6 +49,9 @@ func addToScheme(scheme *runtime.Scheme) {
 var _ http.Handler = &Webhook{}
 
 func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	startTS := time.Now()
+	defer metrics.RequestLatency.WithLabelValues(wh.Name).Observe(time.Now().Sub(startTS).Seconds())
+
 	var body []byte
 	var err error
 
@@ -55,14 +60,14 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if body, err = ioutil.ReadAll(r.Body); err != nil {
 			log.Error(err, "unable to read the body from the incoming request")
 			reviewResponse = ErrorResponse(http.StatusBadRequest, err)
-			writeResponse(w, reviewResponse)
+			wh.writeResponse(w, reviewResponse)
 			return
 		}
 	} else {
 		err = errors.New("request body is empty")
 		log.Error(err, "bad request")
 		reviewResponse = ErrorResponse(http.StatusBadRequest, err)
-		writeResponse(w, reviewResponse)
+		wh.writeResponse(w, reviewResponse)
 		return
 	}
 
@@ -72,31 +77,39 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		err = fmt.Errorf("contentType=%s, expect application/json", contentType)
 		log.Error(err, "unable to process a request with an unknown content type", "content type", contentType)
 		reviewResponse = ErrorResponse(http.StatusBadRequest, err)
-		writeResponse(w, reviewResponse)
+		wh.writeResponse(w, reviewResponse)
 		return
 	}
 
 	ar := v1beta1.AdmissionReview{}
 	if _, _, err := admissionv1beta1schemecodecs.UniversalDeserializer().Decode(body, nil, &ar); err != nil {
 		log.Error(err, "unable to decode the request")
 		reviewResponse = ErrorResponse(http.StatusBadRequest, err)
-		writeResponse(w, reviewResponse)
+		wh.writeResponse(w, reviewResponse)
 		return
 	}
 
 	// TODO: add panic-recovery for Handle
 	reviewResponse = wh.Handle(context.Background(), types.Request{AdmissionRequest: ar.Request})
-	writeResponse(w, reviewResponse)
+	wh.writeResponse(w, reviewResponse)
 }
 
-func writeResponse(w io.Writer, response types.Response) {
+func (wh *Webhook) writeResponse(w io.Writer, response types.Response) {
+	if response.Response.Result.Code != 0 {
+		if response.Response.Result.Code == http.StatusOK {
+			metrics.TotalRequests.WithLabelValues(wh.Name, "true").Inc()
+		} else {
+			metrics.TotalRequests.WithLabelValues(wh.Name, "false").Inc()
+		}
+	}
+
 	encoder := json.NewEncoder(w)
 	responseAdmissionReview := v1beta1.AdmissionReview{
 		Response: response.Response,
 	}
 	err := encoder.Encode(responseAdmissionReview)
 	if err != nil {
 		log.Error(err, "unable to encode the response")
-		writeResponse(w, ErrorResponse(http.StatusInternalServerError, err))
+		wh.writeResponse(w, ErrorResponse(http.StatusInternalServerError, err))
 	}
 }

pkg/webhook/admission/http_test.go

@@ -134,7 +134,7 @@ var _ = Describe("admission webhook http handler", func() {
 			Type:     types.WebhookTypeValidating,
 			Handlers: []Handler{h},
 		}
-		expected := []byte(`{"response":{"uid":"","allowed":true}}
+		expected := []byte(`{"response":{"uid":"","allowed":true,"status":{"metadata":{},"code":200}}}
 `)
 		It("should return a response successfully", func() {
 			wh.ServeHTTP(w, req)

pkg/webhook/admission/webhook.go

@@ -132,6 +132,7 @@ func (w *Webhook) handleMutating(ctx context.Context, req atypes.Request) atypes
 	for _, handler := range w.Handlers {
 		resp := handler.Handle(ctx, req)
 		if !resp.Response.Allowed {
+			setStatusOKInAdmissionResponse(resp.Response)
 			return resp
 		}
 		if resp.Response.PatchType != nil && *resp.Response.PatchType != admissionv1beta1.PatchTypeJSONPatch {
@@ -148,7 +149,10 @@ func (w *Webhook) handleMutating(ctx context.Context, req atypes.Request) atypes
 	}
 	return atypes.Response{
 		Response: &admissionv1beta1.AdmissionResponse{
-			Allowed:   true,
+			Allowed: true,
+			Result: &metav1.Status{
+				Code: http.StatusOK,
+			},
 			Patch:     marshaledPatch,
 			PatchType: func() *admissionv1beta1.PatchType { pt := admissionv1beta1.PatchTypeJSONPatch; return &pt }(),
 		},
@@ -159,16 +163,32 @@ func (w *Webhook) handleValidating(ctx context.Context, req atypes.Request) atyp
 	for _, handler := range w.Handlers {
 		resp := handler.Handle(ctx, req)
 		if !resp.Response.Allowed {
+			setStatusOKInAdmissionResponse(resp.Response)
 			return resp
 		}
 	}
 	return atypes.Response{
 		Response: &admissionv1beta1.AdmissionResponse{
 			Allowed: true,
+			Result: &metav1.Status{
+				Code: http.StatusOK,
+			},
 		},
 	}
 }
 
+func setStatusOKInAdmissionResponse(resp *admissionv1beta1.AdmissionResponse) {
+	if resp == nil {
+		return
+	}
+	if resp.Result == nil {
+		resp.Result = &metav1.Status{}
+	}
+	if resp.Result.Code == 0 {
+		resp.Result.Code = http.StatusOK
+	}
+}
+
 // GetName returns the name of the webhook.
 func (w *Webhook) GetName() string {
 	w.once.Do(w.setDefaults)

pkg/webhook/admission/webhook_test.go

@@ -83,7 +83,7 @@ var _ = Describe("admission webhook", func() {
 			})
 
 			It("should deny the request", func() {
-				expected := []byte(`{"response":{"uid":"","allowed":false}}
+				expected := []byte(`{"response":{"uid":"","allowed":false,"status":{"metadata":{},"code":200}}}
 `)
 				wh.ServeHTTP(w, req)
 				Expect(w.Body.Bytes()).To(Equal(expected))
@@ -102,7 +102,7 @@ var _ = Describe("admission webhook", func() {
 			})
 
 			It("should deny the request", func() {
-				expected := []byte(`{"response":{"uid":"","allowed":false}}
+				expected := []byte(`{"response":{"uid":"","allowed":false,"status":{"metadata":{},"code":200}}}
 `)
 				wh.ServeHTTP(w, req)
 				Expect(w.Body.Bytes()).To(Equal(expected))
@@ -162,7 +162,8 @@ var _ = Describe("admission webhook", func() {
 				Handlers: []Handler{patcher1, patcher2},
 			}
 			expected := []byte(
-				`{"response":{"uid":"","allowed":true,"patch":"W3sib3AiOiJhZGQiLCJwYXRoIjoiL21ldGFkYXRhL2Fubm90YXRpb2` +
+				`{"response":{"uid":"","allowed":true,"status":{"metadata":{},"code":200},` +
+					`"patch":"W3sib3AiOiJhZGQiLCJwYXRoIjoiL21ldGFkYXRhL2Fubm90YXRpb2` +
 					`4vbmV3LWtleSIsInZhbHVlIjoibmV3LXZhbHVlIn0seyJvcCI6InJlcGxhY2UiLCJwYXRoIjoiL3NwZWMvcmVwbGljYXMiLC` +
 					`J2YWx1ZSI6IjIifSx7Im9wIjoiYWRkIiwicGF0aCI6Ii9tZXRhZGF0YS9hbm5vdGF0aW9uL2hlbGxvIiwidmFsdWUiOiJ3b3JsZCJ9XQ==",` +
 					`"patchType":"JSONPatch"}}
@@ -213,7 +214,7 @@ var _ = Describe("admission webhook", func() {
 				Type:     types.WebhookTypeMutating,
 				Handlers: []Handler{errPatcher},
 			}
-			expected := []byte(`{"response":{"uid":"","allowed":false}}
+			expected := []byte(`{"response":{"uid":"","allowed":false,"status":{"metadata":{},"code":200}}}
 `)
 			It("should deny the request", func() {
 				wh.ServeHTTP(w, req)

pkg/webhook/internal/metrics/metrics.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+var (
+	// TotalRequests is a prometheus metric which counts the total number of requests that
+	// the webhook server has received.
+	TotalRequests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "controller_runtime_webhook_requests_total",
+			Help: "Total number of admission requests",
+		},
+		[]string{"webhook", "succeeded"},
+	)
+
+	// RequestLatency is a prometheus metric which is a histogram of the latency
+	// of processing admission requests.
+	RequestLatency = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "controller_runtime_webhook_latency_seconds",
+			Help: "Histogram of the latency of processing admission requests",
+		},
+		[]string{"webhook"},
+	)
+)
+
+func init() {
+	metrics.Registry.MustRegister(
+		TotalRequests,
+		RequestLatency)
+}