*: add webhooks client config service metrics

In order to better identify, prioritize, and debug webhook latency
issues it is important to have a metric that would point to the resource
it is responsible for.  However, it is not possible to have that
dimension in the metrics exposed by Kubernetes because of the unbound
cardinality that such a label would have.

The name of the webhook could be an alternative since it usually
contains some information about the resource that the webhook targets,
however this is not very practical to use in multi-tenants
environments.

A solution for these kind of platform is to tie a specific webhook to a
namespace in order to be able to know which tenant manages it and take
actions depending on that. This is achieveable by leveraging the client
config information of webhooks configured via WebhookConfiguration
resources since Services are namespaced objects.

With these new metrics, users will be able to split the alerting
severity of webhook latency / rejection rate per namespace on top of
being able to do it based on the webhook name. This is key in
environment where administrators don't have control over the webhooks
installed by the various tenants.

Signed-off-by: Damien Grisonnet <[email protected]>

Author: dgrisonnet

docs/mutatingwebhookconfiguration-metrics.md

@@ -5,3 +5,4 @@
 | kube_mutatingwebhookconfiguration_info | Gauge | `mutatingwebhookconfiguration`=&lt;mutatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;mutatingwebhookconfiguration-namespace&gt; | EXPERIMENTAL |
 | kube_mutatingwebhookconfiguration_created  | Gauge | `mutatingwebhookconfiguration`=&lt;mutatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;mutatingwebhookconfiguration-namespace&gt; | EXPERIMENTAL |
 | kube_mutatingwebhookconfiguration_metadata_resource_version | Gauge | `mutatingwebhookconfiguration`=&lt;mutatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;mutatingwebhookconfiguration-namespace&gt; | EXPERIMENTAL |
+| kube_mutatingwebhookconfiguration_webhook_clientconfig_service | Gauge | `mutatingwebhookconfiguration`=&lt;mutatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;mutatingwebhookconfiguration-namespace&gt; <br> `webhook_name`=&lt;webhook-name&gt; <br> `service_name`=&lt;webhook-service-name&gt; <br> `service_namespace`=&lt;webhook-service-namespace&gt;| EXPERIMENTAL |

docs/validatingwebhookconfiguration-metrics.md

@@ -5,3 +5,4 @@
 | kube_validatingwebhookconfiguration_info | Gauge | `validatingwebhookconfiguration`=&lt;validatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;validatingwebhookconfiguration-namespace&gt; | EXPERIMENTAL |
 | kube_validatingwebhookconfiguration_created  | Gauge | `validatingwebhookconfiguration`=&lt;validatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;validatingwebhookconfiguration-namespace&gt; | EXPERIMENTAL |
 | kube_validatingwebhookconfiguration_metadata_resource_version | Gauge | `validatingwebhookconfiguration`=&lt;validatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;validatingwebhookconfiguration-namespace&gt; | EXPERIMENTAL |
+| kube_validatingwebhookconfiguration_webhook_clientconfig_service | Gauge | `validatingwebhookconfiguration`=&lt;validatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;validatingwebhookconfiguration-namespace&gt; <br> `webhook_name`=&lt;webhook-name&gt; <br> `service_name`=&lt;webhook-service-name&gt; <br> `service_namespace`=&lt;webhook-service-namespace&gt;| EXPERIMENTAL |

internal/store/mutatingwebhookconfiguration.go

@@ -82,6 +82,32 @@ var (
 				}
 			}),
 		),
+		*generator.NewFamilyGeneratorWithStability(
+			"kube_mutatingwebhookconfiguration_webhook_clientconfig_service",
+			"Service used by the apiserver to connect to a mutating webhook.",
+			metric.Gauge,
+			basemetrics.ALPHA,
+			"",
+			wrapMutatingWebhookConfigurationFunc(func(mwc *admissionregistrationv1.MutatingWebhookConfiguration) *metric.Family {
+				ms := []*metric.Metric{}
+				for _, webhook := range mwc.Webhooks {
+					var serviceName, serviceNamespace string
+					if webhook.ClientConfig.Service != nil {
+						serviceName = webhook.ClientConfig.Service.Name
+						serviceNamespace = webhook.ClientConfig.Service.Namespace
+					}
+
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{"webhook_name", "service_name", "service_namespace"},
+						LabelValues: []string{webhook.Name, serviceName, serviceNamespace},
+						Value:       1,
+					})
+				}
+				return &metric.Family{
+					Metrics: ms,
+				}
+			}),
+		),
 	}
 )
 

internal/store/mutatingwebhookconfiguration_test.go

@@ -28,6 +28,7 @@ import (
 func TestMutatingWebhookConfigurationStore(t *testing.T) {
 	startTime := 1501569018
 	metav1StartTime := metav1.Unix(int64(startTime), 0)
+	externalURL := "example.com"
 
 	cases := []generateMetricsTestCase{
 		{
@@ -69,6 +70,37 @@ func TestMutatingWebhookConfigurationStore(t *testing.T) {
 			`,
 			MetricNames: []string{"kube_mutatingwebhookconfiguration_created", "kube_mutatingwebhookconfiguration_info", "kube_mutatingwebhookconfiguration_metadata_resource_version"},
 		},
+		{
+			Obj: &admissionregistrationv1.MutatingWebhookConfiguration{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "mutatingwebhookconfiguration3",
+					Namespace:         "ns3",
+					CreationTimestamp: metav1StartTime,
+					ResourceVersion:   "abcdef",
+				},
+				Webhooks: []admissionregistrationv1.MutatingWebhook{
+					{
+						Name: "webhook_with_service",
+						ClientConfig: admissionregistrationv1.WebhookClientConfig{
+							Service: &admissionregistrationv1.ServiceReference{Name: "svc", Namespace: "ns"},
+						},
+					},
+					{
+						Name: "webhook_with_external_url",
+						ClientConfig: admissionregistrationv1.WebhookClientConfig{
+							URL: &externalURL,
+						},
+					},
+				},
+			},
+			Want: `
+			# HELP kube_mutatingwebhookconfiguration_webhook_clientconfig_service Service used by the apiserver to connect to a mutating webhook.
+			# TYPE kube_mutatingwebhookconfiguration_webhook_clientconfig_service gauge
+			kube_mutatingwebhookconfiguration_webhook_clientconfig_service{webhook_name="webhook_with_external_url",namespace="ns3",service_name="",service_namespace="",mutatingwebhookconfiguration="mutatingwebhookconfiguration3"} 1
+			kube_mutatingwebhookconfiguration_webhook_clientconfig_service{webhook_name="webhook_with_service",namespace="ns3",service_name="svc",service_namespace="ns",mutatingwebhookconfiguration="mutatingwebhookconfiguration3"} 1
+			`,
+			MetricNames: []string{"kube_mutatingwebhookconfiguration_webhook_clientconfig_service"},
+		},
 	}
 	for i, c := range cases {
 		c.Func = generator.ComposeMetricGenFuncs(mutatingWebhookConfigurationMetricFamilies)

internal/store/validatingwebhookconfiguration.go

@@ -82,6 +82,32 @@ var (
 				}
 			}),
 		),
+		*generator.NewFamilyGeneratorWithStability(
+			"kube_validatingwebhookconfiguration_webhook_clientconfig_service",
+			"Service used by the apiserver to connect to a validating webhook.",
+			metric.Gauge,
+			basemetrics.ALPHA,
+			"",
+			wrapValidatingWebhookConfigurationFunc(func(vwc *admissionregistrationv1.ValidatingWebhookConfiguration) *metric.Family {
+				ms := []*metric.Metric{}
+				for _, webhook := range vwc.Webhooks {
+					var serviceName, serviceNamespace string
+					if webhook.ClientConfig.Service != nil {
+						serviceName = webhook.ClientConfig.Service.Name
+						serviceNamespace = webhook.ClientConfig.Service.Namespace
+					}
+
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{"webhook_name", "service_name", "service_namespace"},
+						LabelValues: []string{webhook.Name, serviceName, serviceNamespace},
+						Value:       1,
+					})
+				}
+				return &metric.Family{
+					Metrics: ms,
+				}
+			}),
+		),
 	}
 )
 

internal/store/validatingwebhookconfiguration_test.go

@@ -28,6 +28,7 @@ import (
 func TestValidatingWebhookConfigurationStore(t *testing.T) {
 	startTime := 1501569018
 	metav1StartTime := metav1.Unix(int64(startTime), 0)
+	externalURL := "example.com"
 
 	cases := []generateMetricsTestCase{
 		{
@@ -69,6 +70,37 @@ func TestValidatingWebhookConfigurationStore(t *testing.T) {
 			`,
 			MetricNames: []string{"kube_validatingwebhookconfiguration_created", "kube_validatingwebhookconfiguration_info", "kube_validatingwebhookconfiguration_metadata_resource_version"},
 		},
+		{
+			Obj: &admissionregistrationv1.ValidatingWebhookConfiguration{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "validatingwebhookconfiguration3",
+					Namespace:         "ns3",
+					CreationTimestamp: metav1StartTime,
+					ResourceVersion:   "abcdef",
+				},
+				Webhooks: []admissionregistrationv1.ValidatingWebhook{
+					{
+						Name: "webhook_with_service",
+						ClientConfig: admissionregistrationv1.WebhookClientConfig{
+							Service: &admissionregistrationv1.ServiceReference{Name: "svc", Namespace: "ns"},
+						},
+					},
+					{
+						Name: "webhook_with_external_url",
+						ClientConfig: admissionregistrationv1.WebhookClientConfig{
+							URL: &externalURL,
+						},
+					},
+				},
+			},
+			Want: `
+			# HELP kube_validatingwebhookconfiguration_webhook_clientconfig_service Service used by the apiserver to connect to a validating webhook.
+			# TYPE kube_validatingwebhookconfiguration_webhook_clientconfig_service gauge
+			kube_validatingwebhookconfiguration_webhook_clientconfig_service{webhook_name="webhook_with_external_url",namespace="ns3",service_name="",service_namespace="",validatingwebhookconfiguration="validatingwebhookconfiguration3"} 1
+			kube_validatingwebhookconfiguration_webhook_clientconfig_service{webhook_name="webhook_with_service",namespace="ns3",service_name="svc",service_namespace="ns",validatingwebhookconfiguration="validatingwebhookconfiguration3"} 1
+			`,
+			MetricNames: []string{"kube_validatingwebhookconfiguration_webhook_clientconfig_service"},
+		},
 	}
 	for i, c := range cases {
 		c.Func = generator.ComposeMetricGenFuncs(validatingWebhookConfigurationMetricFamilies)