fix: add readyz endpoint

Discourage the usage of `/metrics` for any probe, and use `/readyz` in
place of the earlier telemetry metrics endpoint to secure the exposition
data.

Signed-off-by: Pranshu Srivastava <[email protected]>

Author: rexagod

README.md

@@ -348,9 +348,11 @@ After running the above, if you see `Clusterrolebinding "cluster-admin-binding"
 
 The following healthcheck endpoints are available, some of which are used to determine the result of the aforementioned probes:
 
-* `/livez`: Returns a 200 status code if the application is not affected by an outage of the Kubernetes API Server. We recommend to use this as a liveness probe.
-* `/metrics`: Returns a 200 status code if the application is able to serve metrics. While this is available for both ports, we recommend to use the telemetry metrics endpoint as a readiness probe.
-* `/healthz`: Returns a 200 status code if the application is running. We recommend to use this as a startup probe.
+* `/healthz`: Returns a 200 status code if the application is running. We recommend to use this for the startup probe.
+* `/livez`: Returns a 200 status code if the application is not affected by an outage of the Kubernetes API Server. We recommend to using this for the liveness probe.
+* `/readyz`: Returns a 200 status code if the application is ready to accept traffic. We recommend using this for the readiness probe.
+
+Note that it is discouraged to use the telemetry metrics endpoint for any probe when proxying the exposition data.
 
 #### Limited privileges environment
 

README.md.tpl

@@ -349,9 +349,11 @@ After running the above, if you see `Clusterrolebinding "cluster-admin-binding"
 
 The following healthcheck endpoints are available, some of which are used to determine the result of the aforementioned probes:
 
-* `/livez`: Returns a 200 status code if the application is not affected by an outage of the Kubernetes API Server. We recommend to use this as a liveness probe.
-* `/metrics`: Returns a 200 status code if the application is able to serve metrics. While this is available for both ports, we recommend to use the telemetry metrics endpoint as a readiness probe.
-* `/healthz`: Returns a 200 status code if the application is running. We recommend to use this as a startup probe.
+* `/healthz`: Returns a 200 status code if the application is running. We recommend to use this for the startup probe.
+* `/livez`: Returns a 200 status code if the application is not affected by an outage of the Kubernetes API Server. We recommend to using this for the liveness probe.
+* `/readyz`: Returns a 200 status code if the application is ready to accept traffic. We recommend using this for the readiness probe.
+
+Note that it is discouraged to use the telemetry metrics endpoint for any probe when proxying the exposition data.
 
 #### Limited privileges environment
 

examples/autosharding/statefulset.yaml

@@ -38,7 +38,7 @@ spec:
         livenessProbe:
           httpGet:
             path: /livez
-            port: 8080
+            port: http-metrics
           initialDelaySeconds: 5
           timeoutSeconds: 5
         name: kube-state-metrics
@@ -49,8 +49,8 @@ spec:
           name: telemetry
         readinessProbe:
           httpGet:
-            path: /metrics
-            port: 8081
+            path: /readyz
+            port: http-metrics
           initialDelaySeconds: 5
           timeoutSeconds: 5
         securityContext:

examples/daemonsetsharding/daemonset.yaml

@@ -33,7 +33,7 @@ spec:
         livenessProbe:
           httpGet:
             path: /livez
-            port: 8080
+            port: http-metrics
           initialDelaySeconds: 5
           timeoutSeconds: 5
         name: kube-state-metrics-shard
@@ -44,8 +44,8 @@ spec:
           name: telemetry
         readinessProbe:
           httpGet:
-            path: /metrics
-            port: 8081
+            path: /readyz
+            port: http-metrics
           initialDelaySeconds: 5
           timeoutSeconds: 5
         securityContext:

examples/daemonsetsharding/deployment-no-node-pods.yaml

@@ -28,7 +28,7 @@ spec:
         livenessProbe:
           httpGet:
             path: /livez
-            port: 8080
+            port: http-metrics
           initialDelaySeconds: 5
           timeoutSeconds: 5
         name: kube-state-metrics
@@ -39,8 +39,8 @@ spec:
           name: telemetry
         readinessProbe:
           httpGet:
-            path: /metrics
-            port: 8081
+            path: /readyz
+            port: http-metrics
           initialDelaySeconds: 5
           timeoutSeconds: 5
         securityContext:

examples/daemonsetsharding/deployment.yaml

@@ -27,7 +27,7 @@ spec:
         livenessProbe:
           httpGet:
             path: /livez
-            port: 8080
+            port: http-metrics
           initialDelaySeconds: 5
           timeoutSeconds: 5
         name: kube-state-metrics
@@ -38,8 +38,8 @@ spec:
           name: telemetry
         readinessProbe:
           httpGet:
-            path: /metrics
-            port: 8081
+            path: /readyz
+            port: http-metrics
           initialDelaySeconds: 5
           timeoutSeconds: 5
         securityContext:

examples/standard/deployment.yaml

@@ -25,7 +25,7 @@ spec:
         livenessProbe:
           httpGet:
             path: /livez
-            port: 8080
+            port: http-metrics
           initialDelaySeconds: 5
           timeoutSeconds: 5
         name: kube-state-metrics
@@ -36,8 +36,8 @@ spec:
           name: telemetry
         readinessProbe:
           httpGet:
-            path: /metrics
-            port: 8081
+            path: /readyz
+            port: http-metrics
           initialDelaySeconds: 5
           timeoutSeconds: 5
         securityContext:

jsonnet/kube-state-metrics/kube-state-metrics.libsonnet

@@ -192,12 +192,12 @@
         seccompProfile: { type: 'RuntimeDefault' },
       },
       livenessProbe: { timeoutSeconds: 5, initialDelaySeconds: 5, httpGet: {
-        port: 8080,
+        port: "http-metrics",
         path: '/livez',
       } },
       readinessProbe: { timeoutSeconds: 5, initialDelaySeconds: 5, httpGet: {
-        port: 8081,
-        path: '/metrics',
+        port: "http-metrics",
+        path: '/readyz',
       } },
     };
 

pkg/app/server.go

@@ -62,6 +62,7 @@ const (
 	metricsPath = "/metrics"
 	healthzPath = "/healthz"
 	livezPath   = "/livez"
+	readyzPath  = "/readyz"
 )
 
 // promLogger implements promhttp.Logger
@@ -396,6 +397,19 @@ func buildTelemetryServer(registry prometheus.Gatherer) *http.ServeMux {
 	return mux
 }
 
+func handleClusterDelegationForProber(client kubernetes.Interface, probeType string) http.HandlerFunc {
+	return func(w http.ResponseWriter, _ *http.Request) {
+		got := client.CoreV1().RESTClient().Get().AbsPath(probeType).Do(context.Background())
+		if got.Error() != nil {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			w.Write([]byte(http.StatusText(http.StatusServiceUnavailable)))
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(http.StatusText(http.StatusOK)))
+	}
+}
+
 func buildMetricsServer(m *metricshandler.MetricsHandler, durationObserver prometheus.ObserverVec, client kubernetes.Interface) *http.ServeMux {
 	mux := http.NewServeMux()
 
@@ -410,24 +424,13 @@ func buildMetricsServer(m *metricshandler.MetricsHandler, durationObserver prome
 	mux.Handle(metricsPath, promhttp.InstrumentHandlerDuration(durationObserver, m))
 
 	// Add livezPath
-	mux.Handle(livezPath, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+	mux.Handle(livezPath, handleClusterDelegationForProber(client, livezPath))
 
-		// Query the Kube API to make sure we are not affected by a network outage.
-		got := client.CoreV1().RESTClient().Get().AbsPath("/livez").Do(context.Background())
-		if got.Error() != nil {
-			w.WriteHeader(http.StatusServiceUnavailable)
-			w.Write([]byte(http.StatusText(http.StatusServiceUnavailable)))
-			return
-		}
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte(http.StatusText(http.StatusOK)))
-	}))
+	// Add readyzPath
+	mux.Handle(readyzPath, handleClusterDelegationForProber(client, readyzPath))
 
 	// Add healthzPath
-	mux.HandleFunc(healthzPath, func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte(http.StatusText(http.StatusOK)))
-	})
+	mux.Handle(healthzPath, handleClusterDelegationForProber(client, healthzPath))
 
 	// Add index
 	landingConfig := web.LandingConfig{