prevents misleading error-level logs

The apiserver and nsqd processes do not gracefully handle the
empty TCP connections generated by kubelet as part of probe
execution, causing misleading error messages to accumulate
in the logs at the probe polling intervals.

For the nsqd process, there exists an HTTP endpoint that can
be used for liveness testing, but it not useful for readiness
testing as the `/ping` endpoint does not reflect readiness
for connections on port 4150. Thus, the liveness test is moved
to an HTTP test of the `/ping` endpoint and the readiness test
is removed.

Furthermore, each access to `/ping` is logged as an INFO level
log in the nsqd log files. While this is not as misleading as
ERROR level logging, it still represents unnecessary logging
and the log level for the nsqd process has been moved up to
WARNING.

For the pgo-apiserver, a new lightweight route is added to
the conventional location of `/healthz` with a simple `ok`
response with the HTTP 200.

As a result, NOAUTH_ROUTES no longer optionally activates
deferred certificate checking - it is required to support
`/healthz` as kubelet does not present a certificate when
performing HTTPS checks.

NOAUTH_ROUTES may still be used to enable unauthenticated
access to the existing whitelist of routes.

Fixes missing probes in Ansible template

[ch6429]
[ch6453]

Author: wilybrace

ansible/roles/pgo-operator/templates/deployment.json.j2

@@ -30,6 +30,24 @@
                                 "containerPort": {{ pgo_apiserver_port }}
                             }
                         ],
+                        "readinessProbe": {
+                            "httpGet": {
+                                "path": "/healthz",
+                                "port": {{ pgo_apiserver_port }},
+                                "scheme": "HTTPS"
+                            },
+                            "initialDelaySeconds": 15,
+                            "periodSeconds": 5
+                        },
+                        "livenessProbe": {
+                            "httpGet": {
+                                "path": "/healthz",
+                                "port": {{ pgo_apiserver_port }},
+                                "scheme": "HTTPS"
+                            },
+                            "initialDelaySeconds": 15,
+                            "periodSeconds": 5
+                        },
                         "env": [
                             {
                                 "name": "CRUNCHY_DEBUG",
@@ -175,6 +193,14 @@
                         "image": "{% if pgo_event_image | default('') != '' %}{{ pgo_event_image }}
                                 {%- else %}{{ pgo_image_prefix }}/pgo-event:{{ pgo_image_tag }}
                                 {%- endif %}",
+                        "livenessProbe": {
+                            "httpGet": {
+                                "path": "/ping",
+                                "port": 4151,
+                            },
+                            "initialDelaySeconds": 15,
+                            "periodSeconds": 5
+                        },
                         "env": [
                             {
                                 "name": "TIMEOUT",

apiserver.go

@@ -109,6 +109,7 @@ func main() {
 	r := mux.NewRouter()
 	r.HandleFunc("/version", versionservice.VersionHandler)
 	r.HandleFunc("/health", versionservice.HealthHandler)
+	r.HandleFunc("/healthz", versionservice.HealthyHandler)
 	r.HandleFunc("/policies", policyservice.CreatePolicyHandler)
 	r.HandleFunc("/showpolicies", policyservice.ShowPolicyHandler).Methods("POST")
 	//here
@@ -185,11 +186,15 @@ func main() {
 	r.HandleFunc("/benchmarkdelete", benchmarkservice.DeleteBenchmarkHandler).Methods("POST")
 	r.HandleFunc("/benchmarkshow", benchmarkservice.ShowBenchmarkHandler).Methods("POST")
 
-	// Optionally lighten mTLS requirement if some paths don't require certs
-	certsVerify := tls.RequireAndVerifyClientCert
-	if !disableTLS && len(skipAuthRoutes) > 0 {
-		certsVerify = tls.VerifyClientCertIfGiven
-		optCertEnforcer, err := apiserver.NewCertEnforcer(strings.Split(skipAuthRoutes, ","))
+	certsVerify := tls.VerifyClientCertIfGiven
+	skipAuth := []string{
+		"/healthz", // Required for kube probes
+	}
+	if !disableTLS {
+		if len(skipAuthRoutes) > 0 {
+			skipAuth = append(skipAuth, strings.Split(skipAuthRoutes, ",")...)
+		}
+		optCertEnforcer, err := apiserver.NewCertEnforcer(skipAuth)
 		if err != nil {
 			log.Fatalf("NOAUTH_ROUTES configured incorrectly: %s", err)
 			os.Exit(2)

apiserver/middleware.go

@@ -33,7 +33,8 @@ type certEnforcer struct {
 func NewCertEnforcer(reqRoutes []string) (*certEnforcer, error) {
 	allowed := map[string]struct{}{
 		// List of allowed routes is part of the published documentation
-		"/health": struct{}{},
+		"/health":  struct{}{},
+		"/healthz": struct{}{},
 	}
 
 	ce := &certEnforcer{

apiserver/versionservice/versionservice.go

@@ -51,3 +51,10 @@ func HealthHandler(w http.ResponseWriter, r *http.Request) {
 
 	json.NewEncoder(w).Encode(resp)
 }
+
+// HealthyHandler follows the health endpoint convention of HTTP/200 and
+// body "ok" used by other cloud services, typically on /healthz
+func HealthyHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte("ok"))
+}

bin/pgo-event/pgo-event.sh

@@ -31,7 +31,7 @@ sleep 3
 
 echo "pgo-event starting nsqd"
 
-/usr/local/bin/nsqd --data-path=/tmp --http-address=0.0.0.0:4151 --tcp-address=0.0.0.0:4150 &
+/usr/local/bin/nsqd --data-path=/tmp --http-address=0.0.0.0:4151 --tcp-address=0.0.0.0:4150 --log-level=warn &
 
 echo "pgo-event waiting till sigterm"
 

deploy/deployment.json

@@ -29,15 +29,19 @@
                             }
                         ],
                         "readinessProbe": {
-                            "tcpSocket": {
-                                "port": $PGO_APISERVER_PORT
+                            "httpGet": {
+                                "path": "/healthz",
+                                "port": $PGO_APISERVER_PORT,
+                                "scheme": "HTTPS"
                             },
                             "initialDelaySeconds": 15,
                             "periodSeconds": 5
                         },
                         "livenessProbe": {
-                            "tcpSocket": {
-                                "port": $PGO_APISERVER_PORT
+                            "httpGet": {
+                                "path": "/healthz",
+                                "port": $PGO_APISERVER_PORT,
+                                "scheme": "HTTPS"
                             },
                             "initialDelaySeconds": 15,
                             "periodSeconds": 5
@@ -183,16 +187,10 @@
                     {
                         "name": "event",
                         "image": "$PGO_IMAGE_PREFIX/pgo-event:$PGO_IMAGE_TAG",
-                        "readinessProbe": {
-                            "tcpSocket": {
-                                "port": 4150
-                            },
-                            "initialDelaySeconds": 15,
-                            "periodSeconds": 5
-                        },
                         "livenessProbe": {
-                            "tcpSocket": {
-                                "port": 4150
+                            "httpGet": {
+                                "path": "/ping",
+                                "port": 4151
                             },
                             "initialDelaySeconds": 15,
                             "periodSeconds": 5

hugo/content/Configuration/configuration.md

@@ -96,9 +96,8 @@ this setting:
 /health
 ```
 
-If the health route has its authentication disabled, the existing readiness
-and liveness probes for the apiserver container could be enhanced by
-configuring them with HTTP-based checks against the health route.
+The `/healthz` route is used by kubernetes probes and has its authentication
+disabed without requiring NOAUTH_ROUTES.
 
 
 ## Security