Merge pull request #485 from kinvolk/imran/smarter-readiness-check

agent: smarter readiness check

Author: k8s-ci-robot

cmd/agent/app/server.go

@@ -17,6 +17,7 @@ limitations under the License.
 package app
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"fmt"
@@ -26,6 +27,7 @@ import (
 	"runtime"
 	runpprof "runtime/pprof"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -36,6 +38,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"sigs.k8s.io/apiserver-network-proxy/cmd/agent/app/options"
+	"sigs.k8s.io/apiserver-network-proxy/pkg/agent"
 	"sigs.k8s.io/apiserver-network-proxy/pkg/util"
 )
 
@@ -63,11 +66,13 @@ func (a *Agent) run(o *options.GrpcProxyAgentOptions) error {
 	}
 
 	stopCh := make(chan struct{})
-	if err := a.runProxyConnection(o, stopCh); err != nil {
+
+	cs, err := a.runProxyConnection(o, stopCh)
+	if err != nil {
 		return fmt.Errorf("failed to run proxy connection with %v", err)
 	}
 
-	if err := a.runHealthServer(o); err != nil {
+	if err := a.runHealthServer(o, cs); err != nil {
 		return fmt.Errorf("failed to run health server with %v", err)
 	}
 
@@ -80,11 +85,11 @@ func (a *Agent) run(o *options.GrpcProxyAgentOptions) error {
 	return nil
 }
 
-func (a *Agent) runProxyConnection(o *options.GrpcProxyAgentOptions, stopCh <-chan struct{}) error {
+func (a *Agent) runProxyConnection(o *options.GrpcProxyAgentOptions, stopCh <-chan struct{}) (agent.ReadinessManager, error) {
 	var tlsConfig *tls.Config
 	var err error
 	if tlsConfig, err = util.GetClientTLSConfig(o.CaCert, o.AgentCert, o.AgentKey, o.ProxyServerHost, o.AlpnProtos); err != nil {
-		return err
+		return nil, err
 	}
 	dialOptions := []grpc.DialOption{
 		grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)),
@@ -97,15 +102,45 @@ func (a *Agent) runProxyConnection(o *options.GrpcProxyAgentOptions, stopCh <-ch
 	cs := cc.NewAgentClientSet(stopCh)
 	cs.Serve()
 
-	return nil
+	return cs, nil
 }
 
-func (a *Agent) runHealthServer(o *options.GrpcProxyAgentOptions) error {
+func (a *Agent) runHealthServer(o *options.GrpcProxyAgentOptions, cs agent.ReadinessManager) error {
 	livenessHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintf(w, "ok")
 	})
+
+	checks := []agent.HealthChecker{agent.Ping, agent.NewServerConnected(cs)}
 	readinessHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "ok")
+		var failedChecks []string
+		var individualCheckOutput bytes.Buffer
+		for _, check := range checks {
+			if err := check.Check(r); err != nil {
+				fmt.Fprintf(&individualCheckOutput, "[-]%s failed: %v\n", check.Name(), err)
+				failedChecks = append(failedChecks, check.Name())
+			} else {
+				fmt.Fprintf(&individualCheckOutput, "[+]%s ok\n", check.Name())
+			}
+		}
+
+		// Always be verbose if the check has failed
+		if len(failedChecks) > 0 {
+			klog.V(0).Infoln("%s check failed: \n%v", strings.Join(failedChecks, ","), individualCheckOutput.String())
+			w.WriteHeader(http.StatusServiceUnavailable)
+			fmt.Fprintf(w, individualCheckOutput.String())
+			return
+		}
+
+		if _, found := r.URL.Query()["verbose"]; !found {
+			w.WriteHeader(http.StatusOK)
+			fmt.Fprint(w, "ok")
+			return
+		}
+
+		fmt.Fprintf(&individualCheckOutput, "check passed\n")
+
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, individualCheckOutput.String())
 	})
 
 	muxHandler := http.NewServeMux()

pkg/agent/readiness.go

@@ -0,0 +1,78 @@
+/*
+copyright 2023 the kubernetes authors.
+
+licensed under the apache license, version 2.0 (the "license");
+you may not use this file except in compliance with the license.
+you may obtain a copy of the license at
+
+    http://www.apache.org/licenses/license-2.0
+
+unless required by applicable law or agreed to in writing, software
+distributed under the license is distributed on an "as is" basis,
+without warranties or conditions of any kind, either express or implied.
+see the license for the specific language governing permissions and
+limitations under the license.
+*/
+
+package agent
+
+import (
+	"fmt"
+	"net/http"
+)
+
+// ReadinessManager supports checking if the agent is ready.
+type ReadinessManager interface {
+	// Ready returns true the proxy server is ready.
+	Ready() bool
+}
+
+var _ ReadinessManager = &ClientSet{}
+
+func (cs *ClientSet) Ready() bool {
+	// Returns true if the agent is connected to at least one server.
+	return cs.HealthyClientsCount() > 0
+}
+
+// HealthChecker represents an entity capable of performing health checks.
+type HealthChecker interface {
+	Name() string
+	Check(req *http.Request) error
+}
+
+// ping implements the simplest possible healthz checker.
+type ping struct{}
+
+var Ping HealthChecker = &ping{}
+
+func (p *ping) Name() string {
+	return "ping"
+}
+
+func (p *ping) Check(_ *http.Request) error {
+	return nil
+}
+
+type serverConnected struct {
+	rm ReadinessManager
+}
+
+var ServerConnected HealthChecker = &serverConnected{}
+
+func NewServerConnected(cs ReadinessManager) HealthChecker {
+	return &serverConnected{
+		rm: cs,
+	}
+}
+
+func (s *serverConnected) Name() string {
+	return "server-connected"
+}
+
+func (s *serverConnected) Check(_ *http.Request) error {
+	if s.rm.Ready() {
+		return nil
+	}
+
+	return fmt.Errorf("no servers connected")
+}

tests/proxy_test.go

@@ -969,6 +969,7 @@ func runAgentWithID(agentID, addr string, stopCh <-chan struct{}) *agent.ClientS
 		ProbeInterval: 100 * time.Millisecond,
 		DialOptions:   []grpc.DialOption{grpc.WithInsecure()},
 	}
+
 	client := cc.NewAgentClientSet(stopCh)
 	client.Serve()
 	return client

tests/readiness_test.go

@@ -16,11 +16,9 @@ limitations under the License.
 
 package tests
 
-import (
-	"testing"
-)
+import "testing"
 
-func TestReadiness(t *testing.T) {
+func TestGRPCServerAndAgentReadiness(t *testing.T) {
 	stopCh := make(chan struct{})
 	defer close(stopCh)
 
@@ -38,8 +36,45 @@ func TestReadiness(t *testing.T) {
 	clientset := runAgent(proxy.agent, stopCh)
 	waitForConnectedServerCount(t, 1, clientset)
 
+	// check the agent connected status
+	isAgentReady := clientset.Ready()
+	if !isAgentReady {
+		t.Fatalf("expected connection status 'true', got: %t", isAgentReady)
+	}
 	ready, _ = server.Readiness.Ready()
 	if !ready {
 		t.Fatalf("expected ready")
 	}
 }
+
+func TestHTTPConnServerAndAgentReadiness(t *testing.T) {
+	stopCh := make(chan struct{})
+	defer close(stopCh)
+
+	proxy, cleanup, err := runHTTPConnProxyServer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+
+	clientset := runAgent(proxy.agent, stopCh)
+	waitForConnectedServerCount(t, 1, clientset)
+
+	// check the agent connected status
+	isAgentReady := clientset.Ready()
+	if !isAgentReady {
+		t.Fatalf("expected connection status 'true', got: %t", isAgentReady)
+	}
+}
+
+func TestAgentReadinessWithoutServer(t *testing.T) {
+	stopCh := make(chan struct{})
+	defer close(stopCh)
+
+	clientset := runAgent("localhost:8080", stopCh)
+	// check the agent connected status
+	isAgentReady := clientset.Ready()
+	if isAgentReady {
+		t.Fatalf("expected connection status 'false', got: %t", isAgentReady)
+	}
+}