Adding support for agent authenticate server by using kubeapi-server ca

certificate.

Author: dberkov

Makefile

@@ -20,9 +20,11 @@ STAGING_REGISTRY := gcr.io/k8s-staging-kas-network-proxy
 
 SERVER_IMAGE_NAME ?= proxy-server
 AGENT_IMAGE_NAME ?= proxy-agent
+TEST_CLIENT_IMAGE_NAME ?= proxy-test-client
 
 SERVER_FULL_IMAGE ?= $(REGISTRY)/$(SERVER_IMAGE_NAME)
 AGENT_FULL_IMAGE ?= $(REGISTRY)/$(AGENT_IMAGE_NAME)
+TEST_CLIENT_FULL_IMAGE ?= $(REGISTRY)/$(TEST_CLIENT_IMAGE_NAME)
 
 TAG ?= $(shell git rev-parse HEAD)
 
@@ -131,10 +133,10 @@ certs: easy-rsa-master cfssl cfssljson
 ## --------------------------------------
 
 .PHONY: docker-build
-docker-build: docker-build/proxy-agent docker-build/proxy-server
+docker-build: docker-build/proxy-agent docker-build/proxy-server docker-build/proxy-test-client
 
 .PHONY: docker-push
-docker-push: docker-push/proxy-agent docker-push/proxy-server
+docker-push: docker-push/proxy-agent docker-push/proxy-server docker-push/proxy-test-client
 
 .PHONY: docker-build/proxy-agent
 docker-build/proxy-agent: cmd/agent/main.go proto/agent/agent.pb.go
@@ -158,17 +160,29 @@ docker-push/proxy-server: docker-build/proxy-server
 	@[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 )
 	${DOCKER_CMD} push ${SERVER_FULL_IMAGE}-$(ARCH):${TAG}
 
+.PHONY: docker-build/proxy-test-client
+docker-build/proxy-test-client: cmd/client/main.go proto/agent/agent.pb.go proto/proxy.pb.go
+	@[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
+	echo "Building proxy-test-client for ${ARCH}"
+	${DOCKER_CMD} build . --build-arg ARCH=$(ARCH) -f artifacts/images/client-build.Dockerfile -t ${TEST_CLIENT_FULL_IMAGE}-$(ARCH):${TAG}
+
+.PHONY: docker-push/proxy-test-client
+docker-push/proxy-test-client: docker-build/proxy-test-client
+	@[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 )
+	${DOCKER_CMD} push ${TEST_CLIENT_FULL_IMAGE}-$(ARCH):${TAG}
+
 ## --------------------------------------
 ## Docker — All ARCH
 ## --------------------------------------
 
 .PHONY: docker-build-all
-docker-build-all: $(addprefix docker-build/proxy-agent-,$(ALL_ARCH)) $(addprefix docker-build/proxy-server-,$(ALL_ARCH))
+docker-build-all: $(addprefix docker-build/proxy-agent-,$(ALL_ARCH)) $(addprefix docker-build/proxy-server-,$(ALL_ARCH)) $(addprefix docker-build/proxy-test-client-,$(ALL_ARCH))
 
 .PHONY: docker-push-all
-docker-push-all: $(addprefix docker-push/proxy-agent-,$(ALL_ARCH)) $(addprefix docker-push/proxy-server-,$(ALL_ARCH))
+docker-push-all: $(addprefix docker-push/proxy-agent-,$(ALL_ARCH)) $(addprefix docker-push/proxy-server-,$(ALL_ARCH)) $(addprefix docker-push/proxy-test-client-,$(ALL_ARCH))
 	$(MAKE) docker-push-manifest/proxy-agent
 	$(MAKE) docker-push-manifest/proxy-server
+	$(MAKE) docker-push-manifest/test-client
 
 docker-build/proxy-agent-%:
 	$(MAKE) ARCH=$* docker-build/proxy-agent
@@ -182,6 +196,13 @@ docker-build/proxy-server-%:
 docker-push/proxy-server-%:
 	$(MAKE) ARCH=$* docker-push/proxy-server
 
+docker-build/proxy-test-client-%:
+	$(MAKE) ARCH=$* docker-build/proxy-test-client
+
+docker-push/proxy-test-client-%:
+	$(MAKE) ARCH=$* docker-push/proxy-test-client
+
+
 .PHONY: docker-push-manifest/proxy-agent
 docker-push-manifest/proxy-agent: ## Push the fat manifest docker image.
 	## Minimum docker version 18.06.0 is required for creating and pushing manifest images.
@@ -196,6 +217,13 @@ docker-push-manifest/proxy-server: ## Push the fat manifest docker image.
 	@for arch in $(ALL_ARCH); do ${DOCKER_CMD} manifest annotate --arch $${arch} ${SERVER_FULL_IMAGE}:${TAG} ${SERVER_FULL_IMAGE}-$${arch}:${TAG}; done
 	${DOCKER_CMD} manifest push --purge $(SERVER_FULL_IMAGE):$(TAG)
 
+.PHONY: docker-push-manifest/proxy-test-client
+docker-push-manifest/proxy-test-client: ## Push the fat manifest docker image.
+	## Minimum docker version 18.06.0 is required for creating and pushing manifest images.
+	${DOCKER_CMD} manifest create --amend $(TEST_CLIENT_FULL_IMAGE):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(TEST_CLIENT_FULL_IMAGE)\-&:$(TAG)~g")
+	@for arch in $(ALL_ARCH); do ${DOCKER_CMD} manifest annotate --arch $${arch} ${TEST_CLIENT_FULL_IMAGE}:${TAG} ${TEST_CLIENT_FULL_IMAGE}-$${arch}:${TAG}; done
+	${DOCKER_CMD} manifest push --purge $(TEST_CLIENT_FULL_IMAGE):$(TAG)
+
 ## --------------------------------------
 ## Release
 ## --------------------------------------
@@ -208,6 +236,7 @@ release-staging: ## Builds and push container images to the staging bucket.
 release-alias-tag: # Adds the tag to the last build tag. BASE_REF comes from the cloudbuild.yaml
 	gcloud container images add-tag $(AGENT_FULL_IMAGE):$(TAG) $(AGENT_FULL_IMAGE):$(BASE_REF)
 	gcloud container images add-tag $(SERVER_FULL_IMAGE):$(TAG) $(SERVER_FULL_IMAGE):$(BASE_REF)
+	gcloud container images add-tag $(TEST_CLIENT_FULL_IMAGE):$(TAG) $(TEST_CLIENT_FULL_IMAGE):$(BASE_REF)
 
 ## --------------------------------------
 ## Cleanup / Verification

README.md

@@ -108,6 +108,9 @@ python -m SimpleHTTPServer
 curl -v -p --proxy-key certs/master/private/proxy-client.key --proxy-cert certs/master/issued/proxy-client.crt --proxy-cacert certs/master/issued/ca.crt --proxy-cert-type PEM -x https://127.0.0.1:8090  http://localhost:8000```
 ```
 
+### Running on kubernetes
+See following [README.md](examples/kubernetes/README.md) 
+
 ## Troubleshoot
 
 ### Undefined ProtoPackageIsVersion3
@@ -118,4 +121,3 @@ protoc-gen-go binary has to be built from the vendored version:
 go install ./vendor/github.com/golang/protobuf/protoc-gen-go
 make gen
 ```
-

artifacts/images/client-build.Dockerfile

@@ -0,0 +1,19 @@
+# Build the client binary
+FROM golang:1.12.1 as builder
+
+# Copy in the go src
+WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
+COPY pkg/    pkg/
+COPY cmd/    cmd/
+COPY proto/  proto/
+COPY vendor/ vendor/
+
+# Build
+ARG ARCH
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -a -ldflags '-extldflags "-static"' -o proxy-test-client sigs.k8s.io/apiserver-network-proxy/cmd/client
+
+# Copy the loader into a thin image
+FROM scratch
+WORKDIR /
+COPY --from=builder /go/src/sigs.k8s.io/apiserver-network-proxy/proxy-test-client .
+ENTRYPOINT ["/proxy-test-client"]

cmd/agent/main.go

@@ -18,10 +18,8 @@ package main
 
 import (
 	"crypto/tls"
-	"crypto/x509"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"os"
 
@@ -158,26 +156,12 @@ func (a *Agent) run(o *GrpcProxyAgentOptions) error {
 }
 
 func (a *Agent) runProxyConnection(o *GrpcProxyAgentOptions) error {
-	agentCert, err := tls.LoadX509KeyPair(o.agentCert, o.agentKey)
-	if err != nil {
-		return fmt.Errorf("failed to load X509 key pair %s and %s: %v", o.agentCert, o.agentKey, err)
-	}
-	certPool := x509.NewCertPool()
-	caCert, err := ioutil.ReadFile(o.caCert)
-	if err != nil {
-		return fmt.Errorf("failed to read agent CA cert %s: %v", o.caCert, err)
-	}
-	ok := certPool.AppendCertsFromPEM(caCert)
-	if !ok {
-		return fmt.Errorf("failed to append CA cert to the cert pool")
+	var tlsConfig *tls.Config
+	var err error
+	if tlsConfig, err = util.GetClientTLSConfig(o.caCert, o.agentCert, o.agentKey, o.proxyServerHost); err != nil {
+		return err
 	}
-
-	transportCreds := credentials.NewTLS(&tls.Config{
-		ServerName:   o.proxyServerHost,
-		Certificates: []tls.Certificate{agentCert},
-		RootCAs:      certPool,
-	})
-	dialOption := grpc.WithTransportCredentials(transportCreds)
+	dialOption := grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
 	client, err := agentclient.NewAgentClient(fmt.Sprintf("%s:%d", o.proxyServerHost, o.proxyServerPort), dialOption)
 	if err != nil {
 		return err

cmd/client/main.go

@@ -20,7 +20,6 @@ import (
 	"bufio"
 	"context"
 	"crypto/tls"
-	"crypto/x509"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -57,6 +56,7 @@ func main() {
 		klog.Flush()
 		os.Exit(1)
 	}
+	klog.Flush()
 }
 
 type GrpcProxyClientOptions struct {
@@ -84,7 +84,7 @@ func (o *GrpcProxyClientOptions) Flags() *pflag.FlagSet {
 	flags.IntVar(&o.requestPort, "request-port", o.requestPort, "The port the request server is listening on.")
 	flags.StringVar(&o.proxyHost, "proxy-host", o.proxyHost, "The host of the proxy server.")
 	flags.IntVar(&o.proxyPort, "proxy-port", o.proxyPort, "The port the proxy server is listening on.")
-	flags.StringVar(&o.proxyUdsName, "proxy-uds", o.proxyHost, "The UDS name to connect to.")
+	flags.StringVar(&o.proxyUdsName, "proxy-uds", o.proxyUdsName, "The UDS name to connect to.")
 	flags.StringVar(&o.mode, "mode", o.mode, "Mode can be either 'grpc' or 'http-connect'.")
 
 	return flags
@@ -142,6 +142,9 @@ func (o *GrpcProxyClientOptions) Validate() error {
 		return fmt.Errorf("please do not try to use reserved port %d for the proxy server port", o.proxyPort)
 	}
 	if o.proxyUdsName != "" {
+		if o.proxyHost != "" {
+			return fmt.Errorf("please do set proxy host when using UDS")
+		}
 		if o.proxyPort != 0 {
 			return fmt.Errorf("please do set proxy server port to 0 not %d when using UDS", o.proxyPort)
 		}
@@ -305,25 +308,12 @@ func (c *Client) getUDSDialer(o *GrpcProxyClientOptions) (func(ctx context.Conte
 }
 
 func (c *Client) getMTLSDialer(o *GrpcProxyClientOptions) (func(ctx context.Context, network, addr string) (net.Conn, error), error) {
-	clientCert, err := tls.LoadX509KeyPair(o.clientCert, o.clientKey)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read key pair %s & %s, got %v", o.clientCert, o.clientKey, err)
-	}
-	certPool := x509.NewCertPool()
-	caCert, err := ioutil.ReadFile(o.caCert)
+	var tlsConfig *tls.Config
+	var err error
+	tlsConfig, err = util.GetClientTLSConfig(o.caCert, o.clientCert, o.clientKey, o.proxyHost)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read cert file %s, got %v", o.caCert, err)
+		return nil, err
 	}
-	ok := certPool.AppendCertsFromPEM(caCert)
-	if !ok {
-		return nil, fmt.Errorf("failed to append CA cert to the cert pool")
-	}
-
-	transportCreds := credentials.NewTLS(&tls.Config{
-		ServerName:   "127.0.0.1",
-		Certificates: []tls.Certificate{clientCert},
-		RootCAs:      certPool,
-	})
 
 	var proxyConn net.Conn
 
@@ -339,6 +329,7 @@ func (c *Client) getMTLSDialer(o *GrpcProxyClientOptions) (func(ctx context.Cont
 
 	switch o.mode {
 	case "grpc":
+		transportCreds := credentials.NewTLS(tlsConfig)
 		dialOption := grpc.WithTransportCredentials(transportCreds)
 		serverAddress := fmt.Sprintf("%s:%d", o.proxyHost, o.proxyPort)
 		tunnel, err := client.CreateGrpcTunnel(serverAddress, dialOption)
@@ -355,13 +346,7 @@ func (c *Client) getMTLSDialer(o *GrpcProxyClientOptions) (func(ctx context.Cont
 		proxyAddress := fmt.Sprintf("%s:%d", o.proxyHost, o.proxyPort)
 		requestAddress := fmt.Sprintf("%s:%d", o.requestHost, o.requestPort)
 
-		proxyConn, err = tls.Dial("tcp", proxyAddress,
-			&tls.Config{
-				ServerName:   o.proxyHost,
-				Certificates: []tls.Certificate{clientCert},
-				RootCAs:      certPool,
-			},
-		)
+		proxyConn, err = tls.Dial("tcp", proxyAddress, tlsConfig)
 		if err != nil {
 			return nil, fmt.Errorf("dialing proxy %q failed: %v", proxyAddress, err)
 		}

cmd/proxy/main.go

@@ -38,7 +38,6 @@ import (
 	"sigs.k8s.io/apiserver-network-proxy/pkg/agent/agentserver"
 	"sigs.k8s.io/apiserver-network-proxy/pkg/util"
 	"sigs.k8s.io/apiserver-network-proxy/proto/agent"
-
 )
 
 func main() {
@@ -329,28 +328,43 @@ func (p *Proxy) runUDSMasterServer(ctx context.Context, o *ProxyRunOptions, serv
 	return stop, nil
 }
 
-func (p *Proxy) runMTLSMasterServer(ctx context.Context, o *ProxyRunOptions, server *agentserver.ProxyServer) (StopFunc, error) {
-	var stop StopFunc
-
-	proxyCert, err := tls.LoadX509KeyPair(o.serverCert, o.serverKey)
+func (p *Proxy) getTLSConfig(caFile, certFile, keyFile string) (*tls.Config, error) {
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
-		return nil, fmt.Errorf("failed to load X509 key pair %s and %s: %v", o.serverCert, o.serverKey, err)
+		return nil, fmt.Errorf("failed to load X509 key pair %s and %s: %v", certFile, keyFile, err)
 	}
+
+	if caFile == "" {
+		return &tls.Config{Certificates: []tls.Certificate{cert}}, nil
+	}
+
 	certPool := x509.NewCertPool()
-	caCert, err := ioutil.ReadFile(o.serverCaCert)
+	caCert, err := ioutil.ReadFile(caFile)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read server CA cert %s: %v", o.serverCaCert, err)
+		return nil, fmt.Errorf("failed to read cluster CA cert %s: %v", caFile, err)
 	}
 	ok := certPool.AppendCertsFromPEM(caCert)
 	if !ok {
-		return nil, fmt.Errorf("failed to append master CA cert to the cert pool")
+		return nil, fmt.Errorf("failed to append cluster CA cert to the cert pool")
 	}
-
 	tlsConfig := &tls.Config{
 		ClientAuth:   tls.RequireAndVerifyClientCert,
-		Certificates: []tls.Certificate{proxyCert},
+		Certificates: []tls.Certificate{cert},
 		ClientCAs:    certPool,
 	}
+
+	return tlsConfig, nil
+}
+
+func (p *Proxy) runMTLSMasterServer(ctx context.Context, o *ProxyRunOptions, server *agentserver.ProxyServer) (StopFunc, error) {
+	var stop StopFunc
+
+	var tlsConfig *tls.Config
+	var err error
+	if tlsConfig, err = p.getTLSConfig(o.serverCaCert, o.serverCert, o.serverKey); err != nil {
+		return nil, err
+	}
+
 	addr := fmt.Sprintf(":%d", o.serverPort)
 
 	if o.mode == "grpc" {
@@ -386,26 +400,13 @@ func (p *Proxy) runMTLSMasterServer(ctx context.Context, o *ProxyRunOptions, ser
 }
 
 func (p *Proxy) runAgentServer(o *ProxyRunOptions, server *agentserver.ProxyServer) error {
-	clusterCert, err := tls.LoadX509KeyPair(o.clusterCert, o.clusterKey)
-	if err != nil {
-		return fmt.Errorf("failed to load X509 key pair %s and %s: %v", o.clusterCert, o.clusterKey, err)
-	}
-	certPool := x509.NewCertPool()
-	caCert, err := ioutil.ReadFile(o.clusterCaCert)
-	if err != nil {
-		return fmt.Errorf("failed to read cluster CA cert %s: %v", o.clusterCaCert, err)
-	}
-	ok := certPool.AppendCertsFromPEM(caCert)
-	if !ok {
-		return fmt.Errorf("failed to append cluster CA cert to the cert pool")
+	var tlsConfig *tls.Config
+	var err error
+	if tlsConfig, err = p.getTLSConfig(o.clusterCaCert, o.clusterCert, o.clusterKey); err != nil {
+		return err
 	}
-	tlsConfig := &tls.Config{
-		ClientAuth:   tls.RequireAndVerifyClientCert,
-		Certificates: []tls.Certificate{clusterCert},
-		ClientCAs:    certPool,
-	}
-	addr := fmt.Sprintf(":%d", o.agentPort)
 
+	addr := fmt.Sprintf(":%d", o.agentPort)
 	serverOption := grpc.Creds(credentials.NewTLS(tlsConfig))
 	grpcServer := grpc.NewServer(serverOption)
 	agent.RegisterAgentServiceServer(grpcServer, server)