Merge pull request #639 from carreter/e2e

Bump go 1.22.2->1.22.5 and add E2E tests

Author: k8s-ci-robot

.github/workflows/e2e.yaml

@@ -48,6 +48,53 @@ jobs:
       with:
         name: konnectivity-agent
         path: _output/konnectivity-agent.tar
+  kind-e2e:
+    name: kind-e2e
+    runs-on: ubuntu-20.04
+    timeout-minutes: 100
+    needs:
+    - build
+    env:
+      REGISTRY: gcr.io/k8s-staging-kas-network-proxy
+      KIND_IMAGE: kindest/node${{ matrix.k8s }}
+      TAG: master
+      CONNECTION_MODE: ${{ matrix.connection-mode }}
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s: [ v1.27.11, v1.28.7, v1.29.2 ]
+        connection-mode: [ grpc, http-connect ]
+    steps:
+    - name: Install kind
+      run: |
+        curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64
+        chmod +x ./kind
+        sudo mv ./kind /usr/local/bin/kind
+    - name: Check out code
+      uses: actions/checkout@v4
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+      id: go
+    - name: Download prebuilt konnectivity-server image
+      uses: actions/download-artifact@v4
+      with:
+        name: konnectivity-server
+    - name: Download prebuilt konnectivity-agent image
+      uses: actions/download-artifact@v4
+      with:
+        name: konnectivity-agent
+    - name: Load prebuilt konnectivity images
+      run: |
+        docker load --input konnectivity-server.tar
+        docker load --input konnectivity-agent.tar
+    - name: Fix konnectivity docker image tags
+      run: |
+        docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-server:master gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:master
+        docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:master
+    - name: Run e2e tests
+      run: make test-e2e-ci
   e2e:
     name: e2e
     runs-on: ubuntu-20.04

Makefile

@@ -24,7 +24,7 @@ ALL_ARCH ?= amd64 arm arm64 ppc64le s390x
 # The output type could either be docker (local), or registry.
 OUTPUT_TYPE ?= docker
 GO_TOOLCHAIN ?= golang
-GO_VERSION ?= 1.22.2
+GO_VERSION ?= 1.22.5
 BASEIMAGE ?= gcr.io/distroless/static-debian11:nonroot
 
 ifeq ($(GOPATH),)
@@ -54,6 +54,9 @@ TAG ?= $(shell git rev-parse HEAD)
 DOCKER_CMD ?= docker
 DOCKER_CLI_EXPERIMENTAL ?= enabled
 PROXY_SERVER_IP ?= 127.0.0.1
+
+KIND_IMAGE ?= kindest/node
+CONNECTION_MODE ?= grpc
 ## --------------------------------------
 ## Testing
 ## --------------------------------------
@@ -67,19 +70,28 @@ mock_gen:
 # Unit tests with faster execution (nicer for development).
 .PHONY: fast-test
 fast-test:
-	go test -mod=vendor -race ./...
+	go test -mod=vendor -race $(shell go list ./... | grep -v -e "/e2e$$" -e "/e2e/.*")
 	cd konnectivity-client && go test -race ./...
 
 # Unit tests with fuller coverage, invoked by CI system.
 .PHONY: test
 test:
-	go test -mod=vendor -race -covermode=atomic -coverprofile=konnectivity.out ./... && go tool cover -html=konnectivity.out -o=konnectivity.html
+	go test -mod=vendor -race -covermode=atomic -coverprofile=konnectivity.out $(shell go list ./... | grep -v -e "/e2e$$" -e "/e2e/.*") && go tool cover -html=konnectivity.out -o=konnectivity.html
 	cd konnectivity-client && go test -race -covermode=atomic -coverprofile=client.out ./... && go tool cover -html=client.out -o=client.html
 
 .PHONY: test-integration
 test-integration: build
 	go test -mod=vendor -race ./tests -agent-path $(PWD)/bin/proxy-agent
 
+.PHONY: test-e2e
+test-e2e: docker-build
+	go test -mod=vendor ./e2e -race -agent-image ${AGENT_FULL_IMAGE}-$(TARGETARCH):${TAG} -server-image ${SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG} -kind-image ${KIND_IMAGE} -mode ${CONNECTION_MODE}
+
+# e2e test runner for continuous integration that does not build a new image.
+.PHONY: test-e2e-ci
+test-e2e-ci:
+	go test -mod=vendor ./e2e -race -agent-image ${AGENT_FULL_IMAGE}-$(TARGETARCH):${TAG} -server-image ${SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG} -kind-image ${KIND_IMAGE} -mode ${CONNECTION_MODE}
+
 ## --------------------------------------
 ## Binaries
 ## --------------------------------------

e2e/README.md

@@ -0,0 +1,23 @@
+# End-to-end tests for konnectivity-network-proxy running in a kind cluster
+
+These e2e tests deploy the KNP agent and server to a local [kind](https://kind.sigs.k8s.io/)
+cluster to verify their functionality.
+
+These can be run automatically using `make e2e-test`.
+
+## Setup in `main_test.go`
+
+Before any of the actual tests are run, the `TestMain()` function
+in `main_test.go` performs the following set up steps:
+
+- Spin up a new kind cluster with the node image provided by the `-kind-image` flag.
+- Sideload the KNP agent and server images provided with `-agent-image` and `-server-image` into the cluster.
+- Deploy the necessary RBAC and service templates for both the KNP agent and server (see `renderAndApplyManifests`).
+
+## The tests
+
+### `static_count_test.go`
+
+These tests deploy the KNP servers and agents to the previously created kind cluster.
+After the deployments are up, the tests check that both the agent and server report
+the correct number of connections on their metrics endpoints.

e2e/main_test.go

@@ -0,0 +1,178 @@
+package e2e
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"testing"
+	"text/template"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
+	"sigs.k8s.io/e2e-framework/support/kind"
+)
+
+var (
+	testenv        env.Environment
+	agentImage     = flag.String("agent-image", "", "The proxy agent's docker image.")
+	serverImage    = flag.String("server-image", "", "The proxy server's docker image.")
+	kindImage      = flag.String("kind-image", "kindest/node", "Image to use for kind nodes.")
+	connectionMode = flag.String("mode", "grpc", "Connection mode to use during e2e tests.")
+)
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+	if *agentImage == "" {
+		log.Fatalf("must provide agent image with -agent-image")
+	}
+	if *serverImage == "" {
+		log.Fatalf("must provide server image with -server-image")
+	}
+
+	scheme.AddToScheme(scheme.Scheme)
+
+	testenv = env.New()
+	kindClusterName := "kind-test"
+	kindCluster := kind.NewCluster(kindClusterName).WithOpts(kind.WithImage(*kindImage))
+
+	testenv.Setup(
+		envfuncs.CreateCluster(kindCluster, kindClusterName),
+		envfuncs.LoadImageToCluster(kindClusterName, *agentImage),
+		envfuncs.LoadImageToCluster(kindClusterName, *serverImage),
+		renderAndApplyManifests,
+	)
+
+	testenv.Finish(envfuncs.DestroyCluster(kindClusterName))
+
+	os.Exit(testenv.Run(m))
+}
+
+// renderTemplate renders a template from e2e/templates into a kubernetes object.
+// Template paths are relative to e2e/templates.
+func renderTemplate(file string, params any) (client.Object, *schema.GroupVersionKind, error) {
+	b := &bytes.Buffer{}
+
+	tmp, err := template.ParseFiles(path.Join("templates/", file))
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not parse template %v: %w", file, err)
+	}
+
+	err = tmp.Execute(b, params)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not execute template %v: %w", file, err)
+	}
+
+	decoder := scheme.Codecs.UniversalDeserializer()
+
+	obj, gvk, err := decoder.Decode(b.Bytes(), nil, nil)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not decode rendered yaml into kubernetes object: %w", err)
+	}
+
+	return obj.(client.Object), gvk, nil
+}
+
+type KeyValue struct {
+	Key   string
+	Value string
+}
+
+type DeploymentConfig struct {
+	Replicas int
+	Image    string
+	Args     []KeyValue
+}
+
+func renderAndApplyManifests(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+	client := cfg.Client()
+
+	// Render agent RBAC and Service templates.
+	agentServiceAccount, _, err := renderTemplate("agent/serviceaccount.yaml", struct{}{})
+	if err != nil {
+		return nil, err
+	}
+	agentClusterRole, _, err := renderTemplate("agent/clusterrole.yaml", struct{}{})
+	if err != nil {
+		return nil, err
+	}
+	agentClusterRoleBinding, _, err := renderTemplate("agent/clusterrolebinding.yaml", struct{}{})
+	if err != nil {
+		return ctx, err
+	}
+	agentService, _, err := renderTemplate("agent/service.yaml", struct{}{})
+	if err != nil {
+		return ctx, err
+	}
+
+	// Submit agent RBAC templates to k8s.
+	err = client.Resources().Create(ctx, agentServiceAccount)
+	if err != nil {
+		return ctx, err
+	}
+	err = client.Resources().Create(ctx, agentClusterRole)
+	if err != nil {
+		return ctx, err
+	}
+	err = client.Resources().Create(ctx, agentClusterRoleBinding)
+	if err != nil {
+		return ctx, err
+	}
+	err = client.Resources().Create(ctx, agentService)
+	if err != nil {
+		return ctx, err
+	}
+
+	// Render server RBAC and Service templates.
+	serverClusterRoleBinding, _, err := renderTemplate("server/clusterrolebinding.yaml", struct{}{})
+	if err != nil {
+		return ctx, err
+	}
+	serverService, _, err := renderTemplate("server/service.yaml", struct{}{})
+	if err != nil {
+		return ctx, err
+	}
+
+	// Submit server templates to k8s.
+	err = client.Resources().Create(ctx, serverClusterRoleBinding)
+	if err != nil {
+		return ctx, err
+	}
+	err = client.Resources().Create(ctx, serverService)
+	if err != nil {
+		return ctx, err
+	}
+
+	return ctx, nil
+}
+
+func deployAndWaitForDeployment(deployment client.Object) func(context.Context, *testing.T, *envconf.Config) context.Context {
+	return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+		client := cfg.Client()
+		err := client.Resources().Create(ctx, deployment)
+		if err != nil {
+			t.Fatalf("could not create Deployment: %v", err)
+		}
+
+		err = wait.For(
+			conditions.New(client.Resources()).DeploymentAvailable(deployment.GetName(), deployment.GetNamespace()),
+			wait.WithTimeout(1*time.Minute),
+			wait.WithInterval(10*time.Second),
+		)
+		if err != nil {
+			t.Fatalf("waiting for Deployment failed: %v", err)
+		}
+
+		return ctx
+	}
+}

e2e/metrics_assertions_test.go

@@ -0,0 +1,85 @@
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/prometheus/common/expfmt"
+	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+)
+
+func getMetricsGaugeValue(url string, name string) (int, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return 0, fmt.Errorf("could not get metrics from url %v: %w", url, err)
+	}
+
+	metricsParser := &expfmt.TextParser{}
+	metricsFamilies, err := metricsParser.TextToMetricFamilies(resp.Body)
+	if err != nil {
+		return 0, fmt.Errorf("could not parse metrics: %w", err)
+	}
+	defer resp.Body.Close()
+
+	metricFamily, exists := metricsFamilies[name]
+	if !exists {
+		return 0, fmt.Errorf("metric %v does not exist", name)
+	}
+	value := int(metricFamily.GetMetric()[0].GetGauge().GetValue())
+	return value, nil
+}
+
+func assertAgentsAreConnected(expectedConnections int, adminPort int) func(context.Context, *testing.T, *envconf.Config) context.Context {
+	return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+		client := cfg.Client()
+
+		var agentPods *corev1.PodList
+		err := client.Resources().List(ctx, agentPods, resources.WithLabelSelector("k8s-app=konnectivity-agent"))
+		if err != nil {
+			t.Fatalf("couldn't get agent pods (label selector 'k8s-app=konnectivity-agent'): %v", err)
+		}
+
+		for _, agentPod := range agentPods.Items {
+			numConnections, err := getMetricsGaugeValue(fmt.Sprintf("%v:%v/metrics", agentPod.Status.PodIP, adminPort), "konnectivity_network_proxy_agent_open_server_connections")
+			if err != nil {
+				t.Fatalf("couldn't get agent metric 'konnectivity_network_proxy_agent_open_server_connections' for pod %v: %v", agentPod.Name, err)
+			}
+
+			if numConnections != expectedConnections {
+				t.Errorf("incorrect number of connected servers (want: %d, got: %d)", expectedConnections, numConnections)
+			}
+		}
+
+		return ctx
+	}
+}
+
+func assertServersAreConnected(expectedConnections int, adminPort int) func(context.Context, *testing.T, *envconf.Config) context.Context {
+	return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+		client := cfg.Client()
+
+		var serverPods *corev1.PodList
+		err := client.Resources().List(ctx, serverPods, resources.WithLabelSelector("k8s-app=konnectivity-server"))
+		if err != nil {
+			t.Fatalf("couldn't get server pods (label selector 'k8s-app=konnectivity-server'): %v", err)
+		}
+
+		for _, serverPod := range serverPods.Items {
+			numConnections, err := getMetricsGaugeValue(fmt.Sprintf("%v:%v/metrics", serverPod.Status.PodIP, adminPort), "konnectivity_network_proxy_server_ready_backend_connections")
+			if err != nil {
+				t.Fatalf("couldn't get agent metric 'konnectivity_network_proxy_server_ready_backend_connections' for pod %v: %v", serverPod.Name, err)
+			}
+
+			if numConnections != expectedConnections {
+				t.Errorf("incorrect number of connected agents (want: %d, got: %d)", expectedConnections, numConnections)
+			}
+		}
+
+		return ctx
+	}
+}