Skip to content

Commit 13ad396

Browse files
carretercarreter
committed
E2E tests
1 parent c977ad8 commit 13ad396

11 files changed

+536
-12
lines changed

‎e2e/main_test.go

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package e2e
2+
3+
import (
4+
"context"
5+
"flag"
6+
"log"
7+
"os"
8+
"testing"
9+
10+
"k8s.io/client-go/kubernetes/scheme"
11+
"sigs.k8s.io/e2e-framework/pkg/env"
12+
"sigs.k8s.io/e2e-framework/pkg/envconf"
13+
"sigs.k8s.io/e2e-framework/pkg/envfuncs"
14+
"sigs.k8s.io/e2e-framework/support/kind"
15+
)
16+
17+
var (
18+
testenv env.Environment
19+
agentImage = flag.String("agent-image", "", "The proxy agent's docker image.")
20+
serverImage = flag.String("server-image", "", "The proxy server's docker image.")
21+
)
22+
23+
func TestMain(m *testing.M) {
24+
flag.Parse()
25+
if *agentImage == "" {
26+
log.Fatalf("must provide agent image with -agent-image")
27+
}
28+
if *serverImage == "" {
29+
log.Fatalf("must provide server image with -server-image")
30+
}
31+
32+
scheme.AddToScheme(scheme.Scheme)
33+
34+
testenv = env.New()
35+
kindClusterName := "kind-test"
36+
kindCluster := kind.NewCluster(kindClusterName)
37+
38+
testenv.Setup(
39+
envfuncs.CreateCluster(kindCluster, kindClusterName),
40+
envfuncs.LoadImageToCluster(kindClusterName, *agentImage),
41+
envfuncs.LoadImageToCluster(kindClusterName, *serverImage),
42+
func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
43+
client := cfg.Client()
44+
45+
// Render agent RBAC templates.
46+
agentServiceAccount, _, err := renderAgentTemplate("serviceaccount.yaml", struct{}{})
47+
if err != nil {
48+
return nil, err
49+
}
50+
agentClusterRole, _, err := renderAgentTemplate("clusterrole.yaml", struct{}{})
51+
if err != nil {
52+
return nil, err
53+
}
54+
agentClusterRoleBinding, _, err := renderAgentTemplate("clusterrolebinding.yaml", struct{}{})
55+
if err != nil {
56+
return ctx, err
57+
}
58+
59+
// Submit agent RBAC templates to k8s.
60+
err = client.Resources().Create(ctx, agentServiceAccount)
61+
if err != nil {
62+
return ctx, err
63+
}
64+
err = client.Resources().Create(ctx, agentClusterRole)
65+
if err != nil {
66+
return ctx, err
67+
}
68+
err = client.Resources().Create(ctx, agentClusterRoleBinding)
69+
if err != nil {
70+
return ctx, err
71+
}
72+
73+
// Render server RBAC and Service templates.
74+
serverClusterRoleBinding, _, err := renderServerTemplate("clusterrolebinding.yaml", struct{}{})
75+
if err != nil {
76+
return ctx, err
77+
}
78+
serverService, _, err := renderServerTemplate("service.yaml", struct{}{})
79+
if err != nil {
80+
return ctx, err
81+
}
82+
83+
// Submit server templates to k8s.
84+
err = client.Resources().Create(ctx, serverClusterRoleBinding)
85+
if err != nil {
86+
return ctx, err
87+
}
88+
err = client.Resources().Create(ctx, serverService)
89+
if err != nil {
90+
return ctx, err
91+
}
92+
93+
return ctx, nil
94+
},
95+
)
96+
97+
testenv.Finish(envfuncs.DestroyCluster(kindClusterName))
98+
99+
os.Exit(testenv.Run(m))
100+
}
101+
102+
func TestSingleAgentAndServer(t *testing.T) {
103+
104+
}

‎e2e/singleserver_singleagent_test.go

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package e2e
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net/http"
7+
"strconv"
8+
"testing"
9+
"time"
10+
11+
"github.com/prometheus/common/expfmt"
12+
"sigs.k8s.io/e2e-framework/klient/wait"
13+
"sigs.k8s.io/e2e-framework/klient/wait/conditions"
14+
"sigs.k8s.io/e2e-framework/pkg/envconf"
15+
"sigs.k8s.io/e2e-framework/pkg/features"
16+
)
17+
18+
func TestSingleServer_SingleAgent_StaticCount(t *testing.T) {
19+
serviceHost := "konnectivity-server.kube-system.svc.cluster.local"
20+
adminPort := 8093
21+
22+
serverDeploymentCfg := DeploymentConfig{
23+
Replicas: 1,
24+
Image: *serverImage,
25+
Args: []KeyValue{
26+
{Key: "log-file", Value: "/var/log/konnectivity-server.log"},
27+
{Key: "logtostderr", Value: "true"},
28+
{Key: "log-file-max-size", Value: "0"},
29+
{Key: "uds-name", Value: "/etc/kubernetes/konnectivity-server/konnectivity-server.socket"},
30+
{Key: "delete-existing-uds-file"},
31+
{Key: "cluster-cert", Value: "/etc/kubernetes/pki/apiserver.crt"},
32+
{Key: "cluster-key", Value: "/etc/kubernetes/pki/apiserver.key"},
33+
{Key: "server-port", Value: "8090"},
34+
{Key: "agent-port", Value: "8091"},
35+
{Key: "health-port", Value: "8092"},
36+
{Key: "admin-port", Value: strconv.Itoa(adminPort)},
37+
{Key: "keepalive-time", Value: "1h"},
38+
{Key: "mode", Value: "grpc"},
39+
{Key: "agent-namespace", Value: "kube-system"},
40+
{Key: "agent-service-account", Value: "konnectivity-agent"},
41+
{Key: "kubeconfig", Value: "/etc/kubernetes/admin.conf"},
42+
{Key: "authentication-audience", Value: "system:konnectivity-server"},
43+
{Key: "server-count", Value: "1"},
44+
},
45+
}
46+
serverDeployment, _, err := renderServerTemplate("deployment.yaml", serverDeploymentCfg)
47+
if err != nil {
48+
t.Fatalf("could not render server deployment: %v", err)
49+
}
50+
51+
agentDeploymentCfg := DeploymentConfig{
52+
Replicas: 1,
53+
Image: *agentImage,
54+
Args: []KeyValue{
55+
{Key: "logtostderr", Value: "true"},
56+
{Key: "ca-cert", Value: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"},
57+
{Key: "proxy-server-host", Value: serviceHost},
58+
{Key: "proxy-server-port", Value: "8091"},
59+
{Key: "sync-interval", Value: "1s"},
60+
{Key: "sync-interval-cap", Value: "10s"},
61+
{Key: "sync-forever"},
62+
{Key: "probe-interval", Value: "1s"},
63+
{Key: "service-account-token-path", Value: "/var/run/secrets/tokens/konnectivity-agent-token"},
64+
{Key: "server-count", Value: "1"},
65+
},
66+
}
67+
agentDeployment, _, err := renderAgentTemplate("deployment.yaml", agentDeploymentCfg)
68+
if err != nil {
69+
t.Fatalf("could not render agent deployment: %v", err)
70+
}
71+
72+
feature := features.New("konnectivity server and agent deployment with single replica for each")
73+
feature.Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
74+
client := cfg.Client()
75+
err := client.Resources().Create(ctx, serverDeployment)
76+
if err != nil {
77+
t.Fatalf("could not create server deployment: %v", err)
78+
}
79+
80+
err = client.Resources().Create(ctx, agentDeployment)
81+
if err != nil {
82+
t.Fatalf("could not create agent deployment: %v", err)
83+
}
84+
85+
err = wait.For(
86+
conditions.New(client.Resources()).DeploymentAvailable(agentDeployment.GetName(), agentDeployment.GetNamespace()),
87+
wait.WithTimeout(1*time.Minute),
88+
wait.WithInterval(10*time.Second),
89+
)
90+
if err != nil {
91+
t.Fatalf("waiting for agent deployment failed: %v", err)
92+
}
93+
94+
err = wait.For(
95+
conditions.New(client.Resources()).DeploymentAvailable(serverDeployment.GetName(), serverDeployment.GetNamespace()),
96+
wait.WithTimeout(1*time.Minute),
97+
wait.WithInterval(10*time.Second),
98+
)
99+
if err != nil {
100+
t.Fatalf("waiting for server deployment failed: %v", err)
101+
}
102+
103+
return ctx
104+
})
105+
feature.Assess("konnectivity server has a connected client", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
106+
resp, err := http.Get(fmt.Sprintf("%v:%v/metrics", serviceHost, adminPort))
107+
if err != nil {
108+
t.Fatalf("could not read server metrics: %v", err)
109+
}
110+
111+
metricsParser := &expfmt.TextParser{}
112+
metricsFamilies, err := metricsParser.TextToMetricFamilies(resp.Body)
113+
defer resp.Body.Close()
114+
if err != nil {
115+
t.Fatalf("could not parse server metrics: %v", err)
116+
}
117+
118+
connectionsMetric, exists := metricsFamilies["konnectivity_network_proxy_server_ready_backend_connections"]
119+
if !exists {
120+
t.Fatalf("couldn't find number of ready backend connections in metrics: %v", metricsFamilies)
121+
}
122+
123+
numConnections := int(connectionsMetric.GetMetric()[0].GetGauge().GetValue())
124+
if numConnections != 1 {
125+
t.Errorf("incorrect number of connected agents (want: 1, got: %v)", numConnections)
126+
}
127+
128+
return ctx
129+
})
130+
}

‎e2e/templates/agent/clusterrole.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRole
3+
metadata:
4+
name: system:konnectivity-agent
5+
labels:
6+
kubernetes.io/cluster-service: "true"
7+
rules:
8+
- apiGroups: ["coordination.k8s.io"]
9+
resources: ["leases"]
10+
verbs: ["get", "watch", "list"]

‎e2e/templates/agent/clusterrolebinding.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apiVersion: rbac.authorization.k8s.io/rbacv1
2+
kind: ClusterRoleBinding
3+
metadata:
4+
name: system:konnectivity-agent
5+
labels:
6+
kubernetes.io/cluster-service: "true"
7+
subjects:
8+
- kind: ServiceAccount
9+
name: konnectivity-agent
10+
namespace: kube-system
11+
roleRef:
12+
kind: ClusterRole
13+
name: system:konnectivity-agent
14+
apiGroup: rbac.authorization.k8s.io

‎e2e/templates/agent/deployment.yaml

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
labels:
5+
k8s-app: konnectivity-agent
6+
namespace: kube-system
7+
name: konnectivity-agent
8+
spec:
9+
replicas: {{ .Replicas }}
10+
selector:
11+
matchLabels:
12+
k8s-app: konnectivity-agent
13+
template:
14+
metadata:
15+
labels:
16+
k8s-app: konnectivity-agent
17+
spec:
18+
containers:
19+
- name: konnectivity-agent-container
20+
image: {{ .Image }}
21+
resources:
22+
requests:
23+
cpu: 50m
24+
limits:
25+
memory: 30Mi
26+
command: [ "/proxy-agent"]
27+
args: [
28+
{{ range .Args }}
29+
"--{{ .Key }}{{if ne .Value ""}}={{ .Value }}{{ end }}",
30+
{{ end }}
31+
]
32+
env:
33+
- name: POD_NAME
34+
valueFrom:
35+
fieldRef:
36+
fieldPath: metadata.name
37+
- name: POD_NAMESPACE
38+
valueFrom:
39+
fieldRef:
40+
fieldPath: metadata.namespace
41+
- name: HOST_IP
42+
valueFrom:
43+
fieldRef:
44+
fieldPath: status.hostIP
45+
livenessProbe:
46+
httpGet:
47+
scheme: HTTP
48+
port: 8093
49+
path: /healthz
50+
initialDelaySeconds: 15
51+
timeoutSeconds: 15
52+
readinessProbe:
53+
httpGet:
54+
scheme: HTTP
55+
port: 8093
56+
path: /readyz
57+
initialDelaySeconds: 15
58+
timeoutSeconds: 15
59+
volumeMounts:
60+
- mountPath: /var/run/secrets/tokens
61+
name: konnectivity-agent-token
62+
serviceAccountName: konnectivity-agent
63+
volumes:
64+
- name: konnectivity-agent-token
65+
projected:
66+
sources:
67+
- serviceAccountToken:
68+
path: konnectivity-agent-token
69+
audience: system:konnectivity-server

‎e2e/templates/server/clusterrolebinding.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRoleBinding
3+
metadata:
4+
name: system:konnectivity-server
5+
labels:
6+
kubernetes.io/cluster-service: "true"
7+
roleRef:
8+
apiGroup: rbac.authorization.k8s.io
9+
kind: ClusterRole
10+
name: system:auth-delegator
11+
subjects:
12+
- apiGroup: rbac.authorization.k8s.io
13+
kind: User
14+
name: system:konnectivity-server

0 commit comments

Comments
 (0)