Skip to content

Commit af61fa0

Browse files
carretercarreter
committed
Add e2e test for multiple servers and multiple agents
1 parent 2c7858a commit af61fa0

File tree

3 files changed

+189
-21
lines changed

3 files changed

+189
-21
lines changed

‎e2e/main_test.go

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,14 @@ package e2e
33
import (
44
"context"
55
"flag"
6+
"fmt"
67
"log"
8+
"net/http"
79
"os"
810
"testing"
911

12+
io_prometheus_client "github.com/prometheus/client_model/go"
13+
"github.com/prometheus/common/expfmt"
1014
"k8s.io/client-go/kubernetes/scheme"
1115
"sigs.k8s.io/e2e-framework/pkg/env"
1216
"sigs.k8s.io/e2e-framework/pkg/envconf"
@@ -42,7 +46,7 @@ func TestMain(m *testing.M) {
4246
func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
4347
client := cfg.Client()
4448

45-
// Render agent RBAC templates.
49+
// Render agent RBAC and Service templates.
4650
agentServiceAccount, _, err := renderAgentTemplate("serviceaccount.yaml", struct{}{})
4751
if err != nil {
4852
return nil, err
@@ -55,6 +59,10 @@ func TestMain(m *testing.M) {
5559
if err != nil {
5660
return ctx, err
5761
}
62+
agentService, _, err := renderAgentTemplate("service.yaml", struct{}{})
63+
if err != nil {
64+
return ctx, err
65+
}
5866

5967
// Submit agent RBAC templates to k8s.
6068
err = client.Resources().Create(ctx, agentServiceAccount)
@@ -69,6 +77,10 @@ func TestMain(m *testing.M) {
6977
if err != nil {
7078
return ctx, err
7179
}
80+
err = client.Resources().Create(ctx, agentService)
81+
if err != nil {
82+
return ctx, err
83+
}
7284

7385
// Render server RBAC and Service templates.
7486
serverClusterRoleBinding, _, err := renderServerTemplate("clusterrolebinding.yaml", struct{}{})
@@ -99,6 +111,18 @@ func TestMain(m *testing.M) {
99111
os.Exit(testenv.Run(m))
100112
}
101113

102-
func TestSingleAgentAndServer(t *testing.T) {
114+
func getMetrics(url string) (map[string]*io_prometheus_client.MetricFamily, error) {
115+
resp, err := http.Get(url)
116+
if err != nil {
117+
return nil, fmt.Errorf("could not get metrics: %w", err)
118+
}
119+
120+
metricsParser := &expfmt.TextParser{}
121+
metricsFamilies, err := metricsParser.TextToMetricFamilies(resp.Body)
122+
defer resp.Body.Close()
123+
if err != nil {
124+
return nil, fmt.Errorf("could not parse metrics: %w", err)
125+
}
103126

127+
return metricsFamilies, nil
104128
}

‎e2e/multiserver_multiagent_test.go

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
package e2e
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strconv"
7+
"testing"
8+
"time"
9+
10+
corev1 "k8s.io/api/core/v1"
11+
"sigs.k8s.io/e2e-framework/klient/k8s/resources"
12+
"sigs.k8s.io/e2e-framework/klient/wait"
13+
"sigs.k8s.io/e2e-framework/klient/wait/conditions"
14+
"sigs.k8s.io/e2e-framework/pkg/envconf"
15+
"sigs.k8s.io/e2e-framework/pkg/features"
16+
)
17+
18+
func TestMultiServer_MultiAgent_StaticCount(t *testing.T) {
19+
serverServiceHost := "konnectivity-server.kube-system.svc.cluster.local"
20+
agentServiceHost := "konnectivity-agent.kube-system.svc.cluster.local"
21+
adminPort := 8093
22+
replicas := 3
23+
24+
serverStatefulSetCfg := StatefulSetConfig{
25+
Replicas: 3,
26+
Image: *serverImage,
27+
Args: []KeyValue{
28+
{Key: "log-file", Value: "/var/log/konnectivity-server.log"},
29+
{Key: "logtostderr", Value: "true"},
30+
{Key: "log-file-max-size", Value: "0"},
31+
{Key: "uds-name", Value: "/etc/kubernetes/konnectivity-server/konnectivity-server.socket"},
32+
{Key: "delete-existing-uds-file"},
33+
{Key: "cluster-cert", Value: "/etc/kubernetes/pki/apiserver.crt"},
34+
{Key: "cluster-key", Value: "/etc/kubernetes/pki/apiserver.key"},
35+
{Key: "server-port", Value: "8090"},
36+
{Key: "agent-port", Value: "8091"},
37+
{Key: "health-port", Value: "8092"},
38+
{Key: "admin-port", Value: strconv.Itoa(adminPort)},
39+
{Key: "keepalive-time", Value: "1h"},
40+
{Key: "mode", Value: "grpc"},
41+
{Key: "agent-namespace", Value: "kube-system"},
42+
{Key: "agent-service-account", Value: "konnectivity-agent"},
43+
{Key: "kubeconfig", Value: "/etc/kubernetes/admin.conf"},
44+
{Key: "authentication-audience", Value: "system:konnectivity-server"},
45+
{Key: "server-count", Value: "1"},
46+
},
47+
}
48+
serverStatefulSet, _, err := renderServerTemplate("statefulset.yaml", serverStatefulSetCfg)
49+
if err != nil {
50+
t.Fatalf("could not render server deployment: %v", err)
51+
}
52+
53+
agentStatefulSetConfig := StatefulSetConfig{
54+
Replicas: 3,
55+
Image: *agentImage,
56+
Args: []KeyValue{
57+
{Key: "logtostderr", Value: "true"},
58+
{Key: "ca-cert", Value: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"},
59+
{Key: "proxy-server-host", Value: serverServiceHost},
60+
{Key: "proxy-server-port", Value: "8091"},
61+
{Key: "sync-interval", Value: "1s"},
62+
{Key: "sync-interval-cap", Value: "10s"},
63+
{Key: "sync-forever"},
64+
{Key: "probe-interval", Value: "1s"},
65+
{Key: "service-account-token-path", Value: "/var/run/secrets/tokens/konnectivity-agent-token"},
66+
{Key: "server-count", Value: "3"},
67+
},
68+
}
69+
agentStatefulSet, _, err := renderAgentTemplate("statefulset.yaml", agentStatefulSetConfig)
70+
if err != nil {
71+
t.Fatalf("could not render agent deployment: %v", err)
72+
}
73+
74+
feature := features.New("konnectivity server and agent stateful set with single replica for each")
75+
feature.Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
76+
client := cfg.Client()
77+
err := client.Resources().Create(ctx, serverStatefulSet)
78+
if err != nil {
79+
t.Fatalf("could not create server deployment: %v", err)
80+
}
81+
82+
err = client.Resources().Create(ctx, agentStatefulSet)
83+
if err != nil {
84+
t.Fatalf("could not create agent deployment: %v", err)
85+
}
86+
87+
err = wait.For(
88+
conditions.New(client.Resources()).DeploymentAvailable(agentStatefulSet.GetName(), agentStatefulSet.GetNamespace()),
89+
wait.WithTimeout(1*time.Minute),
90+
wait.WithInterval(10*time.Second),
91+
)
92+
if err != nil {
93+
t.Fatalf("waiting for agent deployment failed: %v", err)
94+
}
95+
96+
err = wait.For(
97+
conditions.New(client.Resources()).DeploymentAvailable(serverStatefulSet.GetName(), serverStatefulSet.GetNamespace()),
98+
wait.WithTimeout(1*time.Minute),
99+
wait.WithInterval(10*time.Second),
100+
)
101+
if err != nil {
102+
t.Fatalf("waiting for server deployment failed: %v", err)
103+
}
104+
105+
return ctx
106+
})
107+
feature.Assess("all servers connected to all clients", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
108+
client := cfg.Client()
109+
110+
var serverPods *corev1.PodList
111+
err := client.Resources().List(ctx, serverPods, resources.WithLabelSelector("k8s-app=konnectivity-server"))
112+
if err != nil {
113+
t.Fatalf("couldn't get server pods: %v", err)
114+
}
115+
116+
for _, serverPod := range serverPods.Items {
117+
118+
metricsFamilies, err := getMetrics(fmt.Sprintf("%v-%v:%v/metrics", serverPod.Name, serverServiceHost, adminPort))
119+
if err != nil {
120+
t.Fatalf("couldn't get server metrics for pod %v", serverPod.Name)
121+
}
122+
connectionsMetric, exists := metricsFamilies["konnectivity_network_proxy_server_ready_backend_connections"]
123+
if !exists {
124+
t.Fatalf("couldn't find number of ready backend connections in metrics")
125+
}
126+
127+
numConnections := int(connectionsMetric.GetMetric()[0].GetGauge().GetValue())
128+
if numConnections != replicas {
129+
t.Errorf("incorrect number of connected agents (want: %v, got: %v)", replicas, numConnections)
130+
}
131+
}
132+
133+
return ctx
134+
})
135+
feature.Assess("all agents connected to all servers", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
136+
client := cfg.Client()
137+
138+
var agentPods *corev1.PodList
139+
err := client.Resources().List(ctx, agentPods, resources.WithLabelSelector("k8s-app=konnectivity-agent"))
140+
if err != nil {
141+
t.Fatalf("couldn't get agent pods: %v", err)
142+
}
143+
144+
for _, agentPod := range agentPods.Items {
145+
146+
metricsFamilies, err := getMetrics(fmt.Sprintf("%v-%v:%v/metrics", agentPod.Name, agentServiceHost, adminPort))
147+
if err != nil {
148+
t.Fatalf("couldn't get agent metrics for pod %v", agentPod.Name)
149+
}
150+
connectionsMetric, exists := metricsFamilies["konnectivity_network_proxy_agent_open_server_connections"]
151+
if !exists {
152+
t.Fatalf("couldn't find number of ready server connections in metrics")
153+
}
154+
155+
numConnections := int(connectionsMetric.GetMetric()[0].GetGauge().GetValue())
156+
if numConnections != replicas {
157+
t.Errorf("incorrect number of connected servers (want: %v, got: %v)", replicas, numConnections)
158+
}
159+
}
160+
161+
return ctx
162+
})
163+
}

‎e2e/singleserver_singleagent_test.go

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,10 @@ package e2e
33
import (
44
"context"
55
"fmt"
6-
"net/http"
76
"strconv"
87
"testing"
98
"time"
109

11-
io_prometheus_client "github.com/prometheus/client_model/go"
12-
"github.com/prometheus/common/expfmt"
1310
"sigs.k8s.io/e2e-framework/klient/wait"
1411
"sigs.k8s.io/e2e-framework/klient/wait/conditions"
1512
"sigs.k8s.io/e2e-framework/pkg/envconf"
@@ -139,19 +136,3 @@ func TestSingleServer_SingleAgent_StaticCount(t *testing.T) {
139136
return ctx
140137
})
141138
}
142-
143-
func getMetrics(url string) (map[string]*io_prometheus_client.MetricFamily, error) {
144-
resp, err := http.Get(url)
145-
if err != nil {
146-
return nil, fmt.Errorf("could not get metrics: %w", err)
147-
}
148-
149-
metricsParser := &expfmt.TextParser{}
150-
metricsFamilies, err := metricsParser.TextToMetricFamilies(resp.Body)
151-
defer resp.Body.Close()
152-
if err != nil {
153-
return nil, fmt.Errorf("could not parse metrics: %w", err)
154-
}
155-
156-
return metricsFamilies, nil
157-
}

0 commit comments

Comments
 (0)