Skip to content

Commit 28ebff1

Browse files
carretercarreter
committed
Review comments
1 parent af61fa0 commit 28ebff1

File tree

7 files changed

+217
-262
lines changed

7 files changed

+217
-262
lines changed

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ test-integration: build
8282

8383
.PHONY: test-e2e
8484
test-e2e: docker-build
85-
go test -mod=vendor ./e2e -agent-image ${AGENT_FULL_IMAGE}-$(TARGETARCH):${TAG} -server-image ${SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG}
85+
go test -mod=vendor ./e2e -race -agent-image ${AGENT_FULL_IMAGE}-$(TARGETARCH):${TAG} -server-image ${SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG}
8686

8787
## --------------------------------------
8888
## Binaries

‎e2e/assert_agent_server_connections_test.go

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
package e2e
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net/http"
7+
"testing"
8+
9+
corev1 "k8s.io/api/core/v1"
10+
11+
"github.com/prometheus/common/expfmt"
12+
"sigs.k8s.io/e2e-framework/klient/k8s/resources"
13+
"sigs.k8s.io/e2e-framework/pkg/envconf"
14+
)
15+
16+
func getMetricsGaugeValue(url string, name string) (int, error) {
17+
resp, err := http.Get(url)
18+
if err != nil {
19+
return 0, fmt.Errorf("could not get metrics: %w", err)
20+
}
21+
22+
metricsParser := &expfmt.TextParser{}
23+
metricsFamilies, err := metricsParser.TextToMetricFamilies(resp.Body)
24+
defer resp.Body.Close()
25+
if err != nil {
26+
return 0, fmt.Errorf("could not parse metrics: %w", err)
27+
}
28+
29+
metricFamily, exists := metricsFamilies[name]
30+
if !exists {
31+
return 0, fmt.Errorf("metric %v does not exist", name)
32+
}
33+
value := int(metricFamily.GetMetric()[0].GetGauge().GetValue())
34+
return value, nil
35+
}
36+
37+
func assertAgentsAreConnected(expectedConnections int, serviceHost string, adminPort int) func(context.Context, *testing.T, *envconf.Config) context.Context {
38+
return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
39+
client := cfg.Client()
40+
41+
var agentPods *corev1.PodList
42+
err := client.Resources().List(ctx, agentPods, resources.WithLabelSelector("k8s-app=konnectivity-agent"))
43+
if err != nil {
44+
t.Fatalf("couldn't get agent pods: %v", err)
45+
}
46+
47+
for _, agentPod := range agentPods.Items {
48+
numConnections, err := getMetricsGaugeValue(fmt.Sprintf("%v-%v:%v/metrics", agentPod.Name, serviceHost, adminPort), "konnectivity_network_proxy_agent_open_server_connections")
49+
if err != nil {
50+
t.Fatalf("couldn't get agent metric 'konnectivity_network_proxy_agent_open_server_connections' for pod %v", agentPod.Name)
51+
}
52+
53+
if numConnections != expectedConnections {
54+
t.Errorf("incorrect number of connected servers (want: %v, got: %v)", expectedConnections, numConnections)
55+
}
56+
}
57+
58+
return ctx
59+
}
60+
}
61+
62+
func assertServersAreConnected(expectedConnections int, serviceHost string, adminPort int) func(context.Context, *testing.T, *envconf.Config) context.Context {
63+
return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
64+
client := cfg.Client()
65+
66+
var serverPods *corev1.PodList
67+
err := client.Resources().List(ctx, serverPods, resources.WithLabelSelector("k8s-app=konnectivity-server"))
68+
if err != nil {
69+
t.Fatalf("couldn't get server pods: %v", err)
70+
}
71+
72+
for _, serverPod := range serverPods.Items {
73+
numConnections, err := getMetricsGaugeValue(fmt.Sprintf("%v-%v:%v/metrics", serverPod.Name, serviceHost, adminPort), "konnectivity_network_proxy_server_ready_backend_connections")
74+
if err != nil {
75+
t.Fatalf("couldn't get agent metric 'konnectivity_network_proxy_server_ready_backend_connections' for pod %v", serverPod.Name)
76+
}
77+
78+
if numConnections != expectedConnections {
79+
t.Errorf("incorrect number of connected agents (want: %v, got: %v)", expectedConnections, numConnections)
80+
}
81+
}
82+
83+
return ctx
84+
}
85+
}

‎e2e/main_test.go

Lines changed: 96 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,19 @@
11
package e2e
22

33
import (
4+
"bytes"
45
"context"
56
"flag"
67
"fmt"
78
"log"
8-
"net/http"
99
"os"
10+
"path"
1011
"testing"
12+
"text/template"
1113

12-
io_prometheus_client "github.com/prometheus/client_model/go"
13-
"github.com/prometheus/common/expfmt"
14+
"k8s.io/apimachinery/pkg/runtime/schema"
1415
"k8s.io/client-go/kubernetes/scheme"
16+
"sigs.k8s.io/controller-runtime/pkg/client"
1517
"sigs.k8s.io/e2e-framework/pkg/env"
1618
"sigs.k8s.io/e2e-framework/pkg/envconf"
1719
"sigs.k8s.io/e2e-framework/pkg/envfuncs"
@@ -43,86 +45,108 @@ func TestMain(m *testing.M) {
4345
envfuncs.CreateCluster(kindCluster, kindClusterName),
4446
envfuncs.LoadImageToCluster(kindClusterName, *agentImage),
4547
envfuncs.LoadImageToCluster(kindClusterName, *serverImage),
46-
func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
47-
client := cfg.Client()
48-
49-
// Render agent RBAC and Service templates.
50-
agentServiceAccount, _, err := renderAgentTemplate("serviceaccount.yaml", struct{}{})
51-
if err != nil {
52-
return nil, err
53-
}
54-
agentClusterRole, _, err := renderAgentTemplate("clusterrole.yaml", struct{}{})
55-
if err != nil {
56-
return nil, err
57-
}
58-
agentClusterRoleBinding, _, err := renderAgentTemplate("clusterrolebinding.yaml", struct{}{})
59-
if err != nil {
60-
return ctx, err
61-
}
62-
agentService, _, err := renderAgentTemplate("service.yaml", struct{}{})
63-
if err != nil {
64-
return ctx, err
65-
}
66-
67-
// Submit agent RBAC templates to k8s.
68-
err = client.Resources().Create(ctx, agentServiceAccount)
69-
if err != nil {
70-
return ctx, err
71-
}
72-
err = client.Resources().Create(ctx, agentClusterRole)
73-
if err != nil {
74-
return ctx, err
75-
}
76-
err = client.Resources().Create(ctx, agentClusterRoleBinding)
77-
if err != nil {
78-
return ctx, err
79-
}
80-
err = client.Resources().Create(ctx, agentService)
81-
if err != nil {
82-
return ctx, err
83-
}
84-
85-
// Render server RBAC and Service templates.
86-
serverClusterRoleBinding, _, err := renderServerTemplate("clusterrolebinding.yaml", struct{}{})
87-
if err != nil {
88-
return ctx, err
89-
}
90-
serverService, _, err := renderServerTemplate("service.yaml", struct{}{})
91-
if err != nil {
92-
return ctx, err
93-
}
94-
95-
// Submit server templates to k8s.
96-
err = client.Resources().Create(ctx, serverClusterRoleBinding)
97-
if err != nil {
98-
return ctx, err
99-
}
100-
err = client.Resources().Create(ctx, serverService)
101-
if err != nil {
102-
return ctx, err
103-
}
104-
105-
return ctx, nil
106-
},
48+
renderAndApplyManifests,
10749
)
10850

10951
testenv.Finish(envfuncs.DestroyCluster(kindClusterName))
11052

11153
os.Exit(testenv.Run(m))
11254
}
11355

114-
func getMetrics(url string) (map[string]*io_prometheus_client.MetricFamily, error) {
115-
resp, err := http.Get(url)
56+
// renderTemplate renders a template from e2e/templates into a kubernetes object.
57+
// Template paths are relative to e2e/templates.
58+
func renderTemplate(file string, params any) (client.Object, *schema.GroupVersionKind, error) {
59+
b := &bytes.Buffer{}
60+
61+
tmp, err := template.ParseFiles(path.Join("templates/", file))
62+
if err != nil {
63+
return nil, nil, fmt.Errorf("could not parse template %v: %w", file, err)
64+
}
65+
66+
err = tmp.Execute(b, params)
67+
if err != nil {
68+
return nil, nil, fmt.Errorf("could not execute template %v: %w", file, err)
69+
}
70+
71+
decoder := scheme.Codecs.UniversalDeserializer()
72+
73+
obj, gvk, err := decoder.Decode(b.Bytes(), nil, nil)
74+
if err != nil {
75+
return nil, nil, fmt.Errorf("could not decode rendered yaml into kubernetes object: %w", err)
76+
}
77+
78+
return obj.(client.Object), gvk, nil
79+
}
80+
81+
type KeyValue struct {
82+
Key string
83+
Value string
84+
}
85+
86+
type StatefulSetConfig struct {
87+
Replicas int
88+
Image string
89+
Args []KeyValue
90+
}
91+
92+
func renderAndApplyManifests(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
93+
client := cfg.Client()
94+
95+
// Render agent RBAC and Service templates.
96+
agentServiceAccount, _, err := renderTemplate("agent/serviceaccount.yaml", struct{}{})
97+
if err != nil {
98+
return nil, err
99+
}
100+
agentClusterRole, _, err := renderTemplate("agent/clusterrole.yaml", struct{}{})
116101
if err != nil {
117-
return nil, fmt.Errorf("could not get metrics: %w", err)
102+
return nil, err
103+
}
104+
agentClusterRoleBinding, _, err := renderTemplate("agent/clusterrolebinding.yaml", struct{}{})
105+
if err != nil {
106+
return ctx, err
107+
}
108+
agentService, _, err := renderTemplate("agent/service.yaml", struct{}{})
109+
if err != nil {
110+
return ctx, err
111+
}
112+
113+
// Submit agent RBAC templates to k8s.
114+
err = client.Resources().Create(ctx, agentServiceAccount)
115+
if err != nil {
116+
return ctx, err
117+
}
118+
err = client.Resources().Create(ctx, agentClusterRole)
119+
if err != nil {
120+
return ctx, err
121+
}
122+
err = client.Resources().Create(ctx, agentClusterRoleBinding)
123+
if err != nil {
124+
return ctx, err
125+
}
126+
err = client.Resources().Create(ctx, agentService)
127+
if err != nil {
128+
return ctx, err
129+
}
130+
131+
// Render server RBAC and Service templates.
132+
serverClusterRoleBinding, _, err := renderTemplate("server/clusterrolebinding.yaml", struct{}{})
133+
if err != nil {
134+
return ctx, err
135+
}
136+
serverService, _, err := renderTemplate("server/service.yaml", struct{}{})
137+
if err != nil {
138+
return ctx, err
118139
}
119140

120-
metricsParser := &expfmt.TextParser{}
121-
metricsFamilies, err := metricsParser.TextToMetricFamilies(resp.Body)
122-
defer resp.Body.Close()
141+
// Submit server templates to k8s.
142+
err = client.Resources().Create(ctx, serverClusterRoleBinding)
143+
if err != nil {
144+
return ctx, err
145+
}
146+
err = client.Resources().Create(ctx, serverService)
123147
if err != nil {
124-
return nil, fmt.Errorf("could not parse metrics: %w", err)
148+
return ctx, err
125149
}
126150

127-
return metricsFamilies, nil
151+
return ctx, nil
128152
}

‎e2e/multiserver_multiagent_test.go

Lines changed: 4 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,10 @@ package e2e
22

33
import (
44
"context"
5-
"fmt"
65
"strconv"
76
"testing"
87
"time"
98

10-
corev1 "k8s.io/api/core/v1"
11-
"sigs.k8s.io/e2e-framework/klient/k8s/resources"
129
"sigs.k8s.io/e2e-framework/klient/wait"
1310
"sigs.k8s.io/e2e-framework/klient/wait/conditions"
1411
"sigs.k8s.io/e2e-framework/pkg/envconf"
@@ -45,7 +42,7 @@ func TestMultiServer_MultiAgent_StaticCount(t *testing.T) {
4542
{Key: "server-count", Value: "1"},
4643
},
4744
}
48-
serverStatefulSet, _, err := renderServerTemplate("statefulset.yaml", serverStatefulSetCfg)
45+
serverStatefulSet, _, err := renderTemplate("server/statefulset.yaml", serverStatefulSetCfg)
4946
if err != nil {
5047
t.Fatalf("could not render server deployment: %v", err)
5148
}
@@ -66,7 +63,7 @@ func TestMultiServer_MultiAgent_StaticCount(t *testing.T) {
6663
{Key: "server-count", Value: "3"},
6764
},
6865
}
69-
agentStatefulSet, _, err := renderAgentTemplate("statefulset.yaml", agentStatefulSetConfig)
66+
agentStatefulSet, _, err := renderTemplate("agent/statefulset.yaml", agentStatefulSetConfig)
7067
if err != nil {
7168
t.Fatalf("could not render agent deployment: %v", err)
7269
}
@@ -104,60 +101,6 @@ func TestMultiServer_MultiAgent_StaticCount(t *testing.T) {
104101

105102
return ctx
106103
})
107-
feature.Assess("all servers connected to all clients", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
108-
client := cfg.Client()
109-
110-
var serverPods *corev1.PodList
111-
err := client.Resources().List(ctx, serverPods, resources.WithLabelSelector("k8s-app=konnectivity-server"))
112-
if err != nil {
113-
t.Fatalf("couldn't get server pods: %v", err)
114-
}
115-
116-
for _, serverPod := range serverPods.Items {
117-
118-
metricsFamilies, err := getMetrics(fmt.Sprintf("%v-%v:%v/metrics", serverPod.Name, serverServiceHost, adminPort))
119-
if err != nil {
120-
t.Fatalf("couldn't get server metrics for pod %v", serverPod.Name)
121-
}
122-
connectionsMetric, exists := metricsFamilies["konnectivity_network_proxy_server_ready_backend_connections"]
123-
if !exists {
124-
t.Fatalf("couldn't find number of ready backend connections in metrics")
125-
}
126-
127-
numConnections := int(connectionsMetric.GetMetric()[0].GetGauge().GetValue())
128-
if numConnections != replicas {
129-
t.Errorf("incorrect number of connected agents (want: %v, got: %v)", replicas, numConnections)
130-
}
131-
}
132-
133-
return ctx
134-
})
135-
feature.Assess("all agents connected to all servers", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
136-
client := cfg.Client()
137-
138-
var agentPods *corev1.PodList
139-
err := client.Resources().List(ctx, agentPods, resources.WithLabelSelector("k8s-app=konnectivity-agent"))
140-
if err != nil {
141-
t.Fatalf("couldn't get agent pods: %v", err)
142-
}
143-
144-
for _, agentPod := range agentPods.Items {
145-
146-
metricsFamilies, err := getMetrics(fmt.Sprintf("%v-%v:%v/metrics", agentPod.Name, agentServiceHost, adminPort))
147-
if err != nil {
148-
t.Fatalf("couldn't get agent metrics for pod %v", agentPod.Name)
149-
}
150-
connectionsMetric, exists := metricsFamilies["konnectivity_network_proxy_agent_open_server_connections"]
151-
if !exists {
152-
t.Fatalf("couldn't find number of ready server connections in metrics")
153-
}
154-
155-
numConnections := int(connectionsMetric.GetMetric()[0].GetGauge().GetValue())
156-
if numConnections != replicas {
157-
t.Errorf("incorrect number of connected servers (want: %v, got: %v)", replicas, numConnections)
158-
}
159-
}
160-
161-
return ctx
162-
})
104+
feature.Assess("all servers connected to all clients", assertServersAreConnected(replicas, serverServiceHost, adminPort))
105+
feature.Assess("all agents connected to all servers", assertAgentsAreConnected(replicas, agentServiceHost, adminPort))
163106
}

0 commit comments

Comments
 (0)