Skip to content

Commit 4d0d95d

Browse files
carretercarreter
committed
Add e2e tests for lease counting
1 parent c004586 commit 4d0d95d

File tree

2 files changed

+215
-0
lines changed

2 files changed

+215
-0
lines changed

‎e2e/lease_count_test.go

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
package e2e
2+
3+
import (
4+
"context"
5+
"strconv"
6+
"testing"
7+
"time"
8+
9+
"github.com/google/uuid"
10+
coordinationv1 "k8s.io/api/coordination/v1"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"sigs.k8s.io/e2e-framework/pkg/envconf"
13+
"sigs.k8s.io/e2e-framework/pkg/features"
14+
)
15+
16+
type leaseController struct {
17+
labels map[string]string
18+
validLeases []*coordinationv1.Lease
19+
expiredLeases []*coordinationv1.Lease
20+
}
21+
22+
func NewLeaseController(labels map[string]string) *leaseController {
23+
return &leaseController{
24+
labels: labels,
25+
validLeases: []*coordinationv1.Lease{},
26+
expiredLeases: []*coordinationv1.Lease{},
27+
}
28+
}
29+
30+
func (lc *leaseController) PublishValidLease() func(context.Context, *testing.T, *envconf.Config) context.Context {
31+
return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
32+
var duration int32 = 999999999
33+
acquireTime := metav1.NewMicroTime(time.Now())
34+
35+
newLease := &coordinationv1.Lease{
36+
TypeMeta: metav1.TypeMeta{},
37+
ObjectMeta: metav1.ObjectMeta{
38+
Name: uuid.New().String(),
39+
Labels: lc.labels,
40+
},
41+
Spec: coordinationv1.LeaseSpec{
42+
LeaseDurationSeconds: &duration,
43+
AcquireTime: &acquireTime,
44+
},
45+
}
46+
47+
err := cfg.Client().Resources().Create(ctx, newLease)
48+
if err != nil {
49+
t.Fatalf("could not publish valid lease: %v", err)
50+
}
51+
52+
lc.validLeases = append(lc.validLeases, newLease)
53+
return ctx
54+
}
55+
}
56+
57+
func (lc *leaseController) DeleteValidLease() func(context.Context, *testing.T, *envconf.Config) context.Context {
58+
return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
59+
leaseToDelete := lc.validLeases[len(lc.validLeases)-1]
60+
61+
err := cfg.Client().Resources().Delete(ctx, leaseToDelete)
62+
if err != nil {
63+
t.Fatalf("could not delete valid lease: %v", err)
64+
}
65+
66+
lc.validLeases = lc.validLeases[:len(lc.validLeases)-1]
67+
return ctx
68+
}
69+
}
70+
71+
func (lc *leaseController) PublishExpiredLease() func(context.Context, *testing.T, *envconf.Config) context.Context {
72+
return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
73+
var duration int32 = 1
74+
acquireTime := metav1.NewMicroTime(time.Now().Add(-time.Second * 99999999))
75+
76+
newLease := &coordinationv1.Lease{
77+
TypeMeta: metav1.TypeMeta{},
78+
ObjectMeta: metav1.ObjectMeta{
79+
Name: uuid.New().String(),
80+
Labels: lc.labels,
81+
},
82+
Spec: coordinationv1.LeaseSpec{
83+
LeaseDurationSeconds: &duration,
84+
AcquireTime: &acquireTime,
85+
},
86+
}
87+
88+
err := cfg.Client().Resources().Create(ctx, newLease)
89+
if err != nil {
90+
t.Fatalf("could not publish expired lease: %v", err)
91+
}
92+
93+
lc.expiredLeases = append(lc.expiredLeases, newLease)
94+
return ctx
95+
}
96+
}
97+
98+
func (lc *leaseController) DeleteExpiredLease() func(context.Context, *testing.T, *envconf.Config) context.Context {
99+
return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
100+
leaseToDelete := lc.expiredLeases[len(lc.expiredLeases)-1]
101+
102+
err := cfg.Client().Resources().Delete(ctx, leaseToDelete)
103+
if err != nil {
104+
t.Fatalf("could not delete expired lease: %v", err)
105+
}
106+
107+
lc.expiredLeases = lc.expiredLeases[:len(lc.expiredLeases)-1]
108+
return ctx
109+
}
110+
}
111+
112+
func TestLeaseCount(t *testing.T) {
113+
adminPort := 8093
114+
initialServerReplicas := 6
115+
initialAgentReplicas := 2
116+
leaseLabels := map[string]string{"aTestLabel": "aTestValue"}
117+
leaseLabelSelector := "aTestLabel=aTestValue"
118+
119+
serverDeploymentConfig := DeploymentConfig{
120+
Replicas: initialServerReplicas,
121+
Image: *serverImage,
122+
Args: []KeyValue{
123+
{"log-file", "/var/log/konnectivity-server.log"},
124+
{"logtostderr", "true"},
125+
{"log-file-max-size", "0"},
126+
{"uds-name", "/etc/kubernetes/konnectivity-server/konnectivity-server.socket"},
127+
{Key: "delete-existing-uds-file"},
128+
{"cluster-cert", "/etc/kubernetes/pki/apiserver.crt"},
129+
{"cluster-key", "/etc/kubernetes/pki/apiserver.key"},
130+
{"server-port", "8090"},
131+
{"agent-port", "8091"},
132+
{"health-port", "8092"},
133+
{"admin-port", strconv.Itoa(adminPort)},
134+
{"keepalive-time", "1h"},
135+
{"mode", *connectionMode},
136+
{"agent-namespace", "kube-system"},
137+
{"agent-service-account", "konnectivity-agent"},
138+
{"kubeconfig", "/etc/kubernetes/admin.conf"},
139+
{"authentication-audience", "system:konnectivity-server"},
140+
{"server-count", strconv.Itoa(initialServerReplicas)},
141+
},
142+
}
143+
serverDeployment, _, err := renderTemplate("server/deployment.yaml", serverDeploymentConfig)
144+
if err != nil {
145+
t.Fatalf("could not render server deployment: %v", err)
146+
}
147+
148+
agentDeploymentConfig := DeploymentConfig{
149+
Replicas: initialAgentReplicas,
150+
Image: *agentImage,
151+
Args: []KeyValue{
152+
{"logtostderr", "true"},
153+
{"ca-cert", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"},
154+
{"proxy-server-host", "konnectivity-server.kube-system.svc.cluster.local"},
155+
{"proxy-server-port", "8091"},
156+
{"sync-interval", "1s"},
157+
{"sync-interval-cap", "10s"},
158+
{Key: "sync-forever"},
159+
{"probe-interval", "1s"},
160+
{"service-account-token-path", "/var/run/secrets/tokens/konnectivity-agent-token"},
161+
{"server-count-lease-selector", leaseLabelSelector},
162+
},
163+
}
164+
agentDeployment, _, err := renderTemplate("agent/deployment.yaml", agentDeploymentConfig)
165+
if err != nil {
166+
t.Fatalf("could not render agent deployment: %v", err)
167+
}
168+
169+
lc := NewLeaseController(leaseLabels)
170+
171+
feature := features.New("konnectivity agent lease counting system")
172+
feature.Setup(deployAndWaitForDeployment(serverDeployment))
173+
feature.Setup(deployAndWaitForDeployment(agentDeployment))
174+
// We start off by publishing two valid leases and one expired lease.
175+
feature.Setup(lc.PublishValidLease())
176+
feature.Setup(lc.PublishValidLease())
177+
feature.Setup(lc.PublishExpiredLease())
178+
feature.Assess("agents correctly count 2 leases (2 valid, 1 expired)", assertAgentKnownServerCount(2, adminPort))
179+
// Publishing additional expired leases should not change the server count.
180+
feature.Setup(lc.PublishExpiredLease())
181+
feature.Setup(lc.PublishExpiredLease())
182+
feature.Assess("agents correctly count 2 leases (2 valid, 3 expired)", assertAgentKnownServerCount(2, adminPort))
183+
// Publishing additional valid leases should increase the server count.
184+
feature.Setup(lc.PublishValidLease())
185+
feature.Setup(lc.PublishValidLease())
186+
feature.Assess("agents correctly count 4 leases (4 valid, 3 expired)", assertAgentKnownServerCount(4, adminPort))
187+
// Deleting a valid lease should reduce the server count.
188+
feature.Setup(lc.DeleteValidLease())
189+
feature.Assess("agents correctly count 3 leases (3 valid, 3 expired)", assertAgentKnownServerCount(3, adminPort))
190+
}

‎e2e/metrics_assertions_test.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,28 @@ func assertServersAreConnected(expectedConnections int, adminPort int) func(cont
8383
return ctx
8484
}
8585
}
86+
87+
func assertAgentKnownServerCount(expectedServerCount int, adminPort int) func(context.Context, *testing.T, *envconf.Config) context.Context {
88+
return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
89+
client := cfg.Client()
90+
91+
var agentPods *corev1.PodList
92+
err := client.Resources().List(ctx, agentPods, resources.WithLabelSelector("k8s-app=konnectivity-agent"))
93+
if err != nil {
94+
t.Fatalf("couldn't get server pods: %v", err)
95+
}
96+
97+
for _, agentPod := range agentPods.Items {
98+
knownServerCount, err := getMetricsGaugeValue(fmt.Sprintf("%v:%v/metrics", agentPod.Status.PodIP, adminPort), "konnectivity_network_proxy_agent_known_server_count")
99+
if err != nil {
100+
t.Fatalf("couldn't get agent metric 'konnectivity_network_proxy_agent_known_server_count' for pod %v", agentPod.Name)
101+
}
102+
103+
if knownServerCount != expectedServerCount {
104+
t.Errorf("incorrect known server count (want: %v, got: %v)", knownServerCount, expectedServerCount)
105+
}
106+
}
107+
108+
return ctx
109+
}
110+
}

0 commit comments

Comments
 (0)