Skip to content

Commit 5e40d05

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #2950 from jackfrancis/e2e-check-daemonsets-later
E2E: verify daemonset pods after machines
2 parents 9e3506b + 0dcd63c commit 5e40d05

File tree

5 files changed

+165
-43
lines changed

5 files changed

+165
-43
lines changed

‎test/e2e/azure_test.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,15 @@ var _ = Describe("Workload cluster creation", func() {
184184
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
185185
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
186186
},
187+
PostMachinesProvisioned: func() {
188+
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
189+
return DaemonsetsSpecInput{
190+
BootstrapClusterProxy: bootstrapClusterProxy,
191+
Namespace: namespace,
192+
ClusterName: clusterName,
193+
}
194+
})
195+
},
187196
}, result)
188197

189198
By("Creating a private cluster from the management cluster", func() {
@@ -236,6 +245,15 @@ var _ = Describe("Workload cluster creation", func() {
236245
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
237246
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
238247
},
248+
PostMachinesProvisioned: func() {
249+
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
250+
return DaemonsetsSpecInput{
251+
BootstrapClusterProxy: bootstrapClusterProxy,
252+
Namespace: namespace,
253+
ClusterName: clusterName,
254+
}
255+
})
256+
},
239257
}, result)
240258

241259
By("Verifying expected VM extensions are present on the node", func() {
@@ -320,6 +338,15 @@ var _ = Describe("Workload cluster creation", func() {
320338
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
321339
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
322340
},
341+
PostMachinesProvisioned: func() {
342+
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
343+
return DaemonsetsSpecInput{
344+
BootstrapClusterProxy: bootstrapClusterProxy,
345+
Namespace: namespace,
346+
ClusterName: clusterName,
347+
}
348+
})
349+
},
323350
}, result)
324351

325352
By("Verifying expected VM extensions are present on the node", func() {
@@ -379,6 +406,15 @@ var _ = Describe("Workload cluster creation", func() {
379406
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
380407
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
381408
},
409+
PostMachinesProvisioned: func() {
410+
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
411+
return DaemonsetsSpecInput{
412+
BootstrapClusterProxy: bootstrapClusterProxy,
413+
Namespace: namespace,
414+
ClusterName: clusterName,
415+
}
416+
})
417+
},
382418
}, result)
383419

384420
By("Verifying expected VM extensions are present on the node", func() {
@@ -464,6 +500,15 @@ var _ = Describe("Workload cluster creation", func() {
464500
// The workaround is to use server side apply by passing `--server-side` flag to kubectl apply.
465501
// More on server side apply here: https://kubernetes.io/docs/reference/using-api/server-side-apply/
466502
Args: []string{"--server-side"},
503+
PostMachinesProvisioned: func() {
504+
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
505+
return DaemonsetsSpecInput{
506+
BootstrapClusterProxy: bootstrapClusterProxy,
507+
Namespace: namespace,
508+
ClusterName: clusterName,
509+
}
510+
})
511+
},
467512
}, result)
468513

469514
By("Verifying expected VM extensions are present on the node", func() {
@@ -582,6 +627,15 @@ var _ = Describe("Workload cluster creation", func() {
582627
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
583628
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
584629
},
630+
PostMachinesProvisioned: func() {
631+
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
632+
return DaemonsetsSpecInput{
633+
BootstrapClusterProxy: bootstrapClusterProxy,
634+
Namespace: namespace,
635+
ClusterName: clusterName,
636+
}
637+
})
638+
},
585639
}, result)
586640

587641
By("Verifying expected VM extensions are present on the node", func() {
@@ -745,6 +799,15 @@ var _ = Describe("Workload cluster creation", func() {
745799
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
746800
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
747801
},
802+
PostMachinesProvisioned: func() {
803+
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
804+
return DaemonsetsSpecInput{
805+
BootstrapClusterProxy: bootstrapClusterProxy,
806+
Namespace: namespace,
807+
ClusterName: clusterName,
808+
}
809+
})
810+
},
748811
}, result)
749812

750813
By("Verifying expected VM extensions are present on the node", func() {
@@ -818,6 +881,15 @@ var _ = Describe("Workload cluster creation", func() {
818881
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
819882
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
820883
},
884+
PostMachinesProvisioned: func() {
885+
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
886+
return DaemonsetsSpecInput{
887+
BootstrapClusterProxy: bootstrapClusterProxy,
888+
Namespace: namespace,
889+
ClusterName: clusterName,
890+
}
891+
})
892+
},
821893
}, result)
822894

823895
By("Verifying expected VM extensions are present on the node", func() {

‎test/e2e/cloud-provider-azure.go‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,6 @@ func InstallCalicoAndCloudProviderAzureHelmChart(ctx context.Context, input clus
5858
waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName)
5959
WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...)
6060
}
61-
By("Waiting for Ready cloud-node-manager daemonset pods")
62-
for _, ds := range []string{"cloud-node-manager", "cloud-node-manager-windows"} {
63-
waitInput := GetWaitForDaemonsetAvailableInput(ctx, clusterProxy, ds, kubesystem, specName)
64-
WaitForDaemonset(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-daemonset")...)
65-
}
6661
}
6762

6863
// InstallAzureDiskCSIDriverHelmChart installs the official azure-disk CSI driver helm chart
@@ -83,9 +78,4 @@ func InstallAzureDiskCSIDriverHelmChart(ctx context.Context, input clusterctl.Ap
8378
waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName)
8479
WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...)
8580
}
86-
By("Waiting for Running azure-disk-csi node pods")
87-
for _, ds := range []string{"csi-azuredisk-node", "csi-azuredisk-node-win"} {
88-
waitInput := GetWaitForDaemonsetAvailableInput(ctx, clusterProxy, ds, kubesystem, specName)
89-
WaitForDaemonset(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-daemonset")...)
90-
}
9181
}

‎test/e2e/cni.go‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -81,25 +81,6 @@ func InstallCalicoHelmChart(ctx context.Context, input clusterctl.ApplyClusterTe
8181
waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, CalicoAPIServerNamespace, specName)
8282
WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...)
8383
}
84-
By("Waiting for Ready calico-node daemonset pods")
85-
for _, ds := range []string{"calico-node"} {
86-
waitInput := GetWaitForDaemonsetAvailableInput(ctx, clusterProxy, ds, CalicoSystemNamespace, specName)
87-
WaitForDaemonset(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-daemonset")...)
88-
}
89-
// TODO: enable this for all clusters once calico for windows is part of the helm chart.
90-
if hasWindows {
91-
By("Waiting for Ready calico windows pods")
92-
for _, ds := range []string{"calico-node-windows"} {
93-
waitInput := GetWaitForDaemonsetAvailableInput(ctx, clusterProxy, ds, CalicoSystemNamespace, specName)
94-
WaitForDaemonset(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-daemonset")...)
95-
}
96-
97-
By("Waiting for Ready calico windows pods")
98-
for _, ds := range []string{"kube-proxy-windows"} {
99-
waitInput := GetWaitForDaemonsetAvailableInput(ctx, clusterProxy, ds, kubesystem, specName)
100-
WaitForDaemonset(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-daemonset")...)
101-
}
102-
}
10384
}
10485

10586
func getCalicoValues(cidrBlocks []string) *helmVals.Options {

‎test/e2e/daemonsets.go‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
//go:build e2e
2+
// +build e2e
3+
4+
/*
5+
Copyright 2022 The Kubernetes Authors.
6+
7+
Licensed under the Apache License, Version 2.0 (the "License");
8+
you may not use this file except in compliance with the License.
9+
You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
*/
19+
20+
package e2e
21+
22+
import (
23+
"context"
24+
25+
. "github.com/onsi/ginkgo/v2"
26+
. "github.com/onsi/gomega"
27+
corev1 "k8s.io/api/core/v1"
28+
kubeadmv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
29+
"sigs.k8s.io/cluster-api/test/framework"
30+
"sigs.k8s.io/controller-runtime/pkg/client"
31+
)
32+
33+
// DaemonsetsSpecInput is the input for EnsureDaemonsets.
34+
type DaemonsetsSpecInput struct {
35+
BootstrapClusterProxy framework.ClusterProxy
36+
Namespace *corev1.Namespace
37+
ClusterName string
38+
}
39+
40+
// EnsureDaemonsets implements a test that verifies expected Daemonset Pods are running.
41+
func EnsureDaemonsets(ctx context.Context, inputGetter func() DaemonsetsSpecInput) {
42+
var (
43+
specName = "daemonsets"
44+
input DaemonsetsSpecInput
45+
)
46+
47+
Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
48+
49+
input = inputGetter()
50+
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
51+
Expect(input.Namespace).ToNot(BeNil(), "Invalid argument. input.Namespace can't be nil when calling %s spec", specName)
52+
Expect(input.ClusterName).ToNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling %s spec", specName)
53+
54+
mgmtClient := bootstrapClusterProxy.GetClient()
55+
Expect(mgmtClient).NotTo(BeNil())
56+
cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{
57+
Getter: mgmtClient,
58+
Name: input.ClusterName,
59+
Namespace: input.Namespace.Name,
60+
})
61+
kubeadmControlPlane := &kubeadmv1.KubeadmControlPlane{}
62+
key := client.ObjectKey{
63+
Namespace: cluster.Spec.ControlPlaneRef.Namespace,
64+
Name: cluster.Spec.ControlPlaneRef.Name,
65+
}
66+
Eventually(func() error {
67+
return mgmtClient.Get(ctx, key, kubeadmControlPlane)
68+
}, e2eConfig.GetIntervals(specName, "wait-daemonset")...).Should(Succeed(), "Failed to get KubeadmControlPlane object %s/%s", cluster.Spec.ControlPlaneRef.Namespace, cluster.Spec.ControlPlaneRef.Name)
69+
70+
workloadClusterProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, input.Namespace.Name, input.ClusterName)
71+
By("Waiting for all DaemonSet Pods to be Running")
72+
WaitForDaemonsets(ctx, workloadClusterProxy, specName, e2eConfig.GetIntervals(specName, "wait-daemonset")...)
73+
}

‎test/e2e/helpers.go‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -270,33 +270,39 @@ type WaitForDaemonsetInput struct {
270270
func WaitForDaemonset(ctx context.Context, input WaitForDaemonsetInput, intervals ...interface{}) {
271271
start := time.Now()
272272
namespace, name := input.DaemonSet.GetNamespace(), input.DaemonSet.GetName()
273-
Byf("waiting for daemonset %s/%s to be complete", namespace, name)
274-
Logf("waiting for daemonset %s/%s to be complete", namespace, name)
275273
Eventually(func() bool {
276274
key := client.ObjectKey{Namespace: namespace, Name: name}
277275
if err := input.Getter.Get(ctx, key, input.DaemonSet); err == nil {
278-
if input.DaemonSet.Status.DesiredNumberScheduled == input.DaemonSet.Status.NumberReady {
279-
Logf("%d daemonset %s/%s pods are running, took %v", input.DaemonSet.Status.NumberReady, namespace, name, time.Since(start))
276+
if input.DaemonSet.Status.DesiredNumberScheduled > 0 {
277+
Byf("waiting for %d daemonset %s/%s pods to be Running", input.DaemonSet.Status.DesiredNumberScheduled, namespace, name)
278+
if input.DaemonSet.Status.DesiredNumberScheduled == input.DaemonSet.Status.NumberReady {
279+
Logf("%d daemonset %s/%s pods are running, took %v", input.DaemonSet.Status.NumberReady, namespace, name, time.Since(start))
280+
return true
281+
}
282+
} else {
283+
Byf("daemonset %s/%s has no schedulable nodes, will skip", namespace, name)
280284
return true
281285
}
282286
}
283287
return false
284288
}, intervals...).Should(BeTrue(), func() string { return DescribeFailedDaemonset(ctx, input) })
285289
}
286290

287-
// GetWaitForDaemonsetAvailableInput is a convenience func to compose a WaitForDaemonsetInput
288-
func GetWaitForDaemonsetAvailableInput(ctx context.Context, clusterProxy framework.ClusterProxy, name, namespace string, specName string) WaitForDaemonsetInput {
291+
// WaitForDaemonsets retries during E2E until all daemonsets pods are all Running.
292+
func WaitForDaemonsets(ctx context.Context, clusterProxy framework.ClusterProxy, specName string, intervals ...interface{}) {
289293
Expect(clusterProxy).NotTo(BeNil())
290294
cl := clusterProxy.GetClient()
291-
var ds = &appsv1.DaemonSet{}
295+
var dsList = &appsv1.DaemonSetList{}
292296
Eventually(func() error {
293-
return cl.Get(ctx, client.ObjectKey{Name: name, Namespace: namespace}, ds)
294-
}, e2eConfig.GetIntervals(specName, "wait-daemonset")...).Should(Succeed())
295-
clientset := clusterProxy.GetClientSet()
296-
return WaitForDaemonsetInput{
297-
DaemonSet: ds,
298-
Clientset: clientset,
299-
Getter: cl,
297+
return cl.List(ctx, dsList)
298+
}, intervals...).Should(Succeed())
299+
for i := range dsList.Items {
300+
waitForDaemonsetInput := WaitForDaemonsetInput{
301+
DaemonSet: &dsList.Items[i],
302+
Clientset: clusterProxy.GetClientSet(),
303+
Getter: cl,
304+
}
305+
WaitForDaemonset(ctx, waitForDaemonsetInput, intervals...)
300306
}
301307
}
302308

0 commit comments

Comments
 (0)