Skip to content

Commit fc9c034

Browse files
vknabelvknabel
committed
test(upgrade): improved teardown and upgrade process
1 parent 3aceb90 commit fc9c034

File tree

3 files changed

+112
-48
lines changed

3 files changed

+112
-48
lines changed

‎test/e2e/frmwrk/cluster_upgrade_kubernetes_test.go‎

Lines changed: 46 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"path/filepath"
88

99
corev1 "k8s.io/api/core/v1"
10+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1011
"k8s.io/utils/ptr"
1112

1213
. "github.com/onsi/ginkgo/v2" //nolint:staticcheck
@@ -54,21 +55,51 @@ var _ = Describe("Upgrade Kubernetes Cluster Version", Ordered, Label("upgrade")
5455
cfg.Variables["KUBERNETES_VERSION_UPGRADE_TO"] = toKubernetesVersion
5556
})
5657

57-
capi_e2e_ClusterUpgradeConformanceSpec(ctx, func() capi_e2e.ClusterUpgradeConformanceSpecInput {
58-
Expect(cfg.Variables).NotTo(BeNil(), "E2E config variables map must be initialized")
59-
Expect(cfg.Variables).To(HaveKey("CONTROL_PLANE_IP"))
60-
Expect(cfg.Variables).To(HaveKey("METAL_NODE_NETWORK_ID"))
61-
return capi_e2e.ClusterUpgradeConformanceSpecInput{
62-
E2EConfig: cfg,
63-
ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath,
64-
BootstrapClusterProxy: e2eCtx.Environment.Bootstrap,
65-
ArtifactFolder: e2eCtx.Environment.artifactsPath,
66-
SkipCleanup: false,
67-
SkipConformanceTests: true,
68-
ControlPlaneMachineCount: ptr.To[int64](1),
69-
WorkerMachineCount: ptr.To[int64](0),
70-
Flavor: ptr.To("upgrade"),
71-
}
58+
Context("rolling upgrade", func() {
59+
capi_e2e_ClusterUpgradeConformanceSpec(ctx, func() capi_e2e.ClusterUpgradeConformanceSpecInput {
60+
Expect(cfg.Variables).NotTo(BeNil(), "E2E config variables map must be initialized")
61+
Expect(cfg.Variables).To(HaveKey("CONTROL_PLANE_IP"))
62+
Expect(cfg.Variables).To(HaveKey("METAL_NODE_NETWORK_ID"))
63+
return capi_e2e.ClusterUpgradeConformanceSpecInput{
64+
E2EConfig: cfg,
65+
ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath,
66+
BootstrapClusterProxy: e2eCtx.Environment.Bootstrap,
67+
ArtifactFolder: e2eCtx.Environment.artifactsPath,
68+
SkipCleanup: true,
69+
SkipConformanceTests: true,
70+
ControlPlaneMachineCount: ptr.To[int64](1),
71+
WorkerMachineCount: ptr.To[int64](0),
72+
Flavor: ptr.To("upgrade"),
73+
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{},
74+
InfrastructureProvider: new(string),
75+
PostNamespaceCreated: func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string) {
76+
ec.NamespaceName = workloadClusterNamespace
77+
ec.Refs.Namespace = &corev1.Namespace{
78+
ObjectMeta: metav1.ObjectMeta{
79+
Name: workloadClusterNamespace,
80+
},
81+
}
82+
err := ec.E2EContext.Environment.Bootstrap.GetClient().Get(ctx, util.ObjectKey(ec.Refs.Namespace), ec.Refs.Namespace)
83+
Expect(err).NotTo(HaveOccurred(), "Failed to get workload cluster namespace %s", ec.NamespaceName)
84+
85+
ec.Refs.Namespace.SetLabels(map[string]string{
86+
e2eMetalStackProjectIDLabel: ec.E2EContext.Environment.projectID,
87+
})
88+
err = ec.E2EContext.Environment.Bootstrap.GetClient().Update(ctx, ec.Refs.Namespace)
89+
Expect(err).NotTo(HaveOccurred(), "Failed to add e2e project label workload cluster namespace %s", ec.NamespaceName)
90+
},
91+
PreWaitForControlPlaneToBeUpgraded: func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string, workloadClusterName string) {
92+
ec.NamespaceName = workloadClusterNamespace
93+
ec.ClusterName = workloadClusterName
94+
ec.Refs.Workload = ec.E2EContext.Environment.Bootstrap.GetWorkloadCluster(ctx, ec.NamespaceName, ec.ClusterName)
95+
ec.Refs.Cluster = framework.GetClusterByName(ctx, framework.GetClusterByNameInput{
96+
Getter: ec.E2EContext.Environment.Bootstrap.GetClient(),
97+
Name: workloadClusterName,
98+
Namespace: workloadClusterNamespace,
99+
})
100+
},
101+
}
102+
})
72103
})
73104

74105
It("delete cluster", Label("upgrade", "teardown"), func() {

‎test/e2e/frmwrk/shared_cluster.go‎

Lines changed: 54 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ func (e2e *E2ECluster) SetupMetalStackPreconditions(ctx context.Context) {
135135
}
136136

137137
func (e2e *E2ECluster) Teardown(ctx context.Context) {
138+
e2e.teardownAddons(ctx)
138139
e2e.teardownCluster(ctx)
139140
e2e.teardownControlPlaneIP(ctx)
140141
e2e.teardownFirewall(ctx)
@@ -317,7 +318,6 @@ func (e2e *E2ECluster) GenerateAndApplyClusterTemplate(ctx context.Context) {
317318
Flavor: e2e.E2EContext.Environment.Flavor,
318319
LogFolder: path.Join(e2e.E2EContext.Environment.artifactsPath, "clusters", e2e.ClusterName),
319320
ClusterctlVariables: e2e.Variables(),
320-
InfrastructureProvider: "metal-stack:v0.6.2",
321321
})
322322

323323
By("Apply cluster template")
@@ -341,54 +341,75 @@ func (e2e *E2ECluster) GenerateAndApplyClusterTemplate(ctx context.Context) {
341341
Expect(e2e.Refs.Cluster).NotTo(BeNil(), "failed to get cluster")
342342
}
343343

344-
func (e2e *E2ECluster) teardownCluster(ctx context.Context) {
345-
if e2e.Refs.Cluster == nil {
346-
return
347-
}
348-
Expect(e2e.Refs.Cluster).NotTo(BeNil(), "cluster not created yet")
344+
func (e2e *E2ECluster) teardownAddons(ctx context.Context) {
345+
By("Teardown Addons")
349346

350-
resources := framework.GetCAPIResources(ctx, framework.GetCAPIResourcesInput{
351-
Lister: e2e.E2EContext.Environment.Bootstrap.GetClient(),
352-
Namespace: e2e.NamespaceName,
353-
IncludeTypes: []metav1.TypeMeta{
354-
{
355-
Kind: "HelmReleaseProxy",
356-
APIVersion: "addons.cluster.x-k8s.io/v1alpha1",
357-
},
358-
{
359-
Kind: "HelmChartProxy",
360-
APIVersion: "addons.cluster.x-k8s.io/v1alpha1",
361-
},
362-
{
363-
Kind: "ClusterResourceSetBinding",
364-
APIVersion: "addons.cluster.x-k8s.io/v1beta1",
365-
},
366-
{
367-
Kind: "ClusterResourceSet",
368-
APIVersion: "addons.cluster.x-k8s.io/v1beta1",
369-
},
347+
includeTypes := []metav1.TypeMeta{
348+
{
349+
Kind: "HelmChartProxy",
350+
APIVersion: "addons.cluster.x-k8s.io/v1alpha1",
370351
},
371-
})
352+
{
353+
Kind: "HelmReleaseProxy",
354+
APIVersion: "addons.cluster.x-k8s.io/v1alpha1",
355+
},
356+
{
357+
Kind: "ClusterResourceSet",
358+
APIVersion: "addons.cluster.x-k8s.io/v1beta1",
359+
},
360+
{
361+
Kind: "ClusterResourceSetBinding",
362+
APIVersion: "addons.cluster.x-k8s.io/v1beta1",
363+
},
364+
}
365+
366+
resources := []*unstructured.Unstructured{}
367+
for _, typ := range includeTypes {
368+
typeList := new(unstructured.UnstructuredList)
369+
typeList.SetAPIVersion(typ.APIVersion)
370+
typeList.SetKind(typ.Kind)
371+
372+
if err := e2e.E2EContext.Environment.Bootstrap.GetClient().List(ctx, typeList, client.InNamespace(e2e.NamespaceName)); err != nil {
373+
if apierrors.IsNotFound(err) {
374+
continue
375+
}
376+
if apierrors.IsForbidden(err) {
377+
fmt.Printf("Warning: failed to list %s resources due to a rbac issue: %v", typeList.GroupVersionKind(), err)
378+
continue
379+
}
380+
Fail(fmt.Sprintf("failed to list %q resources: %v", typeList.GroupVersionKind(), err))
381+
}
382+
for i := range typeList.Items {
383+
obj := typeList.Items[i]
384+
resources = append(resources, &obj)
385+
}
386+
}
372387

373388
for _, r := range resources {
389+
By(fmt.Sprintf("Deleting resource %s/%s of kind %s", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
374390
err := e2e.E2EContext.Environment.Bootstrap.GetClient().Delete(ctx, r)
375391
Expect(err).To(Or(
376392
Not(HaveOccurred()),
377393
Satisfy(apierrors.IsNotFound)),
378394
fmt.Sprintf("failed to delete resource %s/%s of kind %s", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind),
379395
)
380-
}
381-
382-
for _, r := range resources {
383396
Eventually(func() bool {
397+
By(fmt.Sprintf("Waiting for resource %s/%s of kind %s to be deleted", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
384398
err := e2e.E2EContext.Environment.Bootstrap.GetClient().Get(ctx, client.ObjectKeyFromObject(r), r)
385399
return apierrors.IsNotFound(err)
386400
}, e2e.E2EContext.E2EConfig.GetIntervals("default", "wait-delete-resource")...).Should(BeTrue(),
387401
fmt.Sprintf("resource %s/%s of kind %s still exists", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind),
388402
)
389403
}
404+
}
405+
406+
func (e2e *E2ECluster) teardownCluster(ctx context.Context) {
407+
if e2e.Refs.Cluster == nil {
408+
return
409+
}
410+
Expect(e2e.Refs.Cluster).NotTo(BeNil(), "cluster not created yet")
390411

391-
deleteClusterAndWait(ctx, framework.DeleteClusterAndWaitInput{
412+
capi_e2e_DeleteClusterAndWait(ctx, framework.DeleteClusterAndWaitInput{
392413
ClusterProxy: e2e.E2EContext.Environment.Bootstrap,
393414
ClusterctlConfigPath: e2e.E2EContext.Environment.ClusterctlConfigPath,
394415
ArtifactFolder: e2e.E2EContext.Environment.artifactsPath,
@@ -430,9 +451,9 @@ func (ec *E2ECluster) Dump(ctx context.Context) {
430451
})
431452
}
432453

433-
// deleteClusterAndWait deletes a cluster object and waits for it to be gone.
454+
// capi_e2e_DeleteClusterAndWait deletes a cluster object and waits for it to be gone.
434455
// TODO: remove once cluster expectation has been fixed in framework
435-
func deleteClusterAndWait(ctx context.Context, input framework.DeleteClusterAndWaitInput, intervals ...any) {
456+
func capi_e2e_DeleteClusterAndWait(ctx context.Context, input framework.DeleteClusterAndWaitInput, intervals ...any) {
436457
var (
437458
retryableOperationInterval = 3 * time.Second
438459
// retryableOperationTimeout requires a higher value especially for self-hosted upgrades.

‎test/e2e/frmwrk/shared_context.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,10 +276,22 @@ func (ee *E2EContext) TeardownMetalStackProject(ctx context.Context) {
276276
})
277277

278278
for _, r := range resources {
279+
By(fmt.Sprintf("Deleting resource %s/%s of kind %s", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
279280
err := ee.Environment.Bootstrap.GetClient().Delete(ctx, r)
280281
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to delete resource %s/%s of kind %s", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
281282
}
282283

284+
for _, r := range resources {
285+
By(fmt.Sprintf("Waiting for resource %s/%s of kind %s to be deleted", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
286+
Eventually(ctx, func(g Gomega) {
287+
err := ee.Environment.Bootstrap.GetClient().Get(ctx, client.ObjectKey{
288+
Namespace: r.GetNamespace(),
289+
Name: r.GetName(),
290+
}, r)
291+
g.Expect(client.IgnoreNotFound(err)).NotTo(HaveOccurred(), fmt.Sprintf("failed to get resource %s/%s of kind %s", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
292+
}, "5m", "10s").WithContext(ctx).Should(Succeed(), fmt.Sprintf("timed out waiting for resource %s/%s of kind %s to be deleted", r.GetNamespace(), r.GetName(), r.GetObjectKind().GroupVersionKind().Kind))
293+
}
294+
283295
framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
284296
Deleter: ee.Environment.Bootstrap.GetClient(),
285297
Name: ns.Name,

0 commit comments

Comments
 (0)