Skip to content

Commit 2f57b12

Browse files
simcodvknabel
simcod
and
vknabel
authored
Integration test for HA cluster setup (#103)
Co-authored-by: Valentin Knabel <[email protected]>
1 parent 5082648 commit 2f57b12

File tree

8 files changed

+376
-291
lines changed

8 files changed

+376
-291
lines changed

‎Makefile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ E2E_FIREWALL_IMAGE ?= "firewall-ubuntu-3.0"
122122
E2E_FIREWALL_SIZE ?= "v1-small-x86"
123123
E2E_FIREWALL_NETWORKS ?= "internet-mini-lab"
124124
ARTIFACTS ?= "$(PWD)/_artifacts"
125+
# Can be something like: basic && !move
126+
E2E_LABEL_FILTER ?= ""
125127

126128
.PHONY: test-e2e
127129
test-e2e: manifests generate fmt vet ginkgo
@@ -145,7 +147,7 @@ test-e2e: manifests generate fmt vet ginkgo
145147
FIREWALL_SIZE=$(E2E_FIREWALL_SIZE) \
146148
FIREWALL_NETWORKS=$(E2E_FIREWALL_NETWORKS) \
147149
ARTIFACTS=$(ARTIFACTS) \
148-
$(GINKGO) -vv -r --junit-report="junit.e2e_suite.xml" --output-dir="$(ARTIFACTS)" -timeout 60m ./test/e2e/frmwrk
150+
$(GINKGO) -vv -r --junit-report="junit.e2e_suite.xml" --output-dir="$(ARTIFACTS)" --label-filter="$(E2E_LABEL_FILTER)" -timeout 60m ./test/e2e/frmwrk
149151

150152
.PHONY: lint
151153
lint: golangci-lint ## Run golangci-lint linter

test/e2e/frmwrk/bootstrap_test.go renamed to test/e2e/frmwrk/cluster_basic_test.go

Lines changed: 6 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -7,27 +7,17 @@ import (
77
"path"
88
"strings"
99

10-
"github.com/drone/envsubst/v2"
1110
. "github.com/onsi/ginkgo/v2"
1211
. "github.com/onsi/gomega"
1312

1413
apierrors "k8s.io/apimachinery/pkg/api/errors"
1514
"sigs.k8s.io/controller-runtime/pkg/client"
1615

1716
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
18-
"sigs.k8s.io/cluster-api/test/framework"
1917
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
2018
)
2119

22-
var _ = Describe("Basic Cluster", Ordered, func() {
23-
24-
BeforeAll(func() {
25-
e2eCtx = NewE2EContext()
26-
e2eCtx.ProvideBootstrapCluster()
27-
e2eCtx.CreateClusterctlConfig(context.TODO())
28-
e2eCtx.InitManagementCluster(context.TODO())
29-
})
30-
20+
var _ = Describe("Basic Cluster", Ordered, Label("basic"), func() {
3121
kubernetesVersions := strings.Split(os.Getenv("E2E_KUBERNETES_VERSIONS"), ",")
3222
Expect(kubernetesVersions).ToNot(BeEmpty(), "E2E_KUBERNETES_VERSIONS must be set")
3323

@@ -42,11 +32,11 @@ var _ = Describe("Basic Cluster", Ordered, func() {
4232
ctx = context.Background()
4333
})
4434

45-
It("create new cluster", func() {
35+
It("create new cluster", Label("create"), func() {
4636
ec = createE2ECluster(ctx, e2eCtx, ClusterConfig{
4737
SpecName: "basic-cluster-creation-" + v,
4838
NamespaceName: fmt.Sprintf("e2e-basic-cluster-creation-%d", i),
49-
ClusterName: fmt.Sprintf("simple-%d", i),
39+
ClusterName: fmt.Sprintf("basic-%d", i),
5040
KubernetesVersion: v,
5141
ControlPlaneMachineImage: os.Getenv("E2E_CONTROL_PLANE_MACHINE_IMAGE_PREFIX") + strings.TrimPrefix(v, "v"),
5242
ControlPlaneMachineCount: 1,
@@ -56,7 +46,7 @@ var _ = Describe("Basic Cluster", Ordered, func() {
5646
Expect(ec).ToNot(BeNil())
5747
})
5848

59-
It("move from bootstrap to workload cluster", func() {
49+
It("move from bootstrap to workload cluster", Label("move"), func() {
6050
Expect(ec).NotTo(BeNil(), "e2e cluster required")
6151

6252
clusterctl.InitManagementClusterAndWatchControllerLogs(ctx, clusterctl.InitManagementClusterAndWatchControllerLogsInput{
@@ -89,7 +79,7 @@ var _ = Describe("Basic Cluster", Ordered, func() {
8979
Expect(err).ToNot(HaveOccurred(), "cluster should be present")
9080
})
9181

92-
It("move from workload to bootstrap cluster", func() {
82+
It("move from workload to bootstrap cluster", Label("move"), func() {
9383
Expect(ec).NotTo(BeNil(), "e2e cluster required")
9484

9585
clusterctl.Move(ctx, clusterctl.MoveInput{
@@ -115,63 +105,9 @@ var _ = Describe("Basic Cluster", Ordered, func() {
115105
Expect(err).ToNot(HaveOccurred(), "cluster should be present")
116106
})
117107

118-
It("delete cluster", func() {
108+
It("delete cluster", Label("delete"), func() {
119109
ec.Teardown(ctx)
120-
// TODO: expect resources to be gone
121110
})
122111
})
123112
}
124-
125-
It("teardown management cluster", func() {
126-
e2eCtx.Teardown(context.Background())
127-
})
128113
})
129-
130-
func createE2ECluster(ctx context.Context, e2eCtx *E2EContext, cfg ClusterConfig) *E2ECluster {
131-
ec := e2eCtx.NewE2ECluster(cfg)
132-
133-
ec.SetupMetalStackPreconditions(ctx)
134-
ec.SetupNamespace(ctx)
135-
ec.GenerateAndApplyClusterTemplate(ctx)
136-
137-
By("Wait for cluster")
138-
controlPlane := framework.GetKubeadmControlPlaneByCluster(ctx, framework.GetKubeadmControlPlaneByClusterInput{
139-
Lister: e2eCtx.Environment.Bootstrap.GetClient(),
140-
ClusterName: ec.Refs.Cluster.Name,
141-
Namespace: ec.Refs.Cluster.Namespace,
142-
})
143-
144-
framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{
145-
Getter: e2eCtx.Environment.Bootstrap.GetClient(),
146-
Namespace: ec.Refs.Cluster.Namespace,
147-
Name: ec.Refs.Cluster.Name,
148-
}, e2eCtx.E2EConfig.GetIntervals("default", "wait-cluster")...)
149-
150-
Expect(controlPlane).To(Not(BeNil()))
151-
152-
By("Wait for CNI and CCM")
153-
targetTemplate, err := os.ReadFile(path.Join(e2eCtx.Environment.artifactsPath, "config", "target", "base.yaml"))
154-
Expect(err).ToNot(HaveOccurred())
155-
156-
vars := ec.Variables()
157-
targetResources, err := envsubst.Eval(string(targetTemplate), func(varName string) string {
158-
return vars[varName]
159-
})
160-
Expect(err).ToNot(HaveOccurred())
161-
162-
Eventually(func() error {
163-
return ec.Refs.Workload.CreateOrUpdate(ctx, []byte(targetResources))
164-
}, "5m", "15s").Should(Succeed())
165-
166-
By("Wait for kubeadm control plane")
167-
framework.DiscoveryAndWaitForControlPlaneInitialized(ctx, framework.DiscoveryAndWaitForControlPlaneInitializedInput{
168-
Lister: e2eCtx.Environment.Bootstrap.GetClient(),
169-
Cluster: ec.Refs.Cluster,
170-
}, e2eCtx.E2EConfig.GetIntervals("default", "wait-control-plane")...)
171-
172-
framework.WaitForClusterToProvision(ctx, framework.WaitForClusterToProvisionInput{
173-
Cluster: ec.Refs.Cluster,
174-
Getter: e2eCtx.Environment.Bootstrap.GetClient(),
175-
}, e2eCtx.E2EConfig.GetIntervals("default", "wait-cluster-provisioned")...)
176-
return ec
177-
}

‎test/e2e/frmwrk/cluster_ha_test.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package frmwrk
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"strings"
8+
9+
. "github.com/onsi/ginkgo/v2"
10+
. "github.com/onsi/gomega"
11+
)
12+
13+
var _ = Describe("High Availability Cluster", Ordered, Label("ha"), func() {
14+
15+
kubernetesVersions := strings.Split(os.Getenv("E2E_KUBERNETES_VERSIONS"), ",")
16+
Expect(kubernetesVersions).ToNot(BeEmpty(), "E2E_KUBERNETES_VERSIONS must be set")
17+
18+
for i, v := range kubernetesVersions {
19+
Context(fmt.Sprintf("with kubernetes %s", v), Ordered, func() {
20+
var (
21+
ec *E2ECluster
22+
ctx context.Context
23+
)
24+
25+
BeforeEach(func() {
26+
ctx = context.Background()
27+
})
28+
29+
It("create new cluster", Label("create"), func() {
30+
ec = createE2ECluster(ctx, e2eCtx, ClusterConfig{
31+
SpecName: "ha-cluster-creation-" + v,
32+
NamespaceName: fmt.Sprintf("e2e-ha-cluster-creation-%d", i),
33+
ClusterName: fmt.Sprintf("ha-%d", i),
34+
KubernetesVersion: v,
35+
ControlPlaneMachineImage: os.Getenv("E2E_CONTROL_PLANE_MACHINE_IMAGE_PREFIX") + strings.TrimPrefix(v, "v"),
36+
ControlPlaneMachineCount: 3,
37+
WorkerMachineImage: os.Getenv("E2E_WORKER_MACHINE_IMAGE_PREFIX") + strings.TrimPrefix(v, "v"),
38+
WorkerMachineCount: 0,
39+
})
40+
Expect(ec).ToNot(BeNil())
41+
})
42+
43+
It("delete cluster", Label("delete"), func() {
44+
ec.Teardown(ctx)
45+
})
46+
})
47+
}
48+
})

‎test/e2e/frmwrk/config/capi-e2e-config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ intervals:
136136
default/wait-cluster: ["5m", "10s"]
137137
metal-stack/wait-firewall-allocate: ["5m", "10s"]
138138
default/wait-control-plane: ["10m", "10s"]
139+
default/wait-control-plane-machine: ["5m", "10s"]
139140
default/wait-worker-nodes: ["5m", "10s"]
140141
default/wait-machine-pool-nodes: ["5m", "10s"]
141142
default/wait-delete-cluster: ["3m", "10s"]

‎test/e2e/frmwrk/frmwrk_suite_test.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package frmwrk
22

33
import (
4+
"context"
45
"testing"
56

67
"github.com/onsi/ginkgo/v2"
@@ -20,3 +21,20 @@ func TestE2E(t *testing.T) {
2021
ctrl.SetLogger(klog.Background())
2122
ginkgo.RunSpecs(t, "capms-e2e")
2223
}
24+
25+
var _ = ginkgo.BeforeSuite(func() {
26+
e2eCtx = NewE2EContext()
27+
e2eCtx.ProvideBootstrapCluster()
28+
e2eCtx.CreateClusterctlConfig(context.TODO())
29+
e2eCtx.InitManagementCluster(context.TODO())
30+
})
31+
32+
var _ = ginkgo.AfterSuite(func() {
33+
if ginkgo.CurrentSpecReport().Failed() {
34+
// on failure, we skip cleanup to investigate
35+
return
36+
}
37+
if e2eCtx != nil {
38+
e2eCtx.Teardown(context.TODO())
39+
}
40+
})

‎test/e2e/frmwrk/shared_cases.go‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package frmwrk
2+
3+
import (
4+
"context"
5+
"os"
6+
"path"
7+
8+
. "github.com/onsi/ginkgo/v2" //nolint:staticcheck
9+
. "github.com/onsi/gomega" //nolint:staticcheck
10+
11+
"github.com/drone/envsubst/v2"
12+
"sigs.k8s.io/cluster-api/test/framework"
13+
)
14+
15+
func createE2ECluster(ctx context.Context, e2eCtx *E2EContext, cfg ClusterConfig) *E2ECluster {
16+
ec := e2eCtx.NewE2ECluster(cfg)
17+
18+
ec.SetupMetalStackPreconditions(ctx)
19+
ec.SetupNamespace(ctx)
20+
ec.GenerateAndApplyClusterTemplate(ctx)
21+
22+
By("Wait for cluster")
23+
controlPlane := framework.GetKubeadmControlPlaneByCluster(ctx, framework.GetKubeadmControlPlaneByClusterInput{
24+
Lister: e2eCtx.Environment.Bootstrap.GetClient(),
25+
ClusterName: ec.Refs.Cluster.Name,
26+
Namespace: ec.Refs.Cluster.Namespace,
27+
})
28+
29+
framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{
30+
Getter: e2eCtx.Environment.Bootstrap.GetClient(),
31+
Namespace: ec.Refs.Cluster.Namespace,
32+
Name: ec.Refs.Cluster.Name,
33+
}, e2eCtx.E2EConfig.GetIntervals("default", "wait-cluster")...)
34+
35+
Expect(controlPlane).To(Not(BeNil()))
36+
37+
By("Wait for CNI and CCM")
38+
targetTemplate, err := os.ReadFile(path.Join(e2eCtx.Environment.artifactsPath, "config", "target", "base.yaml"))
39+
Expect(err).ToNot(HaveOccurred())
40+
41+
vars := ec.Variables()
42+
targetResources, err := envsubst.Eval(string(targetTemplate), func(varName string) string {
43+
return vars[varName]
44+
})
45+
Expect(err).ToNot(HaveOccurred())
46+
47+
Eventually(func() error {
48+
return ec.Refs.Workload.CreateOrUpdate(ctx, []byte(targetResources))
49+
}, "10m", "15s").Should(Succeed()) // currently this long delay is required as machines might not be ready yet
50+
51+
By("Wait for kubeadm control plane")
52+
framework.DiscoveryAndWaitForControlPlaneInitialized(ctx, framework.DiscoveryAndWaitForControlPlaneInitializedInput{
53+
Lister: e2eCtx.Environment.Bootstrap.GetClient(),
54+
Cluster: ec.Refs.Cluster,
55+
}, e2eCtx.E2EConfig.GetIntervals("default", "wait-control-plane")...)
56+
57+
framework.WaitForClusterToProvision(ctx, framework.WaitForClusterToProvisionInput{
58+
Cluster: ec.Refs.Cluster,
59+
Getter: e2eCtx.Environment.Bootstrap.GetClient(),
60+
}, e2eCtx.E2EConfig.GetIntervals("default", "wait-cluster-provisioned")...)
61+
return ec
62+
}

0 commit comments

Comments
 (0)