Skip to content

Commit 8228f42

Browse files
deepakm-ntnxdeepakm-ntnx
authored
added modular tests for cluster with topology for all supported k8s versions (#422)
1 parent 0e9a496 commit 8228f42

13 files changed

+1558
-3
lines changed

‎test/e2e/capx_quick_start_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ var _ = Describe("When following the Cluster API quick-start", Label("quickstart
3636
})
3737
})
3838

39-
var _ = Describe("When following the Cluster API quick-start with ClusterClass", Label("quickstart", "capx-feature-test"), func() {
39+
var _ = Describe("When following the Cluster API quick-start with ClusterClass", Label("quickstart", "clusterclass", "capx-feature-test"), func() {
4040
capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput {
4141
return capi_e2e.QuickStartSpecInput{
4242
E2EConfig: e2eConfig,

‎test/e2e/cluster_topology.go‎

Lines changed: 316 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,316 @@
1+
//go:build e2e
2+
3+
/*
4+
Copyright 2024 Nutanix
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package e2e
20+
21+
import (
22+
"context"
23+
"fmt"
24+
"os"
25+
"path/filepath"
26+
27+
corev1 "k8s.io/api/core/v1"
28+
"k8s.io/utils/ptr"
29+
"sigs.k8s.io/cluster-api/test/framework"
30+
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
31+
"sigs.k8s.io/cluster-api/test/framework/kubetest"
32+
)
33+
34+
type NutanixE2ETest struct {
35+
testHelper testHelperInterface
36+
e2eConfig *clusterctl.E2EConfig
37+
bootstrapClusterProxy framework.ClusterProxy
38+
artifactFolder string
39+
clusterctlConfigPath string
40+
flavor string
41+
namespace *corev1.Namespace
42+
testSpecName string
43+
kubetestConfigPath string
44+
}
45+
46+
type NutanixE2ETestOption func(*NutanixE2ETest)
47+
48+
func NewNutanixE2ETest(options ...NutanixE2ETestOption) *NutanixE2ETest {
49+
nutanixE2ETest := &NutanixE2ETest{}
50+
for _, o := range options {
51+
o(nutanixE2ETest)
52+
}
53+
return nutanixE2ETest
54+
}
55+
56+
func WithE2ETestSpecName(testSpecName string) NutanixE2ETestOption {
57+
return func(nutanixE2ETest *NutanixE2ETest) {
58+
nutanixE2ETest.testSpecName = testSpecName
59+
}
60+
}
61+
62+
func WithE2ETestHelper(testHelper testHelperInterface) NutanixE2ETestOption {
63+
return func(nutanixE2ETest *NutanixE2ETest) {
64+
nutanixE2ETest.testHelper = testHelper
65+
}
66+
}
67+
68+
func WithE2ETestConfig(e2eConfig *clusterctl.E2EConfig) NutanixE2ETestOption {
69+
return func(nutanixE2ETest *NutanixE2ETest) {
70+
nutanixE2ETest.e2eConfig = e2eConfig
71+
}
72+
}
73+
74+
func WithE2ETestBootstrapClusterProxy(bootstrapClusterProxy framework.ClusterProxy) NutanixE2ETestOption {
75+
return func(nutanixE2ETest *NutanixE2ETest) {
76+
nutanixE2ETest.bootstrapClusterProxy = bootstrapClusterProxy
77+
}
78+
}
79+
80+
func WithE2ETestArtifactFolder(artifactFolder string) NutanixE2ETestOption {
81+
return func(nutanixE2ETest *NutanixE2ETest) {
82+
nutanixE2ETest.artifactFolder = artifactFolder
83+
}
84+
}
85+
86+
func WithE2ETestClusterctlConfigPath(clusterctlConfigPath string) NutanixE2ETestOption {
87+
return func(nutanixE2ETest *NutanixE2ETest) {
88+
nutanixE2ETest.clusterctlConfigPath = clusterctlConfigPath
89+
}
90+
}
91+
92+
func WithE2ETestKubetestConfigPath(kubetestConfigPath string) NutanixE2ETestOption {
93+
return func(nutanixE2ETest *NutanixE2ETest) {
94+
nutanixE2ETest.kubetestConfigPath = kubetestConfigPath
95+
}
96+
}
97+
98+
func WithE2ETestClusterTemplateFlavor(flavor string) NutanixE2ETestOption {
99+
return func(nutanixE2ETest *NutanixE2ETest) {
100+
nutanixE2ETest.flavor = flavor
101+
}
102+
}
103+
104+
func WithE2ETestNamespace(namespace *corev1.Namespace) NutanixE2ETestOption {
105+
return func(nutanixE2ETest *NutanixE2ETest) {
106+
nutanixE2ETest.namespace = namespace
107+
}
108+
}
109+
110+
func (e2eTest *NutanixE2ETest) CreateCluster(ctx context.Context, clusterTopologyConfig *ClusterTopologyConfig) (*clusterctl.ApplyClusterTemplateAndWaitResult, error) {
111+
configClusterInput := clusterctl.ConfigClusterInput{
112+
LogFolder: filepath.Join(e2eTest.artifactFolder, "clusters", e2eTest.bootstrapClusterProxy.GetName()),
113+
ClusterctlConfigPath: e2eTest.clusterctlConfigPath,
114+
KubeconfigPath: e2eTest.bootstrapClusterProxy.GetKubeconfigPath(),
115+
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
116+
Flavor: e2eTest.flavor,
117+
Namespace: e2eTest.namespace.Name,
118+
ClusterName: clusterTopologyConfig.name,
119+
KubernetesVersion: clusterTopologyConfig.k8sVersion,
120+
ControlPlaneMachineCount: ptr.To(int64(clusterTopologyConfig.cpNodeCount)),
121+
WorkerMachineCount: ptr.To(int64(clusterTopologyConfig.workerNodeCount)),
122+
}
123+
124+
clusterResources := new(clusterctl.ApplyClusterTemplateAndWaitResult)
125+
126+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
127+
ClusterProxy: e2eTest.bootstrapClusterProxy,
128+
ConfigCluster: configClusterInput,
129+
WaitForClusterIntervals: e2eTest.e2eConfig.GetIntervals("", "wait-cluster"),
130+
WaitForControlPlaneIntervals: e2eTest.e2eConfig.GetIntervals("", "wait-control-plane"),
131+
WaitForMachineDeployments: e2eTest.e2eConfig.GetIntervals("", "wait-worker-nodes"),
132+
}, clusterResources)
133+
134+
return clusterResources, nil
135+
}
136+
137+
func (e2eTest *NutanixE2ETest) UpgradeCluster(ctx context.Context, clusterTopologyConfig *ClusterTopologyConfig) (*clusterctl.ApplyClusterTemplateAndWaitResult, error) {
138+
configClusterInput := clusterctl.ConfigClusterInput{
139+
LogFolder: filepath.Join(e2eTest.artifactFolder, "clusters", e2eTest.bootstrapClusterProxy.GetName()),
140+
ClusterctlConfigPath: e2eTest.clusterctlConfigPath,
141+
KubeconfigPath: e2eTest.bootstrapClusterProxy.GetKubeconfigPath(),
142+
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
143+
Flavor: e2eTest.flavor,
144+
Namespace: e2eTest.namespace.Name,
145+
ClusterName: clusterTopologyConfig.name,
146+
KubernetesVersion: clusterTopologyConfig.k8sVersion,
147+
ControlPlaneMachineCount: ptr.To(int64(clusterTopologyConfig.cpNodeCount)),
148+
WorkerMachineCount: ptr.To(int64(clusterTopologyConfig.workerNodeCount)),
149+
}
150+
151+
clusterResources := new(clusterctl.ApplyClusterTemplateAndWaitResult)
152+
153+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
154+
ClusterProxy: e2eTest.bootstrapClusterProxy,
155+
ConfigCluster: configClusterInput,
156+
WaitForClusterIntervals: e2eTest.e2eConfig.GetIntervals("", "wait-cluster"),
157+
WaitForControlPlaneIntervals: e2eTest.e2eConfig.GetIntervals("", "wait-control-plane"),
158+
WaitForMachineDeployments: e2eTest.e2eConfig.GetIntervals("", "wait-worker-nodes"),
159+
}, clusterResources)
160+
161+
return clusterResources, nil
162+
}
163+
164+
func (e2eTest *NutanixE2ETest) WaitForControlPlaneMachinesToBeUpgraded(ctx context.Context, clusterTopologyConfig *ClusterTopologyConfig, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) {
165+
waitForMachinesToBeUpgraded := e2eTest.e2eConfig.GetIntervals("", "wait-machine-upgrade")
166+
mgmtClient := e2eTest.bootstrapClusterProxy.GetClient()
167+
framework.WaitForControlPlaneMachinesToBeUpgraded(ctx, framework.WaitForControlPlaneMachinesToBeUpgradedInput{
168+
Lister: mgmtClient,
169+
Cluster: clusterResources.Cluster,
170+
MachineCount: int(clusterTopologyConfig.cpNodeCount),
171+
KubernetesUpgradeVersion: clusterTopologyConfig.k8sVersion,
172+
}, waitForMachinesToBeUpgraded...)
173+
}
174+
175+
func (e2eTest *NutanixE2ETest) WaitForMachineDeploymentMachinesToBeUpgraded(ctx context.Context, clusterTopologyConfig *ClusterTopologyConfig, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) {
176+
waitForMachinesToBeUpgraded := e2eTest.e2eConfig.GetIntervals("", "wait-machine-upgrade")
177+
mgmtClient := e2eTest.bootstrapClusterProxy.GetClient()
178+
for _, deployment := range clusterResources.MachineDeployments {
179+
if *deployment.Spec.Replicas > 0 {
180+
framework.WaitForMachineDeploymentMachinesToBeUpgraded(ctx, framework.WaitForMachineDeploymentMachinesToBeUpgradedInput{
181+
Lister: mgmtClient,
182+
Cluster: clusterResources.Cluster,
183+
MachineCount: int(*deployment.Spec.Replicas),
184+
KubernetesUpgradeVersion: clusterTopologyConfig.k8sVersion,
185+
MachineDeployment: *deployment,
186+
}, waitForMachinesToBeUpgraded...)
187+
}
188+
}
189+
}
190+
191+
func (e2eTest *NutanixE2ETest) WaitForNodesReady(ctx context.Context, targetKubernetesVersion string, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) {
192+
workloadProxy := e2eTest.bootstrapClusterProxy.GetWorkloadCluster(ctx, e2eTest.namespace.Name, clusterResources.Cluster.Name)
193+
workloadClient := workloadProxy.GetClient()
194+
framework.WaitForNodesReady(ctx, framework.WaitForNodesReadyInput{
195+
Lister: workloadClient,
196+
KubernetesVersion: targetKubernetesVersion,
197+
Count: int(clusterResources.ExpectedTotalNodes()),
198+
WaitForNodesReady: e2eTest.e2eConfig.GetIntervals(e2eTest.testSpecName, "wait-nodes-ready"),
199+
})
200+
}
201+
202+
func (e2eTest *NutanixE2ETest) RunConformanceTest(ctx context.Context, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) error {
203+
workloadProxy := e2eTest.bootstrapClusterProxy.GetWorkloadCluster(ctx, e2eTest.namespace.Name, clusterResources.Cluster.Name)
204+
// Start running the conformance test suite.
205+
return kubetest.Run(
206+
ctx,
207+
kubetest.RunInput{
208+
ClusterProxy: workloadProxy,
209+
NumberOfNodes: int(clusterResources.ExpectedWorkerNodes()),
210+
ArtifactsDirectory: e2eTest.artifactFolder,
211+
ConfigFilePath: e2eTest.kubetestConfigPath,
212+
GinkgoNodes: int(clusterResources.ExpectedWorkerNodes()),
213+
},
214+
)
215+
}
216+
217+
func (e2eTest *NutanixE2ETest) InstallAddonPackage(ctx context.Context, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) error {
218+
return fmt.Errorf("Not implemented yet.")
219+
}
220+
221+
type ClusterTopologyConfig struct {
222+
name string
223+
k8sVersion string
224+
cpNodeCount int
225+
workerNodeCount int
226+
cpImageName string
227+
workerImageName string
228+
machineMemorySize string
229+
machineSystemDiskSize string
230+
machineVCPUSockets int64
231+
machineVCPUSPerSocket int64
232+
}
233+
234+
type ClusterTopologyConfigOption func(*ClusterTopologyConfig)
235+
236+
func NewClusterTopologyConfig(options ...func(*ClusterTopologyConfig)) *ClusterTopologyConfig {
237+
clusterTopologyConfig := &ClusterTopologyConfig{}
238+
for _, o := range options {
239+
o(clusterTopologyConfig)
240+
}
241+
return clusterTopologyConfig
242+
}
243+
244+
// Start
245+
// Option Pattern functions for ClusterTopologyConfig
246+
//
247+
248+
func WithName(name string) ClusterTopologyConfigOption {
249+
return func(clusterTopologyConfig *ClusterTopologyConfig) {
250+
clusterTopologyConfig.name = name
251+
}
252+
}
253+
254+
func WithKubernetesVersion(k8sVersion string) ClusterTopologyConfigOption {
255+
return func(clusterTopologyConfig *ClusterTopologyConfig) {
256+
clusterTopologyConfig.k8sVersion = k8sVersion
257+
}
258+
}
259+
260+
func WithControlPlaneCount(nodeCount int) ClusterTopologyConfigOption {
261+
return func(clusterTopologyConfig *ClusterTopologyConfig) {
262+
clusterTopologyConfig.cpNodeCount = nodeCount
263+
}
264+
}
265+
266+
func WithWorkerNodeCount(nodeCount int) ClusterTopologyConfigOption {
267+
return func(clusterTopologyConfig *ClusterTopologyConfig) {
268+
clusterTopologyConfig.workerNodeCount = nodeCount
269+
}
270+
}
271+
272+
func WithControlPlaneMachineTemplateImage(imageName string) ClusterTopologyConfigOption {
273+
return func(clusterTopologyConfig *ClusterTopologyConfig) {
274+
clusterTopologyConfig.cpImageName = imageName
275+
os.Setenv("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME", imageName)
276+
}
277+
}
278+
279+
func WithWorkerMachineTemplateImage(imageName string) ClusterTopologyConfigOption {
280+
return func(clusterTopologyConfig *ClusterTopologyConfig) {
281+
clusterTopologyConfig.workerImageName = imageName
282+
os.Setenv("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME", imageName)
283+
}
284+
}
285+
286+
func WithMachineMemorySize(machineMemorySize string) ClusterTopologyConfigOption {
287+
return func(clusterTopologyConfig *ClusterTopologyConfig) {
288+
clusterTopologyConfig.machineMemorySize = machineMemorySize
289+
os.Setenv("NUTANIX_MACHINE_MEMORY_SIZE", machineMemorySize)
290+
}
291+
}
292+
293+
func WithMachineSystemDiskSize(machineSystemDiskSize string) ClusterTopologyConfigOption {
294+
return func(clusterTopologyConfig *ClusterTopologyConfig) {
295+
clusterTopologyConfig.machineSystemDiskSize = machineSystemDiskSize
296+
os.Setenv("NUTANIX_SYSTEMDISK_SIZE", machineSystemDiskSize)
297+
}
298+
}
299+
300+
func WithMachineVCPUSockets(machineVCPUSockets int64) ClusterTopologyConfigOption {
301+
return func(clusterTopologyConfig *ClusterTopologyConfig) {
302+
clusterTopologyConfig.machineVCPUSockets = machineVCPUSockets
303+
os.Setenv("NUTANIX_MACHINE_VCPU_SOCKET", fmt.Sprint(machineVCPUSockets))
304+
}
305+
}
306+
307+
func WithMachineVCPUSPerSocket(machineVCPUSPerSocket int64) ClusterTopologyConfigOption {
308+
return func(clusterTopologyConfig *ClusterTopologyConfig) {
309+
clusterTopologyConfig.machineVCPUSPerSocket = machineVCPUSPerSocket
310+
os.Setenv("NUTANIX_MACHINE_VCPU_PER_SOCKET", fmt.Sprint(machineVCPUSPerSocket))
311+
}
312+
}
313+
314+
//
315+
// Option Pattern functions for ClusterTopologyConfig
316+
// End

0 commit comments

Comments
 (0)