Skip to content

Commit 065082d

Browse files
sbueringersbueringer
committed
test/e2e: add field to scale test to deploy Clusters in separate namespaces
1 parent 4d76266 commit 065082d

File tree

1 file changed

+82
-29
lines changed

1 file changed

+82
-29
lines changed

‎test/e2e/scale.go

Lines changed: 82 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@ const (
5353
scaleControlPlaneMachineCount = "CAPI_SCALE_CONTROL_PLANE_MACHINE_COUNT"
5454
scaleWorkerMachineCount = "CAPI_SCALE_WORKER_MACHINE_COUNT"
5555
scaleMachineDeploymentCount = "CAPI_SCALE_MACHINE_DEPLOYMENT_COUNT"
56+
57+
// Note: Names must consist of lower case alphanumeric characters or '-'.
58+
scaleClusterNamePlaceholder = "scale-cluster-name-placeholder"
59+
scaleClusterNamespacePlaceholder = "scale-cluster-namespace-placeholder"
5660
)
5761

5862
// scaleSpecInput is the input for scaleSpec.
@@ -79,6 +83,10 @@ type scaleSpecInput struct {
7983
// Can be overridden by variable CAPI_SCALE_CLUSTER_COUNT.
8084
ClusterCount *int64
8185

86+
// DeployClusterInSeparateNamespaces defines if each cluster should be deployed into its separate namespace.
87+
// In this case The namespace name will be the name of the cluster.
88+
DeployClusterInSeparateNamespaces bool
89+
8290
// Concurrency is the maximum concurrency of each of the scale operations.
8391
// If unspecified it defaults to 5.
8492
// Can be overridden by variable CAPI_SCALE_CONCURRENCY.
@@ -237,15 +245,14 @@ func scaleSpec(ctx context.Context, inputGetter func() scaleSpecInput) {
237245
// Therefore, it is not advised to call this functions across all the concurrency workers.
238246
// To avoid this problem we chose to run ConfigCluster once and reuse its output across all the workers.
239247
log.Logf("Generating YAML for base Cluster and ClusterClass")
240-
baseClusterName := fmt.Sprintf("%s-base", specName)
241248
baseWorkloadClusterTemplate := clusterctl.ConfigCluster(ctx, clusterctl.ConfigClusterInput{
242249
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
243250
ClusterctlConfigPath: input.ClusterctlConfigPath,
244251
KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
245252
InfrastructureProvider: infrastructureProvider,
246253
Flavor: flavor,
247-
Namespace: namespace.Name,
248-
ClusterName: baseClusterName,
254+
Namespace: scaleClusterNamespacePlaceholder,
255+
ClusterName: scaleClusterNamePlaceholder,
249256
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion),
250257
ControlPlaneMachineCount: controlPlaneMachineCount,
251258
WorkerMachineCount: workerMachineCount,
@@ -258,19 +265,23 @@ func scaleSpec(ctx context.Context, inputGetter func() scaleSpecInput) {
258265
// if the resource has to be created or updated before actually executing the operation. If another worker changes
259266
// the status of the cluster during this timeframe the operation will fail.
260267
log.Logf("Extract ClusterClass and Cluster from template YAML")
261-
clusterClassYAML, baseClusterTemplateYAML := extractClusterClassAndClusterFromTemplate(baseWorkloadClusterTemplate)
268+
baseClusterClassYAML, baseClusterTemplateYAML := extractClusterClassAndClusterFromTemplate(baseWorkloadClusterTemplate)
262269

263270
// Modify the baseClusterTemplateYAML so that it has the desired number of machine deployments.
264-
modifiedBaseTemplateYAML := modifyMachineDeployments(baseClusterTemplateYAML, int(*machineDeploymentCount))
265-
266-
if len(clusterClassYAML) > 0 {
267-
// Apply the ClusterClass.
268-
log.Logf("Create ClusterClass")
269-
Eventually(func() error {
270-
return input.BootstrapClusterProxy.Apply(ctx, clusterClassYAML)
271-
}).Should(Succeed())
272-
} else {
273-
log.Logf("ClusterClass already exists. Skipping creation.")
271+
baseClusterTemplateYAML = modifyMachineDeployments(baseClusterTemplateYAML, int(*machineDeploymentCount))
272+
273+
// If all clusters should be deployed in the same namespace (namespace.Name),
274+
// then deploy the ClusterClass in this namespace.
275+
if !input.DeployClusterInSeparateNamespaces {
276+
if len(baseClusterClassYAML) > 0 {
277+
clusterClassYAML := bytes.Replace(baseClusterClassYAML, []byte(scaleClusterNamespacePlaceholder), []byte(namespace.Name), -1)
278+
log.Logf("Apply ClusterClass")
279+
Eventually(func() error {
280+
return input.BootstrapClusterProxy.Apply(ctx, clusterClassYAML)
281+
}, 1*time.Minute).Should(Succeed())
282+
} else {
283+
log.Logf("ClusterClass already exists. Skipping creation.")
284+
}
274285
}
275286

276287
By("Create workload clusters concurrently")
@@ -291,7 +302,6 @@ func scaleSpec(ctx context.Context, inputGetter func() scaleSpecInput) {
291302
// use the "create only" creator function.
292303
creator := getClusterCreateAndWaitFn(clusterctl.ApplyCustomClusterTemplateAndWaitInput{
293304
ClusterProxy: input.BootstrapClusterProxy,
294-
Namespace: namespace.Name,
295305
WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"),
296306
WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
297307
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
@@ -300,15 +310,15 @@ func scaleSpec(ctx context.Context, inputGetter func() scaleSpecInput) {
300310
if !input.SkipCleanup {
301311
log.Logf("WARNING! Using SkipWaitForCreation=true while SkipCleanup=false can lead to workload clusters getting deleted before they are fully provisioned.")
302312
}
303-
creator = getClusterCreateFn(input.BootstrapClusterProxy, namespace.Name)
313+
creator = getClusterCreateFn(input.BootstrapClusterProxy)
304314
}
305315

306316
clusterCreateResults, err := workConcurrentlyAndWait(ctx, workConcurrentlyAndWaitInput{
307317
ClusterNames: clusterNames,
308318
Concurrency: concurrency,
309319
FailFast: input.FailFast,
310320
WorkerFunc: func(ctx context.Context, inputChan chan string, resultChan chan workResult, wg *sync.WaitGroup) {
311-
createClusterWorker(ctx, inputChan, resultChan, wg, modifiedBaseTemplateYAML, baseClusterName, creator)
321+
createClusterWorker(ctx, input.BootstrapClusterProxy, inputChan, resultChan, wg, namespace.Name, input.DeployClusterInSeparateNamespaces, baseClusterClassYAML, baseClusterTemplateYAML, creator)
312322
},
313323
})
314324
if err != nil {
@@ -341,7 +351,7 @@ func scaleSpec(ctx context.Context, inputGetter func() scaleSpecInput) {
341351
Concurrency: concurrency,
342352
FailFast: input.FailFast,
343353
WorkerFunc: func(ctx context.Context, inputChan chan string, resultChan chan workResult, wg *sync.WaitGroup) {
344-
deleteClusterAndWaitWorker(ctx, inputChan, resultChan, wg, input.BootstrapClusterProxy.GetClient(), namespace.Name)
354+
deleteClusterAndWaitWorker(ctx, inputChan, resultChan, wg, input.BootstrapClusterProxy.GetClient(), namespace.Name, input.DeployClusterInSeparateNamespaces)
345355
},
346356
})
347357
if err != nil {
@@ -471,10 +481,10 @@ outer:
471481
return results, kerrors.NewAggregate(errs)
472482
}
473483

474-
type clusterCreator func(ctx context.Context, clusterName string, clusterTemplateYAML []byte)
484+
type clusterCreator func(ctx context.Context, namespace, clusterName string, clusterTemplateYAML []byte)
475485

476486
func getClusterCreateAndWaitFn(input clusterctl.ApplyCustomClusterTemplateAndWaitInput) clusterCreator {
477-
return func(ctx context.Context, clusterName string, clusterTemplateYAML []byte) {
487+
return func(ctx context.Context, namespace, clusterName string, clusterTemplateYAML []byte) {
478488
clusterResources := &clusterctl.ApplyCustomClusterTemplateAndWaitResult{}
479489
// Nb. We cannot directly modify and use `input` in this closure function because this function
480490
// will be called multiple times and this closure will keep modifying the same `input` multiple
@@ -483,7 +493,7 @@ func getClusterCreateAndWaitFn(input clusterctl.ApplyCustomClusterTemplateAndWai
483493
ClusterProxy: input.ClusterProxy,
484494
CustomTemplateYAML: clusterTemplateYAML,
485495
ClusterName: clusterName,
486-
Namespace: input.Namespace,
496+
Namespace: namespace,
487497
CNIManifestPath: input.CNIManifestPath,
488498
WaitForClusterIntervals: input.WaitForClusterIntervals,
489499
WaitForControlPlaneIntervals: input.WaitForControlPlaneIntervals,
@@ -497,16 +507,16 @@ func getClusterCreateAndWaitFn(input clusterctl.ApplyCustomClusterTemplateAndWai
497507
}
498508
}
499509

500-
func getClusterCreateFn(clusterProxy framework.ClusterProxy, namespace string) clusterCreator {
501-
return func(ctx context.Context, clusterName string, clusterTemplateYAML []byte) {
510+
func getClusterCreateFn(clusterProxy framework.ClusterProxy) clusterCreator {
511+
return func(ctx context.Context, namespace, clusterName string, clusterTemplateYAML []byte) {
502512
log.Logf("Applying the cluster template yaml of cluster %s", klog.KRef(namespace, clusterName))
503513
Eventually(func() error {
504514
return clusterProxy.Apply(ctx, clusterTemplateYAML)
505515
}, 1*time.Minute).Should(Succeed(), "Failed to apply the cluster template of cluster %s", klog.KRef(namespace, clusterName))
506516
}
507517
}
508518

509-
func createClusterWorker(ctx context.Context, inputChan <-chan string, resultChan chan<- workResult, wg *sync.WaitGroup, baseTemplate []byte, baseClusterName string, create clusterCreator) {
519+
func createClusterWorker(ctx context.Context, clusterProxy framework.ClusterProxy, inputChan <-chan string, resultChan chan<- workResult, wg *sync.WaitGroup, defaultNamespace string, deployClusterInSeparateNamespaces bool, baseClusterClassYAML, baseClusterTemplateYAML []byte, create clusterCreator) {
510520
defer wg.Done()
511521

512522
for {
@@ -533,9 +543,37 @@ func createClusterWorker(ctx context.Context, inputChan <-chan string, resultCha
533543
}
534544
}()
535545

536-
// Create the cluster template YAML with the target cluster name.
537-
clusterTemplateYAML := bytes.Replace(baseTemplate, []byte(baseClusterName), []byte(clusterName), -1)
538-
create(ctx, clusterName, clusterTemplateYAML)
546+
// Calculate namespace.
547+
namespaceName := defaultNamespace
548+
if deployClusterInSeparateNamespaces {
549+
namespaceName = clusterName
550+
}
551+
552+
// If every cluster should be deployed in a separate namespace:
553+
// * Adjust namespace in ClusterClass YAML.
554+
// * Create new namespace.
555+
// * Deploy ClusterClass in new namespace.
556+
if deployClusterInSeparateNamespaces {
557+
log.Logf("Create namespace %", namespaceName)
558+
_ = framework.CreateNamespace(ctx, framework.CreateNamespaceInput{
559+
Creator: clusterProxy.GetClient(),
560+
Name: namespaceName,
561+
IgnoreAlreadyExists: true,
562+
}, "40s", "10s")
563+
564+
log.Logf("Apply ClusterClass in namespace %", namespaceName)
565+
clusterClassYAML := bytes.Replace(baseClusterClassYAML, []byte(scaleClusterNamespacePlaceholder), []byte(namespaceName), -1)
566+
Eventually(func() error {
567+
return clusterProxy.Apply(ctx, clusterClassYAML)
568+
}, 1*time.Minute).Should(Succeed())
569+
}
570+
571+
// Adjust namespace and name in Cluster YAML
572+
clusterTemplateYAML := bytes.Replace(baseClusterTemplateYAML, []byte(scaleClusterNamespacePlaceholder), []byte(namespaceName), -1)
573+
clusterTemplateYAML = bytes.Replace(clusterTemplateYAML, []byte(scaleClusterNamePlaceholder), []byte(clusterName), -1)
574+
575+
// Deploy Cluster.
576+
create(ctx, namespaceName, clusterName, clusterTemplateYAML)
539577
return false
540578
}
541579
}()
@@ -545,7 +583,7 @@ func createClusterWorker(ctx context.Context, inputChan <-chan string, resultCha
545583
}
546584
}
547585

548-
func deleteClusterAndWaitWorker(ctx context.Context, inputChan <-chan string, resultChan chan<- workResult, wg *sync.WaitGroup, c client.Client, namespace string) {
586+
func deleteClusterAndWaitWorker(ctx context.Context, inputChan <-chan string, resultChan chan<- workResult, wg *sync.WaitGroup, c client.Client, defaultNamespace string, deployClusterInSeparateNamespaces bool) {
549587
defer wg.Done()
550588

551589
for {
@@ -572,10 +610,16 @@ func deleteClusterAndWaitWorker(ctx context.Context, inputChan <-chan string, re
572610
}
573611
}()
574612

613+
// Calculate namespace.
614+
namespaceName := defaultNamespace
615+
if deployClusterInSeparateNamespaces {
616+
namespaceName = clusterName
617+
}
618+
575619
cluster := &clusterv1.Cluster{
576620
ObjectMeta: metav1.ObjectMeta{
577621
Name: clusterName,
578-
Namespace: namespace,
622+
Namespace: namespaceName,
579623
},
580624
}
581625
framework.DeleteCluster(ctx, framework.DeleteClusterInput{
@@ -586,6 +630,15 @@ func deleteClusterAndWaitWorker(ctx context.Context, inputChan <-chan string, re
586630
Getter: c,
587631
Cluster: cluster,
588632
})
633+
634+
// Note: We only delete the namespace in this case because in the case where all clusters are deployed
635+
// to the same namespace deleting the Namespace will lead to deleting all clusters.
636+
if deployClusterInSeparateNamespaces {
637+
framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
638+
Deleter: c,
639+
Name: namespaceName,
640+
})
641+
}
589642
return false
590643
}
591644
}()

0 commit comments

Comments
 (0)