wip - isolating e2e

Signed-off-by: Danil-Grigorev <[email protected]>

Author: Danil-Grigorev

templates/test/ci/cluster-template-prow-clusterclass-ci-rke2.yaml

@@ -6,6 +6,7 @@ resources:
 - ../../../flavors/clusterclass-rke2/rke2-controlplane-template.yaml
 - ../../../flavors/clusterclass-rke2/azure-cluster-template.yaml
 - ../../../flavors/clusterclass-rke2/azure-machine-template-controlplane.yaml
+- ../../../flavors/clusterclass-rke2/azure-machine-template-worker.yaml
 - rke2-config-template.yaml
 - ../../../azure-cluster-identity
 patches:

templates/test/ci/cluster-template-prow-topology-rke2.yaml

@@ -36,13 +36,13 @@ spec:
               path: "/spec/template/spec/files/-"
               valueFrom:
                 template: |
-                  - contentFrom:
-                      secret:
-                        key: control-plane-azure.json
-                        name: "{{ .builtin.controlPlane.machineTemplate.infrastructureRef.name }}-azure-json"
-                    owner: root:root
-                    path: /etc/kubernetes/azure.json
-                    permissions: "0644"
+                  contentFrom:
+                    secret:
+                      key: control-plane-azure.json
+                      name: "{{ .builtin.controlPlane.machineTemplate.infrastructureRef.name }}-azure-json"
+                  owner: root:root
+                  path: /etc/kubernetes/azure.json
+                  permissions: "0644"
     - name: workerAzureJsonSecretName
       definitions:
         - selector:
@@ -174,4 +174,3 @@ spec:
   template:
     spec:
       sshPublicKey: ""
-      vmSize: replace_me

templates/test/ci/prow-clusterclass-ci-rke2/kustomization.yaml

@@ -23,6 +23,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/Azure/azure-service-operator/v2/pkg/common/config"
@@ -137,7 +138,7 @@ var _ = Describe("Workload cluster creation", func() {
 	if os.Getenv("USE_LOCAL_KIND_REGISTRY") != "true" {
 		// This spec expects a user-assigned identity with Contributor role assignment named "cloud-provider-user-identity" in a "capz-ci"
 		// resource group. Override these defaults by setting the USER_IDENTITY and CI_RG environment variables.
-		Context("Creating a private cluster [OPTIONAL]", func() {
+		Context("Creating a private cluster [OPTIONAL-SKIP]", func() {
 			It("Creates a public management cluster in a custom vnet", func() {
 				clusterName = getClusterName(clusterNamePrefix, "public-custom-vnet")
 				By("Creating a custom virtual network", func() {
@@ -355,7 +356,7 @@ var _ = Describe("Workload cluster creation", func() {
 		})
 	})
 
-	Context("Creating a Flatcar cluster [OPTIONAL]", func() {
+	Context("Creating a Flatcar cluster [OPTIONAL-SKIP]", func() {
 		It("With Flatcar control-plane and worker nodes", func() {
 			clusterName = getClusterName(clusterNamePrefix, "flatcar")
 			clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput(
@@ -393,7 +394,7 @@ var _ = Describe("Workload cluster creation", func() {
 		})
 	})
 
-	Context("Creating a Flatcar sysext cluster [OPTIONAL]", func() {
+	Context("Creating a Flatcar sysext cluster [OPTIONAL-SKIP]", func() {
 		It("With Flatcar control-plane and worker nodes", func() {
 			clusterName = getClusterName(clusterNamePrefix, "flatcar-sysext")
 			clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput(
@@ -431,7 +432,7 @@ var _ = Describe("Workload cluster creation", func() {
 		})
 	})
 
-	Context("Creating a cluster with spot vms [OPTIONAL]", func() {
+	Context("Creating a cluster with spot vms [OPTIONAL-SKIP]", func() {
 		It("With spot vm machine deployments", func() {
 			clusterName = getClusterName(clusterNamePrefix, "spot")
 			clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput(
@@ -596,7 +597,7 @@ var _ = Describe("Workload cluster creation", func() {
 	// You can override the default SKU `Standard_NV12s_v3` and `Premium_LRS` storage by setting
 	// the `AZURE_GPU_NODE_MACHINE_TYPE` and `AZURE_GPU_NODE_STORAGE_TYPE` environment variables.
 	// See https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/ for pricing.
-	Context("Creating a GPU-enabled cluster [OPTIONAL]", func() {
+	Context("Creating a GPU-enabled cluster [OPTIONAL-SKIP]", func() {
 		It("with a single control plane node and 1 node", func() {
 			Skip("Skipping since the e2e subscription has no quota for GPU SKUs")
 			clusterName = getClusterName(clusterNamePrefix, "gpu")
@@ -655,7 +656,7 @@ var _ = Describe("Workload cluster creation", func() {
 	})
 
 	// ci-e2e.sh and Prow CI skip this test by default. To include this test, set `GINKGO_SKIP=""`.
-	Context("Creating a cluster with VMSS flex machinepools [OPTIONAL]", func() {
+	Context("Creating a cluster with VMSS flex machinepools [OPTIONAL-SKIP]", func() {
 		It("with 1 control plane node and 1 machinepool", func() {
 			clusterName = getClusterName(clusterNamePrefix, "flex")
 			clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput(
@@ -993,7 +994,7 @@ var _ = Describe("Workload cluster creation", func() {
 	// ci-e2e.sh and Prow CI skip this test by default. To include this test, set `GINKGO_SKIP=""`.
 	// This spec expects a user-assigned identity named "cloud-provider-user-identity" in a "capz-ci"
 	// resource group. Override these defaults by setting the USER_IDENTITY and CI_RG environment variables.
-	Context("Creating a dual-stack cluster [OPTIONAL]", func() {
+	Context("Creating a dual-stack cluster [OPTIONAL-SKIP]", func() {
 		It("With dual-stack worker node", func() {
 			By("using user-assigned identity")
 			clusterName = getClusterName(clusterNamePrefix, "dual-stack")
@@ -1061,7 +1062,7 @@ var _ = Describe("Workload cluster creation", func() {
 		})
 	})
 
-	Context("Creating clusters using clusterclass [OPTIONAL]", func() {
+	Context("Creating clusters using clusterclass [OPTIONAL-SKIP]", func() {
 		It("with a single control plane node, one linux worker node, and one windows worker node", func() {
 			// Use ci-default as the clusterclass name so test infra can find the clusterclass template
 			Expect(os.Setenv("CLUSTER_CLASS_NAME", "ci-default")).To(Succeed())
@@ -1110,19 +1111,30 @@ var _ = Describe("Workload cluster creation", func() {
 
 	Context("Creating RKE2 clusters using clusterclass [OPTIONAL]", func() {
 		It("with 3 control plane node and one linux worker node", func() {
-			// Use ci-default as the clusterclass name so test infra can find the clusterclass template
+			// Use ci-rke2 as the clusterclass name so test infra can find the clusterclass template
 			Expect(os.Setenv("CLUSTER_CLASS_NAME", "ci-rke2")).To(Succeed())
 
 			// Use "cc" as spec name because NAT gateway pip name exceeds limit.
 			clusterName = getClusterName(clusterNamePrefix, "cc")
 
-			// Opt into using windows with prow template
-			Expect(os.Setenv("WINDOWS_WORKER_MACHINE_COUNT", "1")).To(Succeed())
+			// Init rke2 CP and bootstrap providers
+			initInput := clusterctl.InitInput{
+				// pass reference to the management cluster hosting this test
+				KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
+				// pass the clusterctl config file that points to the local provider repository created for this test
+				ClusterctlConfigPath: clusterctlConfigPath,
+				// setup the desired list of providers for a single-tenant management cluster
+				BootstrapProviders:    []string{"rke2"},
+				ControlPlaneProviders: []string{"rke2"},
+				// setup clusterctl logs folder
+				LogFolder: filepath.Join(artifactFolder, "clusters", clusterName),
+			}
+			clusterctl.Init(ctx, initInput)
 
 			// Create a cluster using the cluster class created above
 			clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput(
 				specName,
-				withFlavor("topology"),
+				withFlavor("topology-rke2"),
 				withNamespace(namespace.Name),
 				withClusterName(clusterName),
 				withControlPlaneMachineCount(3),
@@ -1160,7 +1172,7 @@ var _ = Describe("Workload cluster creation", func() {
 	// resource group. Override these defaults by setting the USER_IDENTITY and CI_RG environment variables.
 	// You can also override the default SKU `Standard_DS2_v2` and `Standard_DS4_v2` storage by setting
 	// the `AZURE_EDGEZONE_CONTROL_PLANE_MACHINE_TYPE` and `AZURE_EDGEZONE_NODE_MACHINE_TYPE` environment variables.
-	Context("Creating clusters on public MEC [OPTIONAL]", func() {
+	Context("Creating clusters on public MEC [OPTIONAL-SKIP]", func() {
 		It("with 1 control plane nodes and 1 worker node", func() {
 			Skip("Skipping public MEC test until a new edgezone is available")
 			By("using user-assigned identity")
@@ -1201,7 +1213,7 @@ var _ = Describe("Workload cluster creation", func() {
 		})
 	})
 
-	Context("Creating a self-managed VM based cluster using API Server ILB feature gate using default template [OPTIONAL][API-Server-ILB]", func() {
+	Context("Creating a self-managed VM based cluster using API Server ILB feature gate using default template [OPTIONAL-SKIP][API-Server-ILB]", func() {
 		It("with three controlplane node and three worker nodes", func() {
 			clusterName = getClusterName(clusterNamePrefix, "apiserver-ilb")
 
@@ -1247,7 +1259,7 @@ var _ = Describe("Workload cluster creation", func() {
 		})
 	})
 
-	Context("Creating a self-managed VM based cluster using API Server ILB feature gate and fully spec-ed out APIServer ILB template [OPTIONAL][API-Server-ILB]", func() {
+	Context("Creating a self-managed VM based cluster using API Server ILB feature gate and fully spec-ed out APIServer ILB template [OPTIONAL-SKIP][API-Server-ILB]", func() {
 		It("with three controlplane node and three worker nodes", func() {
 			clusterName = getClusterName(clusterNamePrefix, "apiserver-ilb")
 

templates/test/ci/prow-clusterclass-ci-rke2/patches.yaml

@@ -80,12 +80,6 @@ providers:
       - old: "imagePullPolicy: Always"
         new: "imagePullPolicy: IfNotPresent"
 
-  - name: rke2
-    type: BootstrapProvider
-
-  - name: rke2
-    type: ControlPlaneProvider
-
   - name: azure
     type: InfrastructureProvider
     versions:
@@ -170,6 +164,8 @@ providers:
         targetName: "clusterclass-default.yaml"
       - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-topology.yaml"
         targetName: "cluster-template-topology.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-topology-rke2.yaml"
+        targetName: "cluster-template-topology-rke2.yaml"
       - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-flatcar.yaml"
         targetName: "cluster-template-flatcar.yaml"
       - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-flatcar-sysext.yaml"