Install gpu-operator with CAAPH in e2e

Author: nojnhuh

templates/addons/cluster-api-helm/gpu-operator.yaml

@@ -0,0 +1,12 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: gpu-operator
+spec:
+  clusterSelector:
+    matchLabels:
+      gpu-operator: "true"
+  repoURL: https://helm.ngc.nvidia.com/nvidia
+  chartName: gpu-operator
+  releaseName: nvidia-gpu-operator
+  namespace: default

templates/test/ci/cluster-template-prow-nvidia-gpu.yaml

@@ -0,0 +1,6 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: ${CLUSTER_NAME}
+  labels:
+    gpu-operator: "true"

templates/test/ci/patches/cluster-label-gpu-operator.yaml

@@ -7,6 +7,7 @@ resources:
 - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml
 - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml
 - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml
+- ../../../addons/cluster-api-helm/gpu-operator.yaml
 patches:
 - path: patches/node-storage-type.yaml
   target:
@@ -19,6 +20,7 @@ patches:
 - path: ../patches/azurecluster-gpu.yaml
 - path: ../patches/cluster-label-calico.yaml
 - path: ../patches/cluster-label-cloud-provider-azure.yaml
+- path: ../patches/cluster-label-gpu-operator.yaml
 
 sortOptions:
   order: fifo

templates/test/ci/prow-nvidia-gpu/kustomization.yaml

@@ -620,8 +620,8 @@ var _ = Describe("Workload cluster creation", func() {
 							ClusterName:           clusterName,
 						}
 					})
-					InstallGPUOperator(ctx, func() GPUOperatorSpecInput {
-						return GPUOperatorSpecInput{
+					EnsureGPUOperator(ctx, func() EnsureGPUOperatorInput {
+						return EnsureGPUOperatorInput{
 							BootstrapClusterProxy: bootstrapClusterProxy,
 							Namespace:             namespace,
 							ClusterName:           clusterName,

test/e2e/azure_test.go

@@ -29,24 +29,21 @@ import (
 )
 
 const (
-	nvidiaHelmChartRepoURL           string = "https://helm.ngc.nvidia.com/nvidia"
-	nvidiaGPUOperatorNamespace       string = "default"
-	nvidiaGPUOperatorHelmReleaseName string = "nvidia-gpu-operator"
-	nvidiaGPUOperatorHelmChartName   string = "gpu-operator"
+	nvidiaGPUOperatorNamespace string = "default"
 )
 
-// GPUOperatorSpecInput is the input for InstallGPUOperator.
-type GPUOperatorSpecInput struct {
+// EnsureGPUOperatorInput is the input for InstallGPUOperator.
+type EnsureGPUOperatorInput struct {
 	BootstrapClusterProxy framework.ClusterProxy
 	Namespace             *corev1.Namespace
 	ClusterName           string
 }
 
-// InstallGPUOperator installs the official nvidia/gpu-operator helm chart.
-func InstallGPUOperator(ctx context.Context, inputGetter func() GPUOperatorSpecInput) {
+// EnsureGPUOperator installs the official nvidia/gpu-operator helm chart.
+func EnsureGPUOperator(ctx context.Context, inputGetter func() EnsureGPUOperatorInput) {
 	var (
 		specName = "nvidia-gpu-operator"
-		input    GPUOperatorSpecInput
+		input    EnsureGPUOperatorInput
 	)
 
 	Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
@@ -56,12 +53,12 @@ func InstallGPUOperator(ctx context.Context, inputGetter func() GPUOperatorSpecI
 	Expect(input.Namespace).NotTo(BeNil(), "Invalid argument. input.Namespace can't be nil when calling %s spec", specName)
 	Expect(input.ClusterName).NotTo(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling %s spec", specName)
 	clusterProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, input.Namespace.Name, input.ClusterName)
-	InstallNvidiaGPUOperatorChart(ctx, clusterProxy)
-}
 
-// InstallNvidiaGPUOperatorChart installs the official nvidia/gpu-operator helm chart
-func InstallNvidiaGPUOperatorChart(ctx context.Context, clusterProxy framework.ClusterProxy) {
-	By("Installing nvidia/gpu-operator via helm")
-	values := &HelmOptions{}
-	InstallHelmChart(ctx, clusterProxy, nvidiaGPUOperatorNamespace, nvidiaHelmChartRepoURL, nvidiaGPUOperatorHelmChartName, nvidiaGPUOperatorHelmReleaseName, values, "")
+	By("Ensuring GPU Operator is installed via CAAPH")
+
+	By("Waiting for Ready gpu-operator deployment pods")
+	for _, d := range []string{"gpu-operator"} {
+		waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, nvidiaGPUOperatorNamespace, specName)
+		WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...)
+	}
 }