Merge pull request openshift#8765 from nutanix-cloud-native/nutanix-gpu-datadisks

CORS-3546: Nutanix: add gpus and dataDisks support

Author: openshift-merge-bot[bot]

data/data/install.openshift.io_installconfigs.yaml

@@ -77,6 +77,26 @@ func ValidateForProvisioning(ic *types.InstallConfig) error {
 					fmt.Sprintf("the failure domain %s configured subnet UUID does not correspond to a valid subnet in Prism: %v", fd.Name, err)))
 			}
 		}
+
+		// validate each configured dataSource image exists
+		for _, dsImgRef := range fd.DataSourceImages {
+			switch {
+			case dsImgRef.UUID != "":
+				if _, err = nc.V3.GetImage(ctx, dsImgRef.UUID); err != nil {
+					errMsg := fmt.Sprintf("failureDomain %q: failed to find the dataSource image with uuid %s: %v", fd.Name, dsImgRef.UUID, err)
+					errList = append(errList, field.Invalid(parentPath.Child("failureDomains", "dataSourceImage", "uuid"), dsImgRef.UUID, errMsg))
+				}
+			case dsImgRef.Name != "":
+				if dsImgUUID, err := nutanixtypes.FindImageUUIDByName(ctx, nc, dsImgRef.Name); err != nil {
+					errMsg := fmt.Sprintf("failureDomain %q: failed to find the dataSource image with name %q: %v", fd.Name, dsImgRef.UUID, err)
+					errList = append(errList, field.Invalid(parentPath.Child("failureDomains", "dataSourceImage", "name"), dsImgRef.Name, errMsg))
+				} else {
+					dsImgRef.UUID = *dsImgUUID
+				}
+			default:
+				errList = append(errList, field.Required(parentPath.Child("failureDomains", "dataSourceImage"), "both the dataSourceImage's uuid and name are empty, you need to configure one."))
+			}
+		}
 	}
 
 	return errList.ToAggregate()

pkg/asset/installconfig/nutanix/validation.go

@@ -450,7 +450,7 @@ func (c *ClusterAPI) Generate(ctx context.Context, dependencies asset.Parents) e
 		mpool.NumCPUs = 8
 		mpool.Set(ic.Platform.Nutanix.DefaultMachinePlatform)
 		mpool.Set(pool.Platform.Nutanix)
-		if err = mpool.ValidateConfig(ic.Platform.Nutanix); err != nil {
+		if err = mpool.ValidateConfig(ic.Platform.Nutanix, "master"); err != nil {
 			return fmt.Errorf("failed to generate Cluster API machine manifests for control-plane: %w", err)
 		}
 		pool.Platform.Nutanix = &mpool

pkg/asset/machines/clusterapi.go

@@ -498,7 +498,7 @@ func (m *Master) Generate(ctx context.Context, dependencies asset.Parents) error
 		mpool.NumCPUs = 8
 		mpool.Set(ic.Platform.Nutanix.DefaultMachinePlatform)
 		mpool.Set(pool.Platform.Nutanix)
-		if err = mpool.ValidateConfig(ic.Platform.Nutanix); err != nil {
+		if err = mpool.ValidateConfig(ic.Platform.Nutanix, "master"); err != nil {
 			return errors.Wrap(err, "failed to create master machine objects")
 		}
 		pool.Platform.Nutanix = &mpool

pkg/asset/machines/master.go

@@ -8,6 +8,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/utils/ptr"
 
 	configv1 "github.com/openshift/api/config/v1"
 	machinev1 "github.com/openshift/api/machine/v1"
@@ -181,6 +182,7 @@ func provider(clusterID string, platform *nutanix.Platform, mpool *nutanix.Machi
 			UUID: &peUUID,
 		},
 		SystemDiskSize: resource.MustParse(fmt.Sprintf("%dGi", mpool.OSDisk.DiskSizeGiB)),
+		GPUs:           mpool.GPUs,
 	}
 
 	// FailureDomain
@@ -203,6 +205,49 @@ func provider(clusterID string, platform *nutanix.Platform, mpool *nutanix.Machi
 		providerCfg.Categories = mpool.Categories
 	}
 
+	for _, disk := range mpool.DataDisks {
+		providerDisk := machinev1.NutanixVMDisk{
+			DiskSize:         disk.DiskSize,
+			DeviceProperties: disk.DeviceProperties,
+		}
+
+		if disk.StorageConfig != nil {
+			providerDisk.StorageConfig = &machinev1.NutanixVMStorageConfig{
+				DiskMode: disk.StorageConfig.DiskMode,
+			}
+
+			if disk.StorageConfig.StorageContainer != nil {
+				scRef := disk.StorageConfig.StorageContainer
+				if scRef.ReferenceName != "" && failureDomain != nil {
+					if scRef, err := platform.GetStorageContainerFromFailureDomain(failureDomain.Name, scRef.ReferenceName); err != nil {
+						return nil, fmt.Errorf("not found storage container with reference name %q in failureDomain %q", scRef.ReferenceName, failureDomain.Name)
+					}
+				}
+
+				providerDisk.StorageConfig.StorageContainer = &machinev1.NutanixStorageResourceIdentifier{
+					Type: machinev1.NutanixIdentifierUUID,
+					UUID: ptr.To(scRef.UUID),
+				}
+			}
+		}
+
+		if disk.DataSourceImage != nil {
+			imgRef := disk.DataSourceImage
+			if imgRef.ReferenceName != "" && failureDomain != nil {
+				if imgRef, err := platform.GetDataSourceImageFromFailureDomain(failureDomain.Name, imgRef.ReferenceName); err != nil {
+					return nil, fmt.Errorf("not found dataSource image with reference name %q in failureDomain %q", imgRef.ReferenceName, failureDomain.Name)
+				}
+			}
+
+			providerDisk.DataSource = &machinev1.NutanixResourceIdentifier{
+				Type: machinev1.NutanixIdentifierUUID,
+				UUID: ptr.To(imgRef.UUID),
+			}
+		}
+
+		providerCfg.DataDisks = append(providerCfg.DataDisks, providerDisk)
+	}
+
 	return providerCfg, nil
 }
 

pkg/asset/machines/nutanix/machines.go

@@ -648,8 +648,8 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error
 			mpool := defaultNutanixMachinePoolPlatform()
 			mpool.Set(ic.Platform.Nutanix.DefaultMachinePlatform)
 			mpool.Set(pool.Platform.Nutanix)
-			if err = mpool.ValidateConfig(ic.Platform.Nutanix); err != nil {
-				return errors.Wrap(err, "failed to create master machine objects")
+			if err = mpool.ValidateConfig(ic.Platform.Nutanix, "worker"); err != nil {
+				return errors.Wrap(err, "failed to create worker machine objects")
 			}
 			pool.Platform.Nutanix = &mpool
 			imageName := nutanixtypes.RHCOSImageName(clusterID.InfraID)

pkg/asset/machines/worker.go

@@ -14,6 +14,8 @@ import (
 	"github.com/nutanix-cloud-native/prism-go-client/utils"
 	nutanixclientv3 "github.com/nutanix-cloud-native/prism-go-client/v3"
 	"github.com/pkg/errors"
+
+	machinev1 "github.com/openshift/api/machine/v1"
 )
 
 const (
@@ -194,3 +196,94 @@ func CategoryKey(infraID string) string {
 	categoryKey := fmt.Sprintf("%s%s", categoryKeyPrefix, infraID)
 	return categoryKey
 }
+
+// GetGPUList returns a list of VMGpus for the given list of GPU identifiers in the Prism Element (uuid).
+func GetGPUList(ctx context.Context, client *nutanixclientv3.Client, gpus []machinev1.NutanixGPU, peUUID string) ([]*nutanixclientv3.VMGpu, error) {
+	vmGPUs := make([]*nutanixclientv3.VMGpu, 0)
+
+	if len(gpus) == 0 {
+		return vmGPUs, nil
+	}
+
+	peGPUs, err := GetGPUsForPE(ctx, client, peUUID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve GPUs of the Prism Element cluster (uuid: %v): %w", peUUID, err)
+	}
+	if len(peGPUs) == 0 {
+		return nil, fmt.Errorf("no available GPUs found in Prism Element cluster (uuid: %s)", peUUID)
+	}
+
+	for _, gpu := range gpus {
+		foundGPU, err := GetGPUFromList(ctx, client, gpu, peGPUs)
+		if err != nil {
+			return nil, err
+		}
+		vmGPUs = append(vmGPUs, foundGPU)
+	}
+
+	return vmGPUs, nil
+}
+
+// GetGPUFromList returns the VMGpu matching the input reqirements from the provided list of GPU devices.
+func GetGPUFromList(ctx context.Context, client *nutanixclientv3.Client, gpu machinev1.NutanixGPU, gpuDevices []*nutanixclientv3.GPU) (*nutanixclientv3.VMGpu, error) {
+	for _, gd := range gpuDevices {
+		if gd.Status != "UNUSED" {
+			continue
+		}
+
+		if (gpu.Type == machinev1.NutanixGPUIdentifierDeviceID && gd.DeviceID != nil && *gpu.DeviceID == int32(*gd.DeviceID)) ||
+			(gpu.Type == machinev1.NutanixGPUIdentifierName && *gpu.Name == gd.Name) {
+			return &nutanixclientv3.VMGpu{
+				DeviceID: gd.DeviceID,
+				Mode:     &gd.Mode,
+				Vendor:   &gd.Vendor,
+			}, nil
+		}
+	}
+
+	return nil, fmt.Errorf("no available GPU found that matches required GPU inputs")
+}
+
+// GetGPUsForPE returns all the GPU devices for the given Prism Element (uuid).
+func GetGPUsForPE(ctx context.Context, client *nutanixclientv3.Client, peUUID string) ([]*nutanixclientv3.GPU, error) {
+	gpus := make([]*nutanixclientv3.GPU, 0)
+	hosts, err := client.V3.ListAllHost(ctx)
+	if err != nil {
+		return gpus, fmt.Errorf("failed to get hosts from Prism Central: %w", err)
+	}
+
+	for _, host := range hosts.Entities {
+		if host == nil ||
+			host.Status == nil ||
+			host.Status.ClusterReference == nil ||
+			host.Status.ClusterReference.UUID != peUUID ||
+			host.Status.Resources == nil ||
+			len(host.Status.Resources.GPUList) == 0 {
+			continue
+		}
+
+		for _, peGpu := range host.Status.Resources.GPUList {
+			if peGpu != nil {
+				gpus = append(gpus, peGpu)
+			}
+		}
+	}
+
+	return gpus, nil
+}
+
+// FindImageUUIDByName retrieves the image resource uuid by the given image name from PC.
+func FindImageUUIDByName(ctx context.Context, ntnxclient *nutanixclientv3.Client, imageName string) (*string, error) {
+	res, err := ntnxclient.V3.ListImage(ctx, &nutanixclientv3.DSMetadata{
+		Filter: utils.StringPtr(fmt.Sprintf("name==%s", imageName)),
+	})
+	if err != nil || len(res.Entities) == 0 {
+		return nil, fmt.Errorf("failed to find image by name %q. err: %w", imageName, err)
+	}
+
+	if len(res.Entities) > 1 {
+		return nil, fmt.Errorf("found more than one (%v) images with name %q", len(res.Entities), imageName)
+	}
+
+	return res.Entities[0].Metadata.UUID, nil
+}