feat: add support for datastore clusters (#574)

* feat: add support for datastore clusters

Adds support for the use of datastore clusters in the applicable builders and post-processors.

Signed-off-by: Ryan Johnson <[email protected]>

* refactor: use exported fields for driver context and clients

Signed-off-by: Ryan Johnson <[email protected]>

---------

Signed-off-by: Ryan Johnson <[email protected]>
Signed-off-by: Ryan Johnson <[email protected]>
Co-authored-by: kp2099 <[email protected]>

Author: tenthirtyam

.web-docs/components/builder/vsphere-clone/README.md

@@ -1036,8 +1036,8 @@ wget http://{{ .HTTPIP }}:{{ .HTTPPort }}/foo/bar/preseed.cfg
   Refer to the [Working With Clusters And Hosts](#working-with-clusters-and-hosts)
   section for more details.
 
-- `host` (string) - The ESXi host where the virtual machine is created. A full path must be
-  specified if the ESXi host is in a folder. For example `folder/host`.
+- `host` (string) - The ESX host where the virtual machine is created. A full path must be specified
+  if the ESX host is in a folder. For example `folder/host`.
   Refer to the [Working With Clusters And Hosts](#working-with-clusters-and-hosts)
   section for more details.
 
@@ -1050,9 +1050,17 @@ wget http://{{ .HTTPIP }}:{{ .HTTPPort }}/foo/bar/preseed.cfg
   a nested path might resemble 'rp-packer/rp-linux-images'.
 
 - `datastore` (string) - The datastore where the virtual machine is created.
-  Required if `host` is a cluster, or if `host` has multiple datastores.
+  Required if `host` is a cluster or if `host` has multiple datastores,
+  unless `datastore_cluster` is specified.
+  
+  ~> **Note:** Cannot be used with `datastore_cluster`.
+
+- `datastore_cluster` (string) - The datastore cluster where the virtual machine is created.
+  When specified, Storage DRS will automatically select the optimal datastore.
+  
+  ~> **Note:** Cannot be used with `datastore`.
 
-- `set_host_for_datastore_uploads` (bool) - The ESXI host used for uploading files to the datastore.
+- `set_host_for_datastore_uploads` (bool) - The ESX host used for uploading files to the datastore.
   Defaults to `false`.
 
 <!-- End of code generated from the comments of the LocationConfig struct in builder/vsphere/common/config_location.go; -->

.web-docs/components/builder/vsphere-iso/README.md

@@ -142,8 +142,8 @@ wget http://{{ .HTTPIP }}:{{ .HTTPPort }}/foo/bar/preseed.cfg
   Refer to the [Working With Clusters And Hosts](#working-with-clusters-and-hosts)
   section for more details.
 
-- `host` (string) - The ESXi host where the virtual machine is created. A full path must be
-  specified if the ESXi host is in a folder. For example `folder/host`.
+- `host` (string) - The ESX host where the virtual machine is created. A full path must be specified
+  if the ESX host is in a folder. For example `folder/host`.
   Refer to the [Working With Clusters And Hosts](#working-with-clusters-and-hosts)
   section for more details.
 
@@ -156,9 +156,17 @@ wget http://{{ .HTTPIP }}:{{ .HTTPPort }}/foo/bar/preseed.cfg
   a nested path might resemble 'rp-packer/rp-linux-images'.
 
 - `datastore` (string) - The datastore where the virtual machine is created.
-  Required if `host` is a cluster, or if `host` has multiple datastores.
+  Required if `host` is a cluster or if `host` has multiple datastores,
+  unless `datastore_cluster` is specified.
+  
+  ~> **Note:** Cannot be used with `datastore_cluster`.
+
+- `datastore_cluster` (string) - The datastore cluster where the virtual machine is created.
+  When specified, Storage DRS will automatically select the optimal datastore.
+  
+  ~> **Note:** Cannot be used with `datastore`.
 
-- `set_host_for_datastore_uploads` (bool) - The ESXI host used for uploading files to the datastore.
+- `set_host_for_datastore_uploads` (bool) - The ESX host used for uploading files to the datastore.
   Defaults to `false`.
 
 <!-- End of code generated from the comments of the LocationConfig struct in builder/vsphere/common/config_location.go; -->

.web-docs/components/post-processor/vsphere/README.md

@@ -23,15 +23,13 @@ The following configuration options are available for the post-processor.
 
 <!-- Code generated from the comments of the Config struct in post-processor/vsphere/post-processor.go; DO NOT EDIT MANUALLY -->
 
-- `cluster` (string) - The cluster or ESX host to upload the virtual machine.
-  This can be either the name of the vSphere cluster or the fully qualified domain name (FQDN)
-  or IP address of the ESX host.
+- `cluster` (string) - The cluster or ESX host to upload the virtual machine. This can be either the
+  name of the vSphere cluster or the fully qualified domain name (FQDN) or IP
+  address of the ESX host.
 
 - `datacenter` (string) - The name of the vSphere datacenter object to place the virtual machine.
   This is _not required_ if `resource_pool` is specified.
 
-- `datastore` (string) - The name of the vSphere datastore to place the virtual machine.
-
 - `host` (string) - The fully qualified domain name or IP address of the vCenter instance or ESX host.
 
 - `password` (string) - The password to use to authenticate to the vSphere endpoint.
@@ -45,6 +43,13 @@ The following configuration options are available for the post-processor.
 
 <!-- Code generated from the comments of the Config struct in post-processor/vsphere/post-processor.go; DO NOT EDIT MANUALLY -->
 
+- `datastore` (string) - The name of the vSphere datastore to place the virtual machine.
+  Mutually exclusive with `datastore_cluster`.
+
+- `datastore_cluster` (string) - The name of the vSphere datastore cluster to place the virtual machine.
+  When specified, Storage DRS will automatically select the optimal datastore.
+  Mutually exclusive with `datastore`.
+
 - `disk_mode` (string) - The disk format of the target virtual machine. One of `thin`, `thick`,
 
 - `esxi_host` (string) - The fully qualified domain name or IP address of the ESX host to upload the
@@ -55,7 +60,8 @@ The following configuration options are available for the post-processor.
 - `options` ([]string) - Options to send to `ovftool` when uploading the virtual machine.
   Use `ovftool --help` to list all the options available.
 
-- `overwrite` (bool) - Overwrite existing files. Defaults to `false`.
+- `overwrite` (bool) - Overwrite existing files.
+  If `true`, forces overwrites of existing files. Defaults to `false`.
 
 - `resource_pool` (string) - The name of the resource pool to place the virtual machine.
 
@@ -68,14 +74,15 @@ The following configuration options are available for the post-processor.
 
 - `hardware_version` (string) - The maximum virtual hardware version for the deployed virtual machine.
 
-  It does not upgrade the virtual hardware version of the source VM. Instead, it limits the
-  virtual hardware version of the deployed virtual machine  to the specified version.
-  If the source virtual machine's hardware version is higher than the specified version, the
-  deployed virtual machine's hardware version will be downgraded to the specified version.
+  It does not upgrade the virtual hardware version of the source VM. Instead, it
+  limits the virtual hardware version of the deployed virtual machine to the
+  specified version. If the source virtual machine's hardware version is higher
+  than the specified version, the deployed virtual machine's hardware version will
+  be downgraded to the specified version.
 
-  If the source virtual machine's hardware version is lower than or equal to the specified
-  version, the deployed virtual machine's hardware version will be the same as the source
-  virtual machine's.
+  If the source virtual machine's hardware version is lower than or equal to the
+  specified version, the deployed virtual machine's hardware version will be the
+  same as the source virtual machine's.
 
   This option is useful when deploying to vCenter instance or an ESX host whose
   version is different than the one used to create the artifact.

builder/vsphere/clone/builder.go

@@ -44,6 +44,11 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 		&common.StepConnect{
 			Config: &b.config.ConnectConfig,
 		},
+		&common.StepResolveDatastore{
+			Datastore:        b.config.Datastore,
+			DatastoreCluster: b.config.DatastoreCluster,
+			DiskCount:        len(b.config.StorageConfig.Storage),
+		},
 		&commonsteps.StepCreateCD{
 			Files:   b.config.CDFiles,
 			Content: b.config.CDContent,

builder/vsphere/clone/config.hcl2spec.go

@@ -9,6 +9,7 @@ package clone
 import (
 	"context"
 	"fmt"
+	"log"
 	"path"
 	"strings"
 
@@ -17,6 +18,7 @@ import (
 	"github.com/hashicorp/packer-plugin-sdk/packerbuilderdata"
 	"github.com/hashicorp/packer-plugin-vsphere/builder/vsphere/common"
 	"github.com/hashicorp/packer-plugin-vsphere/builder/vsphere/driver"
+	"github.com/vmware/govmomi/vim25/types"
 )
 
 type vAppConfig struct {
@@ -159,13 +161,61 @@ func (s *StepCloneVM) Run(ctx context.Context, state multistep.StateBag) multist
 		})
 	}
 
+	datastoreName := s.Location.Datastore
+	var primaryDatastore driver.Datastore
+	if ds, ok := state.GetOk("datastore"); ok {
+		primaryDatastore = ds.(driver.Datastore)
+		datastoreName = primaryDatastore.Name()
+	}
+
+	// If no datastore was resolved and no datastore was specified, return an error.
+	if datastoreName == "" && s.Location.DatastoreCluster == "" {
+		state.Put("error", fmt.Errorf("no datastore specified and no datastore resolved from cluster"))
+		return multistep.ActionHalt
+	}
+
+	// Handle multi-disk placement when using a datastore cluster.
+	var datastoreRefs []*types.ManagedObjectReference
+	if s.Location.DatastoreCluster != "" && len(disks) > 1 {
+		if vcDriver, ok := d.(*driver.VCenterDriver); ok {
+			// Request Storage DRS recommendations for all disks at once for optimal placement.
+			ui.Sayf("Requesting Storage DRS recommendations for %d disks...", len(disks))
+
+			diskDatastores, method, err := vcDriver.SelectDatastoresForDisks(s.Location.DatastoreCluster, disks)
+			if err != nil {
+				ui.Errorf("Warning: Failed to get Storage DRS recommendations: %s. Using primary datastore.", err)
+				if primaryDatastore != nil {
+					ref := primaryDatastore.Reference()
+					for i := 0; i < len(disks); i++ {
+						datastoreRefs = append(datastoreRefs, &ref)
+					}
+				}
+			} else {
+				// Use the first disk's datastore as the primary datastore.
+				if len(diskDatastores) > 0 {
+					datastoreName = diskDatastores[0].Name()
+				}
+
+				for i, ds := range diskDatastores {
+					ref := ds.Reference()
+					if method == driver.SelectionMethodDRS {
+						log.Printf("[INFO] Disk %d: Storage DRS selected datastore '%s'", i+1, ds.Name())
+					} else {
+						log.Printf("[INFO] Disk %d: Using first available datastore '%s'", i+1, ds.Name())
+					}
+					datastoreRefs = append(datastoreRefs, &ref)
+				}
+			}
+		}
+	}
+
 	vm, err := template.Clone(ctx, &driver.CloneConfig{
 		Name:            s.Location.VMName,
 		Folder:          s.Location.Folder,
 		Cluster:         s.Location.Cluster,
 		Host:            s.Location.Host,
 		ResourcePool:    s.Location.ResourcePool,
-		Datastore:       s.Location.Datastore,
+		Datastore:       datastoreName,
 		LinkedClone:     s.Config.LinkedClone,
 		Network:         s.Config.Network,
 		MacAddress:      strings.ToLower(s.Config.MacAddress),
@@ -175,6 +225,7 @@ func (s *StepCloneVM) Run(ctx context.Context, state multistep.StateBag) multist
 		StorageConfig: driver.StorageConfig{
 			DiskControllerType: s.Config.StorageConfig.DiskControllerType,
 			Storage:            disks,
+			DatastoreRefs:      datastoreRefs,
 		},
 	})
 	if err != nil {

builder/vsphere/clone/step_clone.go

@@ -21,8 +21,8 @@ type LocationConfig struct {
 	// Refer to the [Working With Clusters And Hosts](#working-with-clusters-and-hosts)
 	// section for more details.
 	Cluster string `mapstructure:"cluster"`
-	// The ESXi host where the virtual machine is created. A full path must be
-	// specified if the ESXi host is in a folder. For example `folder/host`.
+	// The ESX host where the virtual machine is created. A full path must be specified
+	// if the ESX host is in a folder. For example `folder/host`.
 	// Refer to the [Working With Clusters And Hosts](#working-with-clusters-and-hosts)
 	// section for more details.
 	Host string `mapstructure:"host"`
@@ -35,9 +35,17 @@ type LocationConfig struct {
 	// a nested path might resemble 'rp-packer/rp-linux-images'.
 	ResourcePool string `mapstructure:"resource_pool"`
 	// The datastore where the virtual machine is created.
-	// Required if `host` is a cluster, or if `host` has multiple datastores.
+	// Required if `host` is a cluster or if `host` has multiple datastores,
+	// unless `datastore_cluster` is specified.
+	//
+	// ~> **Note:** Cannot be used with `datastore_cluster`.
 	Datastore string `mapstructure:"datastore"`
-	// The ESXI host used for uploading files to the datastore.
+	// The datastore cluster where the virtual machine is created.
+	// When specified, Storage DRS will automatically select the optimal datastore.
+	//
+	// ~> **Note:** Cannot be used with `datastore`.
+	DatastoreCluster string `mapstructure:"datastore_cluster"`
+	// The ESX host used for uploading files to the datastore.
 	// Defaults to `false`.
 	SetHostForDatastoreUploads bool `mapstructure:"set_host_for_datastore_uploads"`
 }
@@ -52,7 +60,10 @@ func (c *LocationConfig) Prepare() []error {
 		errs = append(errs, fmt.Errorf("'host' or 'cluster' is required"))
 	}
 
-	// clean Folder path and remove leading slash as folders are relative within vsphere
+	if c.Datastore != "" && c.DatastoreCluster != "" {
+		errs = append(errs, fmt.Errorf("'datastore' and 'datastore_cluster' are mutually exclusive; specify only one"))
+	}
+
 	c.Folder = path.Clean(c.Folder)
 	c.Folder = strings.TrimLeft(c.Folder, "/")
 

builder/vsphere/common/config_location.go

@@ -63,10 +63,18 @@ func (s *StepAddFloppy) Run(_ context.Context, state multistep.StateBag) multist
 	if floppyPath, ok := state.GetOk("floppy_path"); ok {
 		ui.Say("Uploading floppy image...")
 
-		ds, err := d.FindDatastore(s.Datastore, s.Host)
-		if err != nil {
-			state.Put("error", err)
-			return multistep.ActionHalt
+		var ds driver.Datastore
+		var err error
+
+		// If a datastore was resolved (from datastore or datastore_cluster), use it.
+		if resolvedDs, ok := state.GetOk("datastore"); ok {
+			ds = resolvedDs.(driver.Datastore)
+		} else {
+			ds, err = d.FindDatastore(s.Datastore, s.Host)
+			if err != nil {
+				state.Put("error", err)
+				return multistep.ActionHalt
+			}
 		}
 		vmDir, err := vm.GetDir()
 		if err != nil {
@@ -123,10 +131,18 @@ func (s *StepAddFloppy) Cleanup(state multistep.StateBag) {
 	if UploadedFloppyPath, ok := state.GetOk("uploaded_floppy_path"); ok {
 		ui.Say("Deleting floppy image...")
 
-		ds, err := d.FindDatastore(s.Datastore, s.Host)
-		if err != nil {
-			state.Put("error", err)
-			return
+		var ds driver.Datastore
+		var err error
+
+		// If a datastore was resolved (from datastore or datastore_cluster), use it.
+		if resolvedDs, ok := state.GetOk("datastore"); ok {
+			ds = resolvedDs.(driver.Datastore)
+		} else {
+			ds, err = d.FindDatastore(s.Datastore, s.Host)
+			if err != nil {
+				state.Put("error", err)
+				return
+			}
 		}
 
 		err = ds.Delete(UploadedFloppyPath.(string))

builder/vsphere/common/config_location.hcl2spec.go

@@ -46,7 +46,7 @@ type StepDownload struct {
 }
 
 func (s *StepDownload) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
-	driver := state.Get("driver").(driver.Driver)
+	d := state.Get("driver").(driver.Driver)
 	ui := state.Get("ui").(packersdk.Ui)
 
 	// Set the remote cache datastore. If not set, use the default datastore for the build.
@@ -55,11 +55,19 @@ func (s *StepDownload) Run(ctx context.Context, state multistep.StateBag) multis
 		remoteCacheDatastore = s.RemoteCacheDatastore
 	}
 
-	// Find the datastore to use for the remote cache.
-	ds, err := driver.FindDatastore(remoteCacheDatastore, s.Host)
-	if err != nil {
-		state.Put("error", fmt.Errorf("error finding the datastore: %v", err))
-		return multistep.ActionHalt
+	var ds driver.Datastore
+	var err error
+
+	// If a datastore was resolved (from datastore or datastore_cluster), use it.
+	if resolvedDs, ok := state.GetOk("datastore"); ok && remoteCacheDatastore == s.Datastore {
+		ds = resolvedDs.(driver.Datastore)
+	} else {
+		// Find the datastore to use for the remote cache.
+		ds, err = d.FindDatastore(remoteCacheDatastore, s.Host)
+		if err != nil {
+			state.Put("error", fmt.Errorf("error finding the datastore: %v", err))
+			return multistep.ActionHalt
+		}
 	}
 
 	// Set the remote cache path. If not set, use the default cache path.