Hitachi API for vsphere-xcopy-volume-populator from tatsumir (#1602)

Hitachi Vantara Storage API support for storage offload volume populator

---------

Signed-off-by: Ryosuke Tatsumi <[email protected]>
Signed-off-by: tatsumir <[email protected]>

Author: tatsumir

.github/workflows/build-push-images-hal.yml

@@ -0,0 +1,35 @@
+name: Build and Push Container Image
+
+on:
+  push:
+    branches:
+      - vsphere-xcopy-volume-populator-hal-api 
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up Podman
+        run: |
+          sudo apt update
+          sudo apt install -y podman
+
+      - name: Log in to GitHub Container Registry (GHCR)
+        run: |
+          echo "${{ secrets.GHCR_TOKEN }}" | podman login ghcr.io -u $GITHUB_ACTOR --password-stdin
+
+      - name: Build Container Image
+        run: |
+          make CONTAINER_CMD=$(which podman) build-vsphere-xcopy-volume-populator-image REGISTRY=ghcr.io REGISTRY_ORG=tatsumir
+
+      - name: Push Container Image
+        run: |
+          make CONTAINER_CMD=$(which podman) push-vsphere-xcopy-volume-populator-image REGISTRY=ghcr.io REGISTRY_ORG=tatsumir
+

.gitignore

@@ -40,3 +40,5 @@ vendor/**/Dockerfile.vcsim
 vendor/**/Dockerfile.govc
 
 cmd/vsphere-xcopy-volume-populator/vmkfstools-wrapper/vmkfstools-wrapper.vib
+.github/workflows/build-push-images-hal.yml
+

cmd/vsphere-xcopy-volume-populator/internal/ontap/ontap.go

@@ -31,14 +31,14 @@ func (c *NetappClonner) UnMap(initatorGroup string, targetLUN populator.LUN, _ p
 	return c.api.LunUnmap(context.TODO(), initatorGroup, targetLUN.Name)
 }
 
-func (c *NetappClonner) EnsureClonnerIgroup(initiatorGroup string, clonnerIqn string) (populator.MappingContext, error) {
+func (c *NetappClonner) EnsureClonnerIgroup(initiatorGroup string, clonnerIqn []string) (populator.MappingContext, error) {
 	// esxs needs "vmware" as the group protocol.
 	err := c.api.IgroupCreate(context.Background(), initiatorGroup, "iscsi", "vmware")
 	if err != nil {
 		// TODO ignore if exists error? with ontap there is no error
 		return nil, fmt.Errorf("failed adding igroup %w", err)
 	}
-	err = c.api.EnsureIgroupAdded(context.Background(), initiatorGroup, clonnerIqn)
+	err = c.api.EnsureIgroupAdded(context.Background(), initiatorGroup, clonnerIqn[0])
 	if err != nil {
 		return nil, fmt.Errorf("failed adding host to igroup %w", err)
 	}

cmd/vsphere-xcopy-volume-populator/internal/populator/mocks/storage_mock_client.go

@@ -17,28 +17,37 @@ type Populator interface {
 type LUN struct {
 	//Name is the volume name or just name in the storage system
 	Name string
-	// target's IQN
-	IQN string
-	// the volume handle as set by the CSI driver field spec.volumeHandle
-	VolumeHandle string
 	// naa
 	NAA string
 	// SerialNumber is a representation of the disk. With combination of the
 	// vendor ID it should ve globally unique and can be identified by udev, usually
 	// under /dev/disk/by-id/ with some prefix or postfix, depending on the udev rule
 	// and can also be found by lsblk -o name,serial
 	SerialNumber string
+	// target's IQN
+	IQN string
+	// Storage provider ID in hex
+	ProviderID string
+	// the volume handle as set by the CSI driver field spec.volumeHandle
+	VolumeHandle string
+	//  Logical device ID of the volume
+	LDeviceID string
+	// Storage device Serial Number
+	StorageSerialNumber string
+	// Storage Protocol
+	Protocol string
 }
 
 // VMDisk is the target VMDisk in vmware
 type VMDisk struct {
 	VMName     string
 	Datacenter string
 	VmdkFile   string
+	VmnameDir  string
 }
 
 func (d *VMDisk) Path() string {
-	return fmt.Sprintf("/vmfs/volumes/%s/%s/%s", d.Datacenter, d.VMName, d.VmdkFile)
+	return fmt.Sprintf("/vmfs/volumes/%s/%s/%s", d.Datacenter, d.VmnameDir, d.VmdkFile)
 }
 
 func ParseVmdkPath(vmdkPath string) (VMDisk, error) {
@@ -48,10 +57,14 @@ func ParseVmdkPath(vmdkPath string) (VMDisk, error) {
 	}
 	datastore := strings.TrimPrefix(parts[0], "[")
 	pathParts := strings.SplitN(parts[1], "/", 2)
+
 	if len(pathParts) != 2 {
 		return VMDisk{}, fmt.Errorf("Invalid vmdkPath %q, should be '[datastore] vmname/vmname.vmdk'", vmdkPath)
 	}
-	vmname := pathParts[0]
+
+	vmname_dir := pathParts[0]
 	vmdk := pathParts[1]
-	return VMDisk{VMName: vmname, Datacenter: datastore, VmdkFile: vmdk}, nil
+	vmdkParts := strings.SplitN(vmdk, ".", 2)
+	vmname_sub := vmdkParts[0]
+	return VMDisk{VMName: vmname_sub, Datacenter: datastore, VmdkFile: vmdk, VmnameDir: vmname_dir}, nil
 }

cmd/vsphere-xcopy-volume-populator/internal/populator/populate.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"math/big"
 	"slices"
+	"strings"
 	"time"
 
 	"github.com/kubev2v/forklift/cmd/vsphere-xcopy-volume-populator/internal/vmware"
@@ -59,20 +60,46 @@ func (p *RemoteEsxcliPopulator) Populate(sourceVMDKFile string, volumeHandle str
 	if err != nil {
 		return err
 	}
-	esxIQN := ""
+	uniqueUIDs := make(map[string]bool)
+	hbaUIDs := []string{}
+
+	// Print the adapter information for debugging
+	for i, val := range r {
+		klog.Infof("Adapter [%d]: %+v", i, val)
+		for key, field := range val {
+			klog.Infof("  %s: %v", key, field)
+		}
+	}
+
 	for _, a := range r {
-		// get the first adapter iqn
-		iqnValue, ok := a["UID"]
-		if !ok || len(iqnValue) == 0 {
-			return fmt.Errorf("failed to extract the IQN from the adapter item%s", a)
+		driver, hasDriver := a["Driver"]
+		linkState, hasLink := a["LinkState"]
+		uid, hasUID := a["UID"]
+
+		if !hasDriver || !hasLink || !hasUID || len(driver) == 0 || len(linkState) == 0 || len(uid) == 0 {
+			continue
+		}
+
+		drv := driver[0]
+		link := linkState[0]
+		id := uid[0]
+
+		// Check if the UID is FC, iSCSI or NVMe-oF
+		isTargetUID := strings.HasPrefix(id, "fc.") || strings.HasPrefix(id, "iqn.") || strings.HasPrefix(id, "nqn.")
+
+		if link == "link-up" && isTargetUID {
+			if _, exists := uniqueUIDs[id]; !exists {
+				uniqueUIDs[id] = true
+				hbaUIDs = append(hbaUIDs, id)
+				klog.Infof("Storage Adapter UID: %s (Driver: %s)", id, drv)
+			}
 		}
-		esxIQN = iqnValue[0]
-		klog.Infof("iSCSI adapter IQN %s", esxIQN)
 	}
 
-	mappingContext, err := p.StorageApi.EnsureClonnerIgroup(xcopyInitiatorGroup, esxIQN)
+	mappingContext, err := p.StorageApi.EnsureClonnerIgroup(xcopyInitiatorGroup, hbaUIDs)
+
 	if err != nil {
-		return fmt.Errorf("failed to add the ESX IQN %s to the initiator group %w", esxIQN, err)
+		return fmt.Errorf("failed to add the ESX HBA UID %s to the initiator group %w", hbaUIDs, err)
 	}
 
 	lun, err := p.StorageApi.ResolveVolumeHandleToLUN(volumeHandle)

cmd/vsphere-xcopy-volume-populator/internal/populator/remote_esxcli.go

@@ -10,7 +10,7 @@ type MappingContext map[string]any
 
 type StorageMapper interface {
 	// EnsureClonnerIgroup creates or updates an initiator group with the clonnerIqn
-	EnsureClonnerIgroup(initiatorGroup string, clonnerIqn string) (MappingContext, error)
+	EnsureClonnerIgroup(initiatorGroup string, clonnerIqn []string) (MappingContext, error)
 	// Map is responsible to mapping an initiator group to a LUN
 	Map(initatorGroup string, targetLUN LUN, context MappingContext) (LUN, error)
 	// UnMap is responsible to unmapping an initiator group from a LUN

cmd/vsphere-xcopy-volume-populator/internal/populator/storage.go

@@ -21,8 +21,8 @@ func NewPrimera3ParClonner(storageHostname, storageUsername, storagePassword str
 }
 
 // EnsureClonnerIgroup creates or update an initiator group with the clonnerIqn
-func (c *Primera3ParClonner) EnsureClonnerIgroup(initiatorGroup string, clonnerIqn string) (populator.MappingContext, error) {
-	hostName, err := c.client.EnsureHostWithIqn(clonnerIqn)
+func (c *Primera3ParClonner) EnsureClonnerIgroup(initiatorGroup string, clonnerIqn []string) (populator.MappingContext, error) {
+	hostName, err := c.client.EnsureHostWithIqn(clonnerIqn[0])
 	if err != nil {
 		return nil, fmt.Errorf("failed to ensure host with IQN: %w", err)
 	}

cmd/vsphere-xcopy-volume-populator/internal/primera3par/clonner.go

@@ -20,7 +20,7 @@ func TestPrimera3ParClonner(t *testing.T) {
 	mockClient.Volumes[targetLUN.Name] = targetLUN
 
 	t.Run("Ensure Clonner Igroup", func(t *testing.T) {
-		_, err := clonner.EnsureClonnerIgroup(initiatorGroup, targetLUN.IQN)
+		_, err := clonner.EnsureClonnerIgroup(initiatorGroup, []string{targetLUN.IQN})
 		assert.NoError(t, err, "Expected no error when ensuring Clonner Igroup")
 		_, hostExists := mockClient.Hosts["mock-host-"+targetLUN.IQN]
 		assert.True(t, hostExists, "Expected host to exist")