Fix VF creation race by waiting for VF symlinks before switchdev

Attempting to unbind VFs or switch the PF to switchdev mode
before all VFs are ready causes race conditions, leading to
mode switch failures with "invalid argument" errors.

Signed-off-by: Ashok Pariya <ashok.pariya@ibm.com>

Author: ashokpariya0

pkg/host/internal/sriov/sriov.go

@@ -1154,6 +1154,58 @@ func (s *sriov) setEswitchModeAndNumVFs(pciAddr string, desiredEswitchMode strin
 	return fn(pciAddr, desiredEswitchMode, numVFs)
 }
 
+func (s *sriov) waitForVFLinks(pciAddr string, expectedNum int, maxTimeout time.Duration) error {
+	// VF destruction is asynchronous and safe to ignore
+	if expectedNum == 0 {
+		log.Log.V(2).Info("waitForVFLinks(): skipping wait for cleanup (expected 0 VFs) - async destroy", "device", pciAddr)
+		return nil
+	}
+
+	log.Log.V(2).Info("waitForVFLinks(): waiting for VF symlinks",
+		"device", pciAddr,
+		"expected", expectedNum,
+		"maxTimeout", maxTimeout)
+
+	start := time.Now()
+	sleepDuration := 500 * time.Millisecond
+
+	for time.Since(start) < maxTimeout {
+		vfAddrs, err := s.dputilsLib.GetVFList(pciAddr)
+		if err != nil {
+			log.Log.V(2).Info("waitForVFLinks(): GetVFList failed, retrying", "err", err)
+			time.Sleep(sleepDuration)
+			continue
+		}
+
+		current := len(vfAddrs)
+		if current >= expectedNum {
+			// Check all have physfn symlink
+			allReady := true
+			for _, vfAddr := range vfAddrs {
+				linkPath := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, vfAddr, "physfn")
+				if _, err := os.Lstat(linkPath); err != nil {
+					log.Log.V(2).Info("waitForVFLinks(): physfn symlink missing", "vf", vfAddr, "err", err)
+					allReady = false
+					break
+				}
+			}
+			if allReady {
+				log.Log.V(2).Info("waitForVFLinks(): all expected VF symlinks ready")
+				return nil
+			}
+		}
+
+		time.Sleep(sleepDuration)
+
+		sleepDuration = time.Duration(float64(sleepDuration) * 1.5)
+		if sleepDuration > 5*time.Second {
+			sleepDuration = 5 * time.Second
+		}
+	}
+
+	return fmt.Errorf("timeout waiting for %d VF symlinks on %s (max %v)", expectedNum, pciAddr, maxTimeout)
+}
+
 // setEswitchModeAndNumVFsMlx configures PF eSwitch and sriov_numvfs in the following order:
 // a. set eSwitchMode to legacy
 // b. set the desired number of Virtual Functions
@@ -1182,6 +1234,9 @@ func (s *sriov) setEswitchModeAndNumVFsMlx(pciAddr string, desiredEswitchMode st
 	if err := s.SetSriovNumVfs(pciAddr, numVFs); err != nil {
 		return err
 	}
+	if err := s.waitForVFLinks(pciAddr, numVFs, 120*time.Second); err != nil {
+		return err
+	}
 
 	if desiredEswitchMode == sriovnetworkv1.ESwithModeSwitchDev {
 		if err := s.unbindAllVFsOnPF(pciAddr); err != nil {

pkg/host/internal/sriov/sriov_test.go

@@ -478,7 +478,7 @@ var _ = Describe("SRIOV", func() {
 			hostMock.EXPECT().AddPersistPFNameUdevRule("0000:d8:00.0", "enp216s0f0np0").Return(nil)
 			hostMock.EXPECT().EnableHwTcOffload("enp216s0f0np0").Return(nil)
 			hostMock.EXPECT().GetDevlinkDeviceParam("0000:d8:00.0", "flow_steering_mode").Return("", syscall.EINVAL)
-			dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).Times(2)
+			dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).AnyTimes()
 			pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl)
 			netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil).Times(2)
 			netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false)
@@ -547,7 +547,7 @@ var _ = Describe("SRIOV", func() {
 			hostMock.EXPECT().AddPersistPFNameUdevRule("0000:d8:00.0", "enp216s0f0np0").Return(nil)
 			hostMock.EXPECT().EnableHwTcOffload("enp216s0f0np0").Return(nil)
 			hostMock.EXPECT().GetDevlinkDeviceParam("0000:d8:00.0", "flow_steering_mode").Return("", nil)
-			dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).Times(2)
+			dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).AnyTimes()
 			pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl)
 			netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil).Times(2)
 			netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false)
@@ -616,7 +616,7 @@ var _ = Describe("SRIOV", func() {
 			hostMock.EXPECT().AddPersistPFNameUdevRule("0000:d8:00.0", "enp216s0f0np0").Return(nil)
 			hostMock.EXPECT().EnableHwTcOffload("enp216s0f0np0").Return(nil)
 			hostMock.EXPECT().GetDevlinkDeviceParam("0000:d8:00.0", "flow_steering_mode").Return("smfs", nil)
-			dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).Times(2)
+			dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).AnyTimes()
 			pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl)
 			netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil).Times(2)
 			netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false)
@@ -685,7 +685,7 @@ var _ = Describe("SRIOV", func() {
 			hostMock.EXPECT().AddPersistPFNameUdevRule("0000:d8:00.0", "enp216s0f0np0").Return(nil)
 			hostMock.EXPECT().EnableHwTcOffload("enp216s0f0np0").Return(nil)
 			hostMock.EXPECT().GetDevlinkDeviceParam("0000:d8:00.0", "flow_steering_mode").Return("test", nil)
-			dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).Times(4)
+			dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).AnyTimes()
 			pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl)
 			netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil).Times(2)
 			netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false)