Merge pull request #7 from Seagate/fix/multipath

fix: refresh dm paths, increase attempts, 10s delay plus debug

Author: jskazinski

iscsi/iscsi.go

@@ -16,7 +16,11 @@ import (
 	"time"
 )
 
-const defaultPort = "3260"
+const (
+	defaultPort          = "3260"
+	maxMultipathAttempts = 10
+	multipathDelay       = 10
+)
 
 var (
 	debug           *log.Logger
@@ -158,7 +162,7 @@ func waitForPathToExistImpl(devicePath *string, maxRetries, intervalSeconds int,
 		err = nil
 		if deviceTransport == "tcp" {
 			_, err = osStat(*devicePath)
-			debug.Printf("os stat device: exist %v device %v", !os.IsNotExist(err), *devicePath)
+			debug.Printf("[%d] os stat device: exist %v device %v", i, !os.IsNotExist(err), *devicePath)
 			if err != nil && !os.IsNotExist(err) {
 				debug.Printf("Error attempting to stat device: %s", err.Error())
 				return false, err
@@ -167,7 +171,7 @@ func waitForPathToExistImpl(devicePath *string, maxRetries, intervalSeconds int,
 			}
 
 		} else {
-			debug.Printf("filepathGlob: %s", *devicePath)
+			debug.Printf("[%d] filepathGlob: %s", i, *devicePath)
 			fpath, _ := filepathGlob(*devicePath)
 			if fpath == nil {
 				err = os.ErrNotExist
@@ -187,6 +191,7 @@ func waitForPathToExistImpl(devicePath *string, maxRetries, intervalSeconds int,
 		}
 		time.Sleep(time.Second * time.Duration(intervalSeconds))
 	}
+	debug.Printf("device does NOT exist [%d*%ds] (%v)", maxRetries, intervalSeconds, *devicePath)
 	return false, err
 }
 
@@ -211,37 +216,42 @@ func getMultipathDisk(path string) (string, error) {
 	// check to see if any have an entry under /sys/block/dm-*/slaves matching
 	// the device the symlink was pointing at
 	attempts := 1
-	dmPaths, err := filepath.Glob("/sys/block/dm-*")
-	for attempts < 4 {
-		debug.Printf("[%d] dmPaths=%v", attempts, dmPaths)
-		if err != nil {
-			debug.Printf("Glob error: %s", err)
-			return "", err
-		}
-		if len(dmPaths) > 0 {
-			break
-		}
-		time.Sleep(1 * time.Second)
-		attempts++
+	var dmPaths []string
+
+	for attempts < (maxMultipathAttempts + 1) {
+		var err error
 		dmPaths, err = filepath.Glob("/sys/block/dm-*")
-	}
-	for _, dmPath := range dmPaths {
-		sdevices, err := filepath.Glob(filepath.Join(dmPath, "slaves", "*"))
-		debug.Printf("dmPath=%v/slaves/*, sdevices=%v", dmPath, sdevices)
+		debug.Printf("[%d] refresh dmPaths [%d] %v", attempts, len(dmPaths), dmPaths)
 		if err != nil {
 			debug.Printf("Glob error: %s", err)
+			return "", err
 		}
-		for _, spath := range sdevices {
-			s := filepath.Base(spath)
-			debug.Printf("Basepath: %s", s)
-			if sdevice == s {
-				// We've found a matching entry, return the path for the
-				// dm-* device it was found under
-				p := filepath.Join("/dev", filepath.Base(dmPath))
-				debug.Printf("Found matching multipath device (%s) under dm-* device path (%s), (%v)", sdevice, dmPath, p)
-				return p, nil
+
+		for _, dmPath := range dmPaths {
+			sdevices, err := filepath.Glob(filepath.Join(dmPath, "slaves", "*"))
+			// debug.Printf(".. dmPath=%v, sdevices=[%d]%v", dmPath, len(sdevices), sdevices)
+			if err != nil {
+				debug.Printf("Glob error: %s", err)
+			}
+			for _, spath := range sdevices {
+				s := filepath.Base(spath)
+				// debug.Printf(".. Basepath: %s", s)
+				if sdevice == s {
+					// We've found a matching entry, return the path for the
+					// dm-* device it was found under
+					p := filepath.Join("/dev", filepath.Base(dmPath))
+					debug.Printf("Found matching multipath device (%s) under dm-* device path (%s) p (%v)", sdevice, dmPath, p)
+					return p, nil
+				}
 			}
 		}
+
+		// Force a reload of all existing multipath maps
+		output, err := execCommand("multipath", "-r").CombinedOutput()
+		debug.Printf("## multipath -r: output=%v, err=%v", output, err)
+
+		time.Sleep(multipathDelay * time.Second)
+		attempts++
 	}
 	debug.Printf("Couldn't find dm-* path for path: %s, found non dm-* path: %s", path, devicePath)
 	return "", fmt.Errorf("couldn't find dm-* path for path: %s, found non dm-* path: %s", path, devicePath)
@@ -350,7 +360,7 @@ func Connect(c *Connector) (string, error) {
 	}
 
 	if lastErr != nil {
-		debug.Printf("Last error occured during iscsi init: \n%v", lastErr)
+		debug.Printf("Last error occurred during iscsi init: \n%v", lastErr)
 	}
 
 	for i, path := range devicePaths {
@@ -465,11 +475,14 @@ func PersistConnector(c *Connector, filePath string) error {
 	//file := path.Join("mnt", c.VolumeName+".json")
 	f, err := os.Create(filePath)
 	if err != nil {
+		debug.Printf("ERROR: creating iscsi persistence file %s: %s\n", filePath, err)
 		return fmt.Errorf("error creating iscsi persistence file %s: %s", filePath, err)
 	}
 	defer f.Close()
 	encoder := json.NewEncoder(f)
+	debug.Printf("ConnectorFromFile (write): file=%v, c=%+v\n", filePath, c)
 	if err = encoder.Encode(c); err != nil {
+		debug.Printf("ERROR: error encoding connector: %v\n", err)
 		return fmt.Errorf("error encoding connector: %v", err)
 	}
 	return nil
@@ -490,7 +503,7 @@ func GetConnectorFromFile(filePath string) (*Connector, error) {
 		return &Connector{}, err
 	}
 
-	debug.Printf("data: %+v\n", data)
+	debug.Printf("ConnectorFromFile (read): file=%s, %+v\n", filePath, data)
 
 	return &data, nil
 

iscsi/multipath.go

@@ -108,14 +108,21 @@ func RemoveAndClear(device string) error {
 	debug.Printf("Remove and clear multipath device (%s)\n", device)
 
 	// Remove device-mapper logical device
+	debug.Printf("dmsetup remove -f %s\n", device)
 	if output, err := execCommand("dmsetup", "remove", "-f", device).CombinedOutput(); err != nil {
+		exitErr, ok := err.(*exec.ExitError)
+		if ok && exitErr.ExitCode() != 0 {
+			debug.Printf("ERROR: dmsetup remove -f ExitCode: %d, err=%v\n", exitErr.ExitCode(), err)
+		}
+
 		return fmt.Errorf("device-mapper could not remove device: %s (%v)", output, err)
 	}
 
 	// Clear out device-mapper logical device if it still exists
 	if _, e := os.Stat(device); os.IsNotExist(e) {
 		debug.Printf("device-mapper logical device %q was removed\n", device)
 	} else {
+		debug.Printf("dmsetup clear %s\n", device)
 		if output, err := execCommand("dmsetup", "clear", device).CombinedOutput(); err != nil {
 			return fmt.Errorf("device-mapper could not clear device: %s (%v)", output, err)
 		}