Merge pull request #414 from yussufsh/test

Improve hosts scan time and do not skip if any read error

Author: k8s-ci-robot

adhoc-controllers/controllers/nodeupdate_controller.go

@@ -19,11 +19,12 @@ package controllers
 import (
 	"context"
 	"fmt"
+
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/ibm-powervs-block-csi-driver/pkg/cloud"

go.mod

@@ -19,7 +19,6 @@ require (
 	k8s.io/api v0.26.1
 	k8s.io/apimachinery v0.26.1
 	k8s.io/client-go v1.26.1
-	k8s.io/klog v1.0.0
 	k8s.io/klog/v2 v2.100.1
 	k8s.io/kubernetes v1.26.1
 	k8s.io/mount-utils v0.26.1

go.sum

@@ -139,7 +139,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
-github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
@@ -1016,8 +1015,6 @@ k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4=
 k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU=
 k8s.io/component-helpers v0.26.1 h1:Y5h1OYUJTGyHZlSAsc7mcfNsWF08S/MlrQyF/vn93mU=
 k8s.io/component-helpers v0.26.1/go.mod h1:jxNTnHb1axLe93MyVuvKj9T/+f4nxBVrj/xf01/UNFk=
-k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
-k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
 k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=

pkg/device/device.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"k8s.io/klog/v2"
@@ -31,6 +32,7 @@ import (
 var (
 	lastCleanExecuted      time.Time
 	lastStaleCleanExecuted time.Time
+	scanLock               = &sync.Mutex{}
 )
 
 type LinuxDevice interface {
@@ -81,7 +83,8 @@ func (d *Device) Populate(needActivePath bool) error {
 		tmpPathname := "dm-" + result["Minor"]
 		uuid, err := getUUID(tmpPathname)
 		if err != nil {
-			return err
+			klog.Warning(err)
+			continue
 		}
 		tmpWWID := strings.TrimPrefix(uuid, "mpath-")
 		tmpWWN := tmpWWID[1:] // truncate scsi-id prefix
@@ -122,11 +125,6 @@ func (d *Device) DeleteDevice() (err error) {
 
 // CreateDevice: attach and create linux devices to host
 func (d *Device) CreateDevice() (err error) {
-	if err = scsiHostRescan(); err != nil {
-		return err
-	}
-	// wait for device to appear after rescan
-	time.Sleep(time.Second * 1)
 
 	if err = d.createLinuxDevice(); err != nil {
 		klog.Errorf("unable to create device for wwn %v", d.WWN)
@@ -139,6 +137,35 @@ func (d *Device) CreateDevice() (err error) {
 	return nil
 }
 
+// scsiHostRescanWithLock: scans all scsi hosts with locks
+// This is calling scsiHostRescan
+// which works in a way that only 1 scan will run at a time
+// and other requests will wait till the scan is complete
+// but will not scan again as it is already scanned.
+func scsiHostRescanWithLock() (err error) {
+	start := time.Now()
+	var scan bool = true
+	defer scanLock.Unlock()
+
+	for {
+		if scanLock.TryLock() {
+			if scan {
+				err = scsiHostRescan()
+			}
+			return err
+		} else {
+			if time.Since(start) > time.Minute {
+				// Scanning usually takes < 30s. If wait is more than a min then return.
+				return
+			}
+
+			// Already locked, wait for it to complete and don't scan again.
+			scan = false
+			time.Sleep(5 * time.Second)
+		}
+	}
+}
+
 // createLinuxDevice: attaches and creates a new linux device
 // Try device discovery; retry every 5 sec if no device found or have 0 slaves
 // In between checks we will try to cleanup:
@@ -151,6 +178,12 @@ func (d *Device) CreateDevice() (err error) {
 func (d *Device) createLinuxDevice() (err error) {
 	// Start a Countdown ticker
 	for i := 0; i <= 10; i++ {
+		if err = scsiHostRescanWithLock(); err != nil {
+			return err
+		}
+		// wait for device to appear after rescan
+		time.Sleep(time.Second * 1)
+
 		err := d.Populate(true)
 		if err != nil {
 			return err

pkg/device/multipath.go

@@ -25,7 +25,7 @@ import (
 	"sync"
 	"time"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 const (
@@ -375,7 +375,8 @@ OUTER:
 		wwidPath := scsiPath + f.Name() + "/device/wwid"
 		wwid, err := readFirstLine(wwidPath)
 		if err != nil {
-			return err
+			klog.Warning(err)
+			continue
 		}
 		entries := strings.Split(wwid, ".")
 		if len(entries) <= 1 {

pkg/driver/node.go

@@ -302,7 +302,7 @@ func (d *nodeService) deleteDevice(deviceName string) error {
 	}
 	dev := NewDevice(wwn)
 	if err := dev.Populate(false); err != nil {
-		return fmt.Errorf("failed to delete device %s: %v", deviceName, err)
+		return fmt.Errorf("failed to populate device %s: %v", deviceName, err)
 	}
 	if dev.GetMapper() == "" {
 		return fmt.Errorf("failed to find device %s mapper for wwn %s", deviceName, wwn)