[enhance] attach device with 5 retry; wait /device/block 0.5s -> 5s

Signed-off-by: zhuangbowei.zbw <zhuangbowei.zbw@alibaba-inc.com>

Author: WaberZhuang

pkg/snapshot/overlay.go

@@ -613,7 +613,7 @@ func (o *snapshotter) createMountPoint(ctx context.Context, kind snapshots.Kind,
 			}
 			log.G(ctx).Debugf("attachAndMountBlockDevice (obdID: %s, writeType: %s, fsType %s, targetPath: %s)",
 				obdID, writeType, fsType, o.overlaybdTargetPath(obdID))
-			if err = o.attachAndMountBlockDevice(ctx, obdID, writeType, fsType, parent == ""); err != nil {
+			if err = o.attachWithRetry(ctx, obdID, writeType, fsType, parent == ""); err != nil {
 				log.G(ctx).Errorf("%v", err)
 				return nil, fmt.Errorf("failed to attach and mount for snapshot %v: %w", obdID, err)
 			}
@@ -773,7 +773,7 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) (_ []mount.Mount,
 					fsType = o.defaultFsType
 				}
 			}
-			if err := o.attachAndMountBlockDevice(ctx, parentID, RoDir, fsType, false); err != nil {
+			if err := o.attachWithRetry(ctx, parentID, RoDir, fsType, false); err != nil {
 				return nil, fmt.Errorf("failed to attach and mount for snapshot %v: %w", key, err)
 			}
 			return o.basedOnBlockDeviceMount(ctx, s, RoDir)

pkg/snapshot/storage.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/binary"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -50,7 +51,7 @@ import (
 )
 
 const (
-	maxAttachAttempts = 50
+	maxAttachAttempts = 500
 
 	// hba number used to create tcmu devices in configfs
 	// all overlaybd devices are configured in /sys/kernel/config/target/core/user_999999999/
@@ -478,6 +479,35 @@ func (o *snapshotter) attachAndMountBlockDevice(ctx context.Context, snID string
 			return nil
 		}
 	}
+	return attachRetryError{lastErr}
+}
+
+type attachRetryError struct {
+	err error
+}
+
+func (e attachRetryError) Error() string {
+	return e.err.Error()
+}
+
+func (o *snapshotter) attachWithRetry(ctx context.Context, snID string, writable string, fsType string, mkfs bool) error {
+	maxRetry := 5
+	var lastErr error
+	for retry := 0; retry < maxRetry; retry++ {
+		lastErr = o.attachAndMountBlockDevice(ctx, snID, writable, fsType, mkfs)
+		if errors.As(lastErr, &attachRetryError{}) {
+			log.G(ctx).Warnf("attach failed, retrying(%d/%d)... snID: %s, err: %v", retry+1, maxRetry, snID, lastErr)
+			time.Sleep(1 * time.Second) // Wait for 1 second before retrying
+			continue
+		} else if lastErr != nil {
+			log.G(ctx).Errorf("attach failed, snID: %s, err: %v", snID, lastErr)
+			return lastErr
+		} else {
+			log.G(ctx).Infof("attach success, snID: %s", snID)
+			return nil
+		}
+	}
+	log.G(ctx).Errorf("attach failed, max retry reached")
 	return lastErr
 }