Squashed commit of the following:

commit 92aa40e
Author: weizhichen <[email protected]>
Date:   Fri Mar 24 00:38:44 2023 +0000

    fix ut

commit f1238c6
Author: weizhichen <[email protected]>
Date:   Wed Mar 22 07:57:01 2023 +0000

    fix ut

commit 114cdaf
Author: weizhichen <[email protected]>
Date:   Wed Mar 22 07:46:33 2023 +0000

    fix ut

commit 67440cd
Author: weizhichen <[email protected]>
Date:   Wed Mar 22 07:14:27 2023 +0000

    fix gomod

commit a2144d9
Author: weizhichen <[email protected]>
Date:   Wed Mar 22 07:08:42 2023 +0000

    fix ut

commit 411a3d4
Author: weizhichen <[email protected]>
Date:   Wed Mar 22 06:46:53 2023 +0000

    fix

commit 7c9a5c9
Author: weizhichen <[email protected]>
Date:   Tue Mar 21 06:31:42 2023 +0000

    update vendor

commit fcc9364
Author: weizhichen <[email protected]>
Date:   Tue Mar 21 03:45:44 2023 +0000

    fix ut

commit 9dcc6fa
Author: weizhichen <[email protected]>
Date:   Mon Mar 20 13:52:54 2023 +0000

    fix

commit 0900961
Author: weizhichen <[email protected]>
Date:   Mon Mar 20 11:18:01 2023 +0000

    add ut

commit cd9c5dd
Author: weizhichen <[email protected]>
Date:   Mon Mar 20 06:08:45 2023 +0000

    Revert "fix: add workaround for blobfuse2 mount issue"

    This reverts commit fa9f8a5.

commit 10e8617
Author: weizhichen <[email protected]>
Date:   Mon Mar 20 06:08:28 2023 +0000

    fix : wait for blobfuse2 mount

Author: cvvz

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/Azure/azure-sdk-for-go v67.3.0+incompatible
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
 	github.com/Azure/go-autorest/autorest v0.11.28
-	github.com/Azure/go-autorest/autorest/adal v0.9.22
+	github.com/Azure/go-autorest/autorest/adal v0.9.23
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
 	github.com/container-storage-interface/spec v1.7.0
 	github.com/gofrs/uuid v4.2.0+incompatible // indirect
@@ -17,7 +17,7 @@ require (
 	github.com/onsi/gomega v1.25.0
 	github.com/pborman/uuid v1.2.0
 	github.com/pelletier/go-toml v1.9.4
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.8.2
 	golang.org/x/net v0.7.0
 	google.golang.org/grpc v1.49.0
 	google.golang.org/protobuf v1.28.1
@@ -71,7 +71,7 @@ require (
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
@@ -109,7 +109,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.10.0 // indirect
 	go.opentelemetry.io/otel/trace v1.10.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
-	golang.org/x/crypto v0.5.0 // indirect
+	golang.org/x/crypto v0.6.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
 	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/term v0.5.0 // indirect

go.sum

@@ -51,8 +51,8 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW
 github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM=
 github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA=
 github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
-github.com/Azure/go-autorest/autorest/adal v0.9.22 h1:/GblQdIudfEM3AWWZ0mrYJQSd7JS4S/Mbzh6F0ov0Xc=
-github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk=
+github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8=
+github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c=
 github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
@@ -184,8 +184,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU=
-github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
+github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
@@ -413,8 +413,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -464,8 +464,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
-golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
+golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -540,6 +540,7 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=

pkg/blob/nodeserver.go

@@ -46,6 +46,11 @@ import (
 	mount_azure_blob "sigs.k8s.io/blob-csi-driver/pkg/blobfuse-proxy/pb"
 )
 
+const (
+	waitForMountInterval = 20 * time.Millisecond
+	waitForMountTimeout  = 3 * time.Second
+)
+
 type MountClient struct {
 	service mount_azure_blob.MountServiceClient
 }
@@ -181,11 +186,7 @@ func (d *Driver) mountBlobfuseInsideDriver(args string, protocol string, authEnv
 	cmd.Env = append(os.Environ(), authEnv...)
 	output, err := cmd.CombinedOutput()
 	klog.V(2).Infof("mount output: %s\n", string(output))
-	if err == nil && protocol == Fuse2 {
-		// todo: remove this when https://github.com/Azure/azure-storage-fuse/issues/1079 is fixed
-		klog.V(2).Infof("sleep 2s, waiting for blobfuse2 mount complete")
-		time.Sleep(2 * time.Second)
-	}
+
 	return string(output), err
 }
 
@@ -406,6 +407,12 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
 		return nil, err
 	}
 
+	// wait a few seconds to make sure blobfuse mount is successful
+	// please refer to https://github.com/Azure/azure-storage-fuse/pull/1088 for more details
+	if err := waitForMount(targetPath, waitForMountInterval, waitForMountTimeout); err != nil {
+		return nil, fmt.Errorf("failed to wait for mount: %w", err)
+	}
+
 	klog.V(2).Infof("volume(%s) mount on %q succeeded", volumeID, targetPath)
 	return &csi.NodeStageVolumeResponse{}, nil
 }
@@ -575,3 +582,24 @@ func (d *Driver) ensureMountPoint(target string, perm os.FileMode) (bool, error)
 	}
 	return !notMnt, nil
 }
+
+func waitForMount(path string, intervel, timeout time.Duration) error {
+	timeAfter := time.After(timeout)
+	timeTick := time.Tick(intervel)
+
+	for {
+		select {
+		case <-timeTick:
+			notMount, err := mount.New("").IsLikelyNotMountPoint(path)
+			if err != nil {
+				return err
+			}
+			if !notMount {
+				klog.V(2).Infof("blobfuse mount at %s success", path)
+				return nil
+			}
+		case <-timeAfter:
+			return fmt.Errorf("timeout waiting for mount %s", path)
+		}
+	}
+}

pkg/blob/nodeserver_test.go

@@ -24,8 +24,10 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"strings"
 	"syscall"
 	"testing"
+	"time"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
@@ -754,3 +756,59 @@ func TestMountBlobfuseInsideDriver(t *testing.T) {
 	// the error should be of type exec.ExitError
 	assert.NotNil(t, err)
 }
+
+func Test_waitForMount(t *testing.T) {
+	if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
+		t.Skip("Skipping test on ", runtime.GOOS)
+	}
+
+	tmpDir, err := os.MkdirTemp("", "")
+	assert.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	type args struct {
+		path     string
+		intervel time.Duration
+		timeout  time.Duration
+	}
+
+	tests := []struct {
+		name      string
+		args      args
+		wantErr   bool
+		subErrMsg string
+	}{
+		{
+			name: "test error timeout",
+			args: args{
+				path:     tmpDir,
+				intervel: 1 * time.Millisecond,
+				timeout:  10 * time.Millisecond,
+			},
+			wantErr:   true,
+			subErrMsg: "timeout",
+		},
+		{
+			name: "test error no such file or directory",
+			args: args{
+				path:     "/no/such/file/or/directory",
+				intervel: 1 * time.Millisecond,
+				timeout:  10 * time.Millisecond,
+			},
+			wantErr:   true,
+			subErrMsg: "no such file or directory",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := waitForMount(tt.args.path, tt.args.intervel, tt.args.timeout)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("waitForMount() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if err != nil && !strings.Contains(err.Error(), tt.subErrMsg) {
+				t.Errorf("waitForMount() error = %v, wantErr %v", err, tt.subErrMsg)
+			}
+		})
+	}
+}

pkg/blobfuse-proxy/server/server.go

@@ -23,7 +23,6 @@ import (
 	"os/exec"
 	"strings"
 	"sync"
-	"time"
 
 	"google.golang.org/grpc"
 	"k8s.io/klog/v2"
@@ -96,11 +95,6 @@ func (server *MountServer) MountAzureBlob(ctx context.Context,
 	if err != nil {
 		return &result, fmt.Errorf("%w %s", err, result.Output)
 	}
-	if protocol == blob.Fuse2 || server.blobfuseVersion == BlobfuseV2 {
-		// todo: remove this when https://github.com/Azure/azure-storage-fuse/issues/1079 is fixed
-		klog.V(2).Infof("sleep 2s, waiting for blobfuse2 mount complete\n")
-		time.Sleep(2 * time.Second)
-	}
 	return &result, nil
 }