feat: create blob container using management api by default

fix: dataplane api issue

fix

fix

swtich to cloud provider lib interface

fix: retry if CreateContainer failed with container being deleted

fix

retry when create container failed

enable cloud provider backoff by default

support management api for delete and get container operations

add more validation

add useDataPlaneAPI e2e test

add unit tests

Author: andyzhangx

docs/driver-parameters.md

@@ -20,6 +20,7 @@ allowBlobPublicAccess | Allow or disallow public access to all blobs or containe
 storageEndpointSuffix | specify Azure storage endpoint suffix | `core.windows.net`, `core.chinacloudapi.cn`, etc | No | if empty, driver will use default storage endpoint suffix according to cloud environment
 tags | [tags](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/tag-resources) would be created in newly created storage account | tag format: 'foo=aaa,bar=bbb' | No | ""
 matchTags | whether matching tags when driver tries to find a suitable storage account | `true`,`false` | No | `false`
+useDataPlaneAPI | specify whether use data plane API for blob container create/delete, this could solve the SRP API throltting issue since data plane API has almost no limit, while it would fail when there is firewall or vnet setting on storage account | `true`,`false` | No | `false`
 --- | **Following parameters are only for blobfuse** | --- | --- |
 subscriptionID | specify Azure subscription ID in which blob storage directory will be created | Azure subscription ID | No | if not empty, `resourceGroup` must be provided
 storeAccountKey | whether store account key to k8s secret <br><br> Note:  <br> `false` means driver would leverage kubelet identity to get account key | `true`,`false` | No | `true`

pkg/blob/azure.go

@@ -116,6 +116,7 @@ func getCloudProvider(kubeconfig, nodeID, secretName, secretNamespace, userAgent
 		}
 	} else {
 		config.UserAgent = userAgent
+		config.CloudProviderBackoff = true
 		if err = az.InitializeCloudFromConfig(config, fromSecret, false); err != nil {
 			klog.Warningf("InitializeCloudFromConfig failed with error: %v", err)
 		}

pkg/blob/blob.go

@@ -23,10 +23,11 @@ import (
 	"sync"
 	"time"
 
-	"golang.org/x/net/context"
-
+	azstorage "github.com/Azure/azure-sdk-for-go/storage"
+	az "github.com/Azure/go-autorest/autorest/azure"
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/pborman/uuid"
+	"golang.org/x/net/context"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -88,17 +89,19 @@ const (
 	vnetNameField                = "vnetname"
 	subnetNameField              = "subnetname"
 	mountPermissionsField        = "mountpermissions"
+	useDataPlaneAPIField         = "usedataplaneapi"
 
 	// See https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names
 	containerNameMinLength = 3
 	containerNameMaxLength = 63
 
-	accountNotProvisioned = "StorageAccountIsNotProvisioned"
-	tooManyRequests       = "TooManyRequests"
-	clientThrottled       = "client throttled"
-	containerBeingDeleted = "ContainerBeingDeleted"
-	statusCodeNotFound    = "StatusCode=404"
-	httpCodeNotFound      = "HTTPStatusCode: 404"
+	accountNotProvisioned                   = "StorageAccountIsNotProvisioned"
+	tooManyRequests                         = "TooManyRequests"
+	clientThrottled                         = "client throttled"
+	containerBeingDeletedDataplaneAPIError  = "ContainerBeingDeleted"
+	containerBeingDeletedManagementAPIError = "container is being deleted"
+	statusCodeNotFound                      = "StatusCode=404"
+	httpCodeNotFound                        = "HTTPStatusCode: 404"
 
 	// containerMaxSize is the max size of the blob container. See https://docs.microsoft.com/en-us/azure/storage/blobs/scalability-targets#scale-targets-for-blob-storage
 	containerMaxSize = 100 * util.TiB
@@ -116,7 +119,7 @@ const (
 
 var (
 	supportedProtocolList = []string{fuse, nfs}
-	retriableErrors       = []string{accountNotProvisioned, tooManyRequests, statusCodeNotFound, containerBeingDeleted, clientThrottled}
+	retriableErrors       = []string{accountNotProvisioned, tooManyRequests, statusCodeNotFound, containerBeingDeletedDataplaneAPIError, containerBeingDeletedManagementAPIError, clientThrottled}
 )
 
 // DriverOptions defines driver parameters specified in driver deployment
@@ -166,6 +169,8 @@ type Driver struct {
 	subnetLockMap *util.LockMap
 	// a map storing all volumes created by this driver <volumeName, accountName>
 	volMap sync.Map
+	// a map storing all volumes using data plane API <volumeID, "">, <accountName, "">
+	dataPlaneAPIVolMap sync.Map
 	// a timed cache storing account search history (solve account list throttling issue)
 	accountSearchCache *azcache.TimedCache
 }
@@ -625,6 +630,23 @@ func getStorageAccount(secrets map[string]string) (string, string, error) {
 	return accountName, accountKey, nil
 }
 
+func getContainerReference(containerName string, secrets map[string]string, env az.Environment) (*azstorage.Container, error) {
+	accountName, accountKey, rerr := getStorageAccount(secrets)
+	if rerr != nil {
+		return nil, rerr
+	}
+	client, err := azstorage.NewBasicClientOnSovereignCloud(accountName, accountKey, env)
+	if err != nil {
+		return nil, err
+	}
+	blobClient := client.GetBlobService()
+	container := blobClient.GetContainerReference(containerName)
+	if container == nil {
+		return nil, fmt.Errorf("ContainerReference of %s is nil", containerName)
+	}
+	return container, nil
+}
+
 func setAzureCredentials(kubeClient kubernetes.Interface, accountName, accountKey, secretNamespace string) (string, error) {
 	if kubeClient == nil {
 		klog.Warningf("could not create secret: kubeClient is nil")
@@ -710,6 +732,14 @@ func (d *Driver) getSubnetResourceID() string {
 	return fmt.Sprintf(subnetTemplate, subsID, rg, d.cloud.VnetName, d.cloud.SubnetName)
 }
 
+func (d *Driver) useDataPlaneAPI(volumeID, accountName string) bool {
+	_, useDataPlaneAPI := d.dataPlaneAPIVolMap.Load(volumeID)
+	if !useDataPlaneAPI {
+		_, useDataPlaneAPI = d.dataPlaneAPIVolMap.Load(accountName)
+	}
+	return useDataPlaneAPI
+}
+
 // appendDefaultMountOptions return mount options combined with mountOptions and defaultMountOptions
 func appendDefaultMountOptions(mountOptions []string, tmpPath, containerName string) []string {
 	var defaultMountOptions = map[string]string{
@@ -766,3 +796,10 @@ func chmodIfPermissionMismatch(targetPath string, mode os.FileMode) error {
 	}
 	return nil
 }
+
+func createStorageAccountSecret(account, key string) map[string]string {
+	secret := make(map[string]string)
+	secret[defaultSecretAccountName] = account
+	secret[defaultSecretAccountKey] = key
+	return secret
+}

pkg/blob/blob_test.go

@@ -943,3 +943,10 @@ func getWorkDirPath(dir string) (string, error) {
 	}
 	return fmt.Sprintf("%s%c%s", path, os.PathSeparator, dir), nil
 }
+
+func TestCreateStorageAccountSecret(t *testing.T) {
+	result := createStorageAccountSecret("TestAccountName", "TestAccountKey")
+	if result[defaultSecretAccountName] != "TestAccountName" || result[defaultSecretAccountKey] != "TestAccountKey" {
+		t.Errorf("Expected account name(%s), Actual account name(%s); Expected account key(%s), Actual account key(%s)", "TestAccountName", result[defaultSecretAccountName], "TestAccountKey", result[defaultSecretAccountKey])
+	}
+}