Skip to content

Commit fb1b499

Browse files
authored
Merge pull request #1169 from MartinForReal/shafan/token
Refactor: migrate blobclient to track2 sdk and fix ut cases
2 parents ec61673 + 5c8e59a commit fb1b499

File tree

8 files changed

+424
-145
lines changed

8 files changed

+424
-145
lines changed

go.mod

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,24 @@ toolchain go1.21.4
66

77
require (
88
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
9+
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1
910
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
11+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
12+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
13+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0
14+
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
1015
github.com/Azure/go-autorest/autorest v0.11.29
1116
github.com/Azure/go-autorest/autorest/adal v0.9.23
12-
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
1317
github.com/container-storage-interface/spec v1.8.0
1418
github.com/gofrs/uuid v4.4.0+incompatible // indirect
1519
github.com/golang/protobuf v1.5.3
16-
github.com/imdario/mergo v0.3.9 // indirect
1720
github.com/kubernetes-csi/csi-lib-utils v0.16.0
21+
github.com/onsi/ginkgo/v2 v2.14.0
1822
github.com/onsi/gomega v1.30.0
1923
github.com/pborman/uuid v1.2.1
2024
github.com/pelletier/go-toml v1.9.5
25+
github.com/pkg/errors v0.9.1
26+
github.com/satori/go.uuid v1.2.0
2127
github.com/stretchr/testify v1.8.4
2228
go.uber.org/mock v0.4.0
2329
golang.org/x/net v0.20.0
@@ -37,20 +43,16 @@ require (
3743
)
3844

3945
require (
40-
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1
4146
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect
4247
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.4.0 // indirect
4348
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 // indirect
4449
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.6.0 // indirect
45-
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
4650
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 // indirect
4751
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0 // indirect
48-
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
49-
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 // indirect
50-
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
5152
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
5253
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
5354
github.com/Azure/go-autorest/autorest/mocks v0.4.2 // indirect
55+
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
5456
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
5557
github.com/Azure/go-autorest/logger v0.2.1 // indirect
5658
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
@@ -88,6 +90,7 @@ require (
8890
github.com/google/uuid v1.5.0 // indirect
8991
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
9092
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
93+
github.com/imdario/mergo v0.3.9 // indirect
9194
github.com/inconshreveable/mousetrap v1.1.0 // indirect
9295
github.com/josharian/intern v1.0.0 // indirect
9396
github.com/json-iterator/go v1.1.12 // indirect
@@ -99,17 +102,14 @@ require (
99102
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
100103
github.com/modern-go/reflect2 v1.0.2 // indirect
101104
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
102-
github.com/onsi/ginkgo/v2 v2.14.0
103105
github.com/opencontainers/go-digest v1.0.0 // indirect
104106
github.com/opencontainers/selinux v1.11.0 // indirect
105107
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
106-
github.com/pkg/errors v0.9.1
107108
github.com/pmezard/go-difflib v1.0.0 // indirect
108109
github.com/prometheus/client_golang v1.16.0 // indirect
109110
github.com/prometheus/client_model v0.4.0 // indirect
110111
github.com/prometheus/common v0.44.0 // indirect
111112
github.com/prometheus/procfs v0.10.1 // indirect
112-
github.com/satori/go.uuid v1.2.0
113113
github.com/spf13/cobra v1.8.0 // indirect
114114
github.com/spf13/pflag v1.0.5 // indirect
115115
github.com/stoewer/go-strcase v1.2.0 // indirect

pkg/blob/blob.go

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import (
2727
"sync"
2828
"time"
2929

30-
"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2021-09-01/storage"
30+
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage"
3131
azstorage "github.com/Azure/azure-sdk-for-go/storage"
3232
az "github.com/Azure/go-autorest/autorest/azure"
3333
"github.com/container-storage-interface/spec/lib/go/csi"
@@ -44,6 +44,7 @@ import (
4444

4545
csicommon "sigs.k8s.io/blob-csi-driver/pkg/csi-common"
4646
"sigs.k8s.io/blob-csi-driver/pkg/util"
47+
"sigs.k8s.io/cloud-provider-azure/pkg/azclient"
4748
azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
4849
"sigs.k8s.io/cloud-provider-azure/pkg/provider"
4950
azure "sigs.k8s.io/cloud-provider-azure/pkg/provider"
@@ -203,6 +204,8 @@ type Driver struct {
203204
csicommon.CSIDriver
204205

205206
cloud *azure.Cloud
207+
clientFactory azclient.ClientFactory
208+
networkClientFactory azclient.ClientFactory
206209
KubeClient kubernetes.Interface
207210
blobfuseProxyEndpoint string
208211
// enableBlobMockMount is only for testing, DO NOT set as true in non-testing scenario
@@ -266,6 +269,10 @@ func NewDriver(options *DriverOptions, kubeClient kubernetes.Interface, cloud *p
266269
d.Name = options.DriverName
267270
d.Version = driverVersion
268271
d.NodeID = options.NodeID
272+
if d.cloud != nil {
273+
d.clientFactory = d.cloud.ComputeClientFactory
274+
d.networkClientFactory = d.cloud.NetworkClientFactory
275+
}
269276

270277
var err error
271278
getter := func(key string) (interface{}, error) { return nil, nil }
@@ -761,7 +768,7 @@ func isSupportedAccessTier(accessTier string) bool {
761768
if accessTier == "" {
762769
return true
763770
}
764-
for _, tier := range storage.PossibleAccessTierValues() {
771+
for _, tier := range armstorage.PossibleAccessTierValues() {
765772
if accessTier == string(tier) {
766773
return true
767774
}
@@ -899,11 +906,11 @@ func (d *Driver) GetStorageAccesskey(ctx context.Context, accountOptions *azure.
899906
// GetInfoFromSecret get info from k8s secret
900907
// return <accountName, accountKey, accountSasToken, msiSecret, spnClientSecret, spnClientID, spnTenantID, error>
901908
func (d *Driver) GetInfoFromSecret(ctx context.Context, secretName, secretNamespace string) (string, string, string, string, string, string, string, error) {
902-
if d.cloud.KubeClient == nil {
909+
if d.KubeClient == nil {
903910
return "", "", "", "", "", "", "", fmt.Errorf("could not get account key from secret(%s): KubeClient is nil", secretName)
904911
}
905912

906-
secret, err := d.cloud.KubeClient.CoreV1().Secrets(secretNamespace).Get(ctx, secretName, metav1.GetOptions{})
913+
secret, err := d.KubeClient.CoreV1().Secrets(secretNamespace).Get(ctx, secretName, metav1.GetOptions{})
907914
if err != nil {
908915
return "", "", "", "", "", "", "", fmt.Errorf("could not get secret(%v): %w", secretName, err)
909916
}

pkg/blob/blob_test.go

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -957,7 +957,7 @@ func TestGetContainerReference(t *testing.T) {
957957

958958
d := NewFakeDriver()
959959
d.cloud = azure.GetTestCloud(gomock.NewController(t))
960-
d.cloud.KubeClient = fake.NewSimpleClientset()
960+
d.KubeClient = fake.NewSimpleClientset()
961961

962962
for _, tc := range testCases {
963963
t.Run(tc.name, func(t *testing.T) {
@@ -1076,8 +1076,7 @@ func TestGetStorageAccesskey(t *testing.T) {
10761076
},
10771077
}
10781078
d := NewFakeDriver()
1079-
d.cloud = &azure.Cloud{}
1080-
d.cloud.KubeClient = fake.NewSimpleClientset()
1079+
d.KubeClient = fake.NewSimpleClientset()
10811080
secret := &v1api.Secret{
10821081
ObjectMeta: metav1.ObjectMeta{
10831082
Namespace: secretNamespace,
@@ -1090,7 +1089,7 @@ func TestGetStorageAccesskey(t *testing.T) {
10901089
Type: "Opaque",
10911090
}
10921091
secret.Namespace = secretNamespace
1093-
_, secretCreateErr := d.cloud.KubeClient.CoreV1().Secrets(secretNamespace).Create(context.TODO(), secret, metav1.CreateOptions{})
1092+
_, secretCreateErr := d.KubeClient.CoreV1().Secrets(secretNamespace).Create(context.TODO(), secret, metav1.CreateOptions{})
10941093
if secretCreateErr != nil {
10951094
t.Error("failed to create secret")
10961095
}
@@ -1120,7 +1119,7 @@ func TestGetInfoFromSecret(t *testing.T) {
11201119
testFunc: func(t *testing.T) {
11211120
d := NewFakeDriver()
11221121
d.cloud = &azure.Cloud{}
1123-
d.cloud.KubeClient = nil
1122+
d.KubeClient = nil
11241123
secretName := "foo"
11251124
secretNamespace := "bar"
11261125
_, _, _, _, _, _, _, err := d.GetInfoFromSecret(context.TODO(), secretName, secretNamespace)
@@ -1134,8 +1133,7 @@ func TestGetInfoFromSecret(t *testing.T) {
11341133
name: "Could not get secret",
11351134
testFunc: func(t *testing.T) {
11361135
d := NewFakeDriver()
1137-
d.cloud = &azure.Cloud{}
1138-
d.cloud.KubeClient = fakeClient
1136+
d.KubeClient = fakeClient
11391137
secretName := ""
11401138
secretNamespace := ""
11411139
_, _, _, _, _, _, _, err := d.GetInfoFromSecret(context.TODO(), secretName, secretNamespace)
@@ -1150,8 +1148,7 @@ func TestGetInfoFromSecret(t *testing.T) {
11501148
name: "get account name from secret",
11511149
testFunc: func(t *testing.T) {
11521150
d := NewFakeDriver()
1153-
d.cloud = &azure.Cloud{}
1154-
d.cloud.KubeClient = fakeClient
1151+
d.KubeClient = fakeClient
11551152
secretName := "store_account_name_key"
11561153
secretNamespace := "namespace"
11571154
accountName := "bar"
@@ -1167,7 +1164,7 @@ func TestGetInfoFromSecret(t *testing.T) {
11671164
},
11681165
Type: "Opaque",
11691166
}
1170-
_, secretCreateErr := d.cloud.KubeClient.CoreV1().Secrets(secretNamespace).Create(context.TODO(), secret, metav1.CreateOptions{})
1167+
_, secretCreateErr := d.KubeClient.CoreV1().Secrets(secretNamespace).Create(context.TODO(), secret, metav1.CreateOptions{})
11711168
if secretCreateErr != nil {
11721169
t.Error("failed to create secret")
11731170
}
@@ -1187,7 +1184,7 @@ func TestGetInfoFromSecret(t *testing.T) {
11871184
testFunc: func(t *testing.T) {
11881185
d := NewFakeDriver()
11891186
d.cloud = &azure.Cloud{}
1190-
d.cloud.KubeClient = fakeClient
1187+
d.KubeClient = fakeClient
11911188
secretName := "store_other_info"
11921189
secretNamespace := "namespace"
11931190
accountName := "bar"
@@ -1211,7 +1208,7 @@ func TestGetInfoFromSecret(t *testing.T) {
12111208
},
12121209
Type: "Opaque",
12131210
}
1214-
_, secretCreateErr := d.cloud.KubeClient.CoreV1().Secrets(secretNamespace).Create(context.TODO(), secret, metav1.CreateOptions{})
1211+
_, secretCreateErr := d.KubeClient.CoreV1().Secrets(secretNamespace).Create(context.TODO(), secret, metav1.CreateOptions{})
12151212
if secretCreateErr != nil {
12161213
t.Error("failed to create secret")
12171214
}

pkg/blob/controllerserver.go

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"google.golang.org/grpc/status"
3131

3232
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
33+
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage"
3334
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
3435
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
3536
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
@@ -42,6 +43,7 @@ import (
4243
"k8s.io/utils/pointer"
4344

4445
"sigs.k8s.io/blob-csi-driver/pkg/util"
46+
"sigs.k8s.io/cloud-provider-azure/pkg/azclient/blobcontainerclient"
4547
azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
4648
"sigs.k8s.io/cloud-provider-azure/pkg/metrics"
4749
"sigs.k8s.io/cloud-provider-azure/pkg/provider"
@@ -261,7 +263,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
261263
return nil, status.Errorf(codes.InvalidArgument, "protocol(%s) is not supported, supported protocol list: %v", protocol, supportedProtocolList)
262264
}
263265
if !isSupportedAccessTier(accessTier) {
264-
return nil, status.Errorf(codes.InvalidArgument, "accessTier(%s) is not supported, supported AccessTier list: %v", accessTier, storage.PossibleAccessTierValues())
266+
return nil, status.Errorf(codes.InvalidArgument, "accessTier(%s) is not supported, supported AccessTier list: %v", accessTier, armstorage.PossibleAccessTierValues())
265267
}
266268

267269
if containerName != "" && containerNamePrefix != "" {
@@ -275,7 +277,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
275277
if strings.EqualFold(networkEndpointType, privateEndpoint) {
276278
createPrivateEndpoint = pointer.BoolPtr(true)
277279
}
278-
accountKind := string(storage.KindStorageV2)
280+
accountKind := string(armstorage.KindStorageV2)
279281
if isNFSProtocol(protocol) {
280282
isHnsEnabled = pointer.Bool(true)
281283
enableNfsV3 = pointer.Bool(true)
@@ -293,11 +295,11 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
293295
}
294296

295297
if strings.HasPrefix(strings.ToLower(storageAccountType), "premium") {
296-
accountKind = string(storage.KindBlockBlobStorage)
298+
accountKind = string(armstorage.KindBlockBlobStorage)
297299
}
298300
if IsAzureStackCloud(d.cloud) {
299-
accountKind = string(storage.KindStorage)
300-
if storageAccountType != "" && storageAccountType != string(storage.SkuNameStandardLRS) && storageAccountType != string(storage.SkuNamePremiumLRS) {
301+
accountKind = string(armstorage.KindStorage)
302+
if storageAccountType != "" && storageAccountType != string(armstorage.SKUNameStandardLRS) && storageAccountType != string(armstorage.SKUNamePremiumLRS) {
301303
return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("Invalid skuName value: %s, as Azure Stack only supports %s and %s Storage Account types.", storageAccountType, storage.SkuNamePremiumLRS, storage.SkuNameStandardLRS))
302304
}
303305
}
@@ -449,7 +451,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
449451
}
450452
}
451453

452-
secretName, err := setAzureCredentials(ctx, d.cloud.KubeClient, accountName, accountKey, secretNamespace)
454+
secretName, err := setAzureCredentials(ctx, d.KubeClient, accountName, accountKey, secretNamespace)
453455
if err != nil {
454456
return nil, status.Errorf(codes.Internal, "failed to store storage account key: %v", err)
455457
}
@@ -569,8 +571,12 @@ func (d *Driver) ValidateVolumeCapabilities(ctx context.Context, req *csi.Valida
569571
if resourceGroupName == "" {
570572
resourceGroupName = d.cloud.ResourceGroup
571573
}
572-
blobContainer, retryErr := d.cloud.BlobClient.GetContainer(ctx, subsID, resourceGroupName, accountName, containerName)
573-
err = retryErr.Error()
574+
blobClient, err := d.clientFactory.GetBlobContainerClientForSub(subsID)
575+
if err != nil {
576+
return nil, status.Error(codes.Internal, err.Error())
577+
}
578+
579+
blobContainer, err := blobClient.Get(ctx, resourceGroupName, accountName, containerName)
574580
if err != nil {
575581
return nil, status.Error(codes.Internal, err.Error())
576582
}
@@ -678,12 +684,17 @@ func (d *Driver) CreateBlobContainer(ctx context.Context, subsID, resourceGroupN
678684
}
679685
_, err = container.CreateIfNotExists(&azstorage.CreateContainerOptions{Access: azstorage.ContainerAccessTypePrivate})
680686
} else {
681-
blobContainer := storage.BlobContainer{
682-
ContainerProperties: &storage.ContainerProperties{
683-
PublicAccess: storage.PublicAccessNone,
687+
blobContainer := armstorage.BlobContainer{
688+
ContainerProperties: &armstorage.ContainerProperties{
689+
PublicAccess: to.Ptr(armstorage.PublicAccessNone),
684690
},
685691
}
686-
err = d.cloud.BlobClient.CreateContainer(ctx, subsID, resourceGroupName, accountName, containerName, blobContainer).Error()
692+
var blobClient blobcontainerclient.Interface
693+
blobClient, err = d.clientFactory.GetBlobContainerClientForSub(subsID)
694+
if err != nil {
695+
return true, err
696+
}
697+
_, err = blobClient.CreateContainer(ctx, resourceGroupName, accountName, containerName, blobContainer)
687698
}
688699
if err != nil {
689700
if strings.Contains(err.Error(), containerBeingDeletedDataplaneAPIError) ||
@@ -710,7 +721,12 @@ func (d *Driver) DeleteBlobContainer(ctx context.Context, subsID, resourceGroupN
710721
}
711722
_, err = container.DeleteIfExists(nil)
712723
} else {
713-
err = d.cloud.BlobClient.DeleteContainer(ctx, subsID, resourceGroupName, accountName, containerName).Error()
724+
var blobClient blobcontainerclient.Interface
725+
blobClient, err = d.clientFactory.GetBlobContainerClientForSub(subsID)
726+
if err != nil {
727+
return true, err
728+
}
729+
err = blobClient.DeleteContainer(ctx, resourceGroupName, accountName, containerName)
714730
}
715731
if err != nil {
716732
if strings.Contains(err.Error(), containerBeingDeletedDataplaneAPIError) ||

0 commit comments

Comments
 (0)