Skip to content

Commit 5c6d4d6

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #806 from cvvz/privateendpoint
feat: support privateendpoint
2 parents 6a24508 + 911b4ad commit 5c6d4d6

21 files changed

+526
-247
lines changed

‎docs/driver-parameters.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ location | Azure location | `eastus`, `westus`, etc. | No | if empty, driver wil
1313
resourceGroup | Azure resource group name | existing resource group name | No | if empty, driver will use the same resource group name as current k8s cluster
1414
storageAccount | specify Azure storage account name| STORAGE_ACCOUNT_NAME | - No for blobfuse mount </br> - Yes for NFSv3 mount | - For blobfuse mount: if empty, driver will find a suitable storage account that matches `skuName` in the same resource group; if a storage account name is provided, storage account must exist. </br> - For NFSv3 mount, storage account name must be provided
1515
protocol | specify blobfuse, blobfuse2 or NFSv3 mount (blobfuse2 is still in Preview) | `fuse`, `fuse2`, `nfs` | No | `fuse`
16+
networkEndpointType | specify network endpoint type for the storage account created by driver. If `privateEndpoint` is specified, a private endpoint will be created for the storage account. For other cases, a service endpoint will be created for NFS protocol. | "",`privateEndpoint` | No | ``
17+
storageEndpointSuffix | specify Azure storage endpoint suffix | `core.windows.net`, `core.chinacloudapi.cn`, etc | No | if empty, driver will use default storage endpoint suffix according to cloud environment, e.g. `core.windows.net`
1618
containerName | specify the existing container(directory) name | existing container name | No | if empty, driver will create a new container name, starting with `pvc-fuse` for blobfuse or `pvc-nfs` for NFSv3
1719
containerNamePrefix | specify Azure storage directory prefix created by driver | can only contain lowercase letters, numbers, hyphens, and length should be less than 21 | No |
1820
server | specify Azure storage account server address | existing server address, e.g. `accountname.privatelink.blob.core.windows.net` | No | if empty, driver will use default `accountname.blob.core.windows.net` or other sovereign cloud account address

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ require (
2929
k8s.io/kubernetes v1.26.0
3030
k8s.io/mount-utils v0.26.0
3131
k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
32-
sigs.k8s.io/cloud-provider-azure v1.26.1-0.20221220090543-dea11d70f108
32+
sigs.k8s.io/cloud-provider-azure v1.26.1-0.20221229055728-9863f3ea9b18
3333
sigs.k8s.io/yaml v1.3.0
3434
)
3535

‎go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -850,8 +850,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
850850
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
851851
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.33 h1:LYqFq+6Cj2D0gFfrJvL7iElD4ET6ir3VDdhDdTK7rgc=
852852
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.33/go.mod h1:soWkSNf2tZC7aMibXEqVhCd73GOY5fJikn8qbdzemB0=
853-
sigs.k8s.io/cloud-provider-azure v1.26.1-0.20221220090543-dea11d70f108 h1:Kg/IISal/+xjLdJITsxLY5OydOM3/bMxSRu34Jyhn78=
854-
sigs.k8s.io/cloud-provider-azure v1.26.1-0.20221220090543-dea11d70f108/go.mod h1:7ksoxa026xKQGAc0HYGk1ksucEweJnwxXiuk3krfP4c=
853+
sigs.k8s.io/cloud-provider-azure v1.26.1-0.20221229055728-9863f3ea9b18 h1:QM8faUevzsD24fKkc0AkivnRaRTf8chmmUkwlJa7urY=
854+
sigs.k8s.io/cloud-provider-azure v1.26.1-0.20221229055728-9863f3ea9b18/go.mod h1:7ksoxa026xKQGAc0HYGk1ksucEweJnwxXiuk3krfP4c=
855855
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
856856
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
857857
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=

‎pkg/blob/blob.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ const (
9292
vnetNameField = "vnetname"
9393
subnetNameField = "subnetname"
9494
accessTierField = "accesstier"
95+
networkEndpointTypeField = "networkendpointtype"
9596
mountPermissionsField = "mountpermissions"
9697
useDataPlaneAPIField = "usedataplaneapi"
9798

@@ -122,6 +123,8 @@ const (
122123
pvNameMetadata = "${pv.metadata.name}"
123124

124125
VolumeID = "volumeid"
126+
127+
defaultStorageEndPointSuffix = "core.windows.net"
125128
)
126129

127130
var (

‎pkg/blob/controllerserver.go

Lines changed: 44 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,14 @@ import (
3636
"sigs.k8s.io/blob-csi-driver/pkg/util"
3737
azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
3838
"sigs.k8s.io/cloud-provider-azure/pkg/metrics"
39+
"sigs.k8s.io/cloud-provider-azure/pkg/provider"
3940
azure "sigs.k8s.io/cloud-provider-azure/pkg/provider"
4041
)
4142

43+
const (
44+
privateEndpoint = "privateendpoint"
45+
)
46+
4247
// CreateVolume provisions a volume
4348
func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
4449
if err := d.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
@@ -69,7 +74,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
6974
}
7075
var storageAccountType, subsID, resourceGroup, location, account, containerName, containerNamePrefix, protocol, customTags, secretName, secretNamespace, pvcNamespace string
7176
var isHnsEnabled, requireInfraEncryption *bool
72-
var vnetResourceGroup, vnetName, subnetName, accessTier string
77+
var vnetResourceGroup, vnetName, subnetName, accessTier, networkEndpointType, storageEndpointSuffix string
7378
var matchTags, useDataPlaneAPI bool
7479
// set allowBlobPublicAccess as false by default
7580
allowBlobPublicAccess := pointer.Bool(false)
@@ -135,7 +140,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
135140
case serverNameField:
136141
// no op, only used in NodeStageVolume
137142
case storageEndpointSuffixField:
138-
// no op, only used in NodeStageVolume
143+
storageEndpointSuffix = v
139144
case vnetResourceGroupField:
140145
vnetResourceGroup = v
141146
case vnetNameField:
@@ -144,6 +149,8 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
144149
subnetName = v
145150
case accessTierField:
146151
accessTier = v
152+
case networkEndpointTypeField:
153+
networkEndpointType = v
147154
case mountPermissionsField:
148155
// only do validations here, used in NodeStageVolume, NodePublishVolume
149156
if v != "" {
@@ -201,6 +208,10 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
201208
}
202209

203210
enableHTTPSTrafficOnly := true
211+
createPrivateEndpoint := false
212+
if strings.EqualFold(networkEndpointType, privateEndpoint) {
213+
createPrivateEndpoint = true
214+
}
204215
accountKind := string(storage.KindStorageV2)
205216
var (
206217
vnetResourceIDs []string
@@ -209,15 +220,17 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
209220
if protocol == NFS {
210221
isHnsEnabled = pointer.Bool(true)
211222
enableNfsV3 = pointer.Bool(true)
212-
// set VirtualNetworkResourceIDs for storage account firewall setting
213-
vnetResourceID := d.getSubnetResourceID(vnetResourceGroup, vnetName, subnetName)
214-
klog.V(2).Infof("set vnetResourceID(%s) for NFS protocol", vnetResourceID)
215-
vnetResourceIDs = []string{vnetResourceID}
216-
if err := d.updateSubnetServiceEndpoints(ctx, vnetResourceGroup, vnetName, subnetName); err != nil {
217-
return nil, status.Errorf(codes.Internal, "update service endpoints failed with error: %v", err)
218-
}
219223
// NFS protocol does not need account key
220224
storeAccountKey = false
225+
if !createPrivateEndpoint {
226+
// set VirtualNetworkResourceIDs for storage account firewall setting
227+
vnetResourceID := d.getSubnetResourceID(vnetResourceGroup, vnetName, subnetName)
228+
klog.V(2).Infof("set vnetResourceID(%s) for NFS protocol", vnetResourceID)
229+
vnetResourceIDs = []string{vnetResourceID}
230+
if err := d.updateSubnetServiceEndpoints(ctx, vnetResourceGroup, vnetName, subnetName); err != nil {
231+
return nil, status.Errorf(codes.Internal, "update service endpoints failed with error: %v", err)
232+
}
233+
}
221234
}
222235

223236
if strings.HasPrefix(strings.ToLower(storageAccountType), "premium") {
@@ -235,6 +248,14 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
235248
return nil, status.Errorf(codes.InvalidArgument, err.Error())
236249
}
237250

251+
if strings.TrimSpace(storageEndpointSuffix) == "" {
252+
if d.cloud.Environment.StorageEndpointSuffix != "" {
253+
storageEndpointSuffix = d.cloud.Environment.StorageEndpointSuffix
254+
} else {
255+
storageEndpointSuffix = defaultStorageEndPointSuffix
256+
}
257+
}
258+
238259
accountOptions := &azure.AccountOptions{
239260
Name: account,
240261
Type: storageAccountType,
@@ -254,6 +275,9 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
254275
VNetName: vnetName,
255276
SubnetName: subnetName,
256277
AccessTier: accessTier,
278+
CreatePrivateEndpoint: createPrivateEndpoint,
279+
StorageType: provider.StorageTypeBlob,
280+
StorageEndpointSuffix: storageEndpointSuffix,
257281
}
258282

259283
var accountKey string
@@ -263,7 +287,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
263287
if v, ok := d.volMap.Load(volName); ok {
264288
accountName = v.(string)
265289
} else {
266-
lockKey := fmt.Sprintf("%s%s%s%s%s", storageAccountType, accountKind, resourceGroup, location, protocol)
290+
lockKey := fmt.Sprintf("%s%s%s%s%s%v", storageAccountType, accountKind, resourceGroup, location, protocol, createPrivateEndpoint)
267291
// search in cache first
268292
cache, err := d.accountSearchCache.Get(lockKey, azcache.CacheReadTypeDefault)
269293
if err != nil {
@@ -292,6 +316,16 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
292316
}
293317
}
294318

319+
if createPrivateEndpoint && protocol == NFS {
320+
// As for blobfuse/blobfuse2, serverName, i.e.,AZURE_STORAGE_BLOB_ENDPOINT env variable can't include
321+
// "privatelink", issue: https://github.com/Azure/azure-storage-fuse/issues/1014
322+
//
323+
// And use public endpoint will be befine to blobfuse/blobfuse2, because it will be resolved to private endpoint
324+
// by private dns zone, which includes CNAME record, documented here:
325+
// https://learn.microsoft.com/en-us/azure/storage/common/storage-private-endpoints?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json&bc=%2Fazure%2Fstorage%2Fblobs%2Fbreadcrumb%2Ftoc.json#dns-changes-for-private-endpoints
326+
setKeyValueInMap(parameters, serverNameField, fmt.Sprintf("%s.privatelink.blob.%s", accountName, storageEndpointSuffix))
327+
}
328+
295329
accountOptions.Name = accountName
296330
if len(secrets) == 0 && useDataPlaneAPI {
297331
if accountKey == "" {

‎test/e2e/dynamic_provisioning_test.go

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -606,4 +606,106 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
606606
}
607607
test.Run(cs, ns)
608608
})
609+
610+
ginkgo.It("should create a private endpoint volume on demand", func() {
611+
if isAzureStackCloud {
612+
ginkgo.Skip("test case is not available for Azure Stack")
613+
}
614+
pods := []testsuites.PodDetails{
615+
{
616+
Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
617+
Volumes: []testsuites.VolumeDetails{
618+
{
619+
ClaimSize: "10Gi",
620+
MountOptions: []string{
621+
"-o allow_other",
622+
"--file-cache-timeout-in-seconds=120",
623+
"--cancel-list-on-mount-seconds=0",
624+
},
625+
VolumeMount: testsuites.VolumeMountDetails{
626+
NameGenerate: "test-volume-",
627+
MountPathGenerate: "/mnt/test-",
628+
},
629+
},
630+
},
631+
},
632+
}
633+
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
634+
CSIDriver: testDriver,
635+
Pods: pods,
636+
StorageClassParameters: map[string]string{
637+
"skuName": "Standard_LRS",
638+
"networkEndpointType": "privateEndpoint",
639+
},
640+
}
641+
test.Run(cs, ns)
642+
})
643+
644+
ginkgo.It("should create a private endpoint volume on demand with protocol [fuse2]", func() {
645+
if isAzureStackCloud {
646+
ginkgo.Skip("test case is not available for Azure Stack")
647+
}
648+
pods := []testsuites.PodDetails{
649+
{
650+
Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
651+
Volumes: []testsuites.VolumeDetails{
652+
{
653+
ClaimSize: "10Gi",
654+
MountOptions: []string{
655+
"-o allow_other",
656+
"--virtual-directory=true", // blobfuse2 mount options
657+
},
658+
VolumeMount: testsuites.VolumeMountDetails{
659+
NameGenerate: "test-volume-",
660+
MountPathGenerate: "/mnt/test-",
661+
},
662+
},
663+
},
664+
},
665+
}
666+
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
667+
CSIDriver: testDriver,
668+
Pods: pods,
669+
StorageClassParameters: map[string]string{
670+
"skuName": "Standard_LRS",
671+
"protocol": "fuse2",
672+
"networkEndpointType": "privateEndpoint",
673+
},
674+
}
675+
test.Run(cs, ns)
676+
})
677+
678+
ginkgo.It("should create a private endpoint volume on demand with protocol [nfs]", func() {
679+
if isAzureStackCloud {
680+
ginkgo.Skip("test case is not available for Azure Stack")
681+
}
682+
pods := []testsuites.PodDetails{
683+
{
684+
Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
685+
Volumes: []testsuites.VolumeDetails{
686+
{
687+
ClaimSize: "10Gi",
688+
MountOptions: []string{
689+
"nconnect=8",
690+
},
691+
VolumeMount: testsuites.VolumeMountDetails{
692+
NameGenerate: "test-volume-",
693+
MountPathGenerate: "/mnt/test-",
694+
},
695+
},
696+
},
697+
},
698+
}
699+
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
700+
CSIDriver: testDriver,
701+
Pods: pods,
702+
StorageClassParameters: map[string]string{
703+
"skuName": "Premium_LRS",
704+
"protocol": "nfs",
705+
"mountPermissions": "0755",
706+
"networkEndpointType": "privateEndpoint",
707+
},
708+
}
709+
test.Run(cs, ns)
710+
})
609711
})

‎vendor/modules.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1210,7 +1210,7 @@ k8s.io/utils/trace
12101210
## explicit; go 1.17
12111211
sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client
12121212
sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client
1213-
# sigs.k8s.io/cloud-provider-azure v1.26.1-0.20221220090543-dea11d70f108
1213+
# sigs.k8s.io/cloud-provider-azure v1.26.1-0.20221229055728-9863f3ea9b18
12141214
## explicit; go 1.19
12151215
sigs.k8s.io/cloud-provider-azure/pkg/auth
12161216
sigs.k8s.io/cloud-provider-azure/pkg/azureclients

‎vendor/sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmclient/azure_vmclient.go

Lines changed: 27 additions & 16 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)