Skip to content

Commit dc5cf2f

Browse files
andyzhangxandyzhangx
authored
Merge pull request #1520 from umagnus/release-1.23-fix_volume_cloning
[release-1.23] fix: clone volume content to requested volume
2 parents 5a136f7 + a3ad4c7 commit dc5cf2f

File tree

7 files changed

+286
-255
lines changed

7 files changed

+286
-255
lines changed

‎pkg/blob/controllerserver.go

Lines changed: 103 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,6 @@ const (
5959
MSI = "MSI"
6060
SPN = "SPN"
6161
authorizationPermissionMismatch = "AuthorizationPermissionMismatch"
62-
63-
waitForAzCopyInterval = 2 * time.Second
6462
)
6563

6664
// CreateVolume provisions a volume
@@ -79,19 +77,12 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
7977
return nil, status.Error(codes.InvalidArgument, err.Error())
8078
}
8179

82-
if acquired := d.volumeLocks.TryAcquire(volName); !acquired {
83-
// logging the job status if it's volume cloning
84-
if req.GetVolumeContentSource() != nil {
85-
jobState, percent, err := d.azcopy.GetAzcopyJob(volName, []string{})
86-
klog.V(2).Infof("azcopy job status: %s, copy percent: %s%%, error: %v", jobState, percent, err)
87-
}
88-
return nil, status.Errorf(codes.Aborted, volumeOperationAlreadyExistsFmt, volName)
89-
}
90-
defer d.volumeLocks.Release(volName)
91-
9280
volSizeBytes := int64(req.GetCapacityRange().GetRequiredBytes())
9381
requestGiB := int(util.RoundUpGiB(volSizeBytes))
9482

83+
volContentSource := req.GetVolumeContentSource()
84+
secrets := req.GetSecrets()
85+
9586
parameters := req.GetParameters()
9687
if parameters == nil {
9788
parameters = make(map[string]string)
@@ -341,10 +332,31 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
341332
GetLatestAccountKey: getLatestAccountKey,
342333
}
343334

335+
containerName = replaceWithMap(containerName, containerNameReplaceMap)
336+
validContainerName := containerName
337+
if validContainerName == "" {
338+
validContainerName = volName
339+
if containerNamePrefix != "" {
340+
validContainerName = containerNamePrefix + "-" + volName
341+
}
342+
validContainerName = getValidContainerName(validContainerName, protocol)
343+
setKeyValueInMap(parameters, containerNameField, validContainerName)
344+
}
345+
346+
if acquired := d.volumeLocks.TryAcquire(volName); !acquired {
347+
// logging the job status if it's volume cloning
348+
if volContentSource != nil {
349+
jobState, percent, err := d.azcopy.GetAzcopyJob(validContainerName, []string{})
350+
return nil, status.Errorf(codes.Aborted, volumeOperationAlreadyExistsWithAzcopyFmt, volName, jobState, percent, err)
351+
}
352+
return nil, status.Errorf(codes.Aborted, volumeOperationAlreadyExistsFmt, volName)
353+
}
354+
defer d.volumeLocks.Release(volName)
355+
344356
var volumeID string
345357
requestName := "controller_create_volume"
346-
if req.GetVolumeContentSource() != nil {
347-
switch req.VolumeContentSource.Type.(type) {
358+
if volContentSource != nil {
359+
switch volContentSource.Type.(type) {
348360
case *csi.VolumeContentSource_Snapshot:
349361
requestName = "controller_create_volume_from_snapshot"
350362
case *csi.VolumeContentSource_Volume:
@@ -357,9 +369,39 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
357369
mc.ObserveOperationWithResult(isOperationSucceeded, VolumeID, volumeID)
358370
}()
359371

372+
var srcAzcopyAuthEnv []string
373+
var srcSubscriptionID, srcResourceGroupName, srcAccountName, srcContainerName, srcPath, srcAccountSASToken string
374+
if volContentSource != nil {
375+
switch volContentSource.Type.(type) {
376+
case *csi.VolumeContentSource_Snapshot:
377+
return nil, status.Errorf(codes.InvalidArgument, "VolumeContentSource Snapshot is not yet implemented")
378+
case *csi.VolumeContentSource_Volume:
379+
var srcVolumeID string
380+
if volContentSource.GetVolume() != nil {
381+
srcVolumeID = volContentSource.GetVolume().GetVolumeId()
382+
}
383+
srcResourceGroupName, srcAccountName, srcContainerName, _, srcSubscriptionID, err = GetContainerInfo(srcVolumeID)
384+
if err != nil {
385+
return nil, status.Error(codes.NotFound, err.Error())
386+
}
387+
srcAccountOptions := &azure.AccountOptions{
388+
Name: srcAccountName,
389+
SubscriptionID: srcSubscriptionID,
390+
ResourceGroup: srcResourceGroupName,
391+
GetLatestAccountKey: getLatestAccountKey,
392+
}
393+
srcAccountSASToken, srcAzcopyAuthEnv, err = d.getAzcopyAuth(ctx, srcAccountName, "", storageEndpointSuffix, srcAccountOptions, secrets, secretName, secretNamespace)
394+
if err != nil {
395+
return nil, status.Errorf(codes.Internal, "failed to getAzcopyAuth on account(%s) rg(%s), error: %v", accountOptions.Name, accountOptions.ResourceGroup, err)
396+
}
397+
srcPath = fmt.Sprintf("https://%s.blob.%s/%s", srcAccountName, storageEndpointSuffix, srcContainerName)
398+
default:
399+
return nil, status.Errorf(codes.InvalidArgument, "VolumeContentSource is not recognized: %v", volContentSource)
400+
}
401+
}
402+
360403
var accountKey string
361404
accountName := account
362-
secrets := req.GetSecrets()
363405
if len(secrets) == 0 && accountName == "" {
364406
if v, ok := d.volMap.Load(volName); ok {
365407
accountName = v.(string)
@@ -413,31 +455,26 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
413455
secrets = createStorageAccountSecret(accountName, accountKey)
414456
}
415457

416-
// replace pv/pvc name namespace metadata in subDir
417-
containerName = replaceWithMap(containerName, containerNameReplaceMap)
418-
validContainerName := containerName
419-
if validContainerName == "" {
420-
validContainerName = volName
421-
if containerNamePrefix != "" {
422-
validContainerName = containerNamePrefix + "-" + volName
458+
dstAzcopyAuthEnv := srcAzcopyAuthEnv
459+
dstAccountSASToken := srcAccountSASToken
460+
if volContentSource != nil {
461+
if srcSubscriptionID != subsID || srcResourceGroupName != resourceGroup || srcAccountName != accountName {
462+
if dstAccountSASToken, dstAzcopyAuthEnv, err = d.getAzcopyAuth(ctx, accountName, accountKey, storageEndpointSuffix, accountOptions, secrets, secretName, secretNamespace); err != nil {
463+
return nil, status.Errorf(codes.Internal, "failed to getAzcopyAuth on account(%s) rg(%s), error: %v", accountOptions.Name, accountOptions.ResourceGroup, err)
464+
}
423465
}
424-
validContainerName = getValidContainerName(validContainerName, protocol)
425-
setKeyValueInMap(parameters, containerNameField, validContainerName)
426466
}
427467

428-
if req.GetVolumeContentSource() != nil {
429-
accountSASToken, authAzcopyEnv, err := d.getAzcopyAuth(ctx, accountName, accountKey, storageEndpointSuffix, accountOptions, secrets, secretName, secretNamespace)
430-
if err != nil {
431-
return nil, status.Errorf(codes.Internal, "failed to getAzcopyAuth on account(%s) rg(%s), error: %v", accountOptions.Name, accountOptions.ResourceGroup, err)
432-
}
433-
if err := d.copyVolume(req, accountSASToken, authAzcopyEnv, validContainerName, storageEndpointSuffix); err != nil {
468+
klog.V(2).Infof("begin to create container(%s) on account(%s) type(%s) subsID(%s) rg(%s) location(%s) size(%d)", validContainerName, accountName, storageAccountType, subsID, resourceGroup, location, requestGiB)
469+
if err := d.CreateBlobContainer(ctx, subsID, resourceGroup, accountName, validContainerName, secrets); err != nil {
470+
return nil, status.Errorf(codes.Internal, "failed to create container(%s) on account(%s) type(%s) rg(%s) location(%s) size(%d), error: %v", validContainerName, accountName, storageAccountType, resourceGroup, location, requestGiB, err)
471+
}
472+
473+
if volContentSource != nil {
474+
dstPath := fmt.Sprintf("https://%s.blob.%s/%s", accountName, storageEndpointSuffix, validContainerName)
475+
if err := d.copyBlobContainer(dstAzcopyAuthEnv, srcPath, srcAccountSASToken, dstPath, dstAccountSASToken, validContainerName); err != nil {
434476
return nil, err
435477
}
436-
} else {
437-
klog.V(2).Infof("begin to create container(%s) on account(%s) type(%s) subsID(%s) rg(%s) location(%s) size(%d)", validContainerName, accountName, storageAccountType, subsID, resourceGroup, location, requestGiB)
438-
if err := d.CreateBlobContainer(ctx, subsID, resourceGroup, accountName, validContainerName, secrets); err != nil {
439-
return nil, status.Errorf(codes.Internal, "failed to create container(%s) on account(%s) type(%s) rg(%s) location(%s) size(%d), error: %v", validContainerName, accountName, storageAccountType, resourceGroup, location, requestGiB, err)
440-
}
441478
}
442479

443480
if storeAccountKey && len(req.GetSecrets()) == 0 {
@@ -478,7 +515,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
478515
VolumeId: volumeID,
479516
CapacityBytes: req.GetCapacityRange().GetRequiredBytes(),
480517
VolumeContext: parameters,
481-
ContentSource: req.GetVolumeContentSource(),
518+
ContentSource: volContentSource,
482519
},
483520
}, nil
484521
}
@@ -724,71 +761,45 @@ func (d *Driver) DeleteBlobContainer(ctx context.Context, subsID, resourceGroupN
724761
})
725762
}
726763

727-
// CopyBlobContainer copies a blob container in the same storage account
728-
func (d *Driver) copyBlobContainer(req *csi.CreateVolumeRequest, accountSasToken string, authAzcopyEnv []string, dstContainerName, storageEndpointSuffix string) error {
729-
var sourceVolumeID string
730-
if req.GetVolumeContentSource() != nil && req.GetVolumeContentSource().GetVolume() != nil {
731-
sourceVolumeID = req.GetVolumeContentSource().GetVolume().GetVolumeId()
764+
// copyBlobContainer copies source volume content into a destination volume
765+
func (d *Driver) copyBlobContainer(authAzcopyEnv []string, srcPath string, srcAccountSASToken string, dstPath string, dstAccountSASToken string, dstContainerName string) error {
732766

733-
}
734-
resourceGroupName, accountName, srcContainerName, _, _, err := GetContainerInfo(sourceVolumeID) //nolint:dogsled
735-
if err != nil {
736-
return status.Error(codes.NotFound, err.Error())
737-
}
738-
if srcContainerName == "" || dstContainerName == "" {
739-
return fmt.Errorf("srcContainerName(%s) or dstContainerName(%s) is empty", srcContainerName, dstContainerName)
767+
if srcPath == "" || dstPath == "" || dstContainerName == "" {
768+
return fmt.Errorf("srcPath(%s) or dstPath(%s) or dstContainerName(%s) is empty", srcPath, dstPath, dstContainerName)
740769
}
741770

742-
timeAfter := time.After(time.Duration(d.waitForAzCopyTimeoutMinutes) * time.Minute)
743-
timeTick := time.Tick(waitForAzCopyInterval)
744-
srcPath := fmt.Sprintf("https://%s.blob.%s/%s%s", accountName, storageEndpointSuffix, srcContainerName, accountSasToken)
745-
dstPath := fmt.Sprintf("https://%s.blob.%s/%s%s", accountName, storageEndpointSuffix, dstContainerName, accountSasToken)
746-
747771
jobState, percent, err := d.azcopy.GetAzcopyJob(dstContainerName, authAzcopyEnv)
748772
klog.V(2).Infof("azcopy job status: %s, copy percent: %s%%, error: %v", jobState, percent, err)
749-
if jobState == util.AzcopyJobError || jobState == util.AzcopyJobCompleted {
773+
switch jobState {
774+
case util.AzcopyJobError, util.AzcopyJobCompleted:
750775
return err
751-
}
752-
klog.V(2).Infof("begin to copy blob container %s to %s", srcContainerName, dstContainerName)
753-
for {
754-
select {
755-
case <-timeTick:
756-
jobState, percent, err := d.azcopy.GetAzcopyJob(dstContainerName, authAzcopyEnv)
757-
klog.V(2).Infof("azcopy job status: %s, copy percent: %s%%, error: %v", jobState, percent, err)
758-
switch jobState {
759-
case util.AzcopyJobError, util.AzcopyJobCompleted:
760-
return err
761-
case util.AzcopyJobNotFound:
762-
klog.V(2).Infof("copy blob container %s to %s", srcContainerName, dstContainerName)
763-
cmd := exec.Command("azcopy", "copy", srcPath, dstPath, "--recursive", "--check-length=false")
764-
if len(authAzcopyEnv) > 0 {
765-
cmd.Env = append(os.Environ(), authAzcopyEnv...)
766-
}
767-
out, copyErr := cmd.CombinedOutput()
768-
if copyErr != nil {
769-
klog.Warningf("CopyBlobContainer(%s, %s, %s) failed with error(%v): %v", resourceGroupName, accountName, dstPath, copyErr, string(out))
770-
} else {
771-
klog.V(2).Infof("copied blob container %s to %s successfully", srcContainerName, dstContainerName)
772-
}
773-
return copyErr
776+
case util.AzcopyJobRunning:
777+
return fmt.Errorf("wait for the existing AzCopy job to complete, current copy percentage is %s%%", percent)
778+
case util.AzcopyJobNotFound:
779+
klog.V(2).Infof("copy blob container %s to %s", srcPath, dstContainerName)
780+
execFunc := func() error {
781+
cmd := exec.Command("azcopy", "copy", srcPath+srcAccountSASToken, dstPath+dstAccountSASToken, "--recursive", "--check-length=false", "--s2s-preserve-access-tier=false")
782+
if len(authAzcopyEnv) > 0 {
783+
cmd.Env = append(os.Environ(), authAzcopyEnv...)
784+
}
785+
if out, err := cmd.CombinedOutput(); err != nil {
786+
return fmt.Errorf("exec error: %v, output: %v", err, string(out))
774787
}
775-
case <-timeAfter:
776-
return fmt.Errorf("timeout waiting for copy blob container %s to %s succeed", srcContainerName, dstContainerName)
788+
return nil
777789
}
790+
timeoutFunc := func() error {
791+
_, percent, _ := d.azcopy.GetAzcopyJob(dstContainerName, authAzcopyEnv)
792+
return fmt.Errorf("timeout waiting for copy blob container %s to %s complete, current copy percent: %s%%", srcPath, dstContainerName, percent)
793+
}
794+
copyErr := util.WaitUntilTimeout(time.Duration(d.waitForAzCopyTimeoutMinutes)*time.Minute, execFunc, timeoutFunc)
795+
if copyErr != nil {
796+
klog.Warningf("CopyBlobContainer(%s, %s, %s) failed with error: %v", srcPath, dstPath, dstContainerName, copyErr)
797+
} else {
798+
klog.V(2).Infof("copied blob container %s to %s successfully", srcPath, dstContainerName)
799+
}
800+
return copyErr
778801
}
779-
}
780-
781-
// copyVolume copies a volume form volume or snapshot, snapshot is not supported now
782-
func (d *Driver) copyVolume(req *csi.CreateVolumeRequest, accountSASToken string, authAzcopyEnv []string, dstContainerName, storageEndpointSuffix string) error {
783-
vs := req.VolumeContentSource
784-
switch vs.Type.(type) {
785-
case *csi.VolumeContentSource_Snapshot:
786-
return status.Errorf(codes.InvalidArgument, "copy volume from volumeSnapshot is not supported")
787-
case *csi.VolumeContentSource_Volume:
788-
return d.copyBlobContainer(req, accountSASToken, authAzcopyEnv, dstContainerName, storageEndpointSuffix)
789-
default:
790-
return status.Errorf(codes.InvalidArgument, "%v is not a proper volume source", vs)
791-
}
802+
return err
792803
}
793804

794805
// authorizeAzcopyWithIdentity returns auth env for azcopy using cluster identity

0 commit comments

Comments
 (0)