Skip to content

Commit 29346ec

Browse files
nickliu-msftnickliu-msft
authored
[Storage][DataMovement] NFS & SMB File Share Directory-to-Directory Copy Transfer Preservation (Azure#49349)
* WIP * WIP * WIP * WIP * fixed tests * WIP * Exported API + recorded tests * Re-recorded some tests * Fixed some tests * fixed test + more test re-recordings * re-recorded tests in DataMovement.Blobs.Files.Shares * more recordings * more recording * more recordings * Changed test to use permissionString * tiny test fix * reset recordings * rerecorded DataMovement.Files.shares * renamed method * fixed ShareFileResourceTests * refactored bool logic * fixed to match recording * small fix * Addressed feedback * Exported API * Recorded some tests * Errors.SingleItemContainerNoGetProperties
1 parent 13d9c66 commit 29346ec

29 files changed

+931
-91
lines changed

‎sdk/storage/Azure.Storage.DataMovement.Blobs.Files.Shares/assets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "net",
44
"TagPrefix": "net/storage/Azure.Storage.DataMovement.Blobs.Files.Shares",
5-
"Tag": "net/storage/Azure.Storage.DataMovement.Blobs.Files.Shares_b128b435d6"
5+
"Tag": "net/storage/Azure.Storage.DataMovement.Blobs.Files.Shares_150ed59e4c"
66
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/assets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "net",
44
"TagPrefix": "net/storage/Azure.Storage.DataMovement.Blobs",
5-
"Tag": "net/storage/Azure.Storage.DataMovement.Blobs_7484a44b19"
5+
"Tag": "net/storage/Azure.Storage.DataMovement.Blobs_476246f258"
66
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,5 +239,11 @@ protected override StorageResourceContainer GetChildStorageResourceContainer(str
239239
BlobContainerClient,
240240
options);
241241
}
242+
243+
protected override Task<StorageResourceContainerProperties> GetPropertiesAsync(CancellationToken cancellationToken = default)
244+
{
245+
// Not implemented for now
246+
return Task.FromResult(new StorageResourceContainerProperties());
247+
}
242248
}
243249
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "net",
44
"TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares",
5-
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_a13e75f487"
5+
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_7af87df10f"
66
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs‎

Lines changed: 206 additions & 33 deletions
Large diffs are not rendered by default.

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs‎

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
using System.Runtime.CompilerServices;
88
using System.Threading;
99
using System.Threading.Tasks;
10+
using Azure.Core;
1011
using Azure.Storage.Files.Shares;
1112
using Azure.Storage.Files.Shares.Models;
1213
using Azure.Storage.Files.Shares.Specialized;
@@ -27,7 +28,16 @@ internal class ShareDirectoryStorageResourceContainer : StorageResourceContainer
2728
internal ShareDirectoryStorageResourceContainer(ShareDirectoryClient shareDirectoryClient, ShareFileStorageResourceOptions options)
2829
{
2930
ShareDirectoryClient = shareDirectoryClient;
30-
ResourceOptions = options;
31+
ResourceOptions = options ?? new ShareFileStorageResourceOptions();
32+
}
33+
34+
internal ShareDirectoryStorageResourceContainer(
35+
ShareDirectoryClient shareDirectoryClient,
36+
StorageResourceContainerProperties properties,
37+
ShareFileStorageResourceOptions options = default)
38+
: this(shareDirectoryClient, options)
39+
{
40+
ResourceProperties = properties;
3141
}
3242

3343
protected override StorageResourceItem GetStorageResourceReference(string path, string resourceId)
@@ -47,10 +57,23 @@ protected override async IAsyncEnumerable<StorageResource> GetStorageResourcesAs
4757
[EnumeratorCancellation] CancellationToken cancellationToken = default)
4858
{
4959
// Set the ShareFileTraits to send when listing.
50-
ShareFileTraits traits = ShareFileTraits.Attributes;
51-
if (ResourceOptions?.FilePermissions ?? false)
60+
ShareFileTraits traits = new();
61+
62+
// Copy transfer
63+
if (destinationContainer is ShareDirectoryStorageResourceContainer)
5264
{
53-
traits |= ShareFileTraits.PermissionKey;
65+
ShareDirectoryStorageResourceContainer destinationStorageResourceContainer
66+
= destinationContainer as ShareDirectoryStorageResourceContainer;
67+
ShareFileStorageResourceOptions destinationOptions = destinationStorageResourceContainer.ResourceOptions;
68+
// destination must be SMB
69+
if ((!destinationOptions?.IsNfs ?? true))
70+
{
71+
traits = ShareFileTraits.Attributes;
72+
if (destinationOptions?.FilePermissions ?? false)
73+
{
74+
traits |= ShareFileTraits.PermissionKey;
75+
}
76+
}
5477
}
5578
ShareClient parentDestinationShare = default;
5679
if (destinationContainer != default)
@@ -114,5 +137,44 @@ await ShareDirectoryClient.CreateIfNotExistsAsync(
114137

115138
protected override StorageResourceContainer GetChildStorageResourceContainer(string path)
116139
=> new ShareDirectoryStorageResourceContainer(ShareDirectoryClient.GetSubdirectoryClient(path), ResourceOptions);
140+
141+
protected override async Task<StorageResourceContainerProperties> GetPropertiesAsync(CancellationToken cancellationToken = default)
142+
{
143+
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
144+
Response<ShareDirectoryProperties> response = await ShareDirectoryClient.GetPropertiesAsync(
145+
cancellationToken: cancellationToken).ConfigureAwait(false);
146+
if (ResourceProperties != default)
147+
{
148+
ResourceProperties.AddToStorageResourceContainerProperties(response.Value);
149+
}
150+
else
151+
{
152+
ResourceProperties = response.Value.ToStorageResourceContainerProperties();
153+
}
154+
ResourceProperties.Uri = Uri;
155+
return ResourceProperties;
156+
}
157+
158+
protected override async Task CreateIfNotExistsAsync(
159+
StorageResourceContainerProperties sourceProperties,
160+
CancellationToken cancellationToken = default)
161+
{
162+
IDictionary<string, string> metadata = ResourceOptions?.GetFileMetadata(sourceProperties?.RawProperties);
163+
string filePermission = ResourceOptions?.GetFilePermission(sourceProperties);
164+
FileSmbProperties smbProperties = ResourceOptions?.GetFileSmbProperties(sourceProperties);
165+
FilePosixProperties filePosixProperties = ResourceOptions?.GetFilePosixProperties(sourceProperties);
166+
167+
ShareDirectoryCreateOptions options = new ShareDirectoryCreateOptions
168+
{
169+
Metadata = metadata,
170+
SmbProperties = smbProperties,
171+
FilePermission = new() { Permission = filePermission },
172+
PosixProperties = filePosixProperties
173+
};
174+
175+
await ShareDirectoryClient.CreateIfNotExistsAsync(
176+
options: options,
177+
cancellationToken: cancellationToken).ConfigureAwait(false);
178+
}
117179
}
118180
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ internal async Task CreateAsync(
8080
}
8181
ShareFileHttpHeaders httpHeaders = _options?.GetShareFileHttpHeaders(properties?.RawProperties);
8282
IDictionary<string, string> metadata = _options?.GetFileMetadata(properties?.RawProperties);
83-
string filePermission = _options?.GetFilePermission(properties?.RawProperties);
83+
string filePermission = _options?.GetFilePermission(properties);
8484
FileSmbProperties smbProperties = _options?.GetFileSmbProperties(properties, _destinationPermissionKey);
8585
FilePosixProperties posixProperties = _options?.GetFilePosixProperties(properties);
8686

@@ -119,8 +119,7 @@ protected override async Task CompleteTransferAsync(
119119
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
120120

121121
StorageResourceItemProperties sourceProperties = completeTransferOptions?.SourceProperties;
122-
FileSmbProperties smbProperties = _options?.GetFileSmbProperties(sourceProperties);
123-
FilePosixProperties posixProperties = _options?.GetFilePosixProperties(sourceProperties);
122+
FileSmbProperties smbProperties = _options?.GetFileSmbProperties(sourceProperties, _destinationPermissionKey);
124123
// Call Set Properties
125124
// if transfer is not empty and original File Attribute contains ReadOnly
126125
// or if FileChangedOn is to be preserved or manually set
@@ -132,7 +131,6 @@ await ShareFileClient.SetHttpHeadersAsync(new()
132131
{
133132
HttpHeaders = httpHeaders,
134133
SmbProperties = smbProperties,
135-
PosixProperties = posixProperties
136134
},
137135
cancellationToken: cancellationToken).ConfigureAwait(false);
138136
}
@@ -274,11 +272,13 @@ protected override async Task SetPermissionsAsync(
274272
StorageResourceItemProperties sourceProperties,
275273
CancellationToken cancellationToken = default)
276274
{
275+
// Copy transfer
277276
if (sourceResource is ShareFileStorageResource)
278277
{
279-
if (_options?.FilePermissions ?? false)
278+
ShareFileStorageResource sourceShareFile = (ShareFileStorageResource)sourceResource;
279+
// destination must be SMB and destination FilePermission option must be set.
280+
if ((!_options?.IsNfs ?? true) && (_options?.FilePermissions ?? false))
280281
{
281-
ShareFileStorageResource sourceShareFile = (ShareFileStorageResource)sourceResource;
282282
string permissionsValue = sourceProperties?.RawProperties?.GetPermission();
283283
string destinationPermissionKey = sourceProperties?.RawProperties?.GetDestinationPermissionKey();
284284
// Get / Set the permission key if preserve is set to true,

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResourceOptions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public class ShareFileStorageResourceOptions
4848

4949
/// <summary>
5050
/// Optional. Specifies whether the Share uses NFS or SMB protocol.
51-
/// By default this value is set to false.
51+
/// By default this value is set to false. If true is selected, the account used must support NFS.
5252
/// Applies to copy, upload, and download transfers.
5353
/// </summary>
5454
public bool IsNfs { get; set; } = false;

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/SharesPathScanner.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ public virtual async IAsyncEnumerable<StorageResource> ScanAsync(
6565
toScan.Enqueue(subdir);
6666
yield return new ShareDirectoryStorageResourceContainer(
6767
subdir,
68+
item.ToResourceContainerProperties(destinationPermissionKey),
6869
sourceOptions);
6970
}
7071
else

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public static async Task<DisposingShare> GetTestShareAsync(
6060
return await DisposingShare.CreateAsync(share, metadata);
6161
}
6262

63-
public static async Task<DisposingShare> GetTestShareNfsAsync(
63+
public static async Task<DisposingShare> GetTestShareSasNfsAsync(
6464
this SharesClientBuilder clientBuilder,
6565
ShareServiceClient service = default,
6666
string shareName = default,
@@ -70,9 +70,14 @@ public static async Task<DisposingShare> GetTestShareNfsAsync(
7070
{
7171
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
7272
service ??= clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigPremiumFile, options);
73+
ShareServiceClient sasService = new ShareServiceClient(service.GenerateAccountSasUri(
74+
Sas.AccountSasPermissions.All,
75+
clientBuilder.Recording.UtcNow.AddDays(1),
76+
Sas.AccountSasResourceTypes.All),
77+
clientBuilder.GetOptions());
7378
metadata ??= new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
7479
shareName ??= clientBuilder.GetNewShareName();
75-
ShareClient share = clientBuilder.AzureCoreRecordedTestBase.InstrumentClient(service.GetShareClient(shareName));
80+
ShareClient share = clientBuilder.AzureCoreRecordedTestBase.InstrumentClient(sasService.GetShareClient(shareName));
7681
return await DisposingShare.CreateNfsAsync(share, metadata);
7782
}
7883
}

0 commit comments

Comments
 (0)