Skip to content

Commit bd53f0c

Browse files
nickliu-msftnickliu-msft
authored
[Storage][DataMovement] Protocol Validation for NFS over REST (Azure#49689)
* Initial commit * refactor into ValidateTransferAsync * Added check on the source options * Exported Apis and recorded new tests * Re-recorded some tests * some additional recordings * fixed mocked tests * record * fixed playback issue * removed unused code * Addressed feedback pt1 * addressed feedback pt 2 * fix test * minor fix * another small change * Export apis * Added one more test * removed unused import * Refactor with DataMovementFileShareEventSource * Export api * small change * small change to the authorization failure error message * removed console.log * bubble up original exception for ProtocolValidationAuthorizationFailure
1 parent 46012c7 commit bd53f0c

21 files changed

+685
-28
lines changed

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net6.0.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ public ShareFileStorageResourceOptions() { }
4141
public System.Collections.Generic.IDictionary<string, string> FileMetadata { get { throw null; } set { } }
4242
public bool? FilePermissions { get { throw null; } set { } }
4343
public bool IsNfs { get { throw null; } set { } }
44+
public bool SkipProtocolValidation { get { throw null; } set { } }
4445
public Azure.Storage.Files.Shares.Models.ShareFileRequestConditions SourceConditions { get { throw null; } set { } }
4546
}
4647
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net8.0.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ public ShareFileStorageResourceOptions() { }
4141
public System.Collections.Generic.IDictionary<string, string> FileMetadata { get { throw null; } set { } }
4242
public bool? FilePermissions { get { throw null; } set { } }
4343
public bool IsNfs { get { throw null; } set { } }
44+
public bool SkipProtocolValidation { get { throw null; } set { } }
4445
public Azure.Storage.Files.Shares.Models.ShareFileRequestConditions SourceConditions { get { throw null; } set { } }
4546
}
4647
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.netstandard2.0.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ public ShareFileStorageResourceOptions() { }
4141
public System.Collections.Generic.IDictionary<string, string> FileMetadata { get { throw null; } set { } }
4242
public bool? FilePermissions { get { throw null; } set { } }
4343
public bool IsNfs { get { throw null; } set { } }
44+
public bool SkipProtocolValidation { get { throw null; } set { } }
4445
public Azure.Storage.Files.Shares.Models.ShareFileRequestConditions SourceConditions { get { throw null; } set { } }
4546
}
4647
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "net",
44
"TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares",
5-
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_7af87df10f"
5+
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_5ab02c363e"
66
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/Azure.Storage.DataMovement.Files.Shares.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
<PackageReference Include="System.Threading.Channels" />
2929
</ItemGroup>
3030
<ItemGroup>
31+
<Compile Include="$(AzureCoreSharedSources)AzureEventSource.cs" LinkBase="Shared\Core" />
3132
<Compile Include="$(AzureCoreSharedSources)CancellationHelper.cs" LinkBase="SharedCore" />
3233
<Compile Include="$(AzureCoreSharedSources)HashCodeBuilder.cs" LinkBase="Shared\Core" />
3334
</ItemGroup>

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementFileShareEventSource.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System.Diagnostics.Tracing;
5+
using Azure.Core.Diagnostics;
6+
7+
namespace Azure.Storage.DataMovement.Files.Shares
8+
{
9+
internal class DataMovementFileShareEventSource : AzureEventSource
10+
{
11+
private const string EventSourceName = "Azure-Storage-DataMovement-Files-Shares";
12+
13+
private const int ProtocolValidationSkippedEvent = 1;
14+
15+
private DataMovementFileShareEventSource() : base(EventSourceName) { }
16+
17+
public static DataMovementFileShareEventSource Singleton { get; } = new DataMovementFileShareEventSource();
18+
19+
[Event(ProtocolValidationSkippedEvent, Level = EventLevel.Informational, Message = "Transfer [{0}] Protocol Validation skipped for {1}: Resource={2}")]
20+
public void ProtocolValidationSkipped(string transferId, string endpoint, string resourceUri)
21+
{
22+
WriteEvent(ProtocolValidationSkippedEvent, transferId, endpoint, resourceUri);
23+
}
24+
}
25+
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,11 @@
33

44
using System;
55
using System.Collections.Generic;
6+
using System.Threading.Tasks;
7+
using System.Threading;
68
using Azure.Storage.Files.Shares.Models;
79
using Metadata = System.Collections.Generic.IDictionary<string, string>;
10+
using Azure.Storage.Files.Shares;
811

912
namespace Azure.Storage.DataMovement.Files.Shares
1013
{
@@ -680,6 +683,38 @@ private static string[] ConvertContentPropertyObjectToStringArray(string content
680683
throw Storage.Errors.UnexpectedPropertyType(contentPropertyName, DataMovementConstants.StringTypeStr, DataMovementConstants.StringArrayTypeStr);
681684
}
682685
}
686+
687+
public static async Task ValidateProtocolAsync(
688+
ShareClient parentShareClient,
689+
ShareFileStorageResourceOptions options,
690+
string transferId,
691+
string endpoint,
692+
string resourceUri,
693+
CancellationToken cancellationToken)
694+
{
695+
if (!options?.SkipProtocolValidation ?? true)
696+
{
697+
try
698+
{
699+
ShareProperties properties = await parentShareClient.GetPropertiesAsync(cancellationToken).ConfigureAwait(false);
700+
ShareProtocols expectedProtocol = options.IsNfs ? ShareProtocols.Nfs : ShareProtocols.Smb;
701+
ShareProtocols actualProtocol = properties.Protocols ?? ShareProtocols.Smb;
702+
703+
if (actualProtocol != expectedProtocol)
704+
{
705+
throw Errors.ProtocolSetMismatch(endpoint, expectedProtocol, actualProtocol);
706+
}
707+
}
708+
catch (RequestFailedException ex) when (ex.Status == 403)
709+
{
710+
throw Errors.ProtocolValidationAuthorizationFailure(ex, endpoint);
711+
}
712+
}
713+
else
714+
{
715+
DataMovementFileShareEventSource.Singleton.ProtocolValidationSkipped(transferId, endpoint, resourceUri);
716+
}
717+
}
683718
}
684719

685720
#pragma warning disable SA1402 // File may only contain a single type

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ protected override async IAsyncEnumerable<StorageResource> GetStorageResourcesAs
6565
ShareDirectoryStorageResourceContainer destinationStorageResourceContainer
6666
= destinationContainer as ShareDirectoryStorageResourceContainer;
6767
ShareFileStorageResourceOptions destinationOptions = destinationStorageResourceContainer.ResourceOptions;
68-
// destination must be SMB
69-
if ((!destinationOptions?.IsNfs ?? true))
68+
// both source and destination must be SMB
69+
if ((!ResourceOptions?.IsNfs ?? true) && (!destinationOptions?.IsNfs ?? true))
7070
{
7171
traits = ShareFileTraits.Attributes;
7272
if (destinationOptions?.FilePermissions ?? false)
@@ -176,5 +176,40 @@ await ShareDirectoryClient.CreateIfNotExistsAsync(
176176
options: options,
177177
cancellationToken: cancellationToken).ConfigureAwait(false);
178178
}
179+
180+
protected override async Task ValidateTransferAsync(
181+
string transferId,
182+
StorageResource sourceResource,
183+
CancellationToken cancellationToken = default)
184+
{
185+
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
186+
187+
if (sourceResource is ShareDirectoryStorageResourceContainer sourceShareDirectoryResource)
188+
{
189+
// Ensure the transfer is supported (NFS -> NFS and SMB -> SMB)
190+
if ((ResourceOptions?.IsNfs ?? false) != (sourceShareDirectoryResource.ResourceOptions?.IsNfs ?? false))
191+
{
192+
throw Errors.ShareTransferNotSupported();
193+
}
194+
195+
// Validate the source protocol
196+
await DataMovementSharesExtensions.ValidateProtocolAsync(
197+
sourceShareDirectoryResource.ShareDirectoryClient.GetParentShareClient(),
198+
sourceShareDirectoryResource.ResourceOptions,
199+
transferId,
200+
"source",
201+
sourceResource.Uri.AbsoluteUri,
202+
cancellationToken).ConfigureAwait(false);
203+
}
204+
205+
// Validate the destination protocol
206+
await DataMovementSharesExtensions.ValidateProtocolAsync(
207+
ShareDirectoryClient.GetParentShareClient(),
208+
ResourceOptions,
209+
transferId,
210+
"destination",
211+
Uri.AbsoluteUri,
212+
cancellationToken).ConfigureAwait(false);
213+
}
179214
}
180215
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs‎

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -276,8 +276,8 @@ protected override async Task SetPermissionsAsync(
276276
if (sourceResource is ShareFileStorageResource)
277277
{
278278
ShareFileStorageResource sourceShareFile = (ShareFileStorageResource)sourceResource;
279-
// destination must be SMB and destination FilePermission option must be set.
280-
if ((!_options?.IsNfs ?? true) && (_options?.FilePermissions ?? false))
279+
// both source and destination must be SMB and destination FilePermission option must be set.
280+
if ((!sourceShareFile._options?.IsNfs ?? true) && (!_options?.IsNfs ?? true) && (_options?.FilePermissions ?? false))
281281
{
282282
string permissionsValue = sourceProperties?.RawProperties?.GetPermission();
283283
string destinationPermissionKey = sourceProperties?.RawProperties?.GetDestinationPermissionKey();
@@ -347,6 +347,41 @@ protected override StorageResourceCheckpointDetails GetDestinationCheckpointDeta
347347
isDirectoryMetadataSet: _options?._isDirectoryMetadataSet ?? false,
348348
directoryMetadata: _options?.DirectoryMetadata);
349349
}
350+
351+
protected override async Task ValidateTransferAsync(
352+
string transferId,
353+
StorageResource sourceResource,
354+
CancellationToken cancellationToken = default)
355+
{
356+
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
357+
358+
if (sourceResource is ShareFileStorageResource sourceShareFileResource)
359+
{
360+
// Ensure the transfer is supported (NFS -> NFS and SMB -> SMB)
361+
if ((_options?.IsNfs ?? false) != (sourceShareFileResource._options?.IsNfs ?? false))
362+
{
363+
throw Errors.ShareTransferNotSupported();
364+
}
365+
366+
// Validate the source protocol
367+
await DataMovementSharesExtensions.ValidateProtocolAsync(
368+
sourceShareFileResource.ShareFileClient.GetParentShareClient(),
369+
sourceShareFileResource._options,
370+
transferId,
371+
"source",
372+
sourceResource.Uri.AbsoluteUri,
373+
cancellationToken).ConfigureAwait(false);
374+
}
375+
376+
// Validate the destination protocol
377+
await DataMovementSharesExtensions.ValidateProtocolAsync(
378+
ShareFileClient.GetParentShareClient(),
379+
_options,
380+
transferId,
381+
"destination",
382+
Uri.AbsoluteUri,
383+
cancellationToken).ConfigureAwait(false);
384+
}
350385
}
351386

352387
#pragma warning disable SA1402 // File may only contain a single type
@@ -355,5 +390,16 @@ internal partial class Errors
355390
{
356391
public static InvalidOperationException ShareFileAlreadyExists(string pathName)
357392
=> new InvalidOperationException($"Share File `{pathName}` already exists. Cannot overwrite file.");
393+
394+
public static ArgumentException ProtocolSetMismatch(string endpoint, ShareProtocols setProtocol, ShareProtocols actualProtocol)
395+
=> new ArgumentException($"The Protocol set on the {endpoint} '{setProtocol}' does not match the actual Protocol of the share '{actualProtocol}'.");
396+
397+
public static UnauthorizedAccessException ProtocolValidationAuthorizationFailure(RequestFailedException ex, string endpoint)
398+
=> new UnauthorizedAccessException($"Authorization failure on the {endpoint} when validating the Protocol. " +
399+
$"To skip this validation, please enable SkipProtocolValidation.", ex);
400+
401+
public static NotSupportedException ShareTransferNotSupported()
402+
=> new NotSupportedException("This Share transfer is not supported. " +
403+
"Currently only NFS -> NFS and SMB -> SMB Share transfers are supported");
358404
}
359405
}

‎sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResourceOptions.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,14 @@ public class ShareFileStorageResourceOptions
4646
private Metadata _fileMetadata = default;
4747
internal bool _isFileMetadataSet = false;
4848

49+
/// <summary>
50+
/// Optional. Specifies whether protocol validation for the resource should be skipped before starting the transfer.
51+
/// By default this value is set to false.
52+
/// Applies to copy, upload, and download transfers.
53+
/// Note: Protocol validation requires share-level read access.
54+
/// </summary>
55+
public bool SkipProtocolValidation { get; set; } = false;
56+
4957
/// <summary>
5058
/// Optional. Specifies whether the Share uses NFS or SMB protocol.
5159
/// By default this value is set to false. If true is selected, the account used must support NFS.

0 commit comments

Comments
 (0)