Skip to content

Commit d4e9e97

Browse files
authored
[Storage] Added Fix for Creating Share File before service copy; Added Data Movement E2E service copy tests (Azure#39440)
* WIP * WIP - tests written but CopyAuthorization needs to be done first * Added tests for single sync copy file share; Added minor fixes for creation of file beforehand * Fix recording * Fix to share file resource tests
1 parent 1bc8916 commit d4e9e97

File tree

8 files changed

+1066
-18
lines changed

8 files changed

+1066
-18
lines changed

sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "net",
44
"TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares",
5-
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_29f88351cb"
5+
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_02f417687f"
66
}

sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -158,12 +158,16 @@ protected override async Task CopyFromUriAsync(
158158
CancellationToken cancellationToken = default)
159159
{
160160
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
161-
await ShareFileClient.UploadRangeFromUriAsync(
162-
sourceUri: sourceResource.Uri,
163-
range: new HttpRange(0, completeLength),
164-
sourceRange: new HttpRange(0, completeLength),
165-
options: _options.ToShareFileUploadRangeFromUriOptions(),
166-
cancellationToken: cancellationToken).ConfigureAwait(false);
161+
await CreateAsync(overwrite, completeLength, cancellationToken).ConfigureAwait(false);
162+
if (completeLength > 0)
163+
{
164+
await ShareFileClient.UploadRangeFromUriAsync(
165+
sourceUri: sourceResource.Uri,
166+
range: new HttpRange(0, completeLength),
167+
sourceRange: new HttpRange(0, completeLength),
168+
options: _options.ToShareFileUploadRangeFromUriOptions(),
169+
cancellationToken: cancellationToken).ConfigureAwait(false);
170+
}
167171
}
168172

169173
protected override async Task<bool> DeleteIfExistsAsync(CancellationToken cancellationToken = default)

sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TestEventsRaised.cs" LinkBase="Shared\DataMovement" />
4545
<Compile Include="$(AzureStorageDataMovementTestSharedSources)DisposingLocalDirectory.cs" LinkBase="Shared\DataMovement" />
4646
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferUploadTestBase.cs" LinkBase="Shared\DataMovement" />
47+
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferCopyTestBase.cs" LinkBase="Shared\DataMovement" />
4748
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferDownloadTestBase.cs" LinkBase="Shared\DataMovement" />
4849
</ItemGroup>
4950
<ItemGroup>

sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
using SharesClientBuilder = Azure.Storage.Test.Shared.ClientBuilder<
1111
Azure.Storage.Files.Shares.ShareServiceClient,
1212
Azure.Storage.Files.Shares.ShareClientOptions>;
13+
using Azure.Storage.Files.Shares.Models;
1314

1415
namespace Azure.Storage.DataMovement.Files.Shares.Tests
1516
{
@@ -42,6 +43,17 @@ public static SharesClientBuilder GetNewShareClientBuilder(TenantConfigurationBu
4243
(uri, azureSasCredential, clientOptions) => new ShareServiceClient(uri, azureSasCredential, clientOptions),
4344
() => new ShareClientOptions(serviceVersion));
4445

46+
public static ShareServiceClient GetServiceClient_OAuthAccount_SharedKey(this SharesClientBuilder clientBuilder) =>
47+
clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigOAuth);
48+
49+
public static ShareServiceClient GetServiceClient_OAuth(
50+
this SharesClientBuilder clientBuilder, ShareClientOptions options = default)
51+
{
52+
options ??= clientBuilder.GetOptions();
53+
options.ShareTokenIntent = ShareTokenIntent.Backup;
54+
return clientBuilder.GetServiceClientFromOauthConfig(clientBuilder.Tenants.TestConfigOAuth, options);
55+
}
56+
4557
public static async Task<DisposingShare> GetTestShareAsync(
4658
this SharesClientBuilder clientBuilder,
4759
ShareServiceClient service = default,

sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileResourceTests.cs

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,22 @@ public async Task CopyFromUriAsync()
313313
contentHash: default,
314314
isServerEncrypted: false),
315315
new MockResponse(200))));
316+
mockDestination.Setup(b => b.ExistsAsync(It.IsAny<CancellationToken>()))
317+
.Returns(Task.FromResult(Response.FromValue(false, new MockResponse(200))));
318+
mockDestination.Setup(b => b.CreateAsync(It.IsAny<long>(), It.IsAny<ShareFileHttpHeaders>(), It.IsAny<Dictionary<string, string>>(), It.IsAny<FileSmbProperties>(), It.IsAny<string>(), It.IsAny<ShareFileRequestConditions>(), It.IsAny<CancellationToken>()))
319+
.Returns(Task.FromResult(Response.FromValue(
320+
FilesModelFactory.StorageFileInfo(
321+
eTag: new ETag("eTag"),
322+
lastModified: DateTimeOffset.UtcNow,
323+
isServerEncrypted: false,
324+
filePermissionKey: "rw",
325+
fileAttributes: "Archive|ReadOnly",
326+
fileCreationTime: DateTimeOffset.UtcNow,
327+
fileLastWriteTime: DateTimeOffset.UtcNow,
328+
fileChangeTime: DateTimeOffset.UtcNow,
329+
fileId: "48903841",
330+
fileParentId: "93024923"),
331+
new MockResponse(200))));
316332
ShareFileStorageResource destinationResource = new ShareFileStorageResource(mockDestination.Object);
317333

318334
int length = 1024;
@@ -327,6 +343,18 @@ public async Task CopyFromUriAsync()
327343
It.IsAny<ShareFileUploadRangeFromUriOptions>(),
328344
It.IsAny<CancellationToken>()),
329345
Times.Once());
346+
mockDestination.Verify(b => b.CreateAsync(
347+
length,
348+
It.IsAny<ShareFileHttpHeaders>(),
349+
It.IsAny<Dictionary<string, string>>(),
350+
It.IsAny<FileSmbProperties>(),
351+
It.IsAny<string>(),
352+
It.IsAny<ShareFileRequestConditions>(),
353+
It.IsAny<CancellationToken>()),
354+
Times.Once());
355+
mockDestination.Verify(b => b.ExistsAsync(
356+
It.IsAny<CancellationToken>()),
357+
Times.Once());
330358
mockDestination.VerifyNoOtherCalls();
331359
}
332360

@@ -342,28 +370,24 @@ public async Task CopyFromUriAsync_Error()
342370
new Uri("https://storageaccount.file.core.windows.net/container/destinationfile"),
343371
new ShareClientOptions());
344372

345-
mockDestination.Setup(b => b.UploadRangeFromUriAsync(It.IsAny<Uri>(), It.IsAny<HttpRange>(), It.IsAny<HttpRange>(), It.IsAny<ShareFileUploadRangeFromUriOptions>(), It.IsAny<CancellationToken>()))
346-
.Throws(new RequestFailedException(status: 404, message: "The specified resource does not exist.", errorCode: "ResourceNotFound", default));
373+
mockDestination.Setup(b => b.ExistsAsync(It.IsAny<CancellationToken>()))
374+
.Returns(Task.FromResult(Response.FromValue(true, new MockResponse(200))));
347375
ShareFileStorageResource destinationResource = new ShareFileStorageResource(mockDestination.Object);
348376

349377
// Act
350378
int length = 1024;
351-
await TestHelper.AssertExpectedExceptionAsync<RequestFailedException>(
352-
destinationResource.CopyFromUriInternalAsync(sourceResource.Object, false, length),
379+
await TestHelper.AssertExpectedExceptionAsync<InvalidOperationException>(
380+
destinationResource.CopyBlockFromUriInternalAsync(sourceResource.Object, new HttpRange(0, length), false, length),
353381
e =>
354382
{
355-
Assert.AreEqual("ResourceNotFound", e.ErrorCode);
383+
Assert.IsTrue(e.Message.Contains("Cannot overwrite file."));
356384
});
357385

358-
sourceResource.Verify(b => b.Uri, Times.Once());
359386
sourceResource.VerifyNoOtherCalls();
360-
mockDestination.Verify(b => b.UploadRangeFromUriAsync(
361-
sourceResource.Object.Uri,
362-
new HttpRange(0, length),
363-
new HttpRange(0, length),
364-
It.IsAny<ShareFileUploadRangeFromUriOptions>(),
387+
mockDestination.Verify(b => b.ExistsAsync(
365388
It.IsAny<CancellationToken>()),
366389
Times.Once());
390+
mockDestination.Verify(b => b.Path, Times.Once());
367391
mockDestination.VerifyNoOtherCalls();
368392
}
369393

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.Threading.Tasks;
6+
using Azure.Storage.Test.Shared;
7+
using Azure.Storage.DataMovement.Tests;
8+
using Azure.Storage.Files.Shares;
9+
using Azure.Storage.Files.Shares.Tests;
10+
using System.IO;
11+
using Azure.Core;
12+
using Azure.Core.TestFramework;
13+
14+
namespace Azure.Storage.DataMovement.Files.Shares.Tests
15+
{
16+
[ShareClientTestFixture]
17+
public class ShareFileStartTransferCopyTests : StartTransferCopyTestBase
18+
<ShareServiceClient,
19+
ShareClient,
20+
ShareFileClient,
21+
ShareClientOptions,
22+
ShareServiceClient,
23+
ShareClient,
24+
ShareFileClient,
25+
ShareClientOptions,
26+
StorageTestEnvironment>
27+
{
28+
private const string _fileResourcePrefix = "test-file-";
29+
private const string _expectedOverwriteExceptionMessage = "Cannot overwrite file.";
30+
protected readonly ShareClientOptions.ServiceVersion _serviceVersion;
31+
32+
public ShareFileStartTransferCopyTests(bool async, ShareClientOptions.ServiceVersion serviceVersion)
33+
: base(async, _expectedOverwriteExceptionMessage, _fileResourcePrefix, null /* RecordedTestMode.Record /* to re-record */)
34+
{
35+
_serviceVersion = serviceVersion;
36+
SourceClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion);
37+
DestinationClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion);
38+
}
39+
40+
protected override async Task<bool> SourceExistsAsync(ShareFileClient objectClient)
41+
=> await objectClient.ExistsAsync();
42+
43+
protected override async Task<bool> DestinationExistsAsync(ShareFileClient objectClient)
44+
=> await objectClient.ExistsAsync();
45+
46+
protected override async Task<IDisposingContainer<ShareClient>> GetSourceDisposingContainerAsync(ShareServiceClient service = null, string containerName = null)
47+
=> await SourceClientBuilder.GetTestShareAsync(service, containerName);
48+
49+
protected override async Task<IDisposingContainer<ShareClient>> GetDestinationDisposingContainerAsync(ShareServiceClient service = null, string containerName = null)
50+
=> await DestinationClientBuilder.GetTestShareAsync(service, containerName);
51+
52+
private async Task<ShareFileClient> CreateFileClientAsync(
53+
ShareClient container,
54+
long? objectLength = null,
55+
bool createResource = false,
56+
string objectName = null,
57+
ShareClientOptions options = null,
58+
Stream contents = null)
59+
{
60+
objectName ??= GetNewObjectName();
61+
ShareFileClient fileClient = container.GetRootDirectoryClient().GetFileClient(objectName);
62+
if (createResource)
63+
{
64+
if (!objectLength.HasValue)
65+
{
66+
throw new InvalidOperationException($"Cannot create share file without size specified. Either set {nameof(createResource)} to false or specify a {nameof(objectLength)}.");
67+
}
68+
await fileClient.CreateAsync(objectLength.Value);
69+
70+
if (contents != default)
71+
{
72+
await fileClient.UploadAsync(contents);
73+
}
74+
}
75+
Uri sourceUri = fileClient.GenerateSasUri(Sas.ShareFileSasPermissions.All, Recording.UtcNow.AddDays(1));
76+
return InstrumentClient(new ShareFileClient(sourceUri, GetOptions()));
77+
}
78+
79+
protected override Task<ShareFileClient> GetSourceObjectClientAsync(
80+
ShareClient container,
81+
long? objectLength = null,
82+
bool createResource = false,
83+
string objectName = null,
84+
ShareClientOptions options = null,
85+
Stream contents = null)
86+
=> CreateFileClientAsync(
87+
container,
88+
objectLength,
89+
createResource,
90+
objectName,
91+
options,
92+
contents);
93+
94+
protected override StorageResourceItem GetSourceStorageResourceItem(ShareFileClient objectClient)
95+
=> new ShareFileStorageResource(objectClient);
96+
97+
protected override Task<Stream> SourceOpenReadAsync(ShareFileClient objectClient)
98+
=> objectClient.OpenReadAsync();
99+
100+
protected override Task<ShareFileClient> GetDestinationObjectClientAsync(
101+
ShareClient container,
102+
long? objectLength = null,
103+
bool createResource = false,
104+
string objectName = null,
105+
ShareClientOptions options = null,
106+
Stream contents = null)
107+
=> CreateFileClientAsync(
108+
container,
109+
objectLength,
110+
createResource,
111+
objectName,
112+
options,
113+
contents);
114+
115+
protected override StorageResourceItem GetDestinationStorageResourceItem(ShareFileClient objectClient)
116+
=> new ShareFileStorageResource(objectClient);
117+
118+
protected override Task<Stream> DestinationOpenReadAsync(ShareFileClient objectClient)
119+
=> objectClient.OpenReadAsync();
120+
121+
public ShareClientOptions GetOptions()
122+
{
123+
var options = new ShareClientOptions(_serviceVersion)
124+
{
125+
Diagnostics = { IsLoggingEnabled = true },
126+
Retry =
127+
{
128+
Mode = RetryMode.Exponential,
129+
MaxRetries = Constants.MaxReliabilityRetries,
130+
Delay = TimeSpan.FromSeconds(Mode == RecordedTestMode.Playback ? 0.01 : 1),
131+
MaxDelay = TimeSpan.FromSeconds(Mode == RecordedTestMode.Playback ? 0.1 : 60)
132+
},
133+
};
134+
if (Mode != RecordedTestMode.Live)
135+
{
136+
options.AddPolicy(new RecordedClientRequestIdPolicy(Recording), HttpPipelinePosition.PerCall);
137+
}
138+
139+
return InstrumentClientOptions(options);
140+
}
141+
}
142+
}

sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,12 @@ await _destinationResource.CopyFromUriAsync(
210210
{
211211
await InvokeSkippedArg().ConfigureAwait(false);
212212
}
213+
catch (InvalidOperationException ex)
214+
when (_createMode == StorageResourceCreationPreference.SkipIfExists
215+
&& ex.Message.Contains("Cannot overwrite file."))
216+
{
217+
await InvokeSkippedArg().ConfigureAwait(false);
218+
}
213219
catch (Exception ex)
214220
{
215221
await InvokeFailedArg(ex).ConfigureAwait(false);

0 commit comments

Comments
 (0)