Skip to content

Commit ab26547

Browse files
nickliu-msftnickliu-msft
authored
[Storage][DataMovement][Tests] Added test coverage for upload/download directory for each blob type (Azure#46826)
* WIP - Finished directory downloads tests * WIP - Removed old block blob test * WIP - Recorded download directories tests + beginnings of directory upload * Added Upload tests + recordings * small refactor * removed old upload directory test class * recorded dm.files.shares * assert Inconclusive for page blobs * reverted test case back to 12345 size + new rcording * re-recorded dm-files.shares
1 parent ea8d2b3 commit ab26547

14 files changed

+427
-1647
lines changed

‎sdk/storage/Azure.Storage.DataMovement.Blobs/assets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "net",
44
"TagPrefix": "net/storage/Azure.Storage.DataMovement.Blobs",
5-
"Tag": "net/storage/Azure.Storage.DataMovement.Blobs_c66f15b319"
5+
"Tag": "net/storage/Azure.Storage.DataMovement.Blobs_acf7c63811"
66
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobDirectoryStartTransferDownloadTests.cs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
extern alias DMBlobs;
5+
extern alias BaseBlobs;
6+
7+
using System.IO;
8+
using System.Threading;
9+
using System.Threading.Tasks;
10+
using BaseBlobs::Azure.Storage.Blobs;
11+
using BaseBlobs::Azure.Storage.Blobs.Models;
12+
using BaseBlobs::Azure.Storage.Blobs.Specialized;
13+
14+
namespace Azure.Storage.DataMovement.Blobs.Tests
15+
{
16+
[DataMovementBlobsClientTestFixture]
17+
public class AppendBlobDirectoryStartTransferDownloadTests
18+
: BlobDirectoryStartTransferDownloadTestBase
19+
{
20+
public AppendBlobDirectoryStartTransferDownloadTests(
21+
bool async,
22+
BlobClientOptions.ServiceVersion serviceVersion)
23+
: base(async, serviceVersion)
24+
{
25+
}
26+
27+
protected override async Task CreateBlobClientAsync(
28+
BlobContainerClient container, string blobName, Stream contents, CancellationToken cancellationToken = default)
29+
{
30+
AppendBlobClient appendBlobClient = container.GetAppendBlobClient(blobName);
31+
await appendBlobClient.CreateIfNotExistsAsync();
32+
if (contents != default)
33+
{
34+
if (contents.Length != 0)
35+
{
36+
// cannot append a stream that has a Content-Length of 0
37+
await appendBlobClient.AppendBlockAsync(contents, cancellationToken: cancellationToken);
38+
}
39+
}
40+
}
41+
42+
protected override BlobType GetBlobType()
43+
=> BlobType.Append;
44+
}
45+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobDirectoryStartTransferUploadTests.cs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
extern alias DMBlobs;
5+
extern alias BaseBlobs;
6+
7+
using System.IO;
8+
using System.Threading;
9+
using System.Threading.Tasks;
10+
using BaseBlobs::Azure.Storage.Blobs;
11+
using BaseBlobs::Azure.Storage.Blobs.Models;
12+
using BaseBlobs::Azure.Storage.Blobs.Specialized;
13+
14+
namespace Azure.Storage.DataMovement.Blobs.Tests
15+
{
16+
[DataMovementBlobsClientTestFixture]
17+
public class AppendBlobDirectoryStartTransferUploadTests
18+
: BlobDirectoryStartTransferUploadTestBase<AppendBlobClient>
19+
{
20+
public AppendBlobDirectoryStartTransferUploadTests(
21+
bool async,
22+
BlobClientOptions.ServiceVersion serviceVersion)
23+
: base(async, serviceVersion)
24+
{
25+
}
26+
27+
protected override async Task CreateBlobClientAsync(
28+
BlobContainerClient container, string blobName, Stream contents, CancellationToken cancellationToken = default)
29+
{
30+
AppendBlobClient appendBlobClient = container.GetAppendBlobClient(blobName);
31+
await appendBlobClient.CreateIfNotExistsAsync();
32+
if (contents != default)
33+
{
34+
if (contents.Length != 0)
35+
{
36+
// cannot append a stream that has a Content-Length of 0
37+
await appendBlobClient.AppendBlockAsync(contents, cancellationToken: cancellationToken);
38+
}
39+
}
40+
}
41+
42+
protected override BlobType GetBlobType()
43+
=> BlobType.Append;
44+
}
45+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@
4747
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferCopyTestBase.cs" LinkBase="Shared\DataMovement" />
4848
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferDownloadTestBase.cs" LinkBase="Shared\DataMovement" />
4949
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferDirectoryCopyTestBase.cs" LinkBase="Shared\DataMovement" />
50+
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferDirectoryDownloadTestBase.cs" LinkBase="Shared\DataMovement" />
51+
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferUploadDirectoryTestBase.cs" LinkBase="Shared\DataMovement" />
5052
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TransferValidator.cs" LinkBase="Shared\DataMovement" />
5153
<Compile Include="$(AzureStorageDataMovementTestSharedSources)DataMovementTestBase.cs" LinkBase="Shared\DataMovement" />
5254
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TransferDirection.cs" LinkBase="Shared\DataMovement" />

‎sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlobDirectoryStartTransferDownloadTestBase.cs‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
extern alias DMBlobs;
5+
extern alias BaseBlobs;
6+
7+
using System.Collections.Generic;
8+
using System.IO;
9+
using System.Threading;
10+
using System.Threading.Tasks;
11+
using BaseBlobs::Azure.Storage.Blobs;
12+
using BaseBlobs::Azure.Storage.Blobs.Models;
13+
using Azure.Storage.DataMovement.Tests;
14+
using Azure.Storage.Test.Shared;
15+
using DMBlobs::Azure.Storage.DataMovement.Blobs;
16+
17+
namespace Azure.Storage.DataMovement.Blobs.Tests
18+
{
19+
[DataMovementBlobsClientTestFixture]
20+
public abstract class BlobDirectoryStartTransferDownloadTestBase
21+
: StartTransferDirectoryDownloadTestBase<
22+
BlobServiceClient,
23+
BlobContainerClient,
24+
BlobClientOptions,
25+
StorageTestEnvironment>
26+
{
27+
private const string _fileResourcePrefix = "test-file-";
28+
private const string _dirResourcePrefix = "test-file-";
29+
private const string _expectedOverwriteExceptionMessage = "Cannot overwrite file.";
30+
31+
public BlobDirectoryStartTransferDownloadTestBase(
32+
bool async,
33+
BlobClientOptions.ServiceVersion serviceVersion)
34+
: base(async, _expectedOverwriteExceptionMessage, _fileResourcePrefix, _dirResourcePrefix, null /* RecordedTestMode.Record /* to re-record */)
35+
{
36+
ClientBuilder = ClientBuilderExtensions.GetNewBlobsClientBuilder(Tenants, serviceVersion);
37+
}
38+
39+
protected abstract Task CreateBlobClientAsync(BlobContainerClient container, string blobName, Stream contents, CancellationToken cancellationToken = default);
40+
41+
protected abstract BlobType GetBlobType();
42+
43+
protected override async Task<IDisposingContainer<BlobContainerClient>> GetDisposingContainerAsync(BlobServiceClient service = null, string containerName = null)
44+
=> await ClientBuilder.GetTestContainerAsync(service, containerName);
45+
46+
protected override StorageResourceContainer GetStorageResourceContainer(BlobContainerClient container, string directoryPath)
47+
=> new BlobStorageResourceContainer(container, new() { BlobType = new(GetBlobType()), BlobDirectoryPrefix = directoryPath });
48+
49+
protected override TransferValidator.ListFilesAsync GetSourceLister(BlobContainerClient container, string prefix)
50+
=> TransferValidator.GetBlobLister(container, prefix);
51+
52+
protected override async Task CreateObjectClientAsync(
53+
BlobContainerClient container,
54+
long? objectLength,
55+
string blobName,
56+
bool createResource = false,
57+
BlobClientOptions options = null,
58+
Stream contents = null,
59+
CancellationToken cancellationToken = default)
60+
{
61+
blobName ??= GetNewObjectName();
62+
await CreateBlobClientAsync(container, blobName, contents, cancellationToken);
63+
}
64+
65+
protected override async Task SetupSourceDirectoryAsync(
66+
BlobContainerClient container,
67+
string directoryPath,
68+
List<(string PathName, int Size)> fileSizes,
69+
CancellationToken cancellationToken)
70+
{
71+
foreach ((string blobName, int size) in fileSizes)
72+
{
73+
await CreateBlobClientAsync(container, blobName, new MemoryStream(GetRandomBuffer(size)), cancellationToken);
74+
}
75+
}
76+
}
77+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlobDirectoryStartTransferUploadTestBase.cs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
extern alias BaseBlobs;
5+
extern alias DMBlobs;
6+
using System.Collections.Generic;
7+
using System.IO;
8+
using System.Threading;
9+
using System.Threading.Tasks;
10+
using Azure.Storage.DataMovement.Tests;
11+
using Azure.Storage.Test.Shared;
12+
using BaseBlobs::Azure.Storage.Blobs;
13+
using BaseBlobs::Azure.Storage.Blobs.Models;
14+
using BaseBlobs::Azure.Storage.Blobs.Specialized;
15+
using DMBlobs::Azure.Storage.DataMovement.Blobs;
16+
17+
namespace Azure.Storage.DataMovement.Blobs.Tests
18+
{
19+
[DataMovementBlobsClientTestFixture]
20+
public abstract class BlobDirectoryStartTransferUploadTestBase<TObjectClient>
21+
: StartTransferUploadDirectoryTestBase<
22+
BlobServiceClient,
23+
BlobContainerClient,
24+
TObjectClient,
25+
BlobClientOptions,
26+
StorageTestEnvironment>
27+
where TObjectClient : BlobBaseClient
28+
{
29+
public BlobDirectoryStartTransferUploadTestBase(bool async, BlobClientOptions.ServiceVersion serviceVersion, bool isPageBlob = false)
30+
: base(async, null, isPageBlob)
31+
{
32+
ClientBuilder = ClientBuilderExtensions.GetNewBlobsClientBuilder(Tenants, serviceVersion);
33+
}
34+
35+
protected abstract Task CreateBlobClientAsync(BlobContainerClient container, string blobName, Stream contents, CancellationToken cancellationToken = default);
36+
37+
protected abstract BlobType GetBlobType();
38+
39+
protected override async Task<IDisposingContainer<BlobContainerClient>> GetDisposingContainerAsync(
40+
BlobServiceClient service = default,
41+
string containerName = default)
42+
=> await ClientBuilder.GetTestContainerAsync(service, containerName);
43+
44+
protected override TransferValidator.ListFilesAsync GetStorageResourceLister(BlobContainerClient containerClient)
45+
=> TransferValidator.GetBlobLister(containerClient, "");
46+
47+
protected override async Task InitializeDestinationDataAsync(
48+
BlobContainerClient containerClient,
49+
List<(string FilePath, long Size)> fileSizes,
50+
CancellationToken cancellationToken)
51+
{
52+
foreach ((string blobName, long size) in fileSizes)
53+
{
54+
await CreateBlobClientAsync(containerClient, blobName, new MemoryStream(GetRandomBuffer(size)), cancellationToken);
55+
}
56+
}
57+
58+
protected override StorageResourceContainer GetStorageResourceContainer(BlobContainerClient containerClient)
59+
{
60+
BlobStorageResourceContainerOptions options = new();
61+
options.BlobType = new(GetBlobType());
62+
return new BlobStorageResourceContainer(containerClient, options);
63+
}
64+
}
65+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobDirectoryStartTransferDownloadTests.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
extern alias DMBlobs;
5+
extern alias BaseBlobs;
6+
7+
using System.IO;
8+
using System.Threading;
9+
using System.Threading.Tasks;
10+
using BaseBlobs::Azure.Storage.Blobs;
11+
using BaseBlobs::Azure.Storage.Blobs.Models;
12+
using BaseBlobs::Azure.Storage.Blobs.Specialized;
13+
14+
namespace Azure.Storage.DataMovement.Blobs.Tests
15+
{
16+
[DataMovementBlobsClientTestFixture]
17+
public class BlockBlobDirectoryStartTransferDownloadTests
18+
: BlobDirectoryStartTransferDownloadTestBase
19+
{
20+
public BlockBlobDirectoryStartTransferDownloadTests(
21+
bool async,
22+
BlobClientOptions.ServiceVersion serviceVersion)
23+
: base(async, serviceVersion)
24+
{
25+
}
26+
27+
protected override async Task CreateBlobClientAsync(
28+
BlobContainerClient container, string blobName, Stream contents, CancellationToken cancellationToken = default)
29+
{
30+
BlockBlobClient blockBlobClient = container.GetBlockBlobClient(blobName);
31+
if (contents != default)
32+
{
33+
await blockBlobClient.UploadAsync(contents, cancellationToken: cancellationToken);
34+
}
35+
}
36+
37+
protected override BlobType GetBlobType()
38+
=> BlobType.Block;
39+
}
40+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobDirectoryStartTransferUploadTests.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
extern alias DMBlobs;
5+
extern alias BaseBlobs;
6+
7+
using System.IO;
8+
using System.Threading;
9+
using System.Threading.Tasks;
10+
using BaseBlobs::Azure.Storage.Blobs;
11+
using BaseBlobs::Azure.Storage.Blobs.Models;
12+
using BaseBlobs::Azure.Storage.Blobs.Specialized;
13+
14+
namespace Azure.Storage.DataMovement.Blobs.Tests
15+
{
16+
[DataMovementBlobsClientTestFixture]
17+
public class BlockBlobDirectoryStartTransferUploadTests
18+
: BlobDirectoryStartTransferUploadTestBase<BlockBlobClient>
19+
{
20+
public BlockBlobDirectoryStartTransferUploadTests(
21+
bool async,
22+
BlobClientOptions.ServiceVersion serviceVersion)
23+
: base(async, serviceVersion)
24+
{
25+
}
26+
27+
protected override async Task CreateBlobClientAsync(
28+
BlobContainerClient container, string blobName, Stream contents, CancellationToken cancellationToken = default)
29+
{
30+
BlockBlobClient blockBlobClient = container.GetBlockBlobClient(blobName);
31+
if (contents != default)
32+
{
33+
await blockBlobClient.UploadAsync(contents, cancellationToken: cancellationToken);
34+
}
35+
}
36+
37+
protected override BlobType GetBlobType()
38+
=> BlobType.Block;
39+
}
40+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobDirectoryStartTransferDownloadTests.cs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
extern alias DMBlobs;
5+
extern alias BaseBlobs;
6+
7+
using System.IO;
8+
using System.Threading;
9+
using System.Threading.Tasks;
10+
using BaseBlobs::Azure.Storage.Blobs;
11+
using BaseBlobs::Azure.Storage.Blobs.Models;
12+
using BaseBlobs::Azure.Storage.Blobs.Specialized;
13+
using NUnit.Framework;
14+
15+
namespace Azure.Storage.DataMovement.Blobs.Tests
16+
{
17+
[DataMovementBlobsClientTestFixture]
18+
public class PageBlobDirectoryStartTransferDownloadTests
19+
: BlobDirectoryStartTransferDownloadTestBase
20+
{
21+
public PageBlobDirectoryStartTransferDownloadTests(
22+
bool async,
23+
BlobClientOptions.ServiceVersion serviceVersion)
24+
: base(async, serviceVersion)
25+
{
26+
}
27+
28+
protected override async Task CreateBlobClientAsync(
29+
BlobContainerClient container, string blobName, Stream contents, CancellationToken cancellationToken = default)
30+
{
31+
PageBlobClient pageBlobClient = container.GetPageBlobClient(blobName);
32+
await pageBlobClient.CreateAsync(Constants.KB * 5);
33+
if (contents != default)
34+
{
35+
if (contents.Length % (Constants.KB/2) != 0)
36+
{
37+
Assert.Inconclusive("Cannot upload a page that has a Content-Length not an increment of 512");
38+
}
39+
if (contents.Length != 0)
40+
{
41+
// cannot upload a page that has a Content-Length of 0
42+
await pageBlobClient.UploadPagesAsync(contents, 0, cancellationToken: cancellationToken);
43+
}
44+
}
45+
}
46+
47+
protected override BlobType GetBlobType()
48+
=> BlobType.Page;
49+
}
50+
}

0 commit comments

Comments
 (0)