Skip to content

Commit f5d4fe2

Browse files
amnguyeamnguye
authored
[Storage] [DataMovement] [Blobs] Stress Scenarios (#46400)
* Added separate sln file for Storage Data Movement and start of stress testing for DM Blobs * Fix to Stress Test README * Fix path for .slnf file * WIP * WIP * WIP * WIP * WIP * WIP * WIP - builds * WIP - docker issues * WIP - build issue in docker * WIP - builds locally, not in Docker * WIP - runs until hitting excuting yaml file * WIP - helm won't commit run * WIP - fixing upload scenario test * WIP - helm still not pushing commit; added more options for scenarios * WIP * Cleanup * WIP * More cleanup and some PR Comments * Part 2 PR comments * WIP * WIP * WIP - Created all scenario types and implementation * Cleanup * Update script to include all new scenarios * Cleanup * PR Comments: Updated Create Blob Helper methods; removed file close; short hand * PR comments: Removed unnecessary init transfermanager; reapplied options
1 parent 1a1b91b commit f5d4fe2

36 files changed

+1461
-139
lines changed

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/generatedValues.yaml‎

Lines changed: 86 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,92 @@
11
scenarios:
22
- image: Dockerfile
33
imageBuildDir: ../../../..
4-
testScenario: uploadsingleblockblob
54
Scenario: uploadsingleblockblob
5+
testScenario: uploadsingleblockblob
6+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
7+
- image: Dockerfile
8+
imageBuildDir: ../../../..
9+
Scenario: uploaddirectoryblockBlob
10+
testScenario: uploaddirectoryblockBlob
11+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
12+
- image: Dockerfile
13+
imageBuildDir: ../../../..
14+
Scenario: downloadsingleblockblob
15+
testScenario: downloadsingleblockblob
16+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
17+
- image: Dockerfile
18+
imageBuildDir: ../../../..
19+
Scenario: downloaddirectoryblockblob
20+
testScenario: downloaddirectoryblockblob
21+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
22+
- image: Dockerfile
23+
imageBuildDir: ../../../..
24+
Scenario: copysingleblockblob
25+
testScenario: copysingleblockblob
26+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
27+
- image: Dockerfile
28+
imageBuildDir: ../../../..
29+
Scenario: copydirectoryblockblob
30+
testScenario: copydirectoryblockblob
31+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
32+
- image: Dockerfile
33+
imageBuildDir: ../../../..
34+
Scenario: uploadsingleappendblob
35+
testScenario: uploadsingleappendblob
36+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
37+
- image: Dockerfile
38+
imageBuildDir: ../../../..
39+
Scenario: uploaddirectoryappendblob
40+
testScenario: uploaddirectoryappendblob
41+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
42+
- image: Dockerfile
43+
imageBuildDir: ../../../..
44+
Scenario: downloadsingleappendblob
45+
testScenario: downloadsingleappendblob
46+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
47+
- image: Dockerfile
48+
imageBuildDir: ../../../..
49+
Scenario: downloaddirectoryappendblob
50+
testScenario: downloaddirectoryappendblob
51+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
52+
- image: Dockerfile
53+
imageBuildDir: ../../../..
54+
Scenario: copysingleappendblob
55+
testScenario: copysingleappendblob
56+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
57+
- image: Dockerfile
58+
imageBuildDir: ../../../..
59+
Scenario: copydirectoryappendblob
60+
testScenario: copydirectoryappendblob
61+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
62+
- image: Dockerfile
63+
imageBuildDir: ../../../..
64+
Scenario: uploadsinglepageblob
65+
testScenario: uploadsinglepageblob
66+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
67+
- image: Dockerfile
68+
imageBuildDir: ../../../..
69+
Scenario: uploaddirectorypageblob
70+
testScenario: uploaddirectorypageblob
71+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
72+
- image: Dockerfile
73+
imageBuildDir: ../../../..
74+
Scenario: downloadsinglepageblob
75+
testScenario: downloadsinglepageblob
76+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
77+
- image: Dockerfile
78+
imageBuildDir: ../../../..
79+
Scenario: downloaddirectorypageblob
80+
testScenario: downloaddirectorypageblob
81+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
82+
- image: Dockerfile
83+
imageBuildDir: ../../../..
84+
Scenario: copysinglepageblob
85+
testScenario: copysinglepageblob
86+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
87+
- image: Dockerfile
88+
imageBuildDir: ../../../..
89+
Scenario: copydirectorypageblob
90+
testScenario: copydirectorypageblob
691
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
792

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/scenarios-matrix.yaml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,72 @@ matrix:
66
uploadsingleblockblob:
77
testScenario: uploadsingleblockblob
88
image: Dockerfile
9+
imageBuildDir: "../../../.."
10+
uploaddirectoryblockblob:
11+
testScenario: uploaddirectoryblockblob
12+
image: Dockerfile
13+
imageBuildDir: "../../../.."
14+
downloadsingleblockblob:
15+
testScenario: downloadsingleblockblob
16+
image: Dockerfile
17+
imageBuildDir: "../../../.."
18+
downloaddirectoryblockblob:
19+
testScenario: downloaddirectoryblockblob
20+
image: Dockerfile
21+
imageBuildDir: "../../../.."
22+
copysingleblockblob:
23+
testScenario: copysingleblockblob
24+
image: Dockerfile
25+
imageBuildDir: "../../../.."
26+
copydirectoryblockblob:
27+
testScenario: copydirectoryblockblob
28+
image: Dockerfile
29+
imageBuildDir: "../../../.."
30+
uploadsingleappendblob:
31+
testScenario: uploadsingleappendblob
32+
image: Dockerfile
33+
imageBuildDir: "../../../.."
34+
uploaddirectoryappendblob:
35+
testScenario: uploaddirectoryappendblob
36+
image: Dockerfile
37+
imageBuildDir: "../../../.."
38+
downloadsingleappendblob:
39+
testScenario: downloadsingleappendblob
40+
image: Dockerfile
41+
imageBuildDir: "../../../.."
42+
downloaddirectoryappendblob:
43+
testScenario: downloaddirectoryappendblob
44+
image: Dockerfile
45+
imageBuildDir: "../../../.."
46+
copysingleappendblob:
47+
testScenario: copysingleappendblob
48+
image: Dockerfile
49+
imageBuildDir: "../../../.."
50+
copydirectoryappendblob:
51+
testScenario: copydirectoryappendblob
52+
image: Dockerfile
53+
imageBuildDir: "../../../.."
54+
uploadsinglepageblob:
55+
testScenario: uploadsinglepageblob
56+
image: Dockerfile
57+
imageBuildDir: "../../../.."
58+
uploaddirectorypageblob:
59+
testScenario: uploaddirectorypageblob
60+
image: Dockerfile
61+
imageBuildDir: "../../../.."
62+
downloadsinglepageblob:
63+
testScenario: downloadsinglepageblob
64+
image: Dockerfile
65+
imageBuildDir: "../../../.."
66+
downloaddirectorypageblob:
67+
testScenario: downloaddirectorypageblob
68+
image: Dockerfile
69+
imageBuildDir: "../../../.."
70+
copysinglepageblob:
71+
testScenario: copysinglepageblob
72+
image: Dockerfile
73+
imageBuildDir: "../../../.."
74+
copydirectorypageblob:
75+
testScenario: copydirectorypageblob
76+
image: Dockerfile
977
imageBuildDir: "../../../.."

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Azure.Storage.DataMovement.Blobs.Stress.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
</ItemGroup>
3535
<ItemGroup>
3636
<Compile Include="$(AzureStorageSharedSources)Constants.cs" Link="Shared\%(RecursiveDir)\%(Filename)%(Extension)" />
37+
<Compile Include="$(AzureStorageSharedSources)WindowStream.cs" LinkBase="Shared\Storage" />
3738
</ItemGroup>
3839
<ItemGroup>
3940
<Compile Include="$(AzureStorageSharedTestSources)TemporaryFileStream.cs" LinkBase="SharedSource\Storage" />

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Infrastructure/BlobTestSetupHelper.cs‎

Lines changed: 68 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,25 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33

4+
using System;
45
using System.IO;
56
using System.Threading;
67
using System.Threading.Tasks;
8+
using Azure.Storage.Shared;
79
using Azure.Storage.Blobs;
810
using Azure.Storage.Blobs.Specialized;
11+
using NUnit.Framework;
12+
using Azure.Storage.Blobs.Models;
913

1014
namespace Azure.Storage.DataMovement.Blobs.Stress
1115
{
1216
public static class BlobTestSetupHelper
1317
{
14-
public static async Task CreateDirectoryBlockBlobsAsync(
18+
private const int DefaultBufferSize = 4 * Constants.KB * Constants.KB;
19+
20+
public static async Task CreateBlobsInDirectoryAsync(
1521
BlobContainerClient container,
22+
BlobType blobType,
1623
string pathPrefix,
1724
int blobCount = 2,
1825
long objectLength = Constants.KB * 4,
@@ -21,10 +28,27 @@ public static async Task CreateDirectoryBlockBlobsAsync(
2128
for (int i = 0; i < blobCount; i++)
2229
{
2330
string blobName = $"{pathPrefix}/{TestSetupHelper.Randomize("blob")}";
24-
await CreateBlockBlobAsync(
31+
if (blobType == BlobType.Append)
32+
{
33+
await CreateAppendBlobAsync(
34+
container.GetAppendBlobClient(blobName),
35+
objectLength,
36+
cancellationToken);
37+
}
38+
else if (blobType == BlobType.Page)
39+
{
40+
await CreatePageBlobAsync(
41+
container.GetPageBlobClient(blobName),
42+
objectLength,
43+
cancellationToken);
44+
}
45+
else
46+
{
47+
await CreateBlockBlobAsync(
2548
container.GetBlockBlobClient(blobName),
2649
objectLength,
2750
cancellationToken);
51+
}
2852
}
2953
}
3054

@@ -34,18 +58,52 @@ public static async Task CreateBlockBlobAsync(
3458
CancellationToken cancellationToken = default)
3559
{
3660
using Stream originalStream = await TestSetupHelper.CreateLimitedMemoryStream(objectLength.Value, cancellationToken: cancellationToken);
37-
if (originalStream != default)
61+
var data = new byte[0];
62+
using (var stream = new MemoryStream(data))
3863
{
39-
await blockBlobClient.UploadAsync(originalStream, cancellationToken: cancellationToken);
64+
await blockBlobClient.UploadAsync(
65+
content: stream,
66+
cancellationToken: cancellationToken);
4067
}
41-
else
68+
}
69+
70+
public static async Task CreateAppendBlobAsync(
71+
AppendBlobClient appendBlobClient,
72+
long? objectLength = Constants.KB * 4,
73+
CancellationToken cancellationToken = default)
74+
{
75+
await appendBlobClient.CreateIfNotExistsAsync();
76+
if (objectLength.Value > 0)
77+
{
78+
using Stream originalStream = await TestSetupHelper.CreateLimitedMemoryStream(objectLength.Value, cancellationToken: cancellationToken);
79+
long offset = 0;
80+
long blockSize = Math.Min(DefaultBufferSize, objectLength.Value);
81+
while (offset < objectLength.Value)
82+
{
83+
Stream partStream = WindowStream.GetWindow(originalStream, blockSize);
84+
await appendBlobClient.AppendBlockAsync(partStream);
85+
offset += blockSize;
86+
}
87+
}
88+
}
89+
90+
public static async Task CreatePageBlobAsync(
91+
PageBlobClient pageBlobClient,
92+
long? objectLength = Constants.KB * 4,
93+
CancellationToken cancellationToken = default)
94+
{
95+
Assert.IsTrue(objectLength.Value % (Constants.KB / 2) == 0, "Cannot create page blob that's not a multiple of 512");
96+
await pageBlobClient.CreateIfNotExistsAsync(objectLength.Value);
97+
if (objectLength.Value > 0)
4298
{
43-
var data = new byte[0];
44-
using (var stream = new MemoryStream(data))
99+
using Stream originalStream = await TestSetupHelper.CreateLimitedMemoryStream(objectLength.Value, cancellationToken: cancellationToken);
100+
long offset = 0;
101+
long blockSize = Math.Min(DefaultBufferSize, objectLength.Value);
102+
while (offset < objectLength.Value)
45103
{
46-
await blockBlobClient.UploadAsync(
47-
content: stream,
48-
cancellationToken: cancellationToken);
104+
Stream partStream = WindowStream.GetWindow(originalStream, blockSize);
105+
await pageBlobClient.UploadPagesAsync(partStream, offset);
106+
offset += blockSize;
49107
}
50108
}
51109
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Infrastructure/TestSetupHelper.cs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,8 @@ public static async Task<StorageResource> GetTemporaryFileStorageResourceAsync(
7272
string localSourceFile = Path.Combine(prefixPath, fileName);
7373
// create a new file and copy contents of stream into it, and then close the FileStream
7474
// so the StagedUploadAsync call is not prevented from reading using its FileStream.
75-
Console.Out.Write($"Creating File: {localSourceFile}..");
7675
using (FileStream fileStream = File.Create(localSourceFile))
7776
{
78-
Console.Out.WriteLine("Copying Stream to File..");
7977
await originalStream.CopyToAsync(
8078
fileStream,
8179
bufferSize,

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Scenarios/BlobScenarioBase.cs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,29 +14,32 @@ namespace Azure.Storage.DataMovement.Blobs.Stress
1414
public abstract class BlobScenarioBase : TestScenarioBase
1515
{
1616
protected internal readonly Uri _destinationBlobUri;
17+
protected internal int _blobSize;
1718
protected internal readonly TokenCredential _tokenCredential;
1819
protected internal BlobsStorageResourceProvider _blobsStorageResourceProvider;
1920
protected internal LocalFilesStorageResourceProvider _localFilesStorageResourceProvider;
20-
protected internal BlobServiceClient _destinationServiceClient;
21+
protected internal BlobServiceClient _blobServiceClient;
2122
protected internal readonly TransferManagerOptions _transferManagerOptions;
2223
protected internal readonly DataTransferOptions _dataTransferOptions;
2324

2425
public BlobScenarioBase(
25-
Uri destinationBlobUri,
26+
Uri blobUri,
27+
int? blobSize,
2628
TransferManagerOptions transferManagerOptions,
2729
DataTransferOptions dataTransferOptions,
2830
TokenCredential tokenCredential,
2931
Metrics metrics,
3032
string testRunId)
3133
: base(metrics, testRunId)
3234
{
33-
_destinationBlobUri = destinationBlobUri;
35+
_destinationBlobUri = blobUri;
36+
_blobSize = blobSize != default ? blobSize.Value : DataMovementBlobStressConstants.DefaultObjectSize;
3437
_transferManagerOptions = transferManagerOptions;
3538
_dataTransferOptions = dataTransferOptions;
3639
_tokenCredential = tokenCredential;
3740
_blobsStorageResourceProvider = new BlobsStorageResourceProvider(tokenCredential);
3841
_localFilesStorageResourceProvider = new LocalFilesStorageResourceProvider();
39-
_destinationServiceClient = new BlobServiceClient(destinationBlobUri, tokenCredential);
42+
_blobServiceClient = new BlobServiceClient(blobUri, tokenCredential);
4043
}
4144
}
4245
}

0 commit comments

Comments
 (0)