Skip to content

Commit a620967

Browse files
jalauzon-msftjalauzon-msft
authored
[Storage][DataMovement] Add basic perf tests for DataMovement Blobs (#46424)
1 parent acc57c6 commit a620967

File tree

12 files changed

+404
-18
lines changed

12 files changed

+404
-18
lines changed

‎sdk/storage/Azure.Storage.Common/tests/Shared/RepeatingStream.cs‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,7 @@
22
// Licensed under the MIT License.
33

44
using System;
5-
using System.Buffers;
65
using System.IO;
7-
using Azure.Core;
8-
using Azure.Storage.Common;
96

107
namespace Azure.Storage.Tests.Shared
118
{
@@ -45,9 +42,14 @@ public override void Flush()
4542

4643
public override int Read(byte[] buffer, int offset, int count)
4744
{
48-
Argument.AssertNotNull(buffer, nameof(buffer));
49-
Argument.AssertInRange(offset, 0, buffer.Length - 1, nameof(offset));
50-
Argument.AssertInRange(count, 0, buffer.Length - offset, nameof(count));
45+
if (offset < 0 || offset > buffer.Length - 1)
46+
{
47+
throw new ArgumentOutOfRangeException(nameof(offset));
48+
}
49+
if (count < 0 || count > buffer.Length - offset)
50+
{
51+
throw new ArgumentOutOfRangeException(nameof(count));
52+
}
5153

5254
int dataOffset = (int) (Position % _data.Length);
5355
int toRead = (int)Math.Min(Math.Min(count, _length - Position), _data.Length - dataOffset);

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Azure.Storage.DataMovement.Blobs.Perf.csproj‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<OutputType>Exe</OutputType>
4+
</PropertyGroup>
5+
6+
<ItemGroup>
7+
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\src\Azure.Storage.DataMovement.Blobs.csproj" />
8+
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj" />
9+
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj" />
10+
</ItemGroup>
11+
12+
<ItemGroup>
13+
<Compile Include="$(AzureStorageSharedTestSources)\RepeatingStream.cs" LinkBase="Shared" />
14+
</ItemGroup>
15+
</Project>

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Infrastructure/DirectoryTransferTest.cs‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.IO;
6+
using System.Linq;
7+
using System.Threading;
8+
using System.Threading.Tasks;
9+
using Azure.Storage.Blobs;
10+
using Azure.Storage.Tests.Shared;
11+
using Azure.Test.Perf;
12+
13+
namespace Azure.Storage.DataMovement.Blobs.Perf
14+
{
15+
public abstract class DirectoryTransferTest<TOptions> : PerfTest<TOptions> where TOptions : DirectoryTransferOptions
16+
{
17+
protected Random Random { get; }
18+
protected BlobServiceClient BlobServiceClient { get; }
19+
protected LocalFilesStorageResourceProvider LocalFileResourceProvider { get; }
20+
protected BlobsStorageResourceProvider BlobResourceProvider { get; }
21+
22+
public DirectoryTransferTest(TOptions options) : base(options)
23+
{
24+
Random = new Random();
25+
BlobServiceClient = new BlobServiceClient(PerfTestEnvironment.Instance.BlobStorageEndpoint, PerfTestEnvironment.Instance.Credential);
26+
LocalFileResourceProvider = new LocalFilesStorageResourceProvider();
27+
BlobResourceProvider = new BlobsStorageResourceProvider(PerfTestEnvironment.Instance.Credential);
28+
}
29+
30+
protected string CreateLocalDirectory(bool populate = false)
31+
{
32+
string directory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
33+
Directory.CreateDirectory(directory);
34+
35+
if (populate)
36+
{
37+
foreach (int i in Enumerable.Range(0, Options.Count))
38+
{
39+
string filePath = Path.Combine(directory, $"file{i}");
40+
using (RepeatingStream stream = new(1024 * 1024, Options.Size, true))
41+
using (FileStream file = File.Open(filePath, FileMode.Create))
42+
{
43+
stream.CopyTo(file);
44+
}
45+
}
46+
}
47+
48+
return directory;
49+
}
50+
51+
protected async Task<BlobContainerClient> CreateBlobContainerAsync(bool populate = false)
52+
{
53+
string containerName = $"test-{Guid.NewGuid()}".ToLowerInvariant();
54+
BlobContainerClient container = BlobServiceClient.GetBlobContainerClient(containerName);
55+
await container.CreateIfNotExistsAsync();
56+
57+
if (populate)
58+
{
59+
foreach (int i in Enumerable.Range(0, Options.Count))
60+
{
61+
BlobClient blob = container.GetBlobClient($"blob{i}");
62+
using (RepeatingStream stream = new(1024 * 1024, Options.Size, true))
63+
{
64+
await blob.UploadAsync(stream);
65+
}
66+
}
67+
}
68+
69+
return container;
70+
}
71+
72+
protected async Task RunAndVerifyTransferAsync(
73+
StorageResource source,
74+
StorageResource destination,
75+
CancellationToken cancellationToken)
76+
{
77+
TransferManagerOptions managerOptions = new()
78+
{
79+
ErrorHandling = DataTransferErrorMode.StopOnAnyFailure,
80+
CheckpointerOptions = Options.DisableCheckpointer ? TransferCheckpointStoreOptions.Disabled() : default
81+
};
82+
TransferManager transferManager = new(managerOptions);
83+
84+
DataTransferOptions options = new()
85+
{
86+
CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
87+
InitialTransferSize = Options.InitialTransferSize,
88+
MaximumTransferChunkSize = Options.ChunkSize,
89+
};
90+
options.ItemTransferFailed += HandleFailure;
91+
DataTransfer transfer = await transferManager.StartTransferAsync(
92+
source, destination, options, cancellationToken);
93+
94+
await transfer.WaitForCompletionAsync(cancellationToken);
95+
if (!transfer.TransferStatus.HasCompletedSuccessfully)
96+
{
97+
throw new Exception("A failure occurred during the transfer.");
98+
}
99+
}
100+
101+
private Task HandleFailure(TransferItemFailedEventArgs args)
102+
{
103+
Console.WriteLine(args.Exception);
104+
return Task.CompletedTask;
105+
}
106+
}
107+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Infrastructure/PerfTestEnvironment.cs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using Azure.Core.TestFramework;
6+
7+
namespace Azure.Storage.DataMovement.Blobs.Perf
8+
{
9+
/// <summary>
10+
/// Represents the ambient environment in which the test suite is being run, offering access to information such as environment variables.
11+
/// </summary>
12+
internal sealed class PerfTestEnvironment : TestEnvironment
13+
{
14+
/// <summary>
15+
/// The shared instance of the <see cref="PerfTestEnvironment"/> to be used during test runs.
16+
/// </summary>
17+
public static PerfTestEnvironment Instance { get; } = new PerfTestEnvironment();
18+
19+
/// <summary>
20+
/// The storage account endpoint suffix of the cloud to use for testing.
21+
/// </summary>
22+
public new string StorageEndpointSuffix => base.StorageEndpointSuffix ?? "core.windows.net";
23+
24+
/// <summary>
25+
/// The name of the Blob storage account to test against.
26+
/// </summary>
27+
/// <value>The Blob storage account name, read from the "AZURE_STORAGE_ACCOUNT_NAME" environment variable.</value>
28+
public string BlobStorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
29+
30+
/// <summary>
31+
/// The Blob storage endpoint.
32+
/// </summary>
33+
public Uri BlobStorageEndpoint { get; }
34+
35+
/// <summary>
36+
/// Initializes a new instance of the <see cref="PerfTestEnvironment"/> class.
37+
/// </summary>
38+
public PerfTestEnvironment()
39+
{
40+
BlobStorageEndpoint = new Uri($"https://{BlobStorageAccountName}.blob.{StorageEndpointSuffix}");
41+
}
42+
}
43+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Options/DirectoryTransferOptions.cs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using Azure.Test.Perf;
5+
using CommandLine;
6+
7+
namespace Azure.Storage.DataMovement.Blobs.Perf
8+
{
9+
public class DirectoryTransferOptions : PerfOptions
10+
{
11+
[Option('c', "count", Default = 10, HelpText = "Number of items in each transfer.")]
12+
public int Count { get; set; }
13+
14+
[Option('s', "size", Default = 1024, HelpText = "Size of each file (in bytes)")]
15+
public long Size { get; set; }
16+
17+
[Option("initial-transfer-size", HelpText = "The initial size to use during transfers (in bytes)")]
18+
public long? InitialTransferSize { get; set; }
19+
20+
[Option("chunk-size", HelpText = "The chunk/block size to use during transfers (in bytes)")]
21+
public long? ChunkSize { get; set; }
22+
23+
[Option("disable-checkpointer", HelpText = "Set to disable checkpointing.")]
24+
public bool DisableCheckpointer { get; set; }
25+
26+
// Override warmup to set default to 0
27+
[Option('w', "warmup", Default = 0, HelpText = "Duration of warmup in seconds")]
28+
public new int Warmup { get; set; }
29+
}
30+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Program.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System.Reflection;
5+
using Azure.Test.Perf;
6+
7+
await PerfProgram.Main(Assembly.GetEntryAssembly(), args);

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Scenarios/CopyDirectory.cs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.Threading;
6+
using System.Threading.Tasks;
7+
using Azure.Storage.Blobs;
8+
9+
namespace Azure.Storage.DataMovement.Blobs.Perf
10+
{
11+
public class CopyDirectory : DirectoryTransferTest<DirectoryTransferOptions>
12+
{
13+
private BlobContainerClient _sourceContainer;
14+
private BlobContainerClient _destinationContainer;
15+
16+
public CopyDirectory(DirectoryTransferOptions options) : base(options)
17+
{
18+
}
19+
20+
public override async Task GlobalSetupAsync()
21+
{
22+
await base.GlobalSetupAsync();
23+
_sourceContainer = await CreateBlobContainerAsync(populate: true);
24+
_destinationContainer = await CreateBlobContainerAsync();
25+
}
26+
27+
public override async Task GlobalCleanupAsync()
28+
{
29+
await _sourceContainer.DeleteIfExistsAsync();
30+
await _destinationContainer.DeleteIfExistsAsync();
31+
await base.GlobalCleanupAsync();
32+
}
33+
34+
public override void Run(CancellationToken cancellationToken)
35+
{
36+
throw new NotImplementedException();
37+
}
38+
39+
public override async Task RunAsync(CancellationToken cancellationToken)
40+
{
41+
StorageResource source = BlobResourceProvider.FromContainer(_sourceContainer.Uri);
42+
StorageResource destination = BlobResourceProvider.FromContainer(_destinationContainer.Uri);
43+
44+
await RunAndVerifyTransferAsync(source, destination, cancellationToken);
45+
}
46+
}
47+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Scenarios/DownloadDirectory.cs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System.IO;
5+
using System;
6+
using System.Threading;
7+
using System.Threading.Tasks;
8+
using Azure.Storage.Blobs;
9+
10+
namespace Azure.Storage.DataMovement.Blobs.Perf
11+
{
12+
public class DownloadDirectory : DirectoryTransferTest<DirectoryTransferOptions>
13+
{
14+
private BlobContainerClient _sourceContainer;
15+
private string _destinationDirectory;
16+
17+
public DownloadDirectory(DirectoryTransferOptions options) : base(options)
18+
{
19+
}
20+
21+
public override async Task GlobalSetupAsync()
22+
{
23+
await base.GlobalSetupAsync();
24+
_sourceContainer = await CreateBlobContainerAsync(populate: true);
25+
_destinationDirectory = CreateLocalDirectory();
26+
}
27+
28+
public override async Task GlobalCleanupAsync()
29+
{
30+
await _sourceContainer.DeleteIfExistsAsync();
31+
Directory.Delete(_destinationDirectory, true);
32+
await base.GlobalCleanupAsync();
33+
}
34+
35+
public override void Run(CancellationToken cancellationToken)
36+
{
37+
throw new NotImplementedException();
38+
}
39+
40+
public override async Task RunAsync(CancellationToken cancellationToken)
41+
{
42+
StorageResource source = BlobResourceProvider.FromContainer(_sourceContainer.Uri);
43+
StorageResource destination = LocalFileResourceProvider.FromDirectory(_destinationDirectory);
44+
45+
await RunAndVerifyTransferAsync(source, destination, cancellationToken);
46+
}
47+
}
48+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Scenarios/UploadDirectory.cs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.IO;
6+
using System.Threading;
7+
using System.Threading.Tasks;
8+
using Azure.Storage.Blobs;
9+
10+
namespace Azure.Storage.DataMovement.Blobs.Perf
11+
{
12+
public class UploadDirectory : DirectoryTransferTest<DirectoryTransferOptions>
13+
{
14+
private string _sourceDirectory;
15+
private BlobContainerClient _destinationContainer;
16+
17+
public UploadDirectory(DirectoryTransferOptions options) : base(options)
18+
{
19+
}
20+
21+
public override async Task GlobalSetupAsync()
22+
{
23+
await base.GlobalSetupAsync();
24+
_sourceDirectory = CreateLocalDirectory(populate: true);
25+
_destinationContainer = await CreateBlobContainerAsync();
26+
}
27+
28+
public override async Task GlobalCleanupAsync()
29+
{
30+
Directory.Delete(_sourceDirectory, true);
31+
await _destinationContainer.DeleteIfExistsAsync();
32+
await base.GlobalCleanupAsync();
33+
}
34+
35+
public override void Run(CancellationToken cancellationToken)
36+
{
37+
throw new NotImplementedException();
38+
}
39+
40+
public override async Task RunAsync(CancellationToken cancellationToken)
41+
{
42+
StorageResource source = LocalFileResourceProvider.FromDirectory(_sourceDirectory);
43+
StorageResource destination = BlobResourceProvider.FromContainer(_destinationContainer.Uri);
44+
45+
await RunAndVerifyTransferAsync(source, destination, cancellationToken);
46+
}
47+
}
48+
}

0 commit comments

Comments
 (0)