Skip to content

Commit 26e83cc

Browse files
amnguyeamnguye
authored
DataMovement Stress Testing Part 1 (Azure#46287)
* Added separate sln file for Storage Data Movement and start of stress testing for DM Blobs * Fix to Stress Test README * Fix path for .slnf file * WIP * WIP * WIP * WIP * WIP * WIP * WIP - builds * WIP - docker issues * WIP - build issue in docker * WIP - builds locally, not in Docker * WIP - runs until hitting excuting yaml file * WIP - helm won't commit run * WIP - fixing upload scenario test * WIP - helm still not pushing commit; added more options for scenarios * WIP * Cleanup * WIP * More cleanup and some PR Comments * Part 2 PR comments
1 parent 1b1a2b4 commit 26e83cc

29 files changed

+1609
-29
lines changed

‎sdk/storage/Azure.Storage.Common/src/Azure.Storage.Common.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
<Compile Remove="Shared\*.cs" />
3333
<Compile Remove="Shared\ClientsideEncryption\*.cs;Shared\ClientsideEncryption\Models\*.cs" />
3434
<Compile Remove="Shared\AesGcm\**\*.cs" />
35+
<Compile Remove="stress\**" />
3536
<Compile Include="Shared\Argument.cs" />
3637
<Compile Include="Shared\AzureSasCredentialSynchronousPolicy.cs" />
3738
<!--we're defining the Shared Source in Common, but we only want to build in the files that Common names use of-->

‎sdk/storage/Azure.Storage.Common/src/stress/Shared/Metrics.cs‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using Microsoft.ApplicationInsights;
6+
using Microsoft.ApplicationInsights.Extensibility;
7+
8+
namespace Azure.Storage.Stress;
9+
10+
/// <summary>
11+
/// The set of metrics and the <see cref="TelemetryClient" /> used to collect information
12+
/// from test runs and send them to Application Insights.
13+
/// </summary>
14+
///
15+
public class Metrics
16+
{
17+
/// <summary>
18+
/// The Client used to communicate with Application Insights.
19+
/// </summary>
20+
///
21+
public TelemetryClient Client { get; }
22+
23+
/// <summary>
24+
/// This is the metric name used to collect metrics on generation zero
25+
/// garbage collection.
26+
/// </summary>
27+
///
28+
public const string GenerationZeroCollections = "GenerationZeroCollections";
29+
30+
/// <summary>
31+
/// This is the metric name used to collect metrics on generation one
32+
/// garbage collection.
33+
/// </summary>
34+
///
35+
public const string GenerationOneCollections = "GenerationOneCollections";
36+
37+
/// <summary>
38+
/// This is the metric name used to collect metrics on generation two
39+
/// garbage collection.
40+
/// </summary>
41+
///
42+
public const string GenerationTwoCollections = "GenerationTwoCollections";
43+
44+
/// <summary>
45+
/// This is the metric name used to collect how many objects (e.g. local file, blob, share file) were transferred.
46+
/// </summary>
47+
///
48+
public const string ObjectsTransferred = "ObjectsTransferred";
49+
50+
/// <summary>
51+
/// This is the metric name used to collect metrics on how many times
52+
/// the individual transfer part was restarted or retried.
53+
/// </summary>
54+
///
55+
public const string TransferRetriedRestarted = "TransferRestarted";
56+
57+
/// <summary>
58+
/// This is the metric name used to collect metrics on the total number of bytes that have been
59+
/// Sent to the Storage Service over the course of the test run.
60+
/// </summary>
61+
///
62+
public const string TotalSentSizeBytes = "TotalSentSizeBytes";
63+
64+
/// <summary>
65+
/// This is the dimension name used to hold which identifier is associated with a
66+
/// given processor.
67+
/// </summary>
68+
///
69+
public const string Identifier = "Identifier";
70+
71+
/// <summary>
72+
/// Initializes a new instance of the <see cref="Metrics" /> class.
73+
/// </summary>
74+
///
75+
/// <param name="instrumentationKey">The instrumentation key associated with the Application Insights resource.</param>
76+
///
77+
public Metrics(string instrumentationKey)
78+
{
79+
var configuration = TelemetryConfiguration.CreateDefault();
80+
configuration.InstrumentationKey = instrumentationKey;
81+
82+
Client = new TelemetryClient(configuration);
83+
}
84+
85+
/// <summary>
86+
/// Collects garbage collection environment metrics and sends them to Application Insights.
87+
/// </summary>
88+
///
89+
public void UpdateEnvironmentStatistics()
90+
{
91+
Client.GetMetric(Metrics.GenerationZeroCollections).TrackValue(GC.CollectionCount(0));
92+
Client.GetMetric(Metrics.GenerationOneCollections).TrackValue(GC.CollectionCount(1));
93+
Client.GetMetric(Metrics.GenerationTwoCollections).TrackValue(GC.CollectionCount(2));
94+
}
95+
}

‎sdk/storage/Azure.Storage.Common/src/stress/Shared/TestScenarioBase.cs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System.Threading;
5+
using System.Threading.Tasks;
6+
7+
namespace Azure.Storage.Stress;
8+
9+
/// <summary>
10+
/// The test scenario responsible for running all of the roles needed for the send receive test scenario.
11+
/// </summary>
12+
///
13+
public abstract class TestScenarioBase
14+
{
15+
/// <summary> The <see cref="Metrics"/> instance used to send metrics to application insights.</summary>
16+
internal Metrics _metrics;
17+
18+
/// <summary> The name of this test.</summary>
19+
public abstract string Name { get; }
20+
21+
/// <summary>
22+
/// Initializes a new Test instance.
23+
/// </summary>
24+
///\
25+
/// <param name="metrics">The <see cref="Metrics" /> to use to send metrics to Application Insights.</param>
26+
/// <param name="testRunId">Test Run Id to differ between test runs.</param>
27+
///
28+
public TestScenarioBase(Metrics metrics,
29+
string testRunId)
30+
{
31+
_metrics = metrics;
32+
_metrics.Client.Context.GlobalProperties["TestRunID"] = testRunId;
33+
}
34+
35+
/// <summary>
36+
/// Runs all of the roles required for this instance of the send process test scenario.
37+
/// </summary>
38+
///
39+
/// <param name="cancellationToken">A <see cref="CancellationToken" /> instance to signal the request to cancel the operation.</param>
40+
///
41+
public abstract Task RunTestAsync(CancellationToken cancellationToken);
42+
}

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/.helmignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
target/

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/Chart.lock‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
dependencies:
2+
- name: stress-test-addons
3+
repository: https://stresstestcharts.blob.core.windows.net/helm/
4+
version: 0.3.3
5+
digest: sha256:1cffb5ed8ea74953ab7611f9e2de2163af2c3f0918afb9928f71210da9c19a4a
6+
generated: "2024-09-17T12:42:24.6049982-07:00"

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/Chart.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
apiVersion: v2
2-
name: storage-datamovement-blobs-net-stress
2+
name: net-stgdm-blobs
33
version: 0.1.0
44
description: Stress tests for Storage Data Movement Blobs for .NET
55

@@ -11,3 +11,4 @@ dependencies:
1111
annotations:
1212
stressTest: 'true'
1313
namespace: 'net'
14+
environment: 'storage'

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/Dockerfile‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,35 @@
1-
FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build-env
1+
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-env
22

33
# Copy in engineering system needed to build
4-
COPY ./eng/ /app/eng/
5-
COPY ./tools/ /app/tools/
4+
COPY ./eng /app/eng/
5+
#COPY ./tools/ /app/tools/
66
COPY ./*.proj /app/
7-
COPY ./Directory.Build.props/ /app/
8-
COPY ./Directory.Build.targets/ /app/
9-
COPY ./NuGet.Config/ /app/
7+
COPY ./Directory.Build.props /app/
8+
COPY ./Directory.Build.targets /app/
9+
COPY ./NuGet.Config /app/
1010

11-
# Copy in Service Bus Source Code
12-
COPY ./sdk/storage/Azure.Storage.DataMovement.Blobs/ /app/sdk/storage/Azure.Storage.DataMovement.Blobs/
11+
# Copy in DataMovement Source Code
12+
COPY ./sdk/storage/Azure.Storage.Common /app/sdk/storage/Azure.Storage.Common
13+
COPY ./sdk/storage/Azure.Storage.Blobs /app/sdk/storage/Azure.Storage.Blobs
14+
COPY ./sdk/storage/Azure.Storage.Internal.Avro /app/sdk/storage/Azure.Storage.Internal.Avro
15+
COPY ./sdk/storage/Azure.Storage.DataMovement /app/sdk/storage/Azure.Storage.DataMovement
16+
COPY ./sdk/storage/Azure.Storage.DataMovement.Blobs /app/sdk/storage/Azure.Storage.DataMovement.Blobs
17+
COPY ./sdk/storage/Directory.Build.props /app/sdk/storage/
1318

1419
# Copy in Core Source Code
15-
COPY ./sdk/core/ /app/sdk/core/
20+
COPY ./sdk/core /app/sdk/core/
1621

1722
# Run restore and build, copy the .dll files over so they can used in the running container
1823
WORKDIR /app
19-
RUN dotnet build './sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/' --configuration Release
24+
RUN dotnet build './sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src' --configuration Release
2025

2126
# Copy in the dll files to be ready to run
2227
FROM build-env as publish
2328
WORKDIR /app
24-
COPY --from=build-env /app/artifacts/bin/Azure.Storage.DataMovement.Blobs.Stress/Release/net7.0/ /app/artifacts/bin/Azure.Storage.DataMovement.Blobs.Stress/Release/net7.0/
29+
COPY --from=build-env /app/artifacts/bin/Azure.Storage.DataMovement.Blobs.Stress/Release/net8.0/ /app/artifacts/bin/Azure.Storage.DataMovement.Blobs.Stress/Release/net8.0/
2530

26-
WORKDIR /app/artifacts/bin/Azure.Storage.DataMovement.Blobs.Stress/Release/net7.0
31+
WORKDIR /app/artifacts/bin/Azure.Storage.DataMovement.Blobs.Stress/Release/net8.0
2732

28-
# The default is running just the "SendReceiveTest"
33+
# The default is running all the upload tests
2934
ENTRYPOINT ["dotnet Azure.Storage.DataMovement.Blobs.Stress.dll", "--test"]
3035
CMD ["SendRec"]

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/generatedValues.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
scenarios:
2+
- image: Dockerfile
3+
imageBuildDir: ../../../..
4+
testScenario: uploadsingleblockblob
5+
Scenario: uploadsingleblockblob
6+
imageTag: stressstorage4okf44ko4zuos.azurecr.io/amnguye/net-stgdm-blobs/dockerfile:amnguye
7+

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/scenarios-matrix.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
matrix:
2+
# Set subscriptionConfig here ONLY for a global override for all scenarios.
3+
# The value should match the name of the secret in the static secrets keyvault for the cluster.
4+
# subscriptionConfig: <your subscription config secret name>
5+
scenarios:
6+
uploadsingleblockblob:
7+
testScenario: uploadsingleblockblob
8+
image: Dockerfile
9+
imageBuildDir: "../../../.."

‎sdk/storage/Azure.Storage.DataMovement.Blobs/stress/src/Azure.Storage.DataMovement.Blobs.Stress.csproj‎

Lines changed: 54 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,59 @@
11
<Project Sdk="Microsoft.NET.Sdk">
2-
32
<PropertyGroup>
4-
<TargetFramework>net8.0</TargetFramework>
5-
<ImplicitUsings>enable</ImplicitUsings>
6-
<Nullable>enable</Nullable>
3+
<Description>Azure DataMovement Blobs Stress Testing</Description>
4+
<PackageTags>Microsoft Azure Storage DataMovement, DataMovement, Microsoft, Azure, StorageScalable, azureofficial</PackageTags>
5+
<OutputType>Exe</OutputType>
6+
<IsTestSupportProject>true</IsTestSupportProject>
7+
<IsStressProject>false</IsStressProject>
78
</PropertyGroup>
89

10+
<ItemGroup>
11+
<PackageReference Include="Azure.Core" />
12+
<PackageReference Include="Azure.Core.Amqp" />
13+
<PackageReference Include="Azure.Identity" />
14+
<PackageReference Include="Microsoft.Azure.Amqp" />
15+
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" />
16+
<PackageReference Include="System.Memory.Data" />
17+
<PackageReference Include="CommandLineParser" />
18+
<PackageReference Include="Microsoft.ApplicationInsights" />
19+
<PackageReference Include="NUnit" />
20+
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj" />
21+
</ItemGroup>
22+
<ItemGroup>
23+
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\Azure.Storage.DataMovement\src\Azure.Storage.DataMovement.csproj" />
24+
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\Azure.Storage.Blobs\src\Azure.Storage.Blobs.csproj" />
25+
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\src\Azure.Storage.DataMovement.Blobs.csproj" />
26+
</ItemGroup>
27+
<ItemGroup>
28+
<Compile Include="$(AzureStorageSharedTestSources)\TestHelper.cs" LinkBase="Shared\Storage" />
29+
</ItemGroup>
30+
<!-- Import Azure.Core shared source -->
31+
<ItemGroup>
32+
<Compile Include="$(AzureCoreSharedSources)Argument.cs" LinkBase="SharedSource\Azure.Core" />
33+
<Compile Include="$(AzureCoreSharedSources)TaskExtensions.cs" LinkBase="SharedSource\Azure.Core" />
34+
</ItemGroup>
35+
<ItemGroup>
36+
<Compile Include="$(AzureStorageSharedSources)Constants.cs" Link="Shared\%(RecursiveDir)\%(Filename)%(Extension)" />
37+
</ItemGroup>
38+
<ItemGroup>
39+
<Compile Include="$(AzureStorageSharedTestSources)TemporaryFileStream.cs" LinkBase="SharedSource\Storage" />
40+
</ItemGroup>
41+
<ItemGroup>
42+
<Compile Include="$(AzureStorageDataMovementSharedSources)DataTransferStatusInternal.cs" LinkBase="Shared\DataMovement" />
43+
</ItemGroup>
44+
<ItemGroup>
45+
<Compile Include="$(AzureStorageDataMovementTestSharedSources)DisposingLocalDirectory.cs" LinkBase="Shared\DataMovement" />
46+
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TransferValidator.cs" LinkBase="Shared\DataMovement" />
47+
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TransferValidator.Local.cs" LinkBase="Shared\DataMovement" />
48+
<Compile Include="$(AzureStorageDataMovementTestSharedSources)../TransferValidator.Blobs.cs" LinkBase="Shared\DataMovement" />
49+
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TestEventsRaised.cs" LinkBase="Shared\DataMovement" />
50+
</ItemGroup>
51+
<ItemGroup>
52+
<Compile Include="$(AzureStorageStressTestSharedSources)*.cs" LinkBase="Shared\Stress" />
53+
</ItemGroup>
54+
<ItemGroup>
55+
<Content Include="Resources\**">
56+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
57+
</Content>
58+
</ItemGroup>
959
</Project>

0 commit comments

Comments
 (0)