Skip to content

Commit 72a3c27

Browse files
sa7936Serena Akpoyibom-redding
authored
Azure.Core.Perf work (Azure#50534)
Refined benchmarks to make them run using a script that also compares the nuget and local versions. * Mocking the Http calls in the pipelinescenarios. * Removed the .snk files and added Drectory.Build.props files in replacement. * Moved the benchmarks projects into the same folder with azure.core.perf and moved the nested classes into new files. * -Made a pipelinescenario that tests only the send functionality. -Changed the compare script to use a combination of a percentage difference and and absolute threshold of 1000 ns (because sometimes there is a huge percwntage difference but the speed is in ns so the difference doesn't cleanly translate to slow). Still woking on refining script. - Started work to make scenario for dynamicobjectbenchmark. * --- Removed pipeline parts from the yml files and restored them to what they originally were. --- Worked on the DynamicObjectBenchmarks and edited the scripts to run them. ---Modified azure.core.tests classes to be public * Removed random white spaces in the yml files. * pass on different layout * tweak script * add shared source * Updated the files and compare script to save the local and nuget results in separate folders for easy comparison. * --- Renamed pipeline benchmanmrks to be more expressive of what they do. --- Fixed the benchmarks in EventSourceBecnhmarks to test old_implementation with formatting, new_implementation with formatting (of Uri) and new_implementation after formatting. --- Fixed the CustomEventSource to inherit from the AzureEventSource ---Restored Azure.Core tests to being internal --- Removed properties no longer needed from AssemblyInfo. * ---Removed Benchmark.Local and Nuget content from the .sln. ---Renamed private method to go with standards * Edited the old implementation, to have formatting headers as part of the call. * Copilot caught some typos and use of null-forgiving operator that was fixed. * Removed the targetframeworks property (not needed anymore). * Removed targetframeworks change from Sys.clientmodel.tests.internal.perf.csproj --------- Co-authored-by: Serena Akpoyibo <[email protected]> Co-authored-by: Madalyn Redding <[email protected]>
1 parent cfbb576 commit 72a3c27

16 files changed

+608
-165
lines changed

sdk/core/Azure.Core/Azure.Core.sln

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Core.Spatia
2525
EndProject
2626
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Core.Spatial.Tests", "..\Microsoft.Azure.Core.Spatial\tests\Microsoft.Azure.Core.Spatial.Tests.csproj", "{36FF59A9-D6C9-4226-A0FE-47C879F3EB94}"
2727
EndProject
28-
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Core.Perf", "perf\Azure.Core.Perf.csproj", "{B6D7909F-4DE6-4895-BF39-EF892BA64BA3}"
29-
EndProject
3028
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Test.Perf", "..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj", "{96E9F605-9C38-4D77-96F4-679EF8B9390F}"
3129
EndProject
3230
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Core.TestFramework.Tests", "..\Azure.Core.TestFramework\tests\Azure.Core.TestFramework.Tests.csproj", "{33C299B5-ABDA-47BC-838F-973E62C067F9}"
@@ -69,6 +67,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "System.ClientModel.SourceGe
6967
EndProject
7068
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "System.ClientModel.SourceGeneration.Unit.Tests", "..\System.ClientModel\tests\gen.unit\System.ClientModel.SourceGeneration.Unit.Tests.csproj", "{6ED66F0E-C7FA-3F0E-38C8-D576D48C8A58}"
7169
EndProject
70+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Core.Perf", "perf\Azure.Core.Perf.csproj", "{169EE35E-F12E-5230-EAD4-3FBEB1183235}"
71+
EndProject
7272
Global
7373
GlobalSection(SolutionConfigurationPlatforms) = preSolution
7474
Debug|Any CPU = Debug|Any CPU
@@ -119,10 +119,6 @@ Global
119119
{36FF59A9-D6C9-4226-A0FE-47C879F3EB94}.Debug|Any CPU.Build.0 = Debug|Any CPU
120120
{36FF59A9-D6C9-4226-A0FE-47C879F3EB94}.Release|Any CPU.ActiveCfg = Release|Any CPU
121121
{36FF59A9-D6C9-4226-A0FE-47C879F3EB94}.Release|Any CPU.Build.0 = Release|Any CPU
122-
{B6D7909F-4DE6-4895-BF39-EF892BA64BA3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
123-
{B6D7909F-4DE6-4895-BF39-EF892BA64BA3}.Debug|Any CPU.Build.0 = Debug|Any CPU
124-
{B6D7909F-4DE6-4895-BF39-EF892BA64BA3}.Release|Any CPU.ActiveCfg = Release|Any CPU
125-
{B6D7909F-4DE6-4895-BF39-EF892BA64BA3}.Release|Any CPU.Build.0 = Release|Any CPU
126122
{96E9F605-9C38-4D77-96F4-679EF8B9390F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
127123
{96E9F605-9C38-4D77-96F4-679EF8B9390F}.Debug|Any CPU.Build.0 = Debug|Any CPU
128124
{96E9F605-9C38-4D77-96F4-679EF8B9390F}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -179,6 +175,10 @@ Global
179175
{6ED66F0E-C7FA-3F0E-38C8-D576D48C8A58}.Debug|Any CPU.Build.0 = Debug|Any CPU
180176
{6ED66F0E-C7FA-3F0E-38C8-D576D48C8A58}.Release|Any CPU.ActiveCfg = Release|Any CPU
181177
{6ED66F0E-C7FA-3F0E-38C8-D576D48C8A58}.Release|Any CPU.Build.0 = Release|Any CPU
178+
{169EE35E-F12E-5230-EAD4-3FBEB1183235}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
179+
{169EE35E-F12E-5230-EAD4-3FBEB1183235}.Debug|Any CPU.Build.0 = Debug|Any CPU
180+
{169EE35E-F12E-5230-EAD4-3FBEB1183235}.Release|Any CPU.ActiveCfg = Release|Any CPU
181+
{169EE35E-F12E-5230-EAD4-3FBEB1183235}.Release|Any CPU.Build.0 = Release|Any CPU
182182
EndGlobalSection
183183
GlobalSection(SolutionProperties) = preSolution
184184
HideSolutionNode = FALSE

sdk/core/Azure.Core/perf/Azure.Core.Perf.csproj

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,15 @@
44
<OutputType>Exe</OutputType>
55
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
66
</PropertyGroup>
7-
87
<ItemGroup>
98
<ProjectReference Condition="'$(AzureCoreVersion)' == ''" Include="../../Azure.Core/src/Azure.Core.csproj" />
10-
<PackageReference Condition="'$(AzureCoreVersion)' != ''" Include="Azure.Core" VersionOverride="$(AzureCoreVersion)" />
9+
<PackageReference Condition="'$(AzureCoreVersion)' != ''" Include="Azure.Core" />
1110
</ItemGroup>
11+
<PropertyGroup>
12+
<DefineConstants>$(DefineConstants);AZURE_CORE_VERSION_$(AzureCoreVersion)</DefineConstants>
13+
</PropertyGroup>
1214
<ItemGroup>
1315
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj" />
14-
<ProjectReference Include="..\tests\Azure.Core.Tests.csproj" />
15-
<ProjectReference Include="..\tests\public\Azure.Core.Tests.Public.csproj" />
1616
<!--
1717
Explicit references to pull in patched versions of ASP.NET Core packages
1818
-->
@@ -30,5 +30,15 @@
3030
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
3131
</None>
3232
</ItemGroup>
33+
<ItemGroup>
34+
<Compile Include="$(AzureCoreSharedSources)IUtf8JsonSerializable.cs" LinkBase="Shared\Core" />
35+
<Compile Include="..\src\Shared\Utf8JsonWriterExtensions.cs" LinkBase="Shared" />
36+
<Compile Include="..\src\Shared\JsonElementExtensions.cs" LinkBase="Shared" />
37+
<Compile Include="..\src\Shared\Optional.cs" LinkBase="Shared" />
38+
<Compile Include="..\src\Shared\RawRequestUriBuilder.cs" LinkBase="Shared" />
39+
<Compile Include="..\src\Shared\TypeFormatters.cs" LinkBase="Shared" />
40+
<Compile Include="..\src\Shared\ChangeTrackingDictionary.cs" LinkBase="Shared" />
41+
<Compile Include="..\src\Shared\ChangeTrackingList.cs" LinkBase="Shared" />
42+
</ItemGroup>
3343

3444
</Project>
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using Azure.Core;
5+
6+
namespace Azure.Core.Perf
7+
{
8+
/// <summary>
9+
/// Options for configuring the benchmark client.
10+
/// </summary>
11+
internal class BenchmarkClientOptions : ClientOptions
12+
{
13+
}
14+
}
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Compare-Benchmarks.ps1
2+
3+
$overallPassed = $true
4+
5+
function Extract-BaseName($fullName) {
6+
# Extracts the method name without the Local/Nuget prefix
7+
$regex = [regex]'\.(Local|Nuget)[_.]?(.+)$'
8+
$match = $regex.Match($fullName)
9+
if ($match.Success) {
10+
return $match.Groups[2].Value
11+
}
12+
# get the last part after the last dot
13+
return ($fullName -split '\.')[-1]
14+
}
15+
16+
function Get-MeanMap($filename) {
17+
$json = Get-Content $filename -Raw | ConvertFrom-Json
18+
$means = @{}
19+
foreach ($b in $json.Benchmarks) {
20+
$name = $b.FullName
21+
$base = Extract-BaseName $name
22+
$mean = $b.Statistics.Mean
23+
$unit = $b.Statistics.Unit
24+
if (-not $unit -and $b.Unit) { $unit = $b.Unit } # fallback if unit is at root
25+
# Normalize to nanoseconds
26+
switch ($unit) {
27+
"ns" { $mean_ns = $mean }
28+
"us" { $mean_ns = $mean * 1000 }
29+
"ms" { $mean_ns = $mean * 1000 * 1000 }
30+
default { $mean_ns = $mean } # assume ns if unknown
31+
}
32+
$means[$base] = $mean_ns
33+
}
34+
return $means
35+
}
36+
37+
$localDir = "BenchmarkDotNet.Artifacts/results/local/results"
38+
$nugetDir = "BenchmarkDotNet.Artifacts/results/nuget/results"
39+
40+
$localFiles = Get-ChildItem -Path $localDir -Filter "Azure.Core.Perf.*.json"
41+
foreach ($localFile in $localFiles) {
42+
$fileName = $localFile.Name
43+
$nugetFilePath = Join-Path $nugetDir $fileName
44+
if (-not (Test-Path $nugetFilePath)) {
45+
Write-Host "Nuget result file missing for $fileName"
46+
$overallPassed = $false
47+
continue
48+
}
49+
50+
$localMeans = Get-MeanMap $localFile.FullName
51+
$nugetMeans = Get-MeanMap $nugetFilePath
52+
53+
foreach ($base in $localMeans.Keys) {
54+
if (-not $nugetMeans.ContainsKey($base)) {
55+
Write-Host "[$fileName] Nuget result missing for '$base'"
56+
$overallPassed = $false
57+
continue
58+
}
59+
$localMean = $localMeans[$base]
60+
$nugetMean = $nugetMeans[$base]
61+
$percentThreshold = 10
62+
$absoluteThresholdNs = 1000 # 1000 nanoseconds
63+
$percentDiff = (($localMean - $nugetMean) / $nugetMean) * 100
64+
$absoluteDiff = [math]::Abs($localMean - $nugetMean)
65+
if ($percentDiff -gt $percentThreshold -and $absoluteDiff -gt $absoluteThresholdNs) {
66+
Write-Host "[$fileName] FAIL: [$base]: Local is slower by $([math]::Round($percentDiff,2))% ($([math]::Round($absoluteDiff,2)) ns)"
67+
$overallPassed = $false
68+
} else {
69+
Write-Host "[$fileName] PASS: [$base]: Local is within $percentThreshold% or $absoluteThresholdNs ns of Nuget (diff: $([math]::Round($percentDiff,2))%, $([math]::Round($absoluteDiff,2)) ns)"
70+
}
71+
}
72+
}
73+
74+
if (-not $overallPassed) {
75+
exit 1
76+
}
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.Diagnostics.Tracing;
6+
using Azure.Core.Diagnostics;
7+
8+
namespace Azure.Core.Perf;
9+
10+
/// <summary>
11+
/// Custom event source for Azure Core logging.
12+
/// </summary>
13+
internal class CustomEventSource : AzureEventSource
14+
{
15+
/// <summary>
16+
/// Initializes a new instance of the <see cref="CustomEventSource"/> class.
17+
/// </summary>
18+
public CustomEventSource() : base("Azure-Core") { }
19+
20+
/// <summary>
21+
/// Logs a request with the old method.
22+
/// </summary>
23+
/// <param name="strParam">The string parameter.</param>
24+
/// <param name="intParam">The integer parameter.</param>
25+
/// <param name="doubleParam">The double parameter.</param>
26+
/// <param name="bytesParam">The byte array parameter.</param>
27+
[Event(1, Level = EventLevel.Informational)]
28+
public void RequestOld(string strParam, int intParam, double doubleParam, byte[] bytesParam)
29+
{
30+
WriteEvent(1, strParam, intParam, doubleParam, bytesParam);
31+
}
32+
33+
/// <summary>
34+
/// Logs a request with the new method.
35+
/// </summary>
36+
/// <param name="strParam">The string parameter.</param>
37+
/// <param name="intParam">The integer parameter.</param>
38+
/// <param name="doubleParam">The double parameter.</param>
39+
/// <param name="bytesParam">The byte array parameter.</param>
40+
[Event(2, Level = EventLevel.Informational)]
41+
public void RequestNew(string strParam, int intParam, double doubleParam, byte[] bytesParam)
42+
{
43+
WriteEventNew(2, strParam, intParam, doubleParam, bytesParam);
44+
}
45+
46+
/// <summary>
47+
/// Writes an event with the new method.
48+
/// </summary>
49+
/// <param name="eventId">The event ID.</param>
50+
/// <param name="arg0">The string argument.</param>
51+
/// <param name="arg1">The integer parameter.</param>
52+
/// <param name="arg2">The double parameter.</param>
53+
/// <param name="arg3">The byte array parameter.</param>
54+
private unsafe void WriteEventNew(int eventId, string arg0, int arg1, double arg2, byte[] arg3)
55+
{
56+
if (!IsEnabled())
57+
{
58+
return;
59+
}
60+
61+
arg0 ??= string.Empty;
62+
fixed (char* arg0Ptr = arg0)
63+
{
64+
EventData* data = stackalloc EventData[5];
65+
data[0].DataPointer = (IntPtr)arg0Ptr;
66+
data[0].Size = (arg0.Length + 1) * 2;
67+
data[1].DataPointer = (IntPtr)(&arg1);
68+
data[1].Size = 4;
69+
data[2].DataPointer = (IntPtr)(&arg2);
70+
data[2].Size = 8;
71+
72+
var blobSize = arg3.Length;
73+
fixed (byte* blob = &arg3[0])
74+
{
75+
data[3].DataPointer = (IntPtr)(&blobSize);
76+
data[3].Size = 4;
77+
data[4].DataPointer = (IntPtr)blob;
78+
data[4].Size = blobSize;
79+
WriteEventCore(eventId, 4, data);
80+
}
81+
}
82+
}
83+
}
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
2+
<PropertyGroup>
3+
<ExcludeFromCodeCoverage>true</ExcludeFromCodeCoverage>
4+
<ImportRepoCommonSettings>true</ImportRepoCommonSettings>
5+
<!-- Signal that integration projects are building in the repo -->
6+
<IsClientLibrary>true</IsClientLibrary>
7+
<IsPackable>true</IsPackable>
8+
<WarnOnPackingNonPackableProject>false</WarnOnPackingNonPackableProject>
9+
<NoWarn>
10+
$(NoWarn);
11+
NU5104;
12+
</NoWarn>
13+
</PropertyGroup>
14+
15+
<Import Project="..\Directory.Build.props" />
16+
17+
<PropertyGroup>
18+
<DefineConstants>$(DefineConstants);STRONGNAME_SIGNED</DefineConstants>
19+
<InheritDocEnabled>false</InheritDocEnabled>
20+
<IsShippingLibrary>false</IsShippingLibrary>
21+
</PropertyGroup>
22+
</Project>

sdk/core/Azure.Core/perf/DynamicObjectBenchmark.cs

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,19 @@
1212

1313
namespace Azure.Core.Perf
1414
{
15-
[SimpleJob(RuntimeMoniker.NetCoreApp31, baseline: true)]
16-
[SimpleJob(RuntimeMoniker.Net462)]
17-
[SimpleJob(RuntimeMoniker.Net60)]
15+
[SimpleJob(RuntimeMoniker.Net80)]
16+
[MemoryDiagnoser]
1817
public class DynamicObjectBenchmark
1918
{
20-
private static string _fileName = Path.Combine(Path.Combine(Directory.GetParent(Assembly.GetExecutingAssembly().Location).FullName, "TestData", "JsonFormattedString.json"));
2119
private JsonDocument _jsonDocument;
2220
private ModelWithBinaryData _modelWithBinaryData;
2321
private ModelWithObject _modelWithObject;
2422

23+
private static readonly string _fileName = Path.Combine(
24+
Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location),
25+
"TestData",
26+
"JsonFormattedString.json");
27+
2528
[GlobalSetup]
2629
public void SetUp()
2730
{
@@ -65,8 +68,8 @@ public void DeserializeWithObjectAndAccess()
6568
{
6669
var model = ModelWithObject.DeserializeModelWithObject(_jsonDocument.RootElement);
6770
var properties = model.Properties as Dictionary<string, object>;
68-
var innerProperties = properties["innerProperties"] as Dictionary<string, object>;
69-
var innerA = innerProperties["a"] as string;
71+
var innerProperties = properties!["innerProperties"] as Dictionary<string, object>;
72+
var innerA = innerProperties!["a"] as string;
7073
}
7174

7275
[Benchmark]
@@ -87,9 +90,22 @@ public void DeserializeWithBinaryData()
8790
public void DeserializeWithBinaryDataAndAccess()
8891
{
8992
var model = ModelWithBinaryData.DeserializeModelWithBinaryData(_jsonDocument.RootElement);
90-
var properties = model.Properties.ToObjectFromJson() as Dictionary<string, object>;
91-
var innerProperties = properties["innerProperties"] as Dictionary<string, object>;
92-
var innerA = innerProperties["a"] as string;
93+
var properties = model.Properties.ToObjectFromJson<Dictionary<string, object>>();
94+
if (properties == null)
95+
{
96+
throw new InvalidOperationException("Deserialized properties are null.");
97+
}
98+
if (properties.TryGetValue("innerProperties", out var innerPropertiesObj) &&
99+
innerPropertiesObj is JsonElement innerElement &&
100+
innerElement.ValueKind == JsonValueKind.Object)
101+
{
102+
var innerProperties = JsonSerializer.Deserialize<Dictionary<string, object>>(innerElement.GetRawText());
103+
if (innerProperties != null && innerProperties.TryGetValue("a", out var innerAObj))
104+
{
105+
var innerA = innerAObj as string;
106+
// Use innerA as needed
107+
}
108+
}
93109
}
94110

95111
[Benchmark]
@@ -99,5 +115,11 @@ public void SerializeWithBinaryData()
99115
using var writer = new Utf8JsonWriter(ms);
100116
_modelWithBinaryData.Write(writer);
101117
}
118+
119+
[GlobalCleanup]
120+
public void CleanUp()
121+
{
122+
_jsonDocument?.Dispose();
123+
}
102124
}
103125
}

0 commit comments

Comments
 (0)