Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit 586ebfa

Browse files
author
José Rivero
authored
Updating v1.1.2 benchmarks to match 2.0 changes (set matching baseline) (#12397)
This is an test/infrastructure only change to enable running previous release 1.1 with the same benchmarks on master and release 2.0. - Update from the old xUnit Performance runner to the xUnit Performance Api - Update JIT benchmarks - Update `perf-prep.sh` with latest bug fix in master.
1 parent 6db0e39 commit 586ebfa

File tree

160 files changed

+6016
-17184
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

160 files changed

+6016
-17184
lines changed

‎dependencies.props‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@
1313
<!-- Full package version strings that are used in other parts of the build. -->
1414
<PropertyGroup>
1515
<CoreClrPackageVersion>1.1.3-servicing-25423-01</CoreClrPackageVersion>
16-
<XunitPackageVersion>2.1.0</XunitPackageVersion>
16+
<XunitPackageVersion>2.2.0-beta2-build3300</XunitPackageVersion>
1717

18+
<XUnitPerformanceApiVersion>1.0.0-beta-build0007</XUnitPerformanceApiVersion>
1819
<!-- Runtime package versions from projectk-tfs/release/1.0.0 -->
1920
<RuntimeWin7ArmMicrosoftNETCoreTestHostPackageVersion>1.0.0</RuntimeWin7ArmMicrosoftNETCoreTestHostPackageVersion>
2021
<RuntimeWin8ArmMicrosoftNETCoreRuntimeCoreCLRPackageVersion>1.0.4</RuntimeWin8ArmMicrosoftNETCoreRuntimeCoreCLRPackageVersion>
@@ -66,14 +67,12 @@
6667
<Version>$(XunitPackageVersion)</Version>
6768
</StaticDependency>
6869

69-
<XunitPerformanceDependency Include="Microsoft.DotNet.xunit.performance.analysis" />
70-
<XunitPerformanceDependency Include="Microsoft.DotNet.xunit.performance.analysis.cli" />
71-
<XunitPerformanceDependency Include="Microsoft.DotNet.xunit.performance.metrics" />
72-
<XunitPerformanceDependency Include="Microsoft.DotNet.xunit.performance.run.core" />
73-
<XunitPerformanceDependency Include="Microsoft.DotNet.xunit.performance.runner.cli" />
74-
<XunitPerformanceDependency Include="Microsoft.DotNet.xunit.performance.runner.Windows" />
75-
<StaticDependency Include="@(XunitPerformanceDependency)">
76-
<Version>1.0.0-alpha-build0035</Version>
70+
<XUnitPerformanceApiDependency Include="xunit.performance.api" />
71+
<XUnitPerformanceApiDependency Include="xunit.performance.core" />
72+
<XUnitPerformanceApiDependency Include="xunit.performance.execution" />
73+
<XUnitPerformanceApiDependency Include="xunit.performance.metrics" />
74+
<StaticDependency Include="@(XUnitPerformanceApiDependency)">
75+
<Version>$(XUnitPerformanceApiVersion)</Version>
7776
</StaticDependency>
7877

7978
<StaticDependency Include="xunit.console.netcore">

‎perf.groovy‎

Lines changed: 338 additions & 102 deletions
Large diffs are not rendered by default.

‎tests/build.proj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
<!-- Create a collection of all project.json files for dependency updates. -->
3939
<ItemGroup>
4040
<ProjectJsonFiles Include="$(SourceDir)**\project.json" />
41+
<ProjectJsonFiles Include="$(ProjectDir)scripts\**\project.json" />
4142
</ItemGroup>
4243

4344
<Target Name="UpdatePackageDependencyVersion">

‎tests/helix.targets‎

Lines changed: 0 additions & 78 deletions
This file was deleted.

‎tests/runtest.cmd‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
@if not defined __echo @echo off
1+
@if not defined _echo @echo off
22
setlocal EnableDelayedExpansion
33

44
:: Set the default arguments
@@ -297,6 +297,7 @@ set __msbuildLogArgs=^
297297
set __msbuildArgs=%* %__msbuildCommonArgs% %__msbuildLogArgs%
298298

299299
@REM The next line will overwrite the existing log file, if any.
300+
echo %_msbuildexe% %__msbuildArgs%
300301
echo Invoking: %_msbuildexe% %__msbuildArgs% > "%__BuildLog%"
301302

302303
%_msbuildexe% %__msbuildArgs%

‎tests/runtest.proj‎

Lines changed: 3 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ $(_XunitEpilog)
6464
6565
]]>
6666
</_XunitWrapperGen>
67-
<XunitVersionCompiled>2.1.0</XunitVersionCompiled>
67+
<XunitVersionCompiled>2.2.0-beta2-build3300</XunitVersionCompiled>
6868
<XunitWrapperGenCsProj>
6969
<![CDATA[
7070
<?xml version="1.0" encoding="utf-8"?>
@@ -120,7 +120,6 @@ $(_XunitEpilog)
120120
<ProjectLockJson>%24(TestWrappersPackagesConfigFileDirectory)project.lock.json</ProjectLockJson>
121121
</PropertyGroup>
122122
<Import Project="$(SourceDir)dir.targets" />
123-
<Import Project="$(ProjectDir)helix.targets" />
124123
<PropertyGroup>
125124
<OutDir>$(XunitTestBinBase)\$(Category)\</OutDir>
126125
</PropertyGroup>
@@ -307,31 +306,7 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").
307306
</Target>
308307

309308
<Target Name="CreateAllWrappers" DependsOnTargets="GetListOfTestCmds;FindCmdDirectories">
310-
<MSBuild Projects="$(MSBuildProjectFile)" Targets="CreateXunitWrapper" Properties="_CMDDIR=%(TestDirectories.Identity)" />
311-
312-
<!--
313-
Contents of a project.json that will be restored once with errors turned off. This works
314-
around the package being incompatible with dotnet restore. Only the package contents are
315-
required so the compatibility errors can be ignored.
316-
-->
317-
<PropertyGroup>
318-
<XunitRunnerRestoreGenConfig>{ "dependencies": { "xunit.runner.msbuild": "$(XunitPackageVersion)" }, "frameworks": { "net45": {} } }</XunitRunnerRestoreGenConfig>
319-
<XunitRunnerRestoreProjectDir>$(BaseOutputPath)\tempRestoreProject</XunitRunnerRestoreProjectDir>
320-
<XunitRunnerRestoreProjectJsonPath>$(XunitRunnerRestoreProjectDir)\project.json</XunitRunnerRestoreProjectJsonPath>
321-
</PropertyGroup>
322-
323-
<MakeDir Directories="$(XunitRunnerRestoreProjectDir)" />
324-
<WriteLinesToFile File="$(XunitRunnerRestoreProjectJsonPath)"
325-
Lines="$(XunitRunnerRestoreGenConfig)"
326-
Overwrite="true" />
327-
328-
<Exec Command="$(DnuRestoreCommand) &quot;$(XunitRunnerRestoreProjectJsonPath)&quot;"
329-
StandardOutputImportance="Low"
330-
CustomErrorRegularExpression="^Unable to resolve .*"
331-
IgnoreExitCode="true"
332-
IgnoreStandardErrorWarningFormat="true" />
333-
334-
<MSBuild Projects="$(MSBuildProjectFile)" Targets="BuildXunitWrapper" Properties="_CMDDIR=%(TestDirectories.Identity)" />
309+
<MSBuild Projects="$(MSBuildProjectFile)" Targets="CreateXunitWrapper;BuildXunitWrapper" Properties="_CMDDIR=%(TestDirectories.Identity)" />
335310
</Target>
336311

337312
<Target Name="GetListOfTestCmds">
@@ -435,4 +410,4 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").
435410
<Target Name="Clean">
436411
<RemoveDir Condition=" '$(BuildWrappers)'=='true'" Directories="$(XunitWrapperGeneratedCSDirBase);$(XunitWrapperOutputIntermediatedDirBase)" ContinueOnError="WarnAndContinue" />
437412
</Target>
438-
</Project>
413+
</Project>

‎tests/scripts/perf-prep.sh‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/usr/bin/env bash
2+
3+
function print_usage {
4+
echo ''
5+
echo 'CoreCLR perf test environment set up script on Linux.'
6+
echo ''
7+
echo 'Typical command line:'
8+
echo ''
9+
echo 'coreclr/tests/scripts/perf-perp.sh'
10+
echo ' --branch="dotnet_coreclr"'
11+
echo ''
12+
echo 'Required arguments:'
13+
echo ' --branch=<path> : branch where coreclr/corefx/test bits are copied from (e.g. dotnet_coreclr).'
14+
echo 'Optional arguments:'
15+
echo ' --throughput : if we are running setup for a throughput run.'
16+
}
17+
18+
# Exit code constants
19+
readonly EXIT_CODE_SUCCESS=0 # Script ran normally.
20+
21+
# Argument variables
22+
perfArch="x64"
23+
perfConfig="Release"
24+
perfBranch=
25+
throughput=0
26+
27+
for i in "$@"
28+
do
29+
case $i in
30+
-h|--help)
31+
print_usage
32+
exit $EXIT_CODE_SUCCESS
33+
;;
34+
--branch=*)
35+
perfBranch=${i#*=}
36+
;;
37+
-t|--throughput)
38+
throughput=1
39+
;;
40+
*)
41+
echo "Unknown switch: $i"
42+
print_usage
43+
exit $EXIT_CODE_SUCCESS
44+
;;
45+
esac
46+
done
47+
48+
perfBranch="dotnet_coreclr"
49+
echo "branch = $perfBranch"
50+
echo "architecture = $perfArch"
51+
echo "configuration = $perfConfig"
52+
53+
# Since not all perf machines have Mono we cannot run nuget locally to get the Benchview tools
54+
# Instead we curl the package feed and use grep and sed to find the newest package.
55+
# We grep for content type and that returns us strings that contain the path to the nupkg
56+
# Then we match only the last line using '$' and use the s command to replace the entire line
57+
# with what we find inside of the quotes after src=. We then jump to label x on a match and if
58+
# we don't match we delete the line. This returns just the address of the last nupkg to curl.
59+
curl "http://benchviewtestfeed.azurewebsites.net/nuget/FindPackagesById()?id='Microsoft.BenchView.JSONFormat'" | grep "content type" | sed "$ s/.*src=\"\([^\"]*\)\".*/\1/;tx;d;:x" | xargs curl -o benchview.zip
60+
unzip -q -o benchview.zip -d ./tests/scripts/Microsoft.BenchView.JSONFormat
61+
62+
# Install python 3.5.2 to run machinedata.py for machine data collection
63+
python3.5 --version
64+
python3.5 ./tests/scripts/Microsoft.BenchView.JSONFormat/tools/machinedata.py
65+
66+
if [ $throughput -eq 1 ]; then
67+
# Download throughput benchmarks
68+
if [ -d "Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT" ]; then
69+
rm -r -f Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT
70+
fi
71+
mkdir Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT
72+
cd Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT
73+
74+
curl -OL https://dotnet.myget.org/F/dotnet-core/api/v2/package/Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT/1.0.0
75+
mv 1.0.0 1.0.0.zip
76+
unzip -q 1.0.0.zip
77+
78+
else
79+
# Set up the copies
80+
# Coreclr build containing the tests and mscorlib
81+
curl https://ci.dot.net/job/dotnet_coreclr/job/release_1.1.0/job/release_windows_nt/lastSuccessfulBuild/artifact/bin/tests/tests.zip -o tests.zip
82+
83+
# Corefx components. We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
84+
mkdir corefx
85+
curl https://ci.dot.net/job/dotnet_corefx/job/release_1.1.0/job/ubuntu14.04_release/lastSuccessfulBuild/artifact/bin/build.tar.gz -o ./corefx/build.tar.gz
86+
87+
# Unpack the corefx binaries
88+
pushd corefx > /dev/null
89+
tar -xf build.tar.gz
90+
rm build.tar.gz
91+
popd > /dev/null
92+
93+
# Unzip the tests first. Exit with 0
94+
mkdir bin
95+
mkdir bin/tests
96+
unzip -q -o tests.zip -d ./bin/tests/Windows_NT.$perfArch.$perfConfig || exit 0
97+
echo "unzip tests to ./bin/tests/Windows_NT.$perfArch.$perfConfig"
98+
fi

‎tests/scripts/project.json‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
{
22
"dependencies": {
3-
"Microsoft.DotNet.xunit.performance.run.core": "1.0.0-alpha-build0035",
4-
"Microsoft.DotNet.xunit.performance.analysis.cli": "1.0.0-alpha-build0035",
5-
"Microsoft.DotNet.xunit.performance.runner.cli": "1.0.0-alpha-build0035",
6-
"Microsoft.DotNet.xunit.performance":"1.0.0-alpha-build0035",
7-
"xunit.console.netcore": "1.0.3-prerelease-00607-01",
3+
"xunit.console.netcore": "1.0.2-prerelease-00101",
84
"Microsoft.DotNet.BuildTools.TestSuite": "1.0.0-prerelease-00629-04",
5+
6+
"CommandLineParser": "2.1.1-beta",
7+
"Microsoft.3rdpartytools.MarkdownLog": "0.10.0-alpha-experimental",
8+
9+
"xunit.performance.api": "1.0.0-beta-build0007",
10+
"xunit.performance.core": "1.0.0-beta-build0007",
11+
"xunit.performance.execution": "1.0.0-beta-build0007",
12+
"xunit.performance.metrics": "1.0.0-beta-build0007"
913
},
1014
"frameworks": {
1115
"netstandard1.3":{

0 commit comments

Comments
 (0)