Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit 1767112

Browse files
DrewScogginsDrewScoggins
authored
Merge pull request #7838 from DrewScoggins/coreclr-perf-release
Adding CoreCLR perf Jenkins Support
2 parents 95d75b6 + 19e5dd7 commit 1767112

File tree

94 files changed

+857
-531
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

94 files changed

+857
-531
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,3 +301,6 @@ cross/rootfs/*
301301

302302
# JIT32 files
303303
src/jit32
304+
305+
# performance testing sandbox
306+
sandbox

‎perf.groovy‎

Lines changed: 54 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -23,42 +23,61 @@ def static getOSGroup(def os) {
2323
assert osGroup != null : "Could not find os group for ${os}"
2424
return osGroupMap[os]
2525
}
26-
26+
// Setup perflab tests runs
2727
[true, false].each { isPR ->
2828
['Windows_NT'].each { os ->
29-
def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
30-
// Set the label.
31-
label('performance')
32-
steps {
33-
// Batch
34-
batchFile("C:\\tools\\nuget install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory C:\\tools -Prerelease")
35-
batchFile("python C:\\tools\\Microsoft.BenchView.JSONFormat.0.1.0-pre008\\tools\\machinedata.py")
36-
batchFile("set __TestIntermediateDir=int&&build.cmd release x64")
37-
batchFile("tests\\runtest.cmd release x64")
38-
batchFile("tests\\scripts\\run-xunit-perf.cmd")
39-
}
40-
}
29+
['x64'].each { architecture ->
30+
def configuration = 'Release'
31+
def runType = isPR ? 'private' : 'rolling'
32+
def benchViewName = isPR ? 'coreclr private %ghprbPullTitle%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
33+
def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}", isPR)) {
34+
// Set the label.
35+
label('windows_clr_perf')
36+
wrappers {
37+
credentialsBinding {
38+
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
39+
}
40+
}
4141

42-
// Save machinedata.json to /artifact/bin/ Jenkins dir
43-
def archiveSettings = new ArchivalSettings()
44-
archiveSettings.addFiles('sandbox\\perf-*.xml')
45-
archiveSettings.addFiles('machinedata.json')
46-
Utilities.addArchival(newJob, archiveSettings)
42+
steps {
43+
// Batch
44+
45+
batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory C:\\tools -Prerelease -ExcludeVersion")
46+
//Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
47+
//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
48+
batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
49+
"py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py --name " + "\"" + benchViewName + "\"" + " --user " + "\"[email protected]\"\n" +
50+
"py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\build.py git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type " + runType)
51+
batchFile("py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py")
52+
batchFile("set __TestIntermediateDir=int&&build.cmd release ${architecture}")
53+
batchFile("tests\\runtest.cmd release ${architecture} GenerateLayoutOnly")
54+
batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${architecture} -configuration ${configuration} -testBinLoc bin\\tests\\Windows_NT.${architecture}.Release\\performance\\perflab\\Perflab -library -uploadToBenchview C:\\Tools\\Microsoft.Benchview.JSONFormat\\tools -runtype " + runType)
55+
batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${architecture} -configuration ${configuration} -testBinLoc bin\\tests\\Windows_NT.${architecture}.Release\\Jit\\Performance\\CodeQuality -uploadToBenchview C:\\Tools\\Microsoft.Benchview.JSONFormat\\tools -runtype " + runType)
56+
}
57+
}
4758

48-
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
49-
if (isPR) {
50-
TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
51-
builder.setGithubContext("${os} Perf Tests")
52-
builder.triggerOnlyOnComment()
53-
builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
54-
builder.triggerForBranch(branch)
55-
builder.emitTrigger(newJob)
56-
}
57-
else {
58-
// Set a push trigger
59-
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
60-
builder.emitTrigger(newJob)
61-
}
59+
// Save machinedata.json to /artifact/bin/ Jenkins dir
60+
def archiveSettings = new ArchivalSettings()
61+
archiveSettings.addFiles('perf-*.xml')
62+
archiveSettings.addFiles('perf-*.etl')
63+
Utilities.addArchival(newJob, archiveSettings)
64+
65+
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
66+
67+
if (isPR) {
68+
TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
69+
builder.setGithubContext("${os} CoreCLR Perf Tests")
70+
builder.triggerOnlyOnComment()
71+
builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
72+
builder.triggerForBranch(branch)
73+
builder.emitTrigger(newJob)
74+
}
75+
else {
76+
// Set a push trigger
77+
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
78+
builder.emitTrigger(newJob)
79+
}
80+
}
6281
}
6382
}
6483

@@ -122,6 +141,8 @@ def static getOSGroup(def os) {
122141
}
123142
}
124143

144+
Utilities.setMachineAffinity(newJob, os, 'latest-or-auto') // Just run against Linux VM's for now.
145+
125146
// Save machinedata.json to /artifact/bin/ Jenkins dir
126147
def archiveSettings = new ArchivalSettings()
127148
archiveSettings.addFiles('perf-*.xml')
@@ -142,4 +163,4 @@ def static getOSGroup(def os) {
142163
builder.emitTrigger(newJob)
143164
}
144165
} // os
145-
} // isPR
166+
} // isPR

‎tests/scripts/run-xunit-perf.cmd‎

Lines changed: 104 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,25 @@
55
@setlocal
66
@echo off
77

8-
set HERE=%CD%
8+
rem Set defaults for the file extension, architecture and configuration
99
set CORECLR_REPO=%CD%
10-
set CORECLR_OVERLAY=%CORECLR_REPO%\bin\tests\Windows_NT.x64.Release\Tests\Core_Root
11-
set CORECLR_PERF=%CORECLR_REPO%\bin\tests\Windows_NT.x64.Release\Jit\Performance\CodeQuality
12-
set RUNLOG=%HERE%\bin\Logs\perfrun.log
10+
set TEST_FILE_EXT=exe
11+
set TEST_ARCH=x64
12+
set TEST_CONFIG=Release
13+
14+
goto :ARGLOOP
15+
16+
:SETUP
17+
18+
set CORECLR_OVERLAY=%CORECLR_REPO%\bin\tests\Windows_NT.%TEST_ARCH%.%TEST_CONFIG%\Tests\Core_Root
19+
set RUNLOG=%CORECLR_REPO%\bin\Logs\perfrun.log
1320

1421
if NOT EXIST %CORECLR_OVERLAY% (
1522
echo Can't find test overlay directory '%CORECLR_OVERLAY%'
1623
echo Please build and run Release CoreCLR tests
1724
exit /B 1
1825
)
1926

20-
:SETUP
21-
2227
@echo --- setting up sandbox
2328

2429
rd /s /q sandbox
@@ -29,34 +34,115 @@ pushd sandbox
2934

3035
@rem xunit and perf
3136
xcopy /sy %CORECLR_REPO%\packages\Microsoft.DotNet.xunit.performance.runner.Windows\1.0.0-alpha-build0035\tools\* . > %RUNLOG%
32-
xcopy /sy %CORECLR_REPO%\packages\Microsoft.DotNet.xunit.performance.analysis\1.0.0-alpha-build0035\tools\* . > %RUNLOG%
33-
xcopy /sy %CORECLR_REPO%\packages\xunit.console.netcore\1.0.2-prerelease-00101\runtimes\any\native\* . > %RUNLOG%
34-
xcopy /sy %CORECLR_REPO%\bin\tests\Windows_NT.x64.Release\Tests\Core_Root\* . > %RUNLOG%
37+
xcopy /sy %CORECLR_REPO%\packages\Microsoft.DotNet.xunit.performance.analysis\1.0.0-alpha-build0035\tools\* . >> %RUNLOG%
38+
xcopy /sy %CORECLR_REPO%\packages\xunit.console.netcore\1.0.2-prerelease-00101\runtimes\any\native\* . >> %RUNLOG%
39+
xcopy /sy %CORECLR_REPO%\bin\tests\Windows_NT.%TEST_ARCH%.%TEST_CONFIG%\Tests\Core_Root\* . >> %RUNLOG%
3540

3641
@rem find and stage the tests
37-
38-
for /R %CORECLR_PERF% %%T in (*.exe) do (
42+
for /R %CORECLR_PERF% %%T in (*.%TEST_FILE_EXT%) do (
3943
call :DOIT %%T
4044
)
4145

4246
goto :EOF
4347

4448
:DOIT
45-
4649
set BENCHNAME=%~n1
4750
set PERFOUT=perf-%BENCHNAME%
4851
set XMLOUT=%PERFOUT%-summary.xml
4952

5053
echo --- Running %BENCHNAME%
5154

52-
xcopy /s %1 . > %RUNLOG%
55+
xcopy /s %1 . >> %RUNLOG%
5356

54-
set CORE_ROOT=%HERE%\sandbox
57+
set CORE_ROOT=%CORECLR_REPO%\sandbox
5558

56-
xunit.performance.run.exe %BENCHNAME%.exe -runner xunit.console.netcore.exe -runnerhost corerun.exe -verbose -runid %PERFOUT% > %BENCHNAME%.out
59+
xunit.performance.run.exe %BENCHNAME%.%TEST_FILE_EXT% -runner xunit.console.netcore.exe -runnerhost corerun.exe -verbose -runid %PERFOUT% > %BENCHNAME%.out
5760

5861
xunit.performance.analysis.exe %PERFOUT%.xml -xml %XMLOUT% > %BENCHNAME%-analysis.out
5962

60-
type %XMLOUT% | findstr "test name"
61-
type %XMLOUT% | findstr Duration
62-
type %XMLOUT% | findstr InstRetired
63+
@rem optionally upload results to benchview
64+
if not [%BENCHVIEW_PATH%] == [] (
65+
py %BENCHVIEW_PATH%\measurement.py xunit perf-%BENCHNAME%.xml --better desc --drop-first-value
66+
py %BENCHVIEW_PATH%\submission.py measurement.json ^
67+
--build ..\build.json ^
68+
--machine-data ..\machinedata.json ^
69+
--metadata ..\submission-metadata.json ^
70+
--group "CoreCLR" ^
71+
--type "%RUN_TYPE%" ^
72+
--config-name "%TEST_CONFIG%" ^
73+
--config Configuration "%TEST_CONFIG%" ^
74+
--config OS "Windows_NT" ^
75+
--arch "%TEST_ARCH%" ^
76+
--machinepool "PerfSnake"
77+
py %BENCHVIEW_PATH%\upload.py submission.json --container coreclr
78+
REM Save off the results to the root directory for recovery later in Jenkins
79+
xcopy perf-%BENCHNAME%*.xml %CORECLR_REPO%\
80+
xcopy perf-%BENCHNAME%*.etl %CORECLR_REPO%\
81+
) else (
82+
type %XMLOUT% | findstr "test name"
83+
type %XMLOUT% | findstr Duration
84+
type %XMLOUT% | findstr InstRetired
85+
)
86+
87+
goto :EOF
88+
89+
:ARGLOOP
90+
IF /I [%1] == [-testBinLoc] (
91+
set CORECLR_PERF=%CORECLR_REPO%\%2
92+
shift
93+
shift
94+
goto :ARGLOOP
95+
)
96+
IF /I [%1] == [-runtype] (
97+
set RUN_TYPE=%2
98+
shift
99+
shift
100+
goto :ARGLOOP
101+
)
102+
IF /I [%1] == [-library] (
103+
set TEST_FILE_EXT=dll
104+
shift
105+
goto :ARGLOOP
106+
)
107+
IF /I [%1] == [-uploadtobenchview] (
108+
set BENCHVIEW_PATH=%2
109+
shift
110+
shift
111+
goto :ARGLOOP
112+
)
113+
IF /I [%1] == [-arch] (
114+
set TEST_ARCH=%2
115+
shift
116+
shift
117+
goto :ARGLOOP
118+
)
119+
IF /I [%1] == [-configuration] (
120+
set TEST_CONFIG=%2
121+
shift
122+
shift
123+
goto :ARGLOOP
124+
)
125+
if /I [%1] == [-?] (
126+
goto :USAGE
127+
)
128+
if /I [%1] == [-help] (
129+
goto :USAGE
130+
)
131+
132+
if [%CORECLR_PERF%] == [] (
133+
goto :USAGE
134+
)
135+
136+
goto :SETUP
137+
138+
:USAGE
139+
echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-uploadToBenchview] ^<path_to_benchview_tools^> [-runtype] ^<rolling^|private^>
140+
141+
echo For the path to the tests you can pass a parent directory and the script will grovel for
142+
echo all tests in subdirectories and run them.
143+
echo The library flag denotes whether the tests are build as libraries (.dll) or an executable (.exe)
144+
echo Architecture defaults to x64 and configuration defaults to release.
145+
echo -uploadtoBenchview is used to specify a path to the Benchview tooling and when this flag is
146+
echo set we will upload the results of the tests to the coreclr container in benchviewupload.
147+
echo Runtype sets the runtype that we upload to Benchview, rolling for regular runs, and private for
148+
echo PRs.

‎tests/src/JIT/Performance/CodeQuality/BenchF/Adams/Adams.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
<ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
1414
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
1515
<NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
16+
<NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
1617
</PropertyGroup>
1718
<!-- Default configurations to help VS understand the configurations -->
1819
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">

‎tests/src/JIT/Performance/CodeQuality/BenchF/BenchMk2/BenchMk2.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
<ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
1414
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
1515
<NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
16+
<NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
1617
</PropertyGroup>
1718
<!-- Default configurations to help VS understand the configurations -->
1819
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">

‎tests/src/JIT/Performance/CodeQuality/BenchF/BenchMrk/BenchMrk.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
<ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
1414
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
1515
<NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
16+
<NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
1617
</PropertyGroup>
1718
<!-- Default configurations to help VS understand the configurations -->
1819
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">

‎tests/src/JIT/Performance/CodeQuality/BenchF/Bisect/Bisect.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
<ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
1414
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
1515
<NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
16+
<NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
1617
</PropertyGroup>
1718
<!-- Default configurations to help VS understand the configurations -->
1819
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">

‎tests/src/JIT/Performance/CodeQuality/BenchF/DMath/DMath.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
<ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
1414
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
1515
<NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
16+
<NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
1617
</PropertyGroup>
1718
<!-- Default configurations to help VS understand the configurations -->
1819
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">

‎tests/src/JIT/Performance/CodeQuality/BenchF/FFT/FFT.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
<ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
1414
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
1515
<NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
16+
<NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
1617
</PropertyGroup>
1718
<!-- Default configurations to help VS understand the configurations -->
1819
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">

‎tests/src/JIT/Performance/CodeQuality/BenchF/InProd/InProd.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
<ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
1414
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
1515
<NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
16+
<NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
1617
</PropertyGroup>
1718
<!-- Default configurations to help VS understand the configurations -->
1819
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">

0 commit comments

Comments
 (0)