Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit 8d1d348

Browse files
DrewScogginsDrewScoggins
committed
Add CoreCLR perf test support to Jenkins
Made changes to the perf.groovy to add support for running the CoreCLR performance tests in the Perflab directory. Also made changes to the runner script for CoreCLR to allow it to run more than one type of test.
1 parent e15b926 commit 8d1d348

File tree

3 files changed

+169
-25
lines changed

3 files changed

+169
-25
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,3 +301,6 @@ cross/rootfs/*
301301

302302
# JIT32 files
303303
src/jit32
304+
305+
# performance testing sandbox
306+
sandbox

‎perf.groovy‎

Lines changed: 64 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,75 @@ def static getOSGroup(def os) {
2323
assert osGroup != null : "Could not find os group for ${os}"
2424
return osGroupMap[os]
2525
}
26+
// Setup perflab tests runs
27+
[true, false].each { isPR ->
28+
['Windows_NT'].each { os ->
29+
['x64'].each { architecture ->
30+
def configuration = 'Release'
31+
def runType = isPR ? 'private' : 'rolling'
32+
def benchViewName = isPR ? 'coreclr private %ghprbPullTitle%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
33+
def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}", isPR)) {
34+
// Set the label.
35+
label('windows_clr_perf')
36+
wrappers {
37+
credentialsBinding {
38+
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
39+
}
40+
}
41+
42+
steps {
43+
// Batch
44+
45+
batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory C:\\tools -Prerelease -ExcludeVersion")
46+
//Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
47+
//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
48+
batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
49+
"py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py --name " + "\"" + benchViewName + "\"" + " --user " + "\"[email protected]\"\n" +
50+
"py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\build.py git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type " + runType)
51+
batchFile("py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py")
52+
batchFile("set __TestIntermediateDir=int&&build.cmd release ${architecture}")
53+
batchFile("tests\\runtest.cmd release ${architecture} GenerateLayoutOnly")
54+
batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${architecture} -configuration ${configuration} -testBinLoc bin\\tests\\Windows_NT.${architecture}.Release\\performance\\perflab\\Perflab -library -uploadToBenchview C:\\Tools\\Microsoft.Benchview.JSONFormat\\tools -runtype " + runType)
55+
}
56+
}
57+
58+
// Save machinedata.json to /artifact/bin/ Jenkins dir
59+
def archiveSettings = new ArchivalSettings()
60+
archiveSettings.addFiles('sandbox\\perf-*.xml')
61+
Utilities.addArchival(newJob, archiveSettings)
62+
63+
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
64+
65+
if (isPR) {
66+
TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
67+
builder.setGithubContext("${os} CoreCLR Perf Tests")
68+
builder.triggerOnlyOnComment()
69+
builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
70+
builder.triggerForBranch(branch)
71+
builder.emitTrigger(newJob)
72+
}
73+
else {
74+
// Set a push trigger
75+
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
76+
builder.emitTrigger(newJob)
77+
}
78+
}
79+
}
80+
}
2681

2782
[true, false].each { isPR ->
2883
['Windows_NT'].each { os ->
2984
def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
85+
3086
// Set the label.
3187
label('performance')
3288
steps {
3389
// Batch
34-
batchFile("C:\\tools\\nuget install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory C:\\tools -Prerelease")
35-
batchFile("python C:\\tools\\Microsoft.BenchView.JSONFormat.0.1.0-pre008\\tools\\machinedata.py")
36-
batchFile("set __TestIntermediateDir=int&&build.cmd release x64")
37-
batchFile("tests\\runtest.cmd release x64")
38-
batchFile("tests\\scripts\\run-xunit-perf.cmd")
90+
batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory C:\\tools -Prerelease -ExcludeVersion")
91+
batchFile("py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py")
92+
batchFile("set __TestIntermediateDir=int&&build.cmd release ${architecture}")
93+
batchFile("tests\\runtest.cmd release ${architecture}")
94+
batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${architecture} -configuration ${configuration} -testBinLoc bin\\tests\\Windows_NT.x64.Release\\Jit\\Performance\\CodeQuality")
3995
}
4096
}
4197

@@ -122,6 +178,8 @@ def static getOSGroup(def os) {
122178
}
123179
}
124180

181+
Utilities.setMachineAffinity(newJob, os, 'latest-or-auto') // Just run against Linux VM's for now.
182+
125183
// Save machinedata.json to /artifact/bin/ Jenkins dir
126184
def archiveSettings = new ArchivalSettings()
127185
archiveSettings.addFiles('perf-*.xml')
@@ -142,4 +200,4 @@ def static getOSGroup(def os) {
142200
builder.emitTrigger(newJob)
143201
}
144202
} // os
145-
} // isPR
203+
} // isPR

‎tests/scripts/run-xunit-perf.cmd‎

Lines changed: 102 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,25 @@
55
@setlocal
66
@echo off
77

8-
set HERE=%CD%
8+
rem Set defaults for the file extension, architecture and configuration
99
set CORECLR_REPO=%CD%
10-
set CORECLR_OVERLAY=%CORECLR_REPO%\bin\tests\Windows_NT.x64.Release\Tests\Core_Root
11-
set CORECLR_PERF=%CORECLR_REPO%\bin\tests\Windows_NT.x64.Release\Jit\Performance\CodeQuality
12-
set RUNLOG=%HERE%\bin\Logs\perfrun.log
10+
set TEST_FILE_EXT=exe
11+
set TEST_ARCH=x64
12+
set TEST_CONFIG=Release
13+
14+
goto :ARGLOOP
15+
16+
:SETUP
17+
18+
set CORECLR_OVERLAY=%CORECLR_REPO%\bin\tests\Windows_NT.%TEST_ARCH%.%TEST_CONFIG%\Tests\Core_Root
19+
set RUNLOG=%CORECLR_REPO%\bin\Logs\perfrun.log
1320

1421
if NOT EXIST %CORECLR_OVERLAY% (
1522
echo Can't find test overlay directory '%CORECLR_OVERLAY%'
1623
echo Please build and run Release CoreCLR tests
1724
exit /B 1
1825
)
1926

20-
:SETUP
21-
2227
@echo --- setting up sandbox
2328

2429
rd /s /q sandbox
@@ -28,35 +33,113 @@ pushd sandbox
2833
@rem stage stuff we need
2934

3035
@rem xunit and perf
31-
xcopy /sy %CORECLR_REPO%\packages\Microsoft.DotNet.xunit.performance.runner.Windows\1.0.0-alpha-build0035\tools\* . > %RUNLOG%
32-
xcopy /sy %CORECLR_REPO%\packages\Microsoft.DotNet.xunit.performance.analysis\1.0.0-alpha-build0035\tools\* . > %RUNLOG%
33-
xcopy /sy %CORECLR_REPO%\packages\xunit.console.netcore\1.0.2-prerelease-00101\runtimes\any\native\* . > %RUNLOG%
34-
xcopy /sy %CORECLR_REPO%\bin\tests\Windows_NT.x64.Release\Tests\Core_Root\* . > %RUNLOG%
36+
xcopy /sy %CORECLR_REPO%\packages\Microsoft.DotNet.xunit.performance.runner.Windows\1.0.0-alpha-build0040\tools\* . > %RUNLOG%
37+
xcopy /sy %CORECLR_REPO%\packages\Microsoft.DotNet.xunit.performance.analysis\1.0.0-alpha-build0040\tools\* . >> %RUNLOG%
38+
xcopy /sy %CORECLR_REPO%\packages\xunit.console.netcore\1.0.2-prerelease-00177\runtimes\any\native\* . >> %RUNLOG%
39+
xcopy /sy %CORECLR_REPO%\bin\tests\Windows_NT.%TEST_ARCH%.%TEST_CONFIG%\Tests\Core_Root\* . >> %RUNLOG%
3540

3641
@rem find and stage the tests
37-
38-
for /R %CORECLR_PERF% %%T in (*.exe) do (
42+
for /R %CORECLR_PERF% %%T in (*.%TEST_FILE_EXT%) do (
3943
call :DOIT %%T
4044
)
4145

4246
goto :EOF
4347

4448
:DOIT
45-
4649
set BENCHNAME=%~n1
4750
set PERFOUT=perf-%BENCHNAME%
4851
set XMLOUT=%PERFOUT%-summary.xml
4952

5053
echo --- Running %BENCHNAME%
5154

52-
xcopy /s %1 . > %RUNLOG%
55+
xcopy /s %1 . >> %RUNLOG%
5356

54-
set CORE_ROOT=%HERE%\sandbox
57+
set CORE_ROOT=%CORECLR_REPO%\sandbox
5558

56-
xunit.performance.run.exe %BENCHNAME%.exe -runner xunit.console.netcore.exe -runnerhost corerun.exe -verbose -runid %PERFOUT% > %BENCHNAME%.out
59+
xunit.performance.run.exe %BENCHNAME%.%TEST_FILE_EXT% -runner xunit.console.netcore.exe -runnerhost corerun.exe -verbose -runid %PERFOUT% > %BENCHNAME%.out
5760

5861
xunit.performance.analysis.exe %PERFOUT%.xml -xml %XMLOUT% > %BENCHNAME%-analysis.out
5962

60-
type %XMLOUT% | findstr "test name"
61-
type %XMLOUT% | findstr Duration
62-
type %XMLOUT% | findstr InstRetired
63+
@rem optionally upload results to benchview
64+
if not [%BENCHVIEW_PATH%] == [] (
65+
py %BENCHVIEW_PATH%\measurement.py xunit perf-Perflab.xml --better desc --drop-first-value
66+
py %BENCHVIEW_PATH%\submission.py measurement.json ^
67+
--build ..\build.json ^
68+
--machine-data ..\machinedata.json ^
69+
--metadata ..\submission-metadata.json ^
70+
--group "CoreCLR" ^
71+
--type "%RUN_TYPE%" ^
72+
--config-name "%TEST_CONFIG%" ^
73+
--config Configuration "%TEST_CONFIG%" ^
74+
--config OS "Windows_NT" ^
75+
--arch "%TEST_ARCH%" ^
76+
--machinepool "PerfSnake"
77+
py %BENCHVIEW_PATH%\upload.py submission.json --container coreclr
78+
) else (
79+
type %XMLOUT% | findstr "test name"
80+
type %XMLOUT% | findstr Duration
81+
type %XMLOUT% | findstr InstRetired
82+
)
83+
84+
goto :EOF
85+
86+
:ARGLOOP
87+
IF /I [%1] == [-testBinLoc] (
88+
set CORECLR_PERF=%CORECLR_REPO%\%2
89+
shift
90+
shift
91+
goto :ARGLOOP
92+
)
93+
IF /I [%1] == [-runtype] (
94+
set RUN_TYPE=%2
95+
shift
96+
shift
97+
goto :ARGLOOP
98+
)
99+
IF /I [%1] == [-library] (
100+
set TEST_FILE_EXT=dll
101+
shift
102+
goto :ARGLOOP
103+
)
104+
IF /I [%1] == [-uploadtobenchview] (
105+
set BENCHVIEW_PATH=%2
106+
shift
107+
shift
108+
goto :ARGLOOP
109+
)
110+
IF /I [%1] == [-arch] (
111+
set TEST_ARCH=%2
112+
shift
113+
shift
114+
goto :ARGLOOP
115+
)
116+
IF /I [%1] == [-configuration] (
117+
set TEST_CONFIG=%2
118+
shift
119+
shift
120+
goto :ARGLOOP
121+
)
122+
if /I [%1] == [-?] (
123+
goto :USAGE
124+
)
125+
if /I [%1] == [-help] (
126+
goto :USAGE
127+
)
128+
129+
if [%CORECLR_PERF%] == [] (
130+
goto :USAGE
131+
)
132+
133+
goto :SETUP
134+
135+
:USAGE
136+
echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-uploadToBenchview] ^<path_to_benchview_tools^> [-runtype] ^<rolling^|private^>
137+
138+
echo For the path to the tests you can pass a parent directory and the script will grovel for
139+
echo all tests in subdirectories and run them.
140+
echo The library flag denotes whether the tests are build as libraries (.dll) or an executable (.exe)
141+
echo Architecture defaults to x64 and configuration defaults to release.
142+
echo -uploadtoBenchview is used to specify a path to the Benchview tooling and when this flag is
143+
echo set we will upload the results of the tests to the coreclr container in benchviewupload.
144+
echo Runtype sets the runtype that we upload to Benchview, rolling for regular runs, and private for
145+
echo PRs.

0 commit comments

Comments
 (0)