Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit d0d7997

Browse files
sdmacleajashook
sdmaclea
authored and
jashook
committed
Split unix test builds in slices (#17785)
* Split unix test builds in slices Ports #17161 to linux * Address review feedback
1 parent 6db4217 commit d0d7997

File tree

2 files changed

+88
-45
lines changed

2 files changed

+88
-45
lines changed

‎build-test.sh‎

Lines changed: 70 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -295,28 +295,75 @@ build_Tests_internal()
295295
__BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
296296
__BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
297297
__BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
298-
__msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
299-
__msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
300-
__msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
301298

302-
# Generate build command
303-
buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
299+
if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
300+
# Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
301+
# See https://github.com/Microsoft/msbuild/issues/2993
304302

305-
echo "Building step '$stepName' via $buildCommand"
303+
# __SkipPackageRestore and __SkipTargetingPackBuild used to control build by tests/src/dirs.proj
304+
export __SkipPackageRestore=false
305+
export __SkipTargetingPackBuild=false
306+
export __BuildLoopCount=2
307+
export __TestGroupToBuild=1
308+
__AppendToLog=false
306309

307-
# Invoke MSBuild
308-
eval $buildCommand
310+
if [ -n __priority1 ]; then
311+
export __BuildLoopCount=16
312+
export __TestGroupToBuild=2
313+
fi
309314

310-
# Invoke MSBuild
311-
# $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
315+
for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 ))
316+
do
317+
__msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
318+
__msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
319+
__msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
312320

313-
# Make sure everything is OK
314-
if [ $? -ne 0 ]; then
315-
echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
316-
echo " $__BuildLog"
317-
echo " $__BuildWrn"
318-
echo " $__BuildErr"
319-
exit 1
321+
export TestBuildSlice=$slice
322+
323+
# Generate build command
324+
buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
325+
326+
echo "Building step '$stepName' slice=$slice via $buildCommand"
327+
328+
# Invoke MSBuild
329+
eval $buildCommand
330+
331+
# Make sure everything is OK
332+
if [ $? -ne 0 ]; then
333+
echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
334+
echo " $__BuildLog"
335+
echo " $__BuildWrn"
336+
echo " $__BuildErr"
337+
exit 1
338+
fi
339+
export __SkipPackageRestore=true
340+
export __SkipTargetingPackBuild=true
341+
__AppendToLog=true
342+
done
343+
else
344+
__msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
345+
__msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
346+
__msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
347+
348+
# Generate build command
349+
buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
350+
351+
echo "Building step '$stepName' via $buildCommand"
352+
353+
# Invoke MSBuild
354+
eval $buildCommand
355+
356+
# Invoke MSBuild
357+
# $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
358+
359+
# Make sure everything is OK
360+
if [ $? -ne 0 ]; then
361+
echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
362+
echo " $__BuildLog"
363+
echo " $__BuildWrn"
364+
echo " $__BuildErr"
365+
exit 1
366+
fi
320367
fi
321368
}
322369

@@ -342,6 +389,7 @@ usage()
342389
echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
343390
echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
344391
echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
392+
echo "priority1 - include priority=1 tests in the build"
345393
exit 1
346394
}
347395

@@ -461,6 +509,7 @@ __RunTests=0
461509
__RebuildTests=0
462510
__BuildTestWrappers=0
463511
__GenerateLayoutOnly=
512+
__priority1=
464513
CORE_ROOT=
465514

466515
while :; do
@@ -615,6 +664,10 @@ while :; do
615664
msbuildonunsupportedplatform)
616665
__msbuildonunsupportedplatform=1
617666
;;
667+
priority1)
668+
__priority1=1
669+
__UnprocessedBuildArgs="$__UnprocessedBuildArgs -priority=1"
670+
;;
618671
*)
619672
__UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
620673
;;

‎tests/src/dirs.proj‎

Lines changed: 18 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -33,26 +33,16 @@
3333
<DisabledProjects Include="Loader\classloader\generics\GenericMethods\VSW491668.csproj" /> <!-- issue 5501 -->
3434
</ItemGroup>
3535

36-
<!-- Unix builds do not support subgroups -->
37-
<ItemGroup Condition="$(__BuildOS) != 'Windows_NT'" >
38-
<Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)">
39-
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
40-
</Project>
41-
<Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)">
42-
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
43-
</Project>
44-
</ItemGroup>
45-
4636
<!-- Test build is divided in slices which can be created within Test Group
4737
Priority 0 tests are build using Test Group 1 with 2 subgroups or slices -->
4838

49-
<ItemGroup Condition="$(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'">
39+
<ItemGroup Condition="$(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'">
5040
<Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)">
5141
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
5242
</Project>
5343
</ItemGroup>
5444

55-
<ItemGroup Condition="$(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '2'">
45+
<ItemGroup Condition="$(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '2'">
5646
<Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)">
5747
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
5848
</Project>
@@ -61,7 +51,7 @@
6151
<!-- Test build is divided in slices which can be created within Test Group
6252
Priority 1 or higher tests are build using Test Group 2 with 16 subgroups or slices -->
6353

64-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '1')">
54+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '1')">
6555
<Project Include="baseservices\**\*.csproj" Exclude="@(DisabledProjects)">
6656
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
6757
</Project>
@@ -76,7 +66,7 @@
7666
</Project>
7767
</ItemGroup>
7868

79-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '2')">
69+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '2')">
8070
<Project Include="CoreMangLib\**\*.csproj" Exclude="@(DisabledProjects)">
8171
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
8272
</Project>
@@ -85,7 +75,7 @@
8575
</Project>
8676
</ItemGroup>
8777

88-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '3')">
78+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '3')">
8979
<Project Include="E*\**\*.csproj" Exclude="@(DisabledProjects)">
9080
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
9181
</Project>
@@ -112,7 +102,7 @@
112102
</Project>
113103
</ItemGroup>
114104

115-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '4')">
105+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '4')">
116106
<Project Include="JIT\B*\**\*.csproj" Exclude="@(DisabledProjects)">
117107
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
118108
</Project>
@@ -124,7 +114,7 @@
124114
</Project>
125115
</ItemGroup>
126116

127-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '5')">
117+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '5')">
128118
<Project Include="JIT\B*\**\*.ilproj" Exclude="@(DisabledProjects)">
129119
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
130120
</Project>
@@ -136,7 +126,7 @@
136126
</Project>
137127
</ItemGroup>
138128

139-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '6')">
129+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '6')">
140130
<Project Include="JIT\Generics\**\*.csproj" Exclude="@(DisabledProjects)">
141131
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
142132
</Project>
@@ -151,7 +141,7 @@
151141
</Project>
152142
</ItemGroup>
153143

154-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '7')">
144+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '7')">
155145
<Project Include="JIT\IL_Conformance\**\*.csproj" Exclude="@(DisabledProjects)">
156146
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
157147
</Project>
@@ -160,7 +150,7 @@
160150
</Project>
161151
</ItemGroup>
162152

163-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '8')">
153+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '8')">
164154
<Project Include="JIT\jit64\**\*.csproj" Exclude="@(DisabledProjects)">
165155
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
166156
</Project>
@@ -169,19 +159,19 @@
169159
</Project>
170160
</ItemGroup>
171161

172-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '9')">
162+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '9')">
173163
<Project Include="JIT\Methodical\**\*.csproj" Exclude="@(DisabledProjects)">
174164
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
175165
</Project>
176166
</ItemGroup>
177167

178-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '10')">
168+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '10')">
179169
<Project Include="JIT\Methodical\**\*.ilproj" Exclude="@(DisabledProjects)">
180170
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
181171
</Project>
182172
</ItemGroup>
183173

184-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '11')">
174+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '11')">
185175
<Project Include="JIT\opt\**\*.csproj" Exclude="@(DisabledProjects)">
186176
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
187177
</Project>
@@ -202,31 +192,31 @@
202192
</Project>
203193
</ItemGroup>
204194

205-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '12')">
195+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '12')">
206196
<Project Include="JIT\R*\**\*.csproj" Exclude="@(DisabledProjects)">
207197
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
208198
</Project>
209199
</ItemGroup>
210200

211-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '13')">
201+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '13')">
212202
<Project Include="JIT\R*\**\*.ilproj" Exclude="@(DisabledProjects)">
213203
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
214204
</Project>
215205
</ItemGroup>
216206

217-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '14')">
207+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '14')">
218208
<Project Include="Loader\**\*.csproj" Exclude="@(DisabledProjects)">
219209
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
220210
</Project>
221211
</ItemGroup>
222212

223-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '15')">
213+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '15')">
224214
<Project Include="Loader\**\*.ilproj" Exclude="@(DisabledProjects)">
225215
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
226216
</Project>
227217
</ItemGroup>
228218

229-
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '16')">
219+
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '16')">
230220
<Project Include="m*\**\*.csproj" Exclude="@(DisabledProjects)">
231221
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
232222
</Project>

0 commit comments

Comments
 (0)