Skip to content

Commit 3499787

Browse files
kotlarmiloskotlarmilos
authored
[Mobile] Add build time and static linking size measurements for mobile (#4857)
* Add build time measurements for MAUI Android and iOS scenarios * Add AndroidCoreCLR JIT Static Linking scenarios * Add linkingType parameter to performance job configuration * Dynamic linking is default * Handle dynamic linking type correctly * Handle dynamic linking type correctly * Refactor linkingType handling to default to empty string * Fix linkingType handling * Update binlog path * Update binlog path * Fix traces path * Fix copy command
1 parent 4b8cdef commit 3499787

File tree

5 files changed

+40
-6
lines changed

5 files changed

+40
-6
lines changed

‎eng/performance/maui_scenarios_android.proj

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969

7070
<ItemGroup>
7171
<PreparePayloadWorkItem Include="@(MAUIAndroidScenario)">
72-
<Command>$(Python) pre.py publish -f $(PERFLAB_Framework)-android -r android-arm64 --self-contained --msbuild=&quot;$(_MSBuildArgs)&quot; --binlog %(PreparePayloadWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)</Command>
72+
<Command>$(Python) pre.py publish -f $(PERFLAB_Framework)-android -r android-arm64 --self-contained --msbuild=&quot;$(_MSBuildArgs)&quot; --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)\%(PreparePayloadWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)</Command>
7373
<WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
7474
</PreparePayloadWorkItem>
7575
</ItemGroup>
@@ -101,6 +101,10 @@
101101
<PreCommands>echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
102102
<Command>$(Python) test.py devicememoryconsumption --device-type android --package-path pub\%(HelixWorkItem.ApkName).apk --package-name %(HelixWorkItem.PackageName) --scenario-name &quot;%(Identity)&quot; --runtime 30 --test-iteration 2 $(ScenarioArgs)</Command>
103103
</HelixWorkItem>
104+
<HelixWorkItem Include="@(MAUIAndroidScenario -> 'Build Time - %(Identity)')">
105+
<PreCommands>echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y &amp;&amp; mkdir %HELIX_WORKITEM_ROOT%\traces &amp;&amp; copy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog %HELIX_WORKITEM_ROOT%\traces\</PreCommands>
106+
<Command>$(Python) test.py buildtime --scenario-name &quot;%(Identity)&quot; $(ScenarioArgs) --binlog-path .\%(HelixWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog</Command>
107+
</HelixWorkItem>
104108
</ItemGroup>
105109

106110
<Import Project="PreparePayloadWorkItems.targets" />

‎eng/performance/maui_scenarios_ios.proj

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848

4949
<ItemGroup>
5050
<PreparePayloadWorkItem Include="@(MAUIiOSScenario)">
51-
<Command>sudo xcode-select -s /Applications/Xcode_16.3.app; $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(NativeAOTCommandProps) -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName)</Command>
51+
<Command>sudo xcode-select -s /Applications/Xcode_16.3.app; $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(NativeAOTCommandProps) --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)/%(PreparePayloadWorkItem.ScenarioDirectoryName).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName)</Command>
5252
<WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
5353
</PreparePayloadWorkItem>
5454
</ItemGroup>
@@ -62,6 +62,10 @@
6262
<PreCommands>cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub; mv $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).ipa $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; unzip -d $HELIX_WORKITEM_ROOT/pub $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; rm $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip</PreCommands>
6363
<Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot; $(ScenarioArgs)</Command>
6464
</HelixWorkItem>
65+
<HelixWorkItem Include="@(MAUIiOSScenario -> 'Build Time - %(Identity)')">
66+
<PreCommands>cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub; cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/versions.json $HELIX_WORKITEM_ROOT/pub/versions.json; mkdir -p $HELIX_WORKITEM_ROOT/traces; cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName).binlog $HELIX_WORKITEM_ROOT/traces</PreCommands>
67+
<Command>$(Python) test.py buildtime --scenario-name &quot;%(Identity)&quot; $(ScenarioArgs) --binlog-path ./%(HelixWorkItem.ScenarioDirectoryName).binlog</Command>
68+
</HelixWorkItem>
6569
<XHarnessAppBundleToTest Include="Device Startup - iOS .NET Default Template">
6670
<AppBundlePath>$(ScenariosDir)netios.zip</AppBundlePath>
6771
<WorkItemTimeout>00:15:00</WorkItemTimeout>

‎eng/pipelines/runtime-perf-jobs.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,27 @@ jobs:
146146
${{ each parameter in parameters.jobParameters }}:
147147
${{ parameter.key }}: ${{ parameter.value }}
148148

149+
# run android scenarios - CoreCLR JIT Static Linking
150+
- template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }}
151+
parameters:
152+
jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }}
153+
buildConfig: release
154+
runtimeFlavor: coreclr
155+
platforms:
156+
- windows_x64
157+
jobParameters:
158+
runtimeType: AndroidCoreCLR
159+
codeGenType: JIT
160+
linkingType: static
161+
projectFile: $(Build.SourcesDirectory)/eng/testing/performance/android_scenarios.proj
162+
runKind: android_scenarios
163+
isScenario: true
164+
logicalMachine: 'perfpixel4a'
165+
runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }}
166+
performanceRepoAlias: ${{ parameters.performanceRepoAlias }}
167+
${{ each parameter in parameters.jobParameters }}:
168+
${{ parameter.key }}: ${{ parameter.value }}
169+
149170
# run android scenarios - CoreCLR R2R
150171
- template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }}
151172
parameters:

‎eng/pipelines/templates/run-performance-job.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ parameters:
4444
projectFile: '' # optional -- Full path to project file
4545
framework: '' # optional -- Framework to run against (use frameworks). Ignored if frameworks is non-empty.
4646
codeGenType: '' # optional -- CodeGen type (e.g. JIT, AOT)
47+
linkingType: '' # optional -- Linking type (e.g. static, dynamic)
4748
runtimeType: '' # optional -- Runtime type (e.g. mono, wasm, coreclr)
4849
useLocalCommitTime: false # optional -- Whether to use the local commit time or fetch it from the remote repository
4950
isCompare: false # optional -- If the job is comparing two configurations
@@ -60,8 +61,8 @@ jobs:
6061
- template: ${{ parameters.jobTemplate }}
6162
parameters:
6263
enableTelemetry: ${{ parameters.enableTelemetry }}
63-
name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }}
64-
displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), ' NULL', '') }}
64+
name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.linkingType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }}
65+
displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4} {5} {6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.linkingType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), ' NULL', '') }}
6566
timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
6667
workspace:
6768
clean: all

‎eng/pipelines/templates/runtime-perf-job.yml

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ parameters:
1010
liveLibrariesBuildConfig: ''
1111
runtimeType: 'coreclr'
1212
codeGenType: 'JIT'
13+
linkingType: '' # dynamic is default
1314
iOSLlvmBuild: 'False'
1415
iOSStripSymbols: 'False'
1516
hybridGlobalization: 'False'
@@ -171,10 +172,13 @@ jobs:
171172

172173
# AndroidCoreCLR
173174
${{ if eq(parameters.runtimeType, 'AndroidCoreCLR')}}:
174-
${{ if eq(parameters.codeGenType, 'JIT') }}:
175+
${{ if and(eq(parameters.codeGenType, 'JIT'), eq(parameters.linkingType, '')) }}:
175176
artifactFileName: 'AndroidHelloWorldArm64CoreCLR.tar.gz'
176177
artifactName: 'AndroidHelloWorldArm64CoreCLR'
177-
${{ if eq(parameters.codeGenType, 'R2R') }}:
178+
${{ if and(eq(parameters.codeGenType, 'JIT'), eq(parameters.linkingType, 'static')) }}:
179+
artifactFileName: 'AndroidHelloWorldArm64CoreCLRStaticLinking.tar.gz'
180+
artifactName: 'AndroidHelloWorldArm64CoreCLRStaticLinking'
181+
${{ if and(eq(parameters.codeGenType, 'R2R'), eq(parameters.linkingType, '')) }}:
178182
artifactFileName: 'AndroidHelloWorldArm64CoreCLRR2R.tar.gz'
179183
artifactName: 'AndroidHelloWorldArm64CoreCLRR2R'
180184

0 commit comments

Comments
 (0)