Skip to content

Commit 66799e3

Browse files
GregoryComerGregoryComer
authored
[Windows] Add model run CI (#13836)
Add Windows CI jobs to test models. From looking at the existing CI, it looks like our model test jobs are pretty minimal and don't verify much other than it runs without crashing. I'd like to re-visit this in the near future and integrate with the new test harness, but for the purpose this PR, I'm just matching the Windows jobs to what the Linux jobs do.
1 parent 874d3c1 commit 66799e3

File tree

5 files changed

+148
-23
lines changed

5 files changed

+148
-23
lines changed

‎.ci/scripts/setup-windows.ps1‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
param (
2+
[string]$editable = $false
3+
)
4+
5+
conda create --yes --quiet -n et python=3.12
6+
conda activate et
7+
8+
# Activate the VS environment - this is required for Dynamo to work, as it uses MSVC.
9+
# There are a bunch of environment variables that it requires.
10+
# See https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line.
11+
& "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\Tools\Launch-VsDevShell.ps1" -Arch amd64
12+
13+
# Install test dependencies
14+
pip install -r .ci/docker/requirements-ci.txt
15+
16+
if ($editable -eq 'true') {
17+
install_executorch.bat --editable
18+
} else {
19+
install_executorch.bat
20+
}
21+
if ($LASTEXITCODE -ne 0) {
22+
Write-Host "Installation was unsuccessful. Exit code: $LASTEXITCODE."
23+
exit $LASTEXITCODE
24+
}

‎.ci/scripts/test_model.ps1‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
param (
2+
[string]$modelName,
3+
[string]$backend,
4+
[string]$buildDir = "cmake-out",
5+
[bool]$strict = $false
6+
)
7+
8+
Set-PSDebug -Trace 1
9+
$ErrorActionPreference = 'Stop'
10+
$PSNativeCommandUseErrorActionPreference = $true
11+
12+
function ExportModel-Portable {
13+
param (
14+
[string]$model_name,
15+
[bool]$strict
16+
)
17+
18+
$exportParams = "--model_name", "$modelName"
19+
if ($strict) {
20+
$exportParams += "--strict"
21+
}
22+
python -m examples.portable.scripts.export @exportParams | Write-Host
23+
if ($LASTEXITCODE -ne 0) {
24+
Write-Host "Model export failed. Exit code: $LASTEXITCODE."
25+
exit $LASTEXITCODE
26+
}
27+
28+
"$modelName.pte"
29+
}
30+
31+
function ExportModel-Xnnpack {
32+
param (
33+
[string]$model_name,
34+
[bool]$quantize
35+
)
36+
37+
if $(quantize) {
38+
python -m examples.xnnpack.aot_compiler --model_name="${MODEL_NAME}" --delegate --quantize | Write-Host
39+
$modelFile = "$($modelName)_xnnpack_q8.pte"
40+
} else {
41+
python -m examples.xnnpack.aot_compiler --model_name="${MODEL_NAME}" --delegate | Write-Host
42+
$modelFile = "$($modelName)_xnnpack_fp32.pte"
43+
}
44+
if ($LASTEXITCODE -ne 0) {
45+
Write-Host "Model export failed. Exit code: $LASTEXITCODE."
46+
exit $LASTEXITCODE
47+
}
48+
49+
$modelFile
50+
}
51+
52+
# Build the runner
53+
if (Test-Path -Path $buildDir) {
54+
Remove-Item -Path $buildDir -Recurse -Force
55+
}
56+
New-Item -Path $buildDir -ItemType Directory
57+
Push-Location $buildDir
58+
cmake .. --preset windows
59+
cmake --build . -t executor_runner -j16 --config Release
60+
if ($LASTEXITCODE -ne 0) {
61+
Write-Host "Runner build failed. Exit code: $LASTEXITCODE."
62+
exit $LASTEXITCODE
63+
}
64+
$executorBinaryPath = Join-Path -Path $buildDir -ChildPath "Release\executor_runner.exe"
65+
Pop-Location
66+
67+
# Export the model
68+
switch ($backend) {
69+
"portable" {
70+
$model_path = ExportModel-Portable -model_name $modelName -strict $strict
71+
}
72+
"xnnpack-f32" {
73+
$model_path = ExportModel-Xnnpack -model_name $modelName -quantize $false
74+
}
75+
"xnnpack-q8" {
76+
$model_path = ExportModel-Xnnpack -model_name $modelName -quantize $true
77+
}
78+
default {
79+
Write-Host "Unknown backend $backend."
80+
exit 1
81+
}
82+
}
83+
84+
# Run the runner
85+
& "$executorBinaryPath" --model_path="$model_path"
86+
if ($LASTEXITCODE -ne 0) {
87+
Write-Host "Model execution failed. Exit code: $LASTEXITCODE."
88+
exit $LASTEXITCODE
89+
}

‎.ci/scripts/unittest-windows.ps1‎

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,11 @@
11
param (
2-
[string]$editable
2+
[string]$editable = $false
33
)
44

55
Set-PSDebug -Trace 1
66
$ErrorActionPreference = 'Stop'
77
$PSNativeCommandUseErrorActionPreference = $true
88

9-
conda create --yes --quiet -n et python=3.12
10-
conda activate et
11-
12-
# Activate the VS environment - this is required for Dynamo to work, as it uses MSVC.
13-
# There are a bunch of environment variables that it requires.
14-
# See https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line.
15-
& "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\Tools\Launch-VsDevShell.ps1" -Arch amd64
16-
17-
# Install test dependencies
18-
pip install -r .ci/docker/requirements-ci.txt
19-
20-
if ($editable -eq 'true') {
21-
install_executorch.bat --editable
22-
} else {
23-
install_executorch.bat
24-
}
25-
if ($LASTEXITCODE -ne 0) {
26-
Write-Host "Installation was unsuccessful. Exit code: $LASTEXITCODE."
27-
exit $LASTEXITCODE
28-
}
29-
309
# Run pytest with coverage
3110
# pytest -n auto --cov=./ --cov-report=xml
3211
pytest -v --full-trace -c pytest-windows.ini

‎.github/workflows/_unittest.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,4 +63,13 @@ jobs:
6363
timeout: 120
6464
script: |
6565
conda init powershell
66-
powershell .ci/scripts/unittest-windows.ps1 -editable "${{ inputs.editable }}"
66+
67+
powershell -Command "& {
68+
Set-PSDebug -Trace 1
69+
\$ErrorActionPreference = 'Stop'
70+
\$PSNativeCommandUseErrorActionPreference = \$true
71+
72+
.ci/scripts/setup-windows.ps1
73+
74+
powershell .ci/scripts/unittest-windows.ps1 -editable "${{ inputs.editable }}"
75+
}"

‎.github/workflows/trunk.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -979,3 +979,27 @@ jobs:
979979
# Run MCU models
980980
chmod +x examples/arm/run_mcu_models_fvp.sh
981981
examples/arm/run_mcu_models_fvp.sh --target=cortex-m55
982+
983+
test-models-windows:
984+
uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
985+
strategy:
986+
fail-fast: false
987+
matrix:
988+
model: [linear, add, add_mul, ic3, ic4, mv2, mv3, resnet18, resnet50, vit, w2l, mobilebert, emformer_join, emformer_transcribe]
989+
backend: [portable, xnnpack-f32, xnnpack-q8]
990+
with:
991+
submodules: 'recursive'
992+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
993+
timeout: 60
994+
script: |
995+
conda init powershell
996+
997+
powershell -Command "& {
998+
Set-PSDebug -Trace 1
999+
\$ErrorActionPreference = 'Stop'
1000+
\$PSNativeCommandUseErrorActionPreference = \$true
1001+
1002+
.ci/scripts/setup-windows.ps1
1003+
1004+
powershell .ci/scripts/test_model.ps1 -modelName ${{ matrix.model }} -backend ${{ matrix.backend }}
1005+
}"

0 commit comments

Comments
 (0)