Skip to content

Commit 56d5186

Browse files
authored
[Windows] Add CI wheel build jobs for Windows (#13837)
Set up wheel build jobs for Windows (x86-64). This is derived from the torchvision build setup (see https://github.com/pytorch/vision/blob/b208f7f4125e8a9a51638549ccf3e9b163f53dda/.github/workflows/build-wheels-windows.yml#L4).
1 parent dc190f9 commit 56d5186

File tree

8 files changed

+220
-4
lines changed

8 files changed

+220
-4
lines changed

.ci/scripts/test_model.ps1

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ function ExportModel-Xnnpack {
3434
[bool]$quantize
3535
)
3636

37-
if $(quantize) {
37+
if ($quantize) {
3838
python -m examples.xnnpack.aot_compiler --model_name="${MODEL_NAME}" --delegate --quantize | Write-Host
3939
$modelFile = "$($modelName)_xnnpack_q8.pte"
4040
} else {

.ci/scripts/wheel/pre_build_script.sh

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,26 @@ set -euxo pipefail
99

1010
# This script is run before building ExecuTorch binaries
1111

12+
# Clone nested submodules for tokenizers - this is a workaround for recursive
13+
# submodule clone failing due to path length limitations on Windows. Eventually,
14+
# we should update the core job in test-infra to enable long paths before
15+
# checkout to avoid needing to do this.
16+
pushd extension/llm/tokenizers
17+
git submodule update --init
18+
popd
19+
20+
# On Windows, enable symlinks and re-checkout the current revision to create
21+
# the symlinked src/ directory. This is needed to build the wheel.
22+
UNAME_S=$(uname -s)
23+
if [[ $UNAME_S == *"MINGW"* || $UNAME_S == *"MSYS"* ]]; then
24+
echo "Enabling symlinks on Windows"
25+
git config core.symlinks true
26+
git checkout -f HEAD
27+
fi
28+
1229
# Manually install build requirements because `python setup.py bdist_wheel` does
1330
# not install them. TODO(dbort): Switch to using `python -m build --wheel`,
1431
# which does install them. Though we'd need to disable build isolation to be
1532
# able to see the installed torch package.
1633

17-
"${GITHUB_WORKSPACE}/${REPOSITORY}/install_requirements.sh" --example
34+
"${GITHUB_WORKSPACE}/${REPOSITORY}/install_requirements.sh" --example

.ci/scripts/wheel/test_windows.py

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/usr/bin/env python
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
from typing import List
9+
10+
import torch
11+
from executorch.backends.xnnpack.partition.xnnpack_partitioner import XnnpackPartitioner
12+
from executorch.examples.models import Backend, Model, MODEL_NAME_TO_MODEL
13+
from executorch.examples.models.model_factory import EagerModelFactory
14+
from executorch.examples.xnnpack import MODEL_NAME_TO_OPTIONS
15+
from executorch.examples.xnnpack.quantization.utils import quantize as quantize_xnn
16+
from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower
17+
from executorch.extension.pybindings.portable_lib import (
18+
_load_for_executorch_from_buffer,
19+
)
20+
from test_base import ModelTest
21+
22+
23+
def test_model_xnnpack(model: Model, quantize: bool) -> None:
24+
model_instance, example_inputs, _, _ = EagerModelFactory.create_model(
25+
*MODEL_NAME_TO_MODEL[str(model)]
26+
)
27+
28+
model_instance.eval()
29+
ref_outputs = model_instance(*example_inputs)
30+
31+
if quantize:
32+
quant_type = MODEL_NAME_TO_OPTIONS[str(model)].quantization
33+
model_instance = torch.export.export_for_training(
34+
model_instance, example_inputs
35+
)
36+
model_instance = quantize_xnn(
37+
model_instance.module(), example_inputs, quant_type
38+
)
39+
40+
lowered = to_edge_transform_and_lower(
41+
torch.export.export(model_instance, example_inputs),
42+
partitioner=[XnnpackPartitioner()],
43+
compile_config=EdgeCompileConfig(
44+
_check_ir_validity=False,
45+
),
46+
).to_executorch()
47+
48+
loaded_model = _load_for_executorch_from_buffer(lowered.buffer)
49+
et_outputs = loaded_model([*example_inputs])
50+
51+
if isinstance(ref_outputs, torch.Tensor):
52+
ref_outputs = (ref_outputs,)
53+
54+
assert len(ref_outputs) == len(et_outputs)
55+
for i in range(len(ref_outputs)):
56+
torch.testing.assert_close(ref_outputs[i], et_outputs[i], atol=1e-4, rtol=1e-5)
57+
58+
59+
def run_tests(model_tests: List[ModelTest]) -> None:
60+
for model_test in model_tests:
61+
if model_test.backend == Backend.Xnnpack:
62+
test_model_xnnpack(model_test.model, quantize=False)
63+
else:
64+
raise RuntimeError(f"Unsupported backend {model_test.backend}.")
65+
66+
67+
if __name__ == "__main__":
68+
run_tests(
69+
model_tests=[
70+
ModelTest(
71+
model=Model.Mv3,
72+
backend=Backend.Xnnpack,
73+
),
74+
]
75+
)
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
REM This is lightly modified from the torchvision Windows build logic.
2+
REM See https://github.com/pytorch/vision/blob/main/packaging/windows/internal/vc_env_helper.bat
3+
4+
@echo on
5+
6+
set VC_VERSION_LOWER=17
7+
set VC_VERSION_UPPER=18
8+
9+
for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do (
10+
if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
11+
set "VS15INSTALLDIR=%%i"
12+
set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
13+
goto vswhere
14+
)
15+
)
16+
17+
:vswhere
18+
if "%VSDEVCMD_ARGS%" == "" (
19+
call "%VS15VCVARSALL%" x64 || exit /b 1
20+
) else (
21+
call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1
22+
)
23+
24+
@echo on
25+
26+
if "%CU_VERSION%" == "xpu" call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
27+
28+
set DISTUTILS_USE_SDK=1
29+
30+
set args=%1
31+
shift
32+
:start
33+
if [%1] == [] goto done
34+
set args=%args% %1
35+
shift
36+
goto start
37+
38+
:done
39+
if "%args%" == "" (
40+
echo Usage: vc_env_helper.bat [command] [args]
41+
echo e.g. vc_env_helper.bat cl /c test.cpp
42+
)
43+
44+
set work_dir=%CD%
45+
if exist setup.py (
46+
echo "Creating symlink..."
47+
REM Setup a symlink to shorten the path length.
48+
REM Note that the ET directory has to be named "executorch".
49+
cd %GITHUB_WORKSPACE%
50+
if not exist et\ (
51+
mkdir et
52+
)
53+
cd et
54+
echo Work dir: %work_dir%
55+
if not exist executorch\ (
56+
mklink /d executorch %work_dir%
57+
)
58+
cd executorch
59+
)
60+
61+
%args% || exit /b 1
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Build Windows Wheels
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- nightly
8+
- main
9+
- release/*
10+
tags:
11+
# NOTE: Binary build pipelines should only get triggered on release candidate builds
12+
# Release candidate tags look like: v1.11.0-rc1
13+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
14+
workflow_dispatch:
15+
16+
permissions:
17+
id-token: write
18+
contents: read
19+
20+
jobs:
21+
generate-matrix:
22+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
23+
with:
24+
package-type: wheel
25+
os: windows
26+
test-infra-repository: pytorch/test-infra
27+
test-infra-ref: main
28+
with-cuda: disabled
29+
with-rocm: disabled
30+
python-versions: '["3.10", "3.11", "3.12"]'
31+
32+
build:
33+
needs: generate-matrix
34+
strategy:
35+
fail-fast: false
36+
matrix:
37+
include:
38+
- repository: pytorch/executorch
39+
pre-script: .ci\\scripts\\wheel\\pre_build_script.sh
40+
env-script: .ci\\scripts\\wheel\\vc_env_helper.bat
41+
post-script: .ci\\scripts\\wheel\\post_build_script.sh
42+
smoke-test-script: .ci/scripts/wheel/test_windows.py
43+
package-name: executorch
44+
name: ${{ matrix.repository }}
45+
uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main
46+
with:
47+
repository: ${{ matrix.repository }}
48+
ref: ""
49+
test-infra-repository: pytorch/test-infra
50+
test-infra-ref: main
51+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
52+
pre-script: ${{ matrix.pre-script }}
53+
env-script: ${{ matrix.env-script }}
54+
post-script: ${{ matrix.post-script }}
55+
package-name: ${{ matrix.package-name }}
56+
smoke-test-script: ${{ matrix.smoke-test-script }}
57+
trigger-event: ${{ github.event_name }}
58+
wheel-build-params: "--verbose"
59+
submodules: true

examples/models/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ def __str__(self) -> str:
4444

4545

4646
class Backend(str, Enum):
47+
Xnnpack = "xnnpack"
4748
XnnpackQuantizationDelegation = "xnnpack-quantization-delegation"
4849
CoreMlExportOnly = "coreml"
4950
CoreMlExportAndTest = "coreml-test" # AOT export + test with runner

setup.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ def string(cls) -> str:
142142
@classmethod
143143
def write_to_python_file(cls, path: str) -> None:
144144
"""Creates a file similar to PyTorch core's `torch/version.py`."""
145+
145146
lines = [
146147
"from typing import Optional",
147148
'__all__ = ["__version__", "git_version"]',
@@ -773,7 +774,7 @@ def run(self): # noqa C901
773774
# platform-specific files using InstallerBuildExt.
774775
ext_modules=[
775776
BuiltFile(
776-
src_dir="%CMAKE_CACHE_DIR%/third-party/flatc_proj/bin/",
777+
src_dir="%CMAKE_CACHE_DIR%/third-party/flatc_ep/bin/",
777778
src_name="flatc",
778779
dst="executorch/data/bin/",
779780
is_executable=True,

third-party/CMakeLists.txt

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ endif()
2929
# Otherwise, flatc will target the project's toolchain (i.e. iOS, or Android).
3030
ExternalProject_Add(
3131
flatbuffers_ep
32-
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/flatc_proj
32+
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/flatc_ep
33+
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/flatc_ep/src/build
3334
SOURCE_DIR ${PROJECT_SOURCE_DIR}/third-party/flatbuffers
3435
CMAKE_ARGS -DFLATBUFFERS_BUILD_FLATC=ON
3536
-DFLATBUFFERS_INSTALL=ON
@@ -82,6 +83,7 @@ ExternalProject_Add(
8283
flatcc_ep
8384
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/flatcc_ep
8485
SOURCE_DIR ${PROJECT_SOURCE_DIR}/third-party/flatcc
86+
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/flatcc_ep/src/build
8587
CMAKE_ARGS -DFLATCC_RTONLY=OFF
8688
-DFLATCC_TEST=OFF
8789
-DFLATCC_REFLECTION=OFF

0 commit comments

Comments
 (0)