Skip to content

Commit 035c0a1

Browse files
shoumikhinshoumikhin
committed
Update
[ghstack-poisoned]
1 parent e5899c7 commit 035c0a1

5 files changed

Lines changed: 489 additions & 0 deletions

File tree

‎.ci/scripts/wheel/envvar_cuda_linux.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
# This file is sourced into the environment before building a pip wheel. It
8+
# should typically only contain shell variable assignments. Be sure to export
9+
# any variables so that subprocesses will see them.
10+
11+
source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/envvar_base.sh"
12+
13+
# Ask for the CUDA delegate explicitly rather than letting the build detect a toolkit. A
14+
# detected build is fine locally, but a release row states what it is producing, and a row
15+
# that silently produced a CPU wheel because the toolkit was not found would publish under a
16+
# CUDA name.
17+
export EXECUTORCH_BUILD_CUDA=1
18+
export CMAKE_ARGS="${CMAKE_ARGS} -DEXECUTORCH_BUILD_CUDA=ON"
19+
20+
# Fail the build if CUDA is not actually present. Without this the packaging step would look
21+
# for CUDA libraries that were never built and report a confusing missing-file error, several
22+
# minutes after the real problem.
23+
if [ ! -x "${CUDA_HOME:-/usr/local/cuda}/bin/nvcc" ]; then
24+
echo "EXECUTORCH_BUILD_CUDA is set but no nvcc was found. This row cannot build a CUDA wheel." >&2
25+
exit 1
26+
fi

‎.ci/scripts/wheel/test_cuda_linux.py‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
#!/usr/bin/env python
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
"""Smoke test for a CUDA wheel row.
9+
10+
Runs the same checks a CPU wheel gets, then the ones a GPU wheel needs on top. The extra
11+
checks exist because a GPU wheel can install cleanly, import cleanly, and still be unusable:
12+
13+
the CUDA libraries can be absent while the wheel is still named as a CUDA build
14+
the runtime dependency can be undeclared, so a user has nothing to resolve it from
15+
the loader path can point at the build machine's toolkit, which no user has
16+
the device code can cover no GPU the row claims, which only appears when a model runs
17+
18+
The build machines for these rows have no GPU, so this does not execute a model. It verifies
19+
everything that can be checked from the artifact, and the release gate runs a model on real
20+
hardware.
21+
"""
22+
23+
import platform
24+
import subprocess
25+
import tempfile
26+
from pathlib import Path
27+
28+
import test_base
29+
import test_cpp_sdk
30+
import test_shared_libraries
31+
from examples.models import Backend, Model
32+
33+
34+
def _package_dir() -> Path:
35+
import executorch
36+
37+
return Path(executorch.__path__[0])
38+
39+
40+
def test_cuda_libraries_are_shipped() -> None:
41+
"""The row is named for CUDA, so the CUDA libraries have to be in it."""
42+
lib_dir = _package_dir() / "lib"
43+
shipped = {path.name for path in lib_dir.iterdir()} if lib_dir.is_dir() else set()
44+
expected = {
45+
"libexecutorch_backend_cuda.so",
46+
"libexecutorch_extension_cuda.so",
47+
}
48+
missing = sorted(expected - shipped)
49+
assert not missing, (
50+
f"this is a CUDA row but {missing} are not in the wheel, so it would install as a "
51+
f"CUDA build with no CUDA delegate. Shipped: {sorted(shipped)}"
52+
)
53+
print(f"✓ the CUDA libraries ship ({len(expected)} of them)")
54+
55+
56+
def test_cuda_runtime_is_declared() -> None:
57+
"""The wheel links the CUDA runtime without bundling it, so it must declare it.
58+
59+
Without this a user installs the wheel and has nothing to resolve libcudart from, which
60+
surfaces as a loader error at the first import rather than as a resolution failure at
61+
install time.
62+
"""
63+
import importlib.metadata as metadata
64+
65+
requirements = metadata.requires("executorch") or []
66+
cuda = [
67+
requirement
68+
for requirement in requirements
69+
if "nvidia" in requirement.lower() or "cuda" in requirement.lower()
70+
]
71+
assert cuda, (
72+
"this is a CUDA row but the wheel declares no CUDA runtime dependency, so nothing "
73+
"would install the libraries its delegate links"
74+
)
75+
print(f"✓ the CUDA runtime is declared ({len(cuda)} requirements)")
76+
77+
78+
def test_cuda_libraries_resolve_relatively() -> None:
79+
"""Each CUDA library must reach its runtime through a relative path.
80+
81+
An absolute toolkit path names the machine that built the wheel. It resolves there and
82+
nowhere else, so the wheel would work only on a builder.
83+
"""
84+
readelf = test_shared_libraries._tool("readelf")
85+
assert readelf is not None, "readelf is required to inspect the wheel"
86+
87+
lib_dir = _package_dir() / "lib"
88+
names = ("libexecutorch_backend_cuda.so", "libexecutorch_extension_cuda.so")
89+
present = [name for name in names if (lib_dir / name).is_file()]
90+
# Without this the loop below finds nothing on a wheel that ships no CUDA library and
91+
# reports a pass, which is the same as having no check at all.
92+
assert present, (
93+
f"none of {list(names)} is in the wheel, so this check inspected nothing. A CUDA row "
94+
"must ship the libraries it is named for."
95+
)
96+
for name in present:
97+
library = lib_dir / name
98+
output = subprocess.run(
99+
[readelf, "-d", str(library)], capture_output=True, text=True, check=True
100+
).stdout
101+
needs_cuda = any(
102+
"NEEDED" in line and "libcud" in line for line in output.splitlines()
103+
)
104+
if not needs_cuda:
105+
print(f"- {name} does not link the CUDA runtime, nothing to resolve")
106+
continue
107+
entries: list[str] = []
108+
for line in output.splitlines():
109+
if "RPATH" in line or "RUNPATH" in line:
110+
entries = line.split("[", 1)[1].rstrip("]").strip().split(":")
111+
relative = [
112+
entry
113+
for entry in entries
114+
if entry.startswith("$ORIGIN") and "nvidia" in entry
115+
]
116+
assert relative, (
117+
f"{name} links the CUDA runtime but has no relative path to the CUDA wheels "
118+
f"installed beside it, so it can only resolve where the builder had a toolkit: "
119+
f"{entries}"
120+
)
121+
print(f"✓ {name} resolves the CUDA runtime relatively ({relative[0]})")
122+
123+
124+
if __name__ == "__main__":
125+
assert platform.system() == "Linux", "the CUDA rows are Linux only"
126+
127+
test_cuda_libraries_are_shipped()
128+
test_cuda_runtime_is_declared()
129+
test_cuda_libraries_resolve_relatively()
130+
131+
# Everything a CPU wheel is held to still applies: one owner per component, no
132+
# build-tree paths, and a C++ application able to link what the wheel ships.
133+
with tempfile.TemporaryDirectory() as work_dir:
134+
test_shared_libraries.run_tests(Path(work_dir))
135+
with tempfile.TemporaryDirectory() as work_dir:
136+
test_cpp_sdk.run_tests(Path(work_dir))
137+
138+
test_base.run_tests(
139+
model_tests=[
140+
test_base.ModelTest(
141+
model=Model.Mv3,
142+
backend=Backend.XnnpackQuantizationDelegation,
143+
),
144+
]
145+
)

‎.github/scripts/filter_cuda_matrix.py‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
#!/usr/bin/env python3
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
"""Narrow the generated build matrix to the rows a GPU wheel can honestly support.
9+
10+
The shared matrix generator emits every CUDA version and Python version it knows about.
11+
Building all of them would publish wheels for combinations nothing can verify, and a GPU
12+
wheel that installs and then cannot run is worse than one that does not exist: the failure
13+
appears when a model runs, and it looks like a model problem rather than a packaging one.
14+
15+
A row is kept only when all three of these hold:
16+
17+
a GPU exists that the row's device code covers
18+
a PyTorch build is published for that CUDA version and architecture
19+
a machine is available to run a real model before release
20+
21+
The values below are the current answers to those questions. They are written out rather
22+
than derived because each one is an external fact that can change independently.
23+
"""
24+
25+
import argparse
26+
import json
27+
import sys
28+
from typing import Any, Dict, List
29+
30+
# Python versions to skip. 3.14 is excluded because the current CPU wheel rows already fail
31+
# on it for an unrelated reason in the example requirements, so a GPU row would inherit a
32+
# known-broken build. The free-threaded builds are excluded because the CUDA dependencies
33+
# are not published for them.
34+
DISABLED_PYTHON_VERSIONS: List[str] = ["3.13t", "3.14", "3.14t", "3.15", "3.15t"]
35+
36+
# CUDA versions to publish. cu130 first because it is the generator's stable choice, and
37+
# cu126 because it is the floor a consumer pairing with an older PyTorch needs.
38+
SUPPORTED_CUDA_VERSIONS: List[str] = ["cu126", "cu130"]
39+
40+
# The single row built for a pull request. A full matrix on every push would cost hours for
41+
# little signal, and this pair is the one with a machine that can run a model on it.
42+
PR_PYTHON_VERSION: str = "3.12"
43+
PR_CUDA_VERSION: str = "cu130"
44+
45+
# Jetson devices are their own row: a JetPack image, one Python version, and one CUDA
46+
# version. They cannot take a generic aarch64 wheel, because the generic builds carry no
47+
# device code for their GPU architecture and no portable fallback either.
48+
#
49+
# Kept empty on purpose. Published PyTorch stopped shipping sm_87 device code after 2.8.0,
50+
# so a Jetson row today would produce a wheel whose PyTorch dependency cannot execute on the
51+
# device. Populate this when that changes.
52+
JETPACK_PYTHON_VERSIONS: List[str] = []
53+
JETPACK_CUDA_VERSIONS: List[str] = []
54+
JETPACK_CONTAINER_IMAGE: str = "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
55+
56+
# The generic aarch64 builder image predates the glibc that the CUDA wheels need, so the
57+
# aarch64 rows build in a newer manylinux image instead.
58+
SBSA_CONTAINER_IMAGE: str = "quay.io/pypa/manylinux_2_39_aarch64"
59+
60+
61+
def keep(item: Dict[str, Any], is_jetpack: bool) -> bool:
62+
"""Whether this row should be built, adjusting its container image where needed."""
63+
if item["python_version"] in DISABLED_PYTHON_VERSIONS:
64+
return False
65+
66+
if is_jetpack:
67+
if (
68+
item["python_version"] in JETPACK_PYTHON_VERSIONS
69+
and item["desired_cuda"] in JETPACK_CUDA_VERSIONS
70+
):
71+
item["container_image"] = JETPACK_CONTAINER_IMAGE
72+
return True
73+
return False
74+
75+
if item["desired_cuda"] not in SUPPORTED_CUDA_VERSIONS:
76+
return False
77+
78+
if item.get("gpu_arch_type") == "cuda-aarch64":
79+
item["container_image"] = SBSA_CONTAINER_IMAGE
80+
81+
return True
82+
83+
84+
def only_pull_request_row(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
85+
"""One representative row, so a pull request does not build the whole matrix."""
86+
for item in items:
87+
if (
88+
item["python_version"] == PR_PYTHON_VERSION
89+
and item["desired_cuda"] == PR_CUDA_VERSION
90+
):
91+
return [item]
92+
# Falling back to the first row rather than to nothing: an empty matrix would make the
93+
# build job vanish, which reads as a pass.
94+
return items[:1]
95+
96+
97+
def main(argv: List[str]) -> None:
98+
parser = argparse.ArgumentParser()
99+
parser.add_argument("--matrix", required=True, help="the generated matrix, as JSON")
100+
parser.add_argument(
101+
"--jetpack", default="false", help="build the Jetson row instead"
102+
)
103+
parser.add_argument("--limit-pr-builds", default="false", help="build one row only")
104+
args = parser.parse_args(argv)
105+
106+
try:
107+
matrix = json.loads(args.matrix)
108+
except json.JSONDecodeError as error:
109+
print(f"could not parse the matrix: {error}", file=sys.stderr)
110+
sys.exit(1)
111+
112+
is_jetpack = args.jetpack.lower() == "true"
113+
items = [item for item in matrix.get("include", []) if keep(item, is_jetpack)]
114+
115+
if args.limit_pr_builds.lower() == "true" and items:
116+
items = only_pull_request_row(items)
117+
118+
# Fail loudly on an empty result. A silently empty matrix produces a workflow with no
119+
# build job, which shows up as a green check for a build that never happened.
120+
if not items:
121+
print(
122+
"the filter produced no rows to build, so nothing would be verified. "
123+
f"jetpack={is_jetpack}, supported CUDA={SUPPORTED_CUDA_VERSIONS}",
124+
file=sys.stderr,
125+
)
126+
sys.exit(1)
127+
128+
print(json.dumps({"include": items}))
129+
130+
131+
if __name__ == "__main__":
132+
main(sys.argv[1:])

‎.github/workflows/build-wheels-cuda-aarch64-linux.yml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
# From https://github.com/pytorch/test-infra/wiki/Using-Nova-Reusable-Build-Workflows
2+
name: Build Aarch64 Linux CUDA Wheels
3+
4+
on:
5+
pull_request:
6+
paths:
7+
- .ci/**/*
8+
- .github/scripts/filter_cuda_matrix.py
9+
- .github/workflows/build-wheels-cuda-aarch64-linux.yml
10+
- '**/CMakeLists.txt'
11+
- backends/cuda/**/*
12+
- extension/cuda/**/*
13+
- pyproject.toml
14+
- setup.py
15+
- tools/cmake/**/*
16+
push:
17+
branches:
18+
- nightly
19+
- release/*
20+
tags:
21+
# NOTE: Binary build pipelines should only get triggered on release candidate builds
22+
# Release candidate tags look like: v1.11.0-rc1
23+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
24+
- ciflow/binaries/*
25+
workflow_dispatch:
26+
27+
jobs:
28+
generate-matrix:
29+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
30+
with:
31+
package-type: wheel
32+
os: linux-aarch64
33+
test-infra-repository: pytorch/test-infra
34+
test-infra-ref: main
35+
with-cuda: enabled
36+
with-cpu: disabled
37+
with-rocm: disabled
38+
python-versions: '["3.10", "3.11", "3.12", "3.13"]'
39+
40+
# The generator emits every CUDA version it knows about. Publishing all of them would ship
41+
# wheels for combinations nothing can verify, so this keeps only the rows with a GPU to run
42+
# them on. The script fails rather than emitting an empty matrix, because a workflow with no
43+
# build job reads as a pass.
44+
filter-matrix:
45+
needs: generate-matrix
46+
runs-on: ubuntu-latest
47+
outputs:
48+
matrix: ${{ steps.filter.outputs.matrix }}
49+
steps:
50+
- uses: actions/setup-python@v6
51+
with:
52+
python-version: '3.12'
53+
- uses: actions/checkout@v4
54+
- name: Filter the matrix
55+
id: filter
56+
run: |
57+
set -eou pipefail
58+
MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }}
59+
LIMIT_PR=${{ github.event_name == 'pull_request' && 'true' || 'false' }}
60+
MATRIX_BLOB="$(python3 .github/scripts/filter_cuda_matrix.py \
61+
--matrix "${MATRIX_BLOB}" --limit-pr-builds "${LIMIT_PR}")"
62+
echo "${MATRIX_BLOB}"
63+
echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}"
64+
65+
build:
66+
needs: filter-matrix
67+
permissions:
68+
id-token: write
69+
contents: read
70+
strategy:
71+
fail-fast: false
72+
matrix:
73+
include:
74+
- repository: pytorch/executorch
75+
pre-script: .ci/scripts/wheel/pre_build_script.sh
76+
post-script: .ci/scripts/wheel/post_build_script.sh
77+
smoke-test-script: .ci/scripts/wheel/test_cuda_linux.py
78+
package-name: executorch
79+
name: ${{ matrix.repository }}
80+
uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
81+
with:
82+
repository: ${{ matrix.repository }}
83+
ref: ""
84+
test-infra-repository: pytorch/test-infra
85+
test-infra-ref: main
86+
build-matrix: ${{ needs.filter-matrix.outputs.matrix }}
87+
submodules: recursive
88+
env-var-script: .ci/scripts/wheel/envvar_cuda_linux.sh
89+
pre-script: ${{ matrix.pre-script }}
90+
post-script: ${{ matrix.post-script }}
91+
package-name: ${{ matrix.package-name }}
92+
smoke-test-script: ${{ matrix.smoke-test-script }}
93+
trigger-event: ${{ github.event_name }}

0 commit comments

Comments
 (0)