Skip to content

Commit f0086f9

Browse files
lanluo-nvidialanluo-nvidia
authored
Automate release artifacts build: usage pytorch cxx11 builder base image (#2988)
1 parent 68d8c82 commit f0086f9

File tree

5 files changed

+755
-0
lines changed

5 files changed

+755
-0
lines changed

‎.github/scripts/generate-release-matrix.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
import json
5+
import sys
6+
7+
RELEASE_CUDA_VERSION = {
8+
"wheel": ["cu124"],
9+
"tarball": ["cu124"],
10+
}
11+
RELEASE_PYTHON_VERSION = {
12+
"wheel": ["3.8", "3.9", "3.10", "3.11", "3.12"],
13+
"tarball": ["3.10"],
14+
}
15+
16+
CXX11_TARBALL_CONTAINER_IMAGE = {
17+
"cu124": "pytorch/libtorch-cxx11-builder:cuda12.4-main",
18+
}
19+
20+
21+
def main(args: list[str]) -> None:
22+
parser = argparse.ArgumentParser()
23+
parser.add_argument(
24+
"--wheel_matrix",
25+
help="wheel matrix",
26+
type=str,
27+
default="",
28+
)
29+
parser.add_argument(
30+
"--tarball_matrix",
31+
help="tarball matrix",
32+
type=str,
33+
default="",
34+
)
35+
options = parser.parse_args(args)
36+
cuda_versions = []
37+
python_versions = []
38+
39+
if options.tarball_matrix != "":
40+
cuda_versions = RELEASE_CUDA_VERSION["tarball"]
41+
python_versions = RELEASE_PYTHON_VERSION["tarball"]
42+
matrix_dict = json.loads(options.tarball_matrix)
43+
elif options.wheel_matrix != "":
44+
cuda_versions = RELEASE_CUDA_VERSION["wheel"]
45+
python_versions = RELEASE_PYTHON_VERSION["wheel"]
46+
matrix_dict = json.loads(options.wheel_matrix)
47+
else:
48+
raise Exception(
49+
"Either --wheel_matrix or --tarball_matrix needs to be provided"
50+
)
51+
52+
includes = matrix_dict["include"]
53+
filtered_includes = []
54+
for item in includes:
55+
if (
56+
item["desired_cuda"] in cuda_versions
57+
and item["python_version"] in python_versions
58+
):
59+
if options.tarball_matrix != "":
60+
item["container_image"] = CXX11_TARBALL_CONTAINER_IMAGE[
61+
item["desired_cuda"]
62+
]
63+
filtered_includes.append(item)
64+
filtered_matrix_dict = {}
65+
filtered_matrix_dict["include"] = filtered_includes
66+
print(json.dumps(filtered_matrix_dict))
67+
68+
69+
if __name__ == "__main__":
70+
main(sys.argv[1:])

‎.github/workflows/release-linux.yml‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: Release Linux wheels and tarball artifacts
2+
3+
on:
4+
push:
5+
tags:
6+
# NOTE: Binary build pipelines should only get triggered on release candidate builds
7+
# Release candidate tags look like: v1.11.0-rc1
8+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
9+
workflow_dispatch:
10+
11+
permissions:
12+
id-token: write
13+
contents: read
14+
packages: write
15+
16+
jobs:
17+
generate-matrix:
18+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
19+
if: ${{ contains(github.event.pull_request.labels.*.name, 'build-release-artifacts') || startsWith(github.event.ref, 'refs/tags/v') }}
20+
with:
21+
package-type: wheel
22+
os: linux
23+
test-infra-repository: pytorch/test-infra
24+
test-infra-ref: main
25+
with-rocm: false
26+
with-cpu: false
27+
28+
generate-release-tarball-matrix:
29+
needs: [generate-matrix]
30+
outputs:
31+
matrix: ${{ steps.generate.outputs.matrix }}
32+
runs-on: ubuntu-latest
33+
steps:
34+
- uses: actions/setup-python@v4
35+
with:
36+
python-version: '3.10'
37+
- uses: actions/checkout@v3
38+
with:
39+
repository: pytorch/tensorrt
40+
- name: Generate release matrix
41+
id: generate
42+
run: |
43+
set -eou pipefail
44+
MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }}
45+
MATRIX_BLOB="$(python3 .github/scripts/generate-release-matrix.py --tarball_matrix "${MATRIX_BLOB}")"
46+
echo "${MATRIX_BLOB}"
47+
echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}"
48+
49+
release-cxx11-tarball-artifacts:
50+
needs: [generate-release-tarball-matrix]
51+
name: Release torch-tensorrt cxx11 tarball artifacts
52+
strategy:
53+
fail-fast: false
54+
matrix:
55+
include:
56+
- repository: pytorch/tensorrt
57+
package-name: torch_tensorrt
58+
pre-script: packaging/pre_build_script.sh
59+
env-var-script: packaging/env_vars.txt
60+
post-script: packaging/post_build_script.sh
61+
smoke-test-script: packaging/smoke_test_script.sh
62+
cxx11-tarball-release: "true"
63+
uses: ./.github/workflows/release-wheel-linux.yml
64+
with:
65+
repository: ${{ matrix.repository }}
66+
ref: ""
67+
test-infra-repository: pytorch/test-infra
68+
test-infra-ref: main
69+
build-matrix: ${{ needs.generate-release-tarball-matrix.outputs.matrix }}
70+
pre-script: ${{ matrix.pre-script }}
71+
env-var-script: ${{ matrix.env-var-script }}
72+
post-script: ${{ matrix.post-script }}
73+
package-name: ${{ matrix.package-name }}
74+
smoke-test-script: ${{ matrix.smoke-test-script }}
75+
trigger-event: ${{ github.event_name }}
76+
cxx11-tarball-release: "true"
77+
78+
generate-release-wheel-matrix:
79+
needs: [generate-matrix]
80+
outputs:
81+
matrix: ${{ steps.generate.outputs.matrix }}
82+
runs-on: ubuntu-latest
83+
steps:
84+
- uses: actions/setup-python@v4
85+
with:
86+
python-version: '3.10'
87+
- uses: actions/checkout@v3
88+
with:
89+
repository: pytorch/tensorrt
90+
- name: Generate release matrix
91+
id: generate
92+
run: |
93+
set -eou pipefail
94+
MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }}
95+
MATRIX_BLOB="$(python3 .github/scripts/generate-release-matrix.py --wheel_matrix "${MATRIX_BLOB}")"
96+
echo "${MATRIX_BLOB}"
97+
echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}"
98+
99+
release-other-artifacts:
100+
name: Release torch-tensorrt wheel and pre-cxx11 tarball artifacts
101+
needs: [generate-release-wheel-matrix]
102+
strategy:
103+
fail-fast: false
104+
matrix:
105+
include:
106+
- repository: pytorch/tensorrt
107+
package-name: torch_tensorrt
108+
pre-script: packaging/pre_build_script.sh
109+
env-var-script: packaging/env_vars.txt
110+
post-script: packaging/post_build_script.sh
111+
smoke-test-script: packaging/smoke_test_script.sh
112+
cxx11-tarball-release: "true"
113+
uses: ./.github/workflows/release-wheel-linux.yml
114+
with:
115+
repository: ${{ matrix.repository }}
116+
ref: ""
117+
test-infra-repository: pytorch/test-infra
118+
test-infra-ref: main
119+
build-matrix: ${{ needs.generate-release-wheel-matrix.outputs.matrix }}
120+
pre-script: ${{ matrix.pre-script }}
121+
env-var-script: ${{ matrix.env-var-script }}
122+
post-script: ${{ matrix.post-script }}
123+
package-name: ${{ matrix.package-name }}
124+
smoke-test-script: ${{ matrix.smoke-test-script }}
125+
trigger-event: ${{ github.event_name }}
126+
cxx11-tarball-release: "false"
127+
128+
concurrency:
129+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }}
130+
cancel-in-progress: true

0 commit comments

Comments
 (0)