Skip to content

Commit ef3ba78

Browse files
ezyangezyang
authored
0.4.0 packaging (#1212)
* Rewrite torchvision packaging (#1209) Following a similar line of inquiry to pytorch/audio#217 * Packaging fixes (#1214) Add uploading support, make CUDA builds actually work. * 0.4.0 parameters Signed-off-by: Edward Z. Yang <[email protected]> * Actually upload wheels (please port to master) Signed-off-by: Edward Z. Yang <[email protected]> * Put macos binaries in the right place Signed-off-by: Edward Z. Yang <[email protected]> * Propagate more environment variables. Signed-off-by: Edward Z. Yang <[email protected]> * Change the version number Signed-off-by: Edward Z. Yang <[email protected]> * Go time Signed-off-by: Edward Z. Yang <[email protected]>
1 parent d31eafa commit ef3ba78

File tree

16 files changed

+1064
-36
lines changed

16 files changed

+1064
-36
lines changed

‎.circleci/config.yml‎

Lines changed: 530 additions & 0 deletions
Large diffs are not rendered by default.

‎.circleci/config.yml.in‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
version: 2.1
2+
3+
# How to test the Linux jobs:
4+
# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
5+
# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
6+
# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
7+
# Job names are 'name:' key.
8+
9+
binary_common: &binary_common
10+
parameters:
11+
# Edit these defaults to do a release`
12+
build_version:
13+
description: "version number of release binary; by default, build a nightly"
14+
type: string
15+
default: "0.4.0"
16+
pytorch_version:
17+
description: "PyTorch version to build against; by default, use a nightly"
18+
type: string
19+
default: "1.2.0"
20+
# Don't edit these
21+
python_version:
22+
description: "Python version to build against (e.g., 3.7)"
23+
type: string
24+
cu_version:
25+
description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
26+
type: string
27+
unicode_abi:
28+
description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
29+
type: string
30+
default: ""
31+
wheel_docker_image:
32+
description: "Wheel only: what docker image to use"
33+
type: string
34+
default: "soumith/manylinux-cuda100"
35+
environment:
36+
PYTHON_VERSION: << parameters.python_version >>
37+
BUILD_VERSION: << parameters.build_version >>
38+
PYTORCH_VERSION: << parameters.pytorch_version >>
39+
UNICODE_ABI: << parameters.unicode_abi >>
40+
CU_VERSION: << parameters.cu_version >>
41+
42+
jobs:
43+
circleci_consistency:
44+
docker:
45+
- image: circleci/python:3.7
46+
steps:
47+
- checkout
48+
- run:
49+
command: |
50+
pip install --user --progress-bar off jinja2
51+
python .circleci/regenerate.py
52+
git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
53+
54+
binary_linux_wheel:
55+
<<: *binary_common
56+
docker:
57+
- image: << parameters.wheel_docker_image >>
58+
resource_class: 2xlarge+
59+
steps:
60+
- checkout
61+
- run: packaging/build_wheel.sh
62+
- store_artifacts:
63+
path: dist
64+
- persist_to_workspace:
65+
root: dist
66+
paths:
67+
- "*"
68+
69+
binary_linux_conda:
70+
<<: *binary_common
71+
docker:
72+
- image: "soumith/conda-cuda"
73+
resource_class: 2xlarge+
74+
steps:
75+
- checkout
76+
- run: packaging/build_conda.sh
77+
- store_artifacts:
78+
path: /opt/conda/conda-bld/linux-64
79+
- persist_to_workspace:
80+
root: /opt/conda/conda-bld/linux-64
81+
paths:
82+
- "*"
83+
84+
binary_macos_wheel:
85+
<<: *binary_common
86+
macos:
87+
xcode: "9.0"
88+
steps:
89+
- checkout
90+
- run:
91+
# Cannot easily deduplicate this as source'ing activate
92+
# will set environment variables which we need to propagate
93+
# to build_wheel.sh
94+
command: |
95+
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
96+
sh conda.sh -b
97+
source $HOME/miniconda3/bin/activate
98+
packaging/build_wheel.sh
99+
- store_artifacts:
100+
path: dist
101+
- persist_to_workspace:
102+
root: dist
103+
paths:
104+
- "*"
105+
106+
binary_macos_conda:
107+
<<: *binary_common
108+
macos:
109+
xcode: "9.0"
110+
steps:
111+
- checkout
112+
- run:
113+
command: |
114+
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
115+
sh conda.sh -b
116+
source $HOME/miniconda3/bin/activate
117+
conda install -yq conda-build
118+
packaging/build_conda.sh
119+
- store_artifacts:
120+
path: /Users/distiller/miniconda3/conda-bld/osx-64
121+
- persist_to_workspace:
122+
root: /Users/distiller/miniconda3/conda-bld/osx-64
123+
paths:
124+
- "*"
125+
126+
# Requires org-member context
127+
binary_conda_upload:
128+
docker:
129+
- image: continuumio/miniconda
130+
steps:
131+
- attach_workspace:
132+
at: ~/workspace
133+
- run:
134+
command: |
135+
# Prevent credential from leaking
136+
conda install -yq anaconda-client
137+
set +x
138+
anaconda login \
139+
--username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \
140+
--password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD"
141+
set -x
142+
anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force
143+
144+
# Requires org-member context
145+
binary_wheel_upload:
146+
parameters:
147+
subfolder:
148+
description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
149+
type: string
150+
docker:
151+
- image: circleci/python:3.7
152+
steps:
153+
- attach_workspace:
154+
at: ~/workspace
155+
- checkout
156+
- run:
157+
command: |
158+
pip install --user awscli
159+
export PATH="$HOME/.local/bin:$PATH"
160+
# Prevent credential from leaking
161+
set +x
162+
export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
163+
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
164+
set -x
165+
for pkg in ~/workspace/*.whl; do
166+
aws s3 cp "$pkg" "s3://pytorch/whl/nightly/<< parameters.subfolder >>" --acl public-read
167+
done
168+
169+
{%- macro workflow(btype, os, python_version, cu_version, unicode, prefix='', upload=False) %}
170+
- binary_{{os}}_{{btype}}:
171+
name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}}
172+
python_version: "{{python_version}}"
173+
cu_version: "{{cu_version}}"
174+
{%- if unicode %}
175+
unicode_abi: "1"
176+
{%- endif %}
177+
{%- if cu_version == "cu92" %}
178+
wheel_docker_image: "soumith/manylinux-cuda92"
179+
{%- endif %}
180+
181+
{%- if upload %}
182+
- binary_{{btype}}_upload:
183+
name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}}_upload
184+
context: org-member
185+
requires:
186+
- {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}}
187+
{%- if btype == 'wheel' %}
188+
{%- if os == 'macos' %}
189+
subfolder: ""
190+
{%- else %}
191+
subfolder: "{{cu_version}}/"
192+
{%- endif %}
193+
{%- endif %}
194+
{%- endif %}
195+
{%- endmacro %}
196+
197+
{%- macro workflows(prefix='', upload=False) %}
198+
{%- for btype in ["wheel", "conda"] -%}
199+
{%- for os in ["linux", "macos"] -%}
200+
{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] -%}
201+
{%- for cu_version in (["cpu", "cu92", "cu100"] if os == "linux" else ["cpu"]) -%}
202+
{%- for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]) -%}
203+
{{ workflow(btype, os, python_version, cu_version, unicode, prefix=prefix, upload=upload) }}
204+
{%- endfor -%}
205+
{%- endfor -%}
206+
{%- endfor -%}
207+
{%- endfor -%}
208+
{%- endfor %}
209+
{%- endmacro %}
210+
211+
workflows:
212+
build:
213+
{%- if False %}
214+
jobs:
215+
- circleci_consistency
216+
{{ workflows() }}
217+
218+
nightly:
219+
triggers:
220+
- schedule:
221+
cron: "0 9 * * *"
222+
filters:
223+
branches:
224+
only:
225+
- master
226+
{%- endif %}
227+
jobs:
228+
- circleci_consistency
229+
{{ workflows(prefix="nightly_", upload=True) }}

‎.circleci/regenerate.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/usr/bin/env python
2+
3+
import jinja2
4+
import os.path
5+
6+
d = os.path.dirname(__file__)
7+
env = jinja2.Environment(
8+
loader=jinja2.FileSystemLoader(d),
9+
lstrip_blocks=True,
10+
autoescape=False,
11+
)
12+
with open(os.path.join(d, 'config.yml'), 'w') as f:
13+
f.write(env.get_template('config.yml.in').render())

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,4 @@ htmlcov
1919
*/*.dylib*
2020
*.swp
2121
*.swo
22+
gen.yml

‎packaging/build_conda.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
set -ex
3+
4+
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
5+
. "$script_dir/pkg_helpers.bash"
6+
7+
setup_env 0.4.0
8+
export SOURCE_ROOT_DIR="$PWD"
9+
setup_conda_pytorch_constraint
10+
setup_conda_cudatoolkit_constraint
11+
conda build $CONDA_CHANNEL_FLAGS -c defaults -c conda-forge --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision

‎packaging/build_wheel.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
set -ex
3+
4+
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
5+
. "$script_dir/pkg_helpers.bash"
6+
7+
setup_env 0.4.0
8+
setup_wheel_python
9+
pip_install numpy pyyaml future ninja
10+
setup_pip_pytorch_version
11+
python setup.py clean
12+
IS_WHEEL=1 python setup.py bdist_wheel

0 commit comments

Comments
 (0)