Skip to content

Commit 62fb310

Browse files
elpranselprans
committed
ci: Switch to staged release process
Uploading release artifacts directly to PyPI from the build matrix is bad, as this may result in an incomplete or broken release if any of the platforms fail to build. To fix the process, implement a "staged" build process whereby artifacts are built from a dedicated "releases" branch and are uploaded to an S3 bucket. Once all artifacts have been successfully built, the release tag may be pushed to master, and the previously built artifacts will be uploaded to PyPI as a single transaction.
1 parent 04fa0cc commit 62fb310

13 files changed

+408
-129
lines changed

‎.ci/build-manylinux-wheels.sh‎

Lines changed: 14 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,24 @@
22

33
set -e -x
44

5-
yum update -y
6-
yum install -y libtool autoconf automake
7-
8-
PYTHON_VERSIONS="cp35-cp35m"
9-
105
# Compile wheels
11-
for PYTHON_VERSION in ${PYTHON_VERSIONS}; do
12-
PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
13-
PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
14-
${PIP} install --upgrade pip wheel
15-
${PIP} install --upgrade setuptools
16-
${PIP} install -r /io/.ci/requirements.txt
17-
make -C /io/ PYTHON="${PYTHON}" distclean
18-
make -C /io/ PYTHON="${PYTHON}"
19-
${PIP} wheel /io/ -w /io/dist/
20-
done
6+
PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
7+
PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
8+
${PIP} install --upgrade pip wheel
9+
${PIP} install --upgrade setuptools
10+
${PIP} install -r /io/.ci/requirements.txt
11+
make -C /io/ PYTHON="${PYTHON}"
12+
${PIP} wheel /io/ -w /io/dist/
2113

22-
#Bundle external shared libraries into the wheels.
14+
# Bundle external shared libraries into the wheels.
2315
for whl in /io/dist/*.whl; do
2416
auditwheel repair $whl -w /io/dist/
2517
rm /io/dist/*-linux_*.whl
2618
done
2719

28-
for PYTHON_VERSION in ${PYTHON_VERSIONS}; do
29-
PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
30-
PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
31-
${PIP} install ${PYMODULE} --no-index -f file:///io/dist
32-
rm -rf /io/tests/__pycache__
33-
make -C /io/ PYTHON="${PYTHON}" test
34-
rm -rf /io/tests/__pycache__
35-
done
20+
PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
21+
PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
22+
${PIP} install ${PYMODULE} --no-index -f file:///io/dist
23+
rm -rf /io/tests/__pycache__
24+
make -C /io/ PYTHON="${PYTHON}" test
25+
rm -rf /io/tests/__pycache__

‎.ci/package-version.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env python3
2+
3+
4+
import os.path
5+
import sys
6+
7+
8+
def main():
9+
setup_py = os.path.join(os.path.dirname(os.path.dirname(__file__)),
10+
'setup.py')
11+
12+
with open(setup_py, 'r') as f:
13+
for line in f:
14+
if line.startswith('VERSION ='):
15+
_, _, version = line.partition('=')
16+
print(version.strip(" \n'\""))
17+
return 0
18+
19+
print('could not find package version in setup.py', file=sys.stderr)
20+
return 1
21+
22+
23+
if __name__ == '__main__':
24+
sys.exit(main())

‎.ci/pypi-check.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env python3
2+
3+
4+
import argparse
5+
import sys
6+
import xmlrpc.client
7+
8+
9+
def main():
10+
parser = argparse.ArgumentParser(description='PyPI package checker')
11+
parser.add_argument('package_name', metavar='PACKAGE-NAME')
12+
13+
parser.add_argument(
14+
'--pypi-index-url',
15+
help=('PyPI index URL.'),
16+
default='https://pypi.python.org/pypi')
17+
18+
args = parser.parse_args()
19+
20+
pypi = xmlrpc.client.ServerProxy(args.pypi_index_url)
21+
releases = pypi.package_releases(args.package_name)
22+
23+
if releases:
24+
print(next(iter(sorted(releases, reverse=True))))
25+
26+
return 0
27+
28+
29+
if __name__ == '__main__':
30+
sys.exit(main())

‎.ci/requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
cython==0.24.1
22
aiohttp
3+
tinys3
34
twine

‎.ci/s3-download-release.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#!/usr/bin/env python3
2+
3+
4+
import argparse
5+
import os
6+
import os.path
7+
import sys
8+
import urllib.request
9+
10+
import tinys3
11+
12+
13+
def main():
14+
parser = argparse.ArgumentParser(description='S3 File Uploader')
15+
parser.add_argument(
16+
'--s3-bucket',
17+
help=('S3 bucket name (defaults to $S3_UPLOAD_BUCKET)'),
18+
default=os.environ.get('S3_UPLOAD_BUCKET'))
19+
parser.add_argument(
20+
'--s3-region',
21+
help=('S3 region (defaults to $S3_UPLOAD_REGION)'),
22+
default=os.environ.get('S3_UPLOAD_REGION'))
23+
parser.add_argument(
24+
'--s3-username',
25+
help=('S3 username (defaults to $S3_UPLOAD_USERNAME)'),
26+
default=os.environ.get('S3_UPLOAD_USERNAME'))
27+
parser.add_argument(
28+
'--s3-key',
29+
help=('S3 access key (defaults to $S3_UPLOAD_ACCESSKEY)'),
30+
default=os.environ.get('S3_UPLOAD_ACCESSKEY'))
31+
parser.add_argument(
32+
'--s3-secret',
33+
help=('S3 secret (defaults to $S3_UPLOAD_SECRET)'),
34+
default=os.environ.get('S3_UPLOAD_SECRET'))
35+
parser.add_argument(
36+
'--destdir',
37+
help='Destination directory.')
38+
parser.add_argument(
39+
'package', metavar='PACKAGE',
40+
help='Package name and version to download.')
41+
42+
args = parser.parse_args()
43+
44+
if args.s3_region:
45+
endpoint = 's3-{}.amazonaws.com'.format(args.s3_region.lower())
46+
else:
47+
endpoint = 's3.amazonaws.com'
48+
49+
conn = tinys3.Connection(
50+
access_key=args.s3_key,
51+
secret_key=args.s3_secret,
52+
default_bucket=args.s3_bucket,
53+
tls=True,
54+
endpoint=endpoint,
55+
)
56+
57+
files = []
58+
59+
for entry in conn.list(args.package):
60+
files.append(entry['key'])
61+
62+
destdir = args.destdir or os.getpwd()
63+
64+
for file in files:
65+
print('Downloading {}...'.format(file))
66+
url = 'https://{}/{}/{}'.format(endpoint, args.s3_bucket, file)
67+
target = os.path.join(destdir, file)
68+
urllib.request.urlretrieve(url, target)
69+
70+
return 0
71+
72+
73+
if __name__ == '__main__':
74+
sys.exit(main())

‎.ci/s3-upload.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env python3
2+
3+
4+
import argparse
5+
import glob
6+
import os
7+
import os.path
8+
import sys
9+
10+
import tinys3
11+
12+
13+
def main():
14+
parser = argparse.ArgumentParser(description='S3 File Uploader')
15+
parser.add_argument(
16+
'--s3-bucket',
17+
help=('S3 bucket name (defaults to $S3_UPLOAD_BUCKET)'),
18+
default=os.environ.get('S3_UPLOAD_BUCKET'))
19+
parser.add_argument(
20+
'--s3-region',
21+
help=('S3 region (defaults to $S3_UPLOAD_REGION)'),
22+
default=os.environ.get('S3_UPLOAD_REGION'))
23+
parser.add_argument(
24+
'--s3-username',
25+
help=('S3 username (defaults to $S3_UPLOAD_USERNAME)'),
26+
default=os.environ.get('S3_UPLOAD_USERNAME'))
27+
parser.add_argument(
28+
'--s3-key',
29+
help=('S3 access key (defaults to $S3_UPLOAD_ACCESSKEY)'),
30+
default=os.environ.get('S3_UPLOAD_ACCESSKEY'))
31+
parser.add_argument(
32+
'--s3-secret',
33+
help=('S3 secret (defaults to $S3_UPLOAD_SECRET)'),
34+
default=os.environ.get('S3_UPLOAD_SECRET'))
35+
parser.add_argument(
36+
'files', nargs='+', metavar='FILE', help='Files to upload')
37+
38+
args = parser.parse_args()
39+
40+
if args.s3_region:
41+
endpoint = 's3-{}.amazonaws.com'.format(args.s3_region.lower())
42+
else:
43+
endpoint = 's3.amazonaws.com'
44+
45+
conn = tinys3.Connection(
46+
access_key=args.s3_key,
47+
secret_key=args.s3_secret,
48+
default_bucket=args.s3_bucket,
49+
tls=True,
50+
endpoint=endpoint,
51+
)
52+
53+
for pattern in args.files:
54+
for fn in glob.iglob(pattern):
55+
with open(fn, 'rb') as f:
56+
conn.upload(os.path.basename(fn), f)
57+
58+
return 0
59+
60+
61+
if __name__ == '__main__':
62+
sys.exit(main())

‎.ci/travis-build-and-upload.sh‎

Lines changed: 0 additions & 71 deletions
This file was deleted.

‎.ci/travis-build-wheels.sh‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
#!/bin/bash
2+
3+
set -e -x
4+
5+
6+
if [[ "${TRAVIS_BRANCH}" != "releases" || "${BUILD}" != *wheels* ]]; then
7+
# Not a release
8+
exit 0
9+
fi
10+
11+
12+
if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
13+
PYENV_ROOT="$HOME/.pyenv"
14+
PATH="$PYENV_ROOT/bin:$PATH"
15+
eval "$(pyenv init -)"
16+
pyenv local ${PYTHON_VERSION}
17+
fi
18+
19+
PACKAGE_VERSION=$(python ".ci/package-version.py")
20+
PYPI_VERSION=$(python ".ci/pypi-check.py" "${PYMODULE}")
21+
22+
if [ "${PACKAGE_VERSION}" == "${PYPI_VERSION}" ]; then
23+
echo "${PYMODULE}-${PACKAGE_VERSION} is already published on PyPI"
24+
exit 1
25+
fi
26+
27+
28+
pushd $(dirname $0) > /dev/null
29+
_root=$(dirname $(pwd -P))
30+
popd > /dev/null
31+
32+
33+
_upload_wheels() {
34+
python "${_root}/.ci/s3-upload.py" "${_root}/dist"/*.whl
35+
sudo rm -rf "${_root}/dist"/*.whl
36+
}
37+
38+
39+
if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
40+
ML_PYTHON_VERSION=$(python3 -c "import sys; \
41+
print('cp{maj}{min}-cp{maj}{min}m'.format( \
42+
maj=sys.version_info.major, min=sys.version_info.minor))")
43+
44+
PYTHON_VERSION=$(python3 -c "import sys; \
45+
print('{maj}.{min}'.format( \
46+
maj=sys.version_info.major, min=sys.version_info.minor))")
47+
48+
if [[ "${RELEASE_PYTHON_VERSIONS}" != *"${PYTHON_VERSION}"* ]]; then
49+
echo "Skipping release on Python ${PYTHON_VERSION}."
50+
exit 0
51+
fi
52+
53+
for arch in x86_64 i686; do
54+
ML_IMAGE="quay.io/pypa/manylinux1_${arch}"
55+
docker pull "${ML_IMAGE}"
56+
docker run --rm \
57+
-v "${_root}":/io \
58+
-e "PYMODULE=${PYMODULE}" \
59+
-e "PYTHON_VERSION=${ML_PYTHON_VERSION}" \
60+
"${ML_IMAGE}" /io/.ci/build-manylinux-wheels.sh
61+
62+
_upload_wheels
63+
done
64+
65+
elif [ "${TRAVIS_OS_NAME}" == "osx" ]; then
66+
make -C "${_root}"
67+
pip wheel "${_root}" -w "${_root}/dist/"
68+
69+
pip install ${PYMODULE} --no-index -f "file:///${_root}/dist"
70+
pushd / >/dev/null
71+
make -C "${_root}" test
72+
popd >/dev/null
73+
74+
_upload_wheels
75+
76+
else
77+
echo "Cannot build on ${TRAVIS_OS_NAME}."
78+
fi

0 commit comments

Comments
 (0)