Skip to content

Commit 4726a4a

Browse files
oestebanoesteban
committed
ci(CircleCI): Set up Continuous Delivery with Circle
Ref #3.
1 parent a9e85c8 commit 4726a4a

File tree

3 files changed

+409
-0
lines changed

3 files changed

+409
-0
lines changed

‎.circleci/config.yml

Lines changed: 370 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,370 @@
1+
version: 2
2+
jobs:
3+
4+
build:
5+
environment:
6+
TZ: "/usr/share/zoneinfo/America/Los_Angeles"
7+
SCRATCH: "/scratch"
8+
docker:
9+
- image: docker:18.01.0-ce-git
10+
working_directory: /tmp/src/dmriprep
11+
steps:
12+
- checkout
13+
- run:
14+
name: Check whether build should be skipped
15+
command: |
16+
if [[ "$( git log --format='format:%s' -n 1 $CIRCLE_SHA1 | grep -i -E '^docs?(\(\w+\))?:' )" != "" ]]; then
17+
echo "Only docs build"
18+
circleci step halt
19+
fi
20+
21+
- run:
22+
name: Install parallel gzip and python3
23+
command: |
24+
apk add --no-cache pigz python3
25+
- restore_cache:
26+
keys:
27+
- docker-v0-{{ .Branch }}-{{ .Revision }}
28+
- docker-v0-{{ .Branch }}-
29+
- docker-v0-master-
30+
- docker-v0-
31+
paths:
32+
- /tmp/cache/docker.tar.gz
33+
- setup_remote_docker
34+
- run:
35+
name: Load Docker image layer cache
36+
no_output_timeout: 30m
37+
command: |
38+
docker info
39+
set +o pipefail
40+
if [ -f /tmp/cache/docker.tar.gz ]; then
41+
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
42+
docker images
43+
fi
44+
- run:
45+
name: Build Docker image
46+
no_output_timeout: 60m
47+
command: |
48+
# Get version, update files.
49+
THISVERSION=$( python3 get_version.py )
50+
if [[ ${THISVERSION:0:1} == "0" ]] ; then
51+
echo "WARNING: latest git tag could not be found"
52+
echo "Please, make sure you fetch all tags from upstream with"
53+
echo "the command ``git fetch --tags --verbose`` and push"
54+
echo "them to your fork with ``git push origin --tags``"
55+
fi
56+
# Build docker image
57+
e=1 && for i in {1..5}; do
58+
docker build \
59+
--cache-from=nipreps/dmriprep \
60+
--rm=false \
61+
-t nipreps/dmriprep:latest \
62+
--build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
63+
--build-arg VCS_REF=`git rev-parse --short HEAD` \
64+
--build-arg VERSION="${CIRCLE_TAG:-$THISVERSION}" . \
65+
&& e=0 && break || sleep 15
66+
done && [ "$e" -eq "0" ]
67+
# test if development image should be built
68+
if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[build[ _]?devel\]' )" != "" ]]; then
69+
echo "building development container"
70+
docker tag nipreps/dmriprep nipreps/dmriprep:unstable
71+
docker build \
72+
--rm=false \
73+
-t nipreps/dmriprep_devel \
74+
-f Dockerfile_devel .
75+
fi
76+
- run:
77+
name: Smoke test Docker image
78+
command: |
79+
THISVERSION=$( python3 get_version.py )
80+
THISVERSION=${THISVERSION%.dirty*}
81+
THISVERSION=${CIRCLE_TAG:-$THISVERSION}
82+
DOCKERVERSION=$(docker run --rm -it nipreps/dmriprep:latest --version)
83+
DOCKERVERSION=${DOCKERVERSION%$'\r'}
84+
DOCKERVERSION=${DOCKERVERSION#*"dmriprep v"}
85+
echo "VERSION: \"$THISVERSION\""
86+
echo "DOCKERVERSION: \"${DOCKERVERSION}\""
87+
test "$DOCKERVERSION" = "$THISVERSION"
88+
- run:
89+
name: Docker save
90+
no_output_timeout: 40m
91+
command: |
92+
mkdir -p /tmp/cache
93+
docker save ubuntu:xenial-20161213 nipreps/dmriprep:latest \
94+
| pigz -3 > /tmp/cache/docker.tar.gz
95+
96+
- persist_to_workspace:
97+
root: /tmp
98+
paths:
99+
- src/dmriprep
100+
- save_cache:
101+
key: docker-v0-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
102+
paths:
103+
- /tmp/cache/docker.tar.gz
104+
105+
get_data:
106+
machine:
107+
# Ubuntu 14.04 with Docker 17.10.0-ce
108+
image: circleci/classic:201711-01
109+
working_directory: /home/circleci/data
110+
steps:
111+
- restore_cache:
112+
keys:
113+
- data-v0-{{ .Revision }}
114+
- data-v0-
115+
- run:
116+
name: Store FreeSurfer license file
117+
command: |
118+
mkdir -p /tmp/fslicense
119+
cd /tmp/fslicense
120+
echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IGxpY2Vuc2UudHh0Cg==" | base64 -d | sh
121+
- run:
122+
name: Create Nipype config files
123+
command: |
124+
mkdir -p /tmp/config
125+
printf "[execution]\nstop_on_first_crash = true\n" > /tmp/config/nipype.cfg
126+
echo "poll_sleep_duration = 0.01" >> /tmp/config/nipype.cfg
127+
echo "hash_method = content" >> /tmp/config/nipype.cfg
128+
- persist_to_workspace:
129+
root: /tmp
130+
paths:
131+
- fslicense
132+
- config/nipype.cfg
133+
- save_cache:
134+
key: data-v0-{{ .Revision }}-{{ epoch }}
135+
paths:
136+
- /tmp/data
137+
- /tmp/ds005/derivatives/freesurfer
138+
139+
deploy_docker_patches:
140+
machine:
141+
image: circleci/classic:201711-01
142+
working_directory: /tmp/src/dmriprep
143+
steps:
144+
145+
- restore_cache:
146+
keys:
147+
- docker-v0-{{ .Branch }}-{{ .Revision }}
148+
- run:
149+
name: Load Docker image layer cache
150+
no_output_timeout: 30m
151+
command: |
152+
docker info
153+
set +o pipefail
154+
if [ -f /tmp/cache/docker.tar.gz ]; then
155+
sudo apt update && sudo apt -y install pigz
156+
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
157+
docker images
158+
fi
159+
- run:
160+
name: Deploy to Docker Hub
161+
no_output_timeout: 40m
162+
command: |
163+
if [[ -n "$DOCKER_PASS" ]]; then
164+
docker login -u $DOCKER_USER -p $DOCKER_PASS
165+
docker tag nipreps/dmriprep nipreps/dmriprep:${CIRCLE_BRANCH#docker/}
166+
docker push nipreps/dmriprep:${CIRCLE_BRANCH#docker/}
167+
fi
168+
169+
deploy_docker:
170+
machine:
171+
image: circleci/classic:201711-01
172+
working_directory: /tmp/src/dmriprep
173+
steps:
174+
- checkout:
175+
path: /home/circleci/src/dmriprep
176+
- run:
177+
name: Check whether build should be skipped
178+
command: |
179+
cd /home/circleci/src/dmriprep
180+
if [[ "$( git log --format='format:%s' -n 1 $CIRCLE_SHA1 | grep -i -E '^docs?(\(\w+\))?:' )" != "" ]]; then
181+
echo "Only docs build"
182+
circleci step halt
183+
fi
184+
- restore_cache:
185+
keys:
186+
- docker-v0-{{ .Branch }}-{{ .Revision }}
187+
- run:
188+
name: Load Docker image layer cache
189+
no_output_timeout: 30m
190+
command: |
191+
docker info
192+
set +o pipefail
193+
if [ -f /tmp/cache/docker.tar.gz ]; then
194+
sudo apt update && sudo apt -y install pigz
195+
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
196+
docker images
197+
fi
198+
- run:
199+
name: Deploy to Docker Hub
200+
no_output_timeout: 40m
201+
command: |
202+
if [[ -n "$DOCKER_PASS" ]]; then
203+
docker login -u $DOCKER_USER -p $DOCKER_PASS
204+
docker tag nipreps/dmriprep nipreps/dmriprep:unstable
205+
docker push nipreps/dmriprep:unstable
206+
if [[ -n "$CIRCLE_TAG" ]]; then
207+
docker push nipreps/dmriprep:latest
208+
docker tag nipreps/dmriprep nipreps/dmriprep:$CIRCLE_TAG
209+
docker push nipreps/dmriprep:$CIRCLE_TAG
210+
fi
211+
fi
212+
213+
test_deploy_pypi:
214+
machine:
215+
image: circleci/classic:201711-01
216+
working_directory: /tmp/src/dmriprep
217+
steps:
218+
- checkout
219+
- run:
220+
name: Build dMRIPrep
221+
command: |
222+
pyenv local 3.5.2
223+
pip install twine # For use in checking distributions
224+
THISVERSION=$( python get_version.py )
225+
THISVERSION=${THISVERSION%.dirty*}
226+
THISVERSION=${CIRCLE_TAG:-$THISVERSION}
227+
virtualenv --python=python build
228+
source build/bin/activate
229+
pip install --upgrade "pip>=19.1" numpy
230+
echo "${CIRCLE_TAG:-$THISVERSION}" > dmriprep/VERSION
231+
python setup.py sdist
232+
pip wheel --no-deps -w dist/ .
233+
- store_artifacts:
234+
path: /tmp/src/dmriprep/dist
235+
- run:
236+
name: Check sdist distribution
237+
command: |
238+
pyenv local 3.5.2
239+
THISVERSION=$( python get_version.py )
240+
THISVERSION=${THISVERSION%.dirty*}
241+
THISVERSION=${CIRCLE_TAG:-$THISVERSION}
242+
twine check dist/dmriprep*.tar.gz
243+
virtualenv --python=python sdist
244+
source sdist/bin/activate
245+
pip install --upgrade "pip>=19.1" numpy
246+
pip install dist/dmriprep*.tar.gz
247+
which dmriprep | grep sdist\\/bin
248+
INSTALLED_VERSION=$(dmriprep --version)
249+
INSTALLED_VERSION=${INSTALLED_VERSION%$'\r'}
250+
INSTALLED_VERSION=${INSTALLED_VERSION#*"dmriprep v"}
251+
echo "VERSION: \"$THISVERSION\""
252+
echo "INSTALLED: \"$INSTALLED_VERSION\""
253+
test "$INSTALLED_VERSION" = "$THISVERSION"
254+
- run:
255+
name: Check wheel distribution
256+
command: |
257+
pyenv local 3.5.2
258+
THISVERSION=$( python get_version.py )
259+
THISVERSION=${THISVERSION%.dirty*}
260+
THISVERSION=${CIRCLE_TAG:-$THISVERSION}
261+
twine check dist/dmriprep*.whl
262+
virtualenv --python=python wheel
263+
source wheel/bin/activate
264+
pip install dist/dmriprep*.whl
265+
which dmriprep | grep wheel\\/bin
266+
INSTALLED_VERSION=$(dmriprep --version)
267+
INSTALLED_VERSION=${INSTALLED_VERSION%$'\r'}
268+
INSTALLED_VERSION=${INSTALLED_VERSION#*"dmriprep v"}
269+
echo "VERSION: \"$THISVERSION\""
270+
echo "INSTALLED: \"$INSTALLED_VERSION\""
271+
test "$INSTALLED_VERSION" = "$THISVERSION"
272+
- store_artifacts:
273+
path: /tmp/src/dmriprep/dist
274+
275+
deploy_pypi:
276+
machine:
277+
image: circleci/classic:201711-01
278+
working_directory: /tmp/src/dmriprep
279+
steps:
280+
- checkout
281+
- run:
282+
name: Build dMRIPrep
283+
command: |
284+
pyenv local 3.5.2
285+
THISVERSION=$( python get_version.py )
286+
virtualenv --python=python build
287+
source build/bin/activate
288+
pip install --upgrade "pip>=19.1" numpy
289+
echo "${CIRCLE_TAG:-$THISVERSION}" > dmriprep/VERSION
290+
python setup.py sdist
291+
docker run --rm=false -e PLAT='manylinux1_x86_64' -e PKGNAME='dmriprep' -v `pwd`:/io \
292+
quay.io/pypa/manylinux1_x86_64 /io/.circleci/pypi_wheel/build-wheels.sh
293+
- run:
294+
name: Upload packages to PyPI
295+
command: |
296+
pyenv local 3.5.2
297+
pip install twine
298+
twine upload dist/dmriprep*
299+
300+
deployable:
301+
docker:
302+
- image: busybox:latest
303+
steps:
304+
- run: echo Deploying!
305+
306+
workflows:
307+
version: 2
308+
build_test_deploy:
309+
jobs:
310+
- build:
311+
filters:
312+
branches:
313+
ignore:
314+
- /docs?\/.*/
315+
tags:
316+
only: /.*/
317+
318+
- get_data:
319+
filters:
320+
branches:
321+
ignore:
322+
- /docs?\/.*/
323+
- /tests?\/.*/
324+
- /docker\/.*/
325+
tags:
326+
only: /.*/
327+
328+
- test_deploy_pypi:
329+
filters:
330+
branches:
331+
ignore:
332+
- /docs\/.*/
333+
- /docker\/.*/
334+
tags:
335+
only: /.*/
336+
337+
- deploy_docker_patches:
338+
requires:
339+
- build
340+
filters:
341+
branches:
342+
only: /docker\/.*/
343+
344+
- deployable:
345+
requires:
346+
- build
347+
- test_deploy_pypi
348+
filters:
349+
branches:
350+
only: master
351+
tags:
352+
only: /.*/
353+
354+
- deploy_docker:
355+
requires:
356+
- deployable
357+
filters:
358+
branches:
359+
only: master
360+
tags:
361+
only: /.*/
362+
363+
- deploy_pypi:
364+
requires:
365+
- deployable
366+
filters:
367+
branches:
368+
ignore: /.*/
369+
tags:
370+
only: /.*/

‎.circleci/pypi_wheel/build-wheels.sh

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
set -e -x
3+
4+
# Install a system package required by our library
5+
yum install -y atlas-devel
6+
7+
# Compile wheels
8+
for PYBIN in /opt/python/cp3{6,7}*/bin; do
9+
"${PYBIN}/pip" install -U pip
10+
"${PYBIN}/pip" wheel /io/ -w dist/
11+
done
12+
13+
# Bundle external shared libraries into the wheels
14+
for whl in dist/*.whl; do
15+
auditwheel repair "$whl" --plat $PLAT -w /io/dist/
16+
done
17+
18+
# Install packages
19+
################## Disabled bc pybids gives no distribution found
20+
# for PYBIN in /opt/python/cp3{6,7}*/bin; do
21+
# "${PYBIN}/pip" install "$PKGNAME" --no-index -f /io/dist
22+
# done

0 commit comments

Comments
 (0)