Skip to content

Commit 70f7591

Browse files
josephmjejosephmje
committed
Merge remote-tracking branch 'nipreps/master' into nipreps
2 parents 8896975 + c23b30b commit 70f7591

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+5055
-956
lines changed

‎.circleci/config.yml

Lines changed: 370 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,370 @@
1+
version: 2
2+
jobs:
3+
4+
build:
5+
environment:
6+
TZ: "/usr/share/zoneinfo/America/Los_Angeles"
7+
SCRATCH: "/scratch"
8+
docker:
9+
- image: docker:18.01.0-ce-git
10+
working_directory: /tmp/src/dmriprep
11+
steps:
12+
- checkout
13+
- run:
14+
name: Check whether build should be skipped
15+
command: |
16+
if [[ "$( git log --format='format:%s' -n 1 $CIRCLE_SHA1 | grep -i -E '^docs?(\(\w+\))?:' )" != "" ]]; then
17+
echo "Only docs build"
18+
circleci step halt
19+
fi
20+
21+
- run:
22+
name: Install parallel gzip and python3
23+
command: |
24+
apk add --no-cache pigz python3
25+
- restore_cache:
26+
keys:
27+
- docker-v0-{{ .Branch }}-{{ .Revision }}
28+
- docker-v0-{{ .Branch }}-
29+
- docker-v0-master-
30+
- docker-v0-
31+
paths:
32+
- /tmp/cache/docker.tar.gz
33+
- setup_remote_docker
34+
- run:
35+
name: Load Docker image layer cache
36+
no_output_timeout: 30m
37+
command: |
38+
docker info
39+
set +o pipefail
40+
if [ -f /tmp/cache/docker.tar.gz ]; then
41+
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
42+
docker images
43+
fi
44+
- run:
45+
name: Build Docker image
46+
no_output_timeout: 60m
47+
command: |
48+
# Get version, update files.
49+
THISVERSION=$( python3 get_version.py )
50+
if [[ ${THISVERSION:0:2} == "0+" ]] ; then
51+
echo "WARNING: latest git tag could not be found"
52+
echo "Please, make sure you fetch all tags from upstream with"
53+
echo "the command ``git fetch --tags --verbose`` and push"
54+
echo "them to your fork with ``git push origin --tags``"
55+
fi
56+
# Build docker image
57+
e=1 && for i in {1..5}; do
58+
docker build \
59+
--cache-from=nipreps/dmriprep \
60+
--rm=false \
61+
-t nipreps/dmriprep:latest \
62+
--build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
63+
--build-arg VCS_REF=`git rev-parse --short HEAD` \
64+
--build-arg VERSION="${CIRCLE_TAG:-$THISVERSION}" . \
65+
&& e=0 && break || sleep 15
66+
done && [ "$e" -eq "0" ]
67+
# test if development image should be built
68+
if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[build[ _]?devel\]' )" != "" ]]; then
69+
echo "building development container"
70+
docker tag nipreps/dmriprep nipreps/dmriprep:unstable
71+
docker build \
72+
--rm=false \
73+
-t nipreps/dmriprep_devel \
74+
-f Dockerfile_devel .
75+
fi
76+
77+
- run:
78+
name: Docker save
79+
no_output_timeout: 40m
80+
command: |
81+
mkdir -p /tmp/cache
82+
docker save ubuntu:xenial-20161213 nipreps/dmriprep:latest \
83+
| pigz -3 > /tmp/cache/docker.tar.gz
84+
- save_cache:
85+
key: docker-v0-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
86+
paths:
87+
- /tmp/cache/docker.tar.gz
88+
89+
- run:
90+
name: Smoke test Docker image
91+
command: |
92+
THISVERSION=$( python3 get_version.py )
93+
THISVERSION=${THISVERSION%.dirty*}
94+
THISVERSION=${CIRCLE_TAG:-$THISVERSION}
95+
DOCKERVERSION=$(docker run --rm -it nipreps/dmriprep:latest --version)
96+
DOCKERVERSION=${DOCKERVERSION%$'\r'}
97+
DOCKERVERSION=${DOCKERVERSION#*"dmriprep v"}
98+
echo "VERSION: \"$THISVERSION\""
99+
echo "DOCKERVERSION: \"${DOCKERVERSION}\""
100+
test "$DOCKERVERSION" = "$THISVERSION"
101+
102+
- persist_to_workspace:
103+
root: /tmp
104+
paths:
105+
- src/dmriprep
106+
107+
get_data:
108+
machine:
109+
# Ubuntu 14.04 with Docker 17.10.0-ce
110+
image: circleci/classic:201711-01
111+
working_directory: /home/circleci/data
112+
steps:
113+
- restore_cache:
114+
keys:
115+
- data-v0-{{ .Revision }}
116+
- data-v0-
117+
- run:
118+
name: Store FreeSurfer license file
119+
command: |
120+
mkdir -p /tmp/fslicense
121+
cd /tmp/fslicense
122+
echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IGxpY2Vuc2UudHh0Cg==" | base64 -d | sh
123+
- run:
124+
name: Create Nipype config files
125+
command: |
126+
mkdir -p /tmp/config
127+
printf "[execution]\nstop_on_first_crash = true\n" > /tmp/config/nipype.cfg
128+
echo "poll_sleep_duration = 0.01" >> /tmp/config/nipype.cfg
129+
echo "hash_method = content" >> /tmp/config/nipype.cfg
130+
- persist_to_workspace:
131+
root: /tmp
132+
paths:
133+
- fslicense
134+
- config/nipype.cfg
135+
- save_cache:
136+
key: data-v0-{{ .Revision }}-{{ epoch }}
137+
paths:
138+
- /tmp/data
139+
140+
deploy_docker_patches:
141+
machine:
142+
image: circleci/classic:201711-01
143+
working_directory: /tmp/src/dmriprep
144+
steps:
145+
146+
- restore_cache:
147+
keys:
148+
- docker-v0-{{ .Branch }}-{{ .Revision }}
149+
- run:
150+
name: Load Docker image layer cache
151+
no_output_timeout: 30m
152+
command: |
153+
docker info
154+
set +o pipefail
155+
if [ -f /tmp/cache/docker.tar.gz ]; then
156+
sudo apt update && sudo apt -y install pigz
157+
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
158+
docker images
159+
fi
160+
- run:
161+
name: Deploy to Docker Hub
162+
no_output_timeout: 40m
163+
command: |
164+
if [[ -n "$DOCKER_PASS" ]]; then
165+
docker login -u $DOCKER_USER -p $DOCKER_PASS
166+
docker tag nipreps/dmriprep nipreps/dmriprep:${CIRCLE_BRANCH#docker/}
167+
docker push nipreps/dmriprep:${CIRCLE_BRANCH#docker/}
168+
fi
169+
170+
deploy_docker:
171+
machine:
172+
image: circleci/classic:201711-01
173+
working_directory: /tmp/src/dmriprep
174+
steps:
175+
- checkout:
176+
path: /home/circleci/src/dmriprep
177+
- run:
178+
name: Check whether build should be skipped
179+
command: |
180+
cd /home/circleci/src/dmriprep
181+
if [[ "$( git log --format='format:%s' -n 1 $CIRCLE_SHA1 | grep -i -E '^docs?(\(\w+\))?:' )" != "" ]]; then
182+
echo "Only docs build"
183+
circleci step halt
184+
fi
185+
- restore_cache:
186+
keys:
187+
- docker-v0-{{ .Branch }}-{{ .Revision }}
188+
- run:
189+
name: Load Docker image layer cache
190+
no_output_timeout: 30m
191+
command: |
192+
docker info
193+
set +o pipefail
194+
if [ -f /tmp/cache/docker.tar.gz ]; then
195+
sudo apt update && sudo apt -y install pigz
196+
pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
197+
docker images
198+
fi
199+
- run:
200+
name: Deploy to Docker Hub
201+
no_output_timeout: 40m
202+
command: |
203+
if [[ -n "$DOCKER_PASS" ]]; then
204+
docker login -u $DOCKER_USER -p $DOCKER_PASS
205+
docker tag nipreps/dmriprep nipreps/dmriprep:unstable
206+
docker push nipreps/dmriprep:unstable
207+
if [[ -n "$CIRCLE_TAG" ]]; then
208+
docker push nipreps/dmriprep:latest
209+
docker tag nipreps/dmriprep nipreps/dmriprep:$CIRCLE_TAG
210+
docker push nipreps/dmriprep:$CIRCLE_TAG
211+
fi
212+
fi
213+
214+
test_deploy_pypi:
215+
machine:
216+
image: circleci/classic:201711-01
217+
working_directory: /tmp/src/dmriprep
218+
steps:
219+
- checkout
220+
- run:
221+
name: Build dMRIPrep
222+
command: |
223+
pyenv local 3.5.2
224+
pip install twine # For use in checking distributions
225+
THISVERSION=$( python get_version.py )
226+
THISVERSION=${THISVERSION%.dirty*}
227+
THISVERSION=${CIRCLE_TAG:-$THISVERSION}
228+
virtualenv --python=python build
229+
source build/bin/activate
230+
pip install --upgrade "pip>=19.1" numpy
231+
echo "${CIRCLE_TAG:-$THISVERSION}" > dmriprep/VERSION
232+
python setup.py sdist
233+
pip wheel --no-deps -w dist/ .
234+
- store_artifacts:
235+
path: /tmp/src/dmriprep/dist
236+
- run:
237+
name: Check sdist distribution
238+
command: |
239+
pyenv local 3.5.2
240+
THISVERSION=$( python get_version.py )
241+
THISVERSION=${THISVERSION%.dirty*}
242+
THISVERSION=${CIRCLE_TAG:-$THISVERSION}
243+
twine check dist/dmriprep*.tar.gz
244+
virtualenv --python=python sdist
245+
source sdist/bin/activate
246+
pip install --upgrade "pip>=19.1" numpy
247+
pip install dist/dmriprep*.tar.gz
248+
which dmriprep | grep sdist\\/bin
249+
INSTALLED_VERSION=$(dmriprep --version)
250+
INSTALLED_VERSION=${INSTALLED_VERSION%$'\r'}
251+
INSTALLED_VERSION=${INSTALLED_VERSION#*"dmriprep v"}
252+
echo "VERSION: \"$THISVERSION\""
253+
echo "INSTALLED: \"$INSTALLED_VERSION\""
254+
test "$INSTALLED_VERSION" = "$THISVERSION"
255+
- run:
256+
name: Check wheel distribution
257+
command: |
258+
pyenv local 3.5.2
259+
THISVERSION=$( python get_version.py )
260+
THISVERSION=${THISVERSION%.dirty*}
261+
THISVERSION=${CIRCLE_TAG:-$THISVERSION}
262+
twine check dist/dmriprep*.whl
263+
virtualenv --python=python wheel
264+
source wheel/bin/activate
265+
pip install dist/dmriprep*.whl
266+
which dmriprep | grep wheel\\/bin
267+
INSTALLED_VERSION=$(dmriprep --version)
268+
INSTALLED_VERSION=${INSTALLED_VERSION%$'\r'}
269+
INSTALLED_VERSION=${INSTALLED_VERSION#*"dmriprep v"}
270+
echo "VERSION: \"$THISVERSION\""
271+
echo "INSTALLED: \"$INSTALLED_VERSION\""
272+
test "$INSTALLED_VERSION" = "$THISVERSION"
273+
- store_artifacts:
274+
path: /tmp/src/dmriprep/dist
275+
276+
deploy_pypi:
277+
machine:
278+
image: circleci/classic:201711-01
279+
working_directory: /tmp/src/dmriprep
280+
steps:
281+
- checkout
282+
- run:
283+
name: Build dMRIPrep
284+
command: |
285+
sudo setfacl -d -m group:$(id -gn):rwx /tmp/src/dmriprep
286+
sudo setfacl -m group:$(id -gn):rwx /tmp/src/dmriprep
287+
pyenv local 3.5.2
288+
THISVERSION=$( python get_version.py )
289+
echo "${CIRCLE_TAG:-$THISVERSION}" > dmriprep/VERSION
290+
docker run --rm=false --entrypoint=/bin/bash -v `pwd`:/io -w /io python:3.6.9 -c "pip install numpy && python setup.py sdist"
291+
docker run --rm=false -e PLAT='manylinux1_x86_64' -e PKGNAME='dmriprep' -v `pwd`:/io \
292+
quay.io/pypa/manylinux1_x86_64 /io/.circleci/pypi_wheel/build-wheels.sh
293+
- run:
294+
name: Upload packages to PyPI
295+
command: |
296+
pyenv local 3.5.2
297+
pip install twine
298+
twine upload dist/dmriprep*
299+
300+
deployable:
301+
docker:
302+
- image: busybox:latest
303+
steps:
304+
- run: echo Deploying!
305+
306+
workflows:
307+
version: 2
308+
build_test_deploy:
309+
jobs:
310+
- build:
311+
filters:
312+
branches:
313+
ignore:
314+
- /docs?\/.*/
315+
tags:
316+
only: /.*/
317+
318+
- get_data:
319+
filters:
320+
branches:
321+
ignore:
322+
- /docs?\/.*/
323+
- /tests?\/.*/
324+
- /docker\/.*/
325+
tags:
326+
only: /.*/
327+
328+
- test_deploy_pypi:
329+
filters:
330+
branches:
331+
ignore:
332+
- /docs\/.*/
333+
- /docker\/.*/
334+
tags:
335+
only: /.*/
336+
337+
- deploy_docker_patches:
338+
requires:
339+
- build
340+
filters:
341+
branches:
342+
only: /docker\/.*/
343+
344+
- deployable:
345+
requires:
346+
- build
347+
- test_deploy_pypi
348+
filters:
349+
branches:
350+
only: master
351+
tags:
352+
only: /.*/
353+
354+
- deploy_docker:
355+
requires:
356+
- deployable
357+
filters:
358+
branches:
359+
only: master
360+
tags:
361+
only: /.*/
362+
363+
- deploy_pypi:
364+
requires:
365+
- deployable
366+
filters:
367+
branches:
368+
ignore: /.*/
369+
tags:
370+
only: /.*/

‎.circleci/pypi_wheel/build-wheels.sh

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
set -e -x
3+
4+
# Install a system package required by our library
5+
yum install -y atlas-devel
6+
7+
# Compile wheels
8+
for PYBIN in /opt/python/cp3{6,7}*/bin; do
9+
"${PYBIN}/pip" install -U pip
10+
"${PYBIN}/pip" wheel /io/ -w dist/
11+
done
12+
13+
# Bundle external shared libraries into the wheels
14+
for whl in dist/*.whl; do
15+
auditwheel repair "$whl" --plat $PLAT -w /io/dist/
16+
done
17+
18+
# Install packages
19+
################## Disabled bc pybids gives no distribution found
20+
# for PYBIN in /opt/python/cp3{6,7}*/bin; do
21+
# "${PYBIN}/pip" install "$PKGNAME" --no-index -f /io/dist
22+
# done

0 commit comments

Comments
 (0)