Skip to content

Commit 426e77b

Browse files
sgiavasissgiavasis
authored
Merge pull request #1631 from FCP-INDI/develop
Merge v1.8.2 to main branch
2 parents 07d59f6 + 7e1fd06 commit 426e77b

File tree

116 files changed

+4435
-2289
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

116 files changed

+4435
-2289
lines changed

‎.circleci/main.yml‎

Lines changed: 143 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -56,32 +56,38 @@ commands:
5656
steps:
5757
- push-to-docker-hub:
5858
tag: nightly
59-
- when:
60-
condition:
61-
not:
62-
equal: [ << parameters.variant >>, fMRIPrep-LTS ]
63-
steps:
64-
- run:
65-
name: "Starting local registry"
66-
command: docker run -d -p 5000:5000 --restart=always --name registry registry:2
67-
- get-singularity
68-
- run:
69-
name: "Building Singularity image from Docker image"
70-
command: |
71-
docker load < ${DOCKER_TARBALL}.tar.gz
72-
docker tag fcpindi/c-pac:${DOCKER_TAG} localhost:5000/fcpindi/c-pac:${DOCKER_TAG}
73-
docker push localhost:5000/fcpindi/c-pac:${DOCKER_TAG}
74-
SINGULARITY_NOHTTPS=1 singularity build cpac-singularity-image${VARIANT}.simg docker://localhost:5000/fcpindi/c-pac:${DOCKER_TAG}
75-
no_output_timeout: 5h
76-
check-for-built-images:
59+
# - when:
60+
# condition:
61+
# not:
62+
# equal: [ << parameters.variant >>, fMRIPrep-LTS ]
63+
# steps:
64+
# - run:
65+
# name: "Starting local registry"
66+
# command: docker run -d -p 5000:5000 --restart=always --name registry registry:2
67+
# - get-singularity
68+
# - run:
69+
# name: "Building Singularity image from Docker image"
70+
# command: |
71+
# docker load < ${DOCKER_TARBALL}.tar.gz
72+
# docker tag fcpindi/c-pac:${DOCKER_TAG} localhost:5000/fcpindi/c-pac:${DOCKER_TAG}
73+
# docker push localhost:5000/fcpindi/c-pac:${DOCKER_TAG}
74+
# SINGULARITY_NOHTTPS=1 singularity build cpac-singularity-image${VARIANT}.simg docker://localhost:5000/fcpindi/c-pac:${DOCKER_TAG}
75+
# no_output_timeout: 5h
76+
check-updated:
7777
steps:
78+
- restore_cache:
79+
key: updated--{{ .Revision }}
7880
- run:
79-
name: "Checking for locally built images"
80-
command: |
81-
if [[ ! -f ${DOCKER_TARBALL}.tar.gz ]]
82-
then
83-
circleci step halt
84-
fi
81+
name: "Halt if version updated"
82+
command: if [[ $(cat /home/circleci/HALT) == true ]]; then circleci step halt; fi
83+
clear-up-space:
84+
parameters:
85+
filepath:
86+
type: string
87+
steps:
88+
- run:
89+
name: "Delete file we're done with"
90+
command: rm -rf << parameters.filepath >>
8591
combine-coverage:
8692
steps:
8793
- run:
@@ -199,7 +205,7 @@ commands:
199205
DOCKERFILE="variant-<< parameters.variant >>.Dockerfile"
200206
if [[ -f ${DOCKERFILE} ]]
201207
then
202-
sed -i "s|:latest|:${CIRCLE_BRANCH//\//_}|g" ${DOCKERFILE}
208+
sed -i "s|c-pac:latest|c-pac:${CIRCLE_BRANCH//\//_}|g" ${DOCKERFILE}
203209
fi
204210
DOCKER_TARBALL="${DOCKER_TARBALL}-<< parameters.variant >>"
205211
VARIANT="-<< parameters.variant >>"
@@ -217,114 +223,131 @@ commands:
217223
pip install -r dev/circleci_data/requirements.txt
218224
coverage run -m pytest --junitxml=test-results/junit.xml --continue-on-collection-errors dev/circleci_data/test_install.py
219225
220-
jobs:
226+
jobs:
221227
build:
222228
machine:
223229
image: ubuntu-2004:202010-01
224230
steps:
225231
- checkout
232+
- check-updated
226233
- set-python-version
227-
- configure-git-user
228-
- run:
229-
name: "Checking if version needs updated"
230-
# update version if version needs updated, otherwise just move on
231-
command: |
232-
COMMIT_MESSAGE=$(git log -1 --pretty=%B)
233-
if [[ ! "$COMMIT_MESSAGE" == *"Update version to"* ]]
234-
then
235-
cd $HOME/project/CPAC
236-
VERSION=$(python -c "from info import __version__; print('.'.join(('.'.join(__version__[::-1].split('-')[1].split('.')[1:])[::-1], __version__.split('-')[1])) if '-' in __version__ else __version__)")
237-
cd ..
238-
echo "v${VERSION}" > version
239-
sed -i -r "s/^(# [Vv]ersion ).*$/# Version ${VERSION}/g" dev/docker_data/default_pipeline.yml
240-
find ./CPAC/resources/configs -name "*.yml" -exec sed -i -r "s/^(# [Vv]ersion ).*$/# Version ${VERSION}/g" {} \;
241-
git add version CPAC/resources/configs dev/docker_data/default_pipeline.yml
242-
if [[ ! -z $(git diff origin/${CIRCLE_BRANCH}) ]]
243-
then
244-
git commit -m ":bookmark: Update version to ${VERSION} ($COMMIT_MESSAGE)"
245-
git push origin HEAD:${CIRCLE_BRANCH} || true
246-
circleci step halt
247-
fi
248-
cd ..
249-
fi
250234
- build-images
251235
- store_artifacts:
252236
path: cpac-docker-image.tar.gz
253-
- store_artifacts:
254-
path: cpac-singularity-image.simg
237+
# - store_artifacts:
238+
# path: cpac-singularity-image.simg
255239
- persist_to_workspace:
256240
root: /home/circleci/
257241
paths:
258242
- project/cpac-docker-image.tar.gz
259-
- project/cpac-singularity-image.simg
243+
# - project/cpac-singularity-image.simg
260244
build-ABCD-HCP:
261245
machine:
262246
image: ubuntu-2004:202010-01
263247
steps:
264248
- checkout
249+
- check-updated
265250
- set-python-version
266251
- build-images:
267252
variant: ABCD-HCP
268253
- store_artifacts:
269254
path: cpac-docker-image-ABCD-HCP.tar.gz
270-
- store_artifacts:
271-
path: cpac-singularity-image-ABCD-HCP.simg
255+
# - store_artifacts:
256+
# path: cpac-singularity-image-ABCD-HCP.simg
272257
- persist_to_workspace:
273258
root: /home/circleci/
274259
paths:
275260
- project/cpac-docker-image-ABCD-HCP.tar.gz
276-
- project/cpac-singularity-image-ABCD-HCP.simg
261+
# - project/cpac-singularity-image-ABCD-HCP.simg
277262
build-fMRIPrep-LTS:
278263
machine:
279264
image: ubuntu-2004:202010-01
280265
steps:
281266
- checkout
267+
- check-updated
282268
- set-python-version
283269
- build-images:
284270
variant: fMRIPrep-LTS
285271
- store_artifacts:
286272
path: cpac-docker-image-fMRIPrep-LTS.tar.gz
287-
- store_artifacts:
288-
path: cpac-singularity-image-fMRIPrep-LTS.simg
273+
# - store_artifacts:
274+
# path: cpac-singularity-image-fMRIPrep-LTS.simg
289275
- persist_to_workspace:
290276
root: /home/circleci/
291277
paths:
292278
- project/cpac-docker-image-fMRIPrep-LTS.tar.gz
293-
- project/cpac-singularity-image-fMRIPrep-LTS.simg
279+
# - project/cpac-singularity-image-fMRIPrep-LTS.simg
294280
build-lite:
295281
machine:
296282
image: ubuntu-2004:202010-01
297283
steps:
298284
- checkout
285+
- check-updated
299286
- set-python-version
300287
- build-images:
301288
variant: lite
302289
- store_artifacts:
303290
path: cpac-docker-image-lite.tar.gz
304-
- store_artifacts:
305-
path: cpac-singularity-image-lite.simg
291+
# - store_artifacts:
292+
# path: cpac-singularity-image-lite.simg
306293
- persist_to_workspace:
307294
root: /home/circleci/
308295
paths:
309296
- project/cpac-docker-image-lite.tar.gz
310-
- project/cpac-singularity-image-lite.simg
297+
# - project/cpac-singularity-image-lite.simg
298+
check-version:
299+
machine:
300+
image: ubuntu-2004:202010-01
301+
steps:
302+
- checkout
303+
- set-python-version
304+
- configure-git-user
305+
- run:
306+
name: "Checking if version needs updated"
307+
# update version if version needs updated, otherwise just move on
308+
command: |
309+
HALT=false
310+
COMMIT_MESSAGE=$(git log -1 --pretty=%B)
311+
if [[ ! "$COMMIT_MESSAGE" == *"Update version to"* ]]
312+
then
313+
cd $HOME/project/CPAC
314+
VERSION=$(python -c "from info import __version__; print('.'.join(('.'.join(__version__[::-1].split('-')[1].split('.')[1:])[::-1], __version__.split('-')[1])) if '-' in __version__ else __version__)")
315+
cd ..
316+
echo "v${VERSION}" > version
317+
sed -i -r "s/^(# [Vv]ersion ).*$/# Version ${VERSION}/g" dev/docker_data/default_pipeline.yml
318+
find ./CPAC/resources/configs -name "*.yml" -exec sed -i -r "s/^(# [Vv]ersion ).*$/# Version ${VERSION}/g" {} \;
319+
git add version CPAC/resources/configs dev/docker_data/default_pipeline.yml
320+
if [[ ! -z $(git diff origin/${CIRCLE_BRANCH}) ]]
321+
then
322+
git commit -m ":bookmark: Update version to ${VERSION} ($COMMIT_MESSAGE)"
323+
git push origin HEAD:${CIRCLE_BRANCH} || true
324+
HALT=true
325+
fi
326+
cd ..
327+
fi
328+
echo "${HALT}" > /home/circleci/HALT
329+
- save_cache:
330+
key: updated--{{ .Revision }}
331+
paths:
332+
- /home/circleci/HALT
311333
combine-coverage:
312334
machine:
313335
image: ubuntu-2004:202010-01
314336
steps:
315337
- checkout
338+
- check-updated
316339
- restore_cache:
317340
key: coverage-docker--{{ .Revision }}
318-
- restore_cache:
319-
key: coverage-singularity--{{ .Revision }}
341+
# - restore_cache:
342+
# key: coverage-singularity--{{ .Revision }}
320343
- restore_cache:
321344
key: coverage-docker-ABCD-HCP-{{ .Revision }}
322-
- restore_cache:
323-
key: coverage-singularity-ABCD-HCP-{{ .Revision }}
345+
# - restore_cache:
346+
# key: coverage-singularity-ABCD-HCP-{{ .Revision }}
324347
- restore_cache:
325348
key: coverage-docker-lite-{{ .Revision }}
326-
- restore_cache:
327-
key: coverage-singularity-lite-{{ .Revision }}
349+
# - restore_cache:
350+
# key: coverage-singularity-lite-{{ .Revision }}
328351
- set-python-version
329352
- combine-coverage
330353
- store_artifacts:
@@ -338,14 +361,16 @@ jobs:
338361
image: ubuntu-2004:202010-01
339362
steps:
340363
- checkout
341-
- attach_workspace:
342-
at: /home/circleci/
364+
- check-updated
343365
- set-up-variant:
344366
variant: "<< parameters.variant >>"
345-
- check-for-built-images
367+
- attach_workspace:
368+
at: /home/circleci/
346369
- set-python-version
347370
- get-sample-bids-data
348371
- run-pytest-docker
372+
- clear-up-space:
373+
filepath: ${DOCKER_TARBALL}.tar.gz
349374
- store_test_results:
350375
path: test-results
351376
- save_cache:
@@ -362,17 +387,19 @@ jobs:
362387
image: ubuntu-2004:202010-01
363388
steps:
364389
- checkout
365-
- attach_workspace:
366-
at: /home/circleci/
390+
- check-updated
367391
- set-up-variant:
368392
variant: "<< parameters.variant >>"
369-
- check-for-built-images
393+
- attach_workspace:
394+
at: /home/circleci/
370395
- set-python-version
371396
- get-singularity
372397
- test-singularity-installation
373398
- run:
374399
command: "SINGULARITYENV_COVERAGE_FILE=.coverage.singularity${VARIANT} singularity exec -B $PWD:/code cpac-singularity-image${VARIANT}.simg /code/dev/circleci_data/test_in_image.sh"
375400
name: "Running unit tests in Singularity image"
401+
- clear-up-space:
402+
filepath: cpac-singularity-image${VARIANT}.simg
376403
- store_test_results:
377404
path: test-results
378405
- save_cache:
@@ -386,52 +413,62 @@ workflows:
386413
jobs:
387414
- build:
388415
name: "Build images"
389-
- build-ABCD-HCP:
390-
name: "Build ABCD-HCP variant images"
391-
- build-fMRIPrep-LTS:
392-
name: "Build fMRIPrep-LTS variant images"
416+
requires:
417+
- "Checking for updated version"
418+
# - build-ABCD-HCP:
419+
# name: "Build ABCD-HCP variant images"
420+
# requires:
421+
# - "Checking for updated version"
422+
# - build-fMRIPrep-LTS:
423+
# name: "Build fMRIPrep-LTS variant images"
424+
# requires:
425+
# - "Checking for updated version"
393426
- build-lite:
394427
name: "Build lite variant images"
428+
requires:
429+
- "Checking for updated version"
430+
- check-version:
431+
name: "Checking for updated version"
395432
- combine-coverage:
396433
name: "Combine coverage"
397434
requires:
398435
- "Test in Docker"
399-
- "Test in Singularity"
436+
# - "Test in Singularity"
400437
- "Test lite variant in Docker"
401-
- "Test lite variant in Singularity"
402-
- "Test ABCD-HCP variant in Docker"
403-
- "Test ABCD-HCP variant in Singularity"
404-
- "Test fMRIPrep-LTS variant in Docker"
438+
# - "Test lite variant in Singularity"
439+
# - "Test ABCD-HCP variant in Docker"
440+
# - "Test ABCD-HCP variant in Singularity"
441+
# - "Test fMRIPrep-LTS variant in Docker"
405442
- pytest-docker:
406443
name: "Test in Docker"
407444
requires:
408445
- "Build images"
409-
- pytest-docker:
410-
name: "Test ABCD-HCP variant in Docker"
411-
requires:
412-
- "Build ABCD-HCP variant images"
413-
variant: ABCD-HCP
414-
- pytest-docker:
415-
name: "Test fMRIPrep-LTS variant in Docker"
416-
requires:
417-
- "Build fMRIPrep-LTS variant images"
418-
variant: fMRIPrep-LTS
446+
# - pytest-docker:
447+
# name: "Test ABCD-HCP variant in Docker"
448+
# requires:
449+
# - "Build ABCD-HCP variant images"
450+
# variant: ABCD-HCP
451+
# - pytest-docker:
452+
# name: "Test fMRIPrep-LTS variant in Docker"
453+
# requires:
454+
# - "Build fMRIPrep-LTS variant images"
455+
# variant: fMRIPrep-LTS
419456
- pytest-docker:
420457
name: "Test lite variant in Docker"
421458
requires:
422459
- "Build lite variant images"
423460
variant: lite
424-
- pytest-singularity:
425-
name: "Test in Singularity"
426-
requires:
427-
- "Build images"
428-
- pytest-singularity:
429-
name: "Test ABCD-HCP variant in Singularity"
430-
requires:
431-
- "Build ABCD-HCP variant images"
432-
variant: ABCD-HCP
433-
- pytest-singularity:
434-
name: "Test lite variant in Singularity"
435-
requires:
436-
- "Build lite variant images"
437-
variant: lite
461+
# - pytest-singularity:
462+
# name: "Test in Singularity"
463+
# requires:
464+
# - "Build images"
465+
# - pytest-singularity:
466+
# name: "Test ABCD-HCP variant in Singularity"
467+
# requires:
468+
# - "Build ABCD-HCP variant images"
469+
# variant: ABCD-HCP
470+
# - pytest-singularity:
471+
# name: "Test lite variant in Singularity"
472+
# requires:
473+
# - "Build lite variant images"
474+
# variant: lite

0 commit comments

Comments
 (0)