Merge remote-tracking branch 'upstream/master' into fix_tutorials

Author: effigies

.circleci/config.yml

@@ -0,0 +1,364 @@
+_machine_kwds: &machine_kwds
+  image: circleci/classic:201710-02
+
+_store_artifacts_kwds: &store_artifacts_kwds
+  path: /home/circleci/work/tests
+
+_test_environment: &test_environment
+  WORKDIR: /home/circleci/work
+  DOCKER_IMAGE: "nipype/nipype"
+
+_set_pr_number: &set_pr_number
+  name: Set PR number
+  command: |
+      echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
+      source $BASH_ENV
+      echo $CIRCLE_PR_NUMBER
+
+_generate_dockerfiles: &generate_dockerfiles
+  name: Generate Dockerfiles
+  command: |
+    make gen-dockerfiles
+
+_modify_nipype_version: &modify_nipype_version
+  name: Modify Nipype version if necessary
+  command: |
+    if [ "$CIRCLE_TAG" != "" ]; then
+      sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
+    fi
+
+_get_base_image: &get_base_image
+  name: Get base image (pull or build)
+  no_output_timeout: 60m
+  command: |
+    source /tmp/docker/get_base_image.sh
+    if [ "$GET_BASE" == "PULL" ]; then
+      echo "Pulling base image ..."
+      docker pull nipype/nipype:base
+    elif [ "$GET_BASE" == "BUILD" ]; then
+      e=1 && for i in {1..5}; do
+        docker build -t nipype/nipype:base - < docker/Dockerfile.base  && e=0 && break || sleep 15
+      done && [ "$e" -eq "0" ]
+    else
+      echo "Error: method to get base image not understood"
+      exit 1
+    fi
+
+_build_main_image_py36: &build_main_image_py36
+  name: Build main image (py36)
+  no_output_timeout: 60m
+  command: |
+    e=1 && for i in {1..5}; do
+      docker build \
+        --rm=false \
+        --tag nipype/nipype:latest \
+        --tag nipype/nipype:py36 \
+        --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+        --build-arg VCS_REF="$(git rev-parse --short HEAD)" \
+        --build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \
+      && e=0 && break || sleep 15
+    done && [ "$e" -eq "0" ]
+
+_build_main_image_py27: &build_main_image_py27
+  name: Build main image (py27)
+  no_output_timeout: 60m
+  command: |
+    e=1 && for i in {1..5}; do
+      docker build \
+        --rm=false \
+        --tag nipype/nipype:py27 \
+        --build-arg PYTHON_VERSION_MAJOR=2 \
+        --build-arg PYTHON_VERSION_MINOR=7 \
+        --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+        --build-arg VCS_REF="$(git rev-parse --short HEAD)" \
+        --build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \
+      && e=0 && break || sleep 15
+    done && [ "$e" -eq "0" ]
+
+_download_test_data: &_download_test_data
+  name: Download test data
+  no_output_timeout: 20m
+  working_directory: /home/circleci/examples
+  environment:
+    OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage"
+  command: |
+    export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21"
+    curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj
+
+    export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe"
+    curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz
+
+    export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af"
+    curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz
+
+_prepare_working_directory: &prepare_working_directory
+  name: Prepare working directory
+  environment: *test_environment
+  command: |
+    mkdir -p "$WORKDIR"
+    chmod -R 777 "$WORKDIR"
+
+_get_codecov: &_get_codecov
+  name: Get codecov
+  command: |
+    pip install --no-cache-dir codecov
+
+_run_codecov_coverage: &_run_codecov_coverage
+  name: Run codecov (coverage)
+  environment: *test_environment
+  command: |
+    codecov --file $WORKDIR/tests/coverage*.xml --root "$HOME/nipype" --flags unittests -e CIRCLE_JOB
+
+_run_codecov_smoke: &_run_codecov_smoke
+  name: Run codecov (smoke tests)
+  environment: *test_environment
+  command: |
+    codecov --file $WORKDIR/tests/smoketest*.xml --root "$HOME/nipype" --flags smoketests -e CIRCLE_JOB
+
+
+version: 2
+jobs:
+
+  compare_base_dockerfiles:
+    docker:
+      - image: docker:17.10.0-ce-git
+    steps:
+      - checkout:
+          path: /home/circleci/nipype
+      - setup_remote_docker
+      - run:
+          name: Generate and prune base Dockerfile in preparation for cache check
+          working_directory: /home/circleci/nipype/docker
+          command: |
+            mkdir -p /tmp/docker
+            ash ./generate_dockerfiles.sh -b
+
+            # Use the sha256 sum of the pruned Dockerfile as the cache key.
+            ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned
+      - restore_cache:
+          key: dockerfile-cache-v1-master-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }}
+      - run:
+          name: Determine how to get base image
+          command: |
+            if [ -f /tmp/docker/cache/Dockerfile.base-pruned ]; then
+              echo "Cache found. Will pull base image."
+              echo 'export GET_BASE=PULL' > /tmp/docker/get_base_image.sh
+            else
+              echo "Cache not found. Will build base image."
+              echo 'export GET_BASE=BUILD' > /tmp/docker/get_base_image.sh
+            fi
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - docker/Dockerfile.base-pruned
+            - docker/get_base_image.sh
+
+  test_pytest:
+    machine: *machine_kwds
+    working_directory: /home/circleci/nipype
+    steps:
+      - checkout:
+          path: /home/circleci/nipype
+      - attach_workspace:
+          at: /tmp
+      - run: *set_pr_number
+      - run: *generate_dockerfiles
+      - run: *modify_nipype_version
+      - run: *get_base_image
+      - run: *build_main_image_py36
+      - run: *build_main_image_py27
+      - run: *_get_codecov
+      - run: *_download_test_data
+      - run: *prepare_working_directory
+      - run:
+          name: Run pytests (py36)
+          no_output_timeout: 30m
+          environment: *test_environment
+          command: bash -ux /home/circleci/nipype/.circleci/test_py3_pytest.sh
+      - run:
+          name: Run pytests (py27)
+          no_output_timeout: 30m
+          environment: *test_environment
+          command: bash -ux /home/circleci/nipype/.circleci/test_py2_pytest.sh
+      - run: *_run_codecov_coverage
+      - store_artifacts: *store_artifacts_kwds
+      - store_test_results: *store_artifacts_kwds
+      - run:
+          name: Build docs (py36)
+          no_output_timeout: 30m
+          environment: *test_environment
+          command: bash -ux /home/circleci/nipype/.circleci/test_py3_docs.sh
+      - store_artifacts:
+          path: /home/circleci/work/docs
+      - run:
+          name: Save Docker images to workspace if on master
+          no_output_timeout: 60m
+          command: |
+            if [ "$CIRCLE_BRANCH" = "master" -a -z "$CIRCLE_PULL_REQUEST" ]; then
+              docker save nipype/nipype:base \
+                          nipype/nipype:latest \
+                          nipype/nipype:py27 \
+                          nipype/nipype:py36 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz \
+              && du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz
+            fi
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - docker
+
+  test_py3_fmri_fsl_spm:
+    machine: *machine_kwds
+    working_directory: /home/circleci/nipype
+    steps:
+      - checkout:
+          path: /home/circleci/nipype
+      - attach_workspace:
+          at: /tmp
+      - run: *set_pr_number
+      - run: *generate_dockerfiles
+      - run: *modify_nipype_version
+      - run: *get_base_image
+      - run: *build_main_image_py36
+      - run: *_get_codecov
+      - run: *_download_test_data
+      - run: *prepare_working_directory
+      - run:
+          name: Run FSL reuse pipeline (py36)
+          no_output_timeout: 40m
+          environment: *test_environment
+          command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh
+      - run:
+          name: Run SPM test workflow - 3D inputs (py36)
+          no_output_timeout: 40m
+          environment: *test_environment
+          command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_3d.sh
+      - run:
+          name: Run SPM test workflow - 4D inputs (py36)
+          no_output_timeout: 40m
+          environment: *test_environment
+          command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_4d.sh
+      - run: *_run_codecov_smoke
+      - store_artifacts: *store_artifacts_kwds
+
+  test_py3_fmri_spm_dartel_multiproc:
+    machine: *machine_kwds
+    working_directory: /home/circleci/nipype
+    steps:
+      - checkout:
+          path: /home/circleci/nipype
+      - attach_workspace:
+          at: /tmp
+      - run: *set_pr_number
+      - run: *generate_dockerfiles
+      - run: *modify_nipype_version
+      - run: *get_base_image
+      - run: *build_main_image_py36
+      - run: *_get_codecov
+      - run: *_download_test_data
+      - run: *prepare_working_directory
+      - run:
+          name: Run SPM DARTEL Level 1 pipeline (py36)
+          no_output_timeout: 1h
+          environment: *test_environment
+          command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh
+      - run:
+          name: Run SPM DARTEL Level 2 pipeline (py36)
+          no_output_timeout: 30m
+          environment: *test_environment
+          command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh
+      - run: *_run_codecov_smoke
+      - store_artifacts: *store_artifacts_kwds
+
+  test_fmri_spm_nested_fsl_feeds:
+    machine: *machine_kwds
+    working_directory: /home/circleci/nipype
+    steps:
+      - checkout:
+          path: /home/circleci/nipype
+      - attach_workspace:
+          at: /tmp
+      - run: *set_pr_number
+      - run: *generate_dockerfiles
+      - run: *modify_nipype_version
+      - run: *get_base_image
+      - run: *build_main_image_py36
+      - run: *build_main_image_py27
+      - run: *_get_codecov
+      - run: *_download_test_data
+      - run: *prepare_working_directory
+      - run:
+          name: Run SPM Nested Level 1 pipeline (py36)
+          no_output_timeout: 1h
+          environment: *test_environment
+          command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh
+      - run:
+          name: Run SPM Nested Level 2 pipeline (py27)
+          no_output_timeout: 30m
+          environment: *test_environment
+          command: bash -ux /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh
+      - run:
+          name: Run FSL FEEDS pipeline (py36)
+          no_output_timeout: 40m
+          environment: *test_environment
+          command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh
+      - run: *_run_codecov_smoke
+      - store_artifacts: *store_artifacts_kwds
+
+  deploy:
+    docker:
+      - image: docker:17.10.0-ce-git
+    steps:
+      - setup_remote_docker
+      - attach_workspace:
+          at: /tmp
+      - run:
+          name: Load saved Docker images.
+          no_output_timeout: 60m
+          command: |
+            docker load < /tmp/docker/nipype-base-latest-py36-py27.tar.gz
+      - run:
+          name: Push to DockerHub
+          no_output_timeout: 120m
+          command: |
+            echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
+            docker push nipype/nipype:base
+            docker push nipype/nipype:latest
+            docker push nipype/nipype:py36
+            docker push nipype/nipype:py27
+      - run:
+          name: Move pruned Dockerfile to /tmp/docker/cache directory
+          command: |
+            mkdir -p /tmp/docker/cache/
+            mv /tmp/docker/Dockerfile.base-pruned /tmp/docker/cache/Dockerfile.base-pruned
+      - save_cache:
+          paths:
+            - /tmp/docker/cache/Dockerfile.base-pruned
+          key: dockerfile-cache-v1-{{ .Branch }}-{{ checksum "/tmp/docker/cache/Dockerfile.base-pruned" }}
+
+
+workflows:
+  version: 2
+  build_test_deploy:
+    jobs:
+      - compare_base_dockerfiles
+      - test_pytest:
+          requires:
+            - compare_base_dockerfiles
+      - test_py3_fmri_fsl_spm:
+          requires:
+            - compare_base_dockerfiles
+      - test_py3_fmri_spm_dartel_multiproc:
+          requires:
+            - compare_base_dockerfiles
+      - test_fmri_spm_nested_fsl_feeds:
+          requires:
+            - compare_base_dockerfiles
+      - deploy:
+          filters:
+            branches:
+              only: master
+          requires:
+            - test_pytest
+            - test_fmri_spm_nested_fsl_feeds
+            - test_py3_fmri_fsl_spm
+            - test_py3_fmri_spm_dartel_multiproc

.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline

.circleci/test_py2_pytest.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_pytests.sh

.circleci/test_py3_docs.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /src/nipype/doc "${DOCKER_IMAGE}:py36" /usr/bin/run_builddocs.sh

.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline

.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow

.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel MultiProc /data/examples/ level1

.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh fmri_spm_dartel MultiProc /data/examples/ l2pipeline

.circleci/test_py3_fmri_spm_linear_3d.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work "${DOCKER_IMAGE}:py36" /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d