Merge pull request #29 from oesteban/test/add-pepolar-tests

ENH: Revise and add tests for the PEPOLAR correction

Author: oesteban

.circleci/config.yml

@@ -72,9 +72,9 @@ jobs:
     steps:
       - restore_cache:
           keys:
-            - data-v0-{{ .Branch }}-
-            - data-v0-master-
-            - data-v0-
+            - data-v1-{{ .Branch }}-
+            - data-v1-master-
+            - data-v1-
       - run:
           name: Setup git-annex
           command: |
@@ -101,9 +101,9 @@ jobs:
       - run:
           name: Install ds001600
           command: |
-            datalad install -r https://github.com/OpenNeuroDatasets/ds001600.git
-            datalad update ds001600/
-            datalad get -r ds001600/sub-1/fmap/*
+            datalad install https://github.com/OpenNeuroDatasets/ds001600.git
+            datalad update --merge ds001600/
+            datalad get -r ds001600/*
       - run:
           name: Get testdata
           command: |
@@ -113,12 +113,26 @@ jobs:
               unzip testdata.zip -d /tmp/data/testdata
             fi
 
+      - run:
+          name: Store FreeSurfer license file
+          command: |
+            mkdir -p /tmp/data/
+            echo "b2VzdGViYW5Ac3RhbmZvcmQuZWR1CjMwNzU2CiAqQ1MzYkJ5VXMxdTVNCiBGU2kvUGJsejJxR1V3Cg==" | base64 -d > /tmp/data/fslicense.txt
+
+      - run:
+          name: Create Nipype config files
+          command: |
+            printf "[execution]\nstop_on_first_crash = true\n" > /tmp/data/nipype.cfg
+            echo "poll_sleep_duration = 0.01" >> /tmp/data/nipype.cfg
+            echo "hash_method = content" >> /tmp/data/nipype.cfg
+            echo "crashfile_format = txt" >> /tmp/data/nipype.cfg
+
       - save_cache:
-         key: data-v0-{{ .Branch }}-{{ .BuildNum }}
+         key: data-v1-{{ .Branch }}-{{ .BuildNum }}
          paths:
             - "/opt/circleci/.pyenv/versions/3.5.2"
             - /tmp/data
-            - /tmp/cache
+            - /tmp/cache/git-annex-standalone.tar.gz
 
       - persist_to_workspace:
           root: /tmp
@@ -134,6 +148,11 @@ jobs:
           at: /tmp
       - checkout:
           path: /tmp/src/sdcflows
+      - restore_cache:
+          keys:
+            - workdir-v1-{{ .Branch }}-
+            - workdir-v1-master-
+            - workdir-v1-
       - run:
           name: Load Docker image layer cache
           no_output_timeout: 30m
@@ -147,17 +166,27 @@ jobs:
             fi
       - run:
           name: Run tests
+          no_output_timeout: 2h
           command: |
+            mkdir -p /tmp/work
             docker run -it --rm=false -e TEST_DATA_HOME=/data/ -e TEST_OUTPUT_DIR=/out \
-                   -v /tmp/data:/data:ro -v /tmp/src:/src -v /tmp/tests:/out -w /src/sdcflows \
-                   poldracklab/sdcflows:latest \
-                   pytest -n auto --junit-xml=/out/pytest.xml sdcflows
+                   -v /tmp/data/fslicense.txt:/opt/freesurfer/license.txt:ro -e FS_LICENSE=/opt/freesurfer/license.txt \
+                   -v /tmp/work:/work -e TEST_WORK_DIR=/work \
+                   -v /tmp/data/nipype.cfg:/home/sdcflows/.nipype/nipype.cfg \
+                   -v /tmp/data:/data:ro -v /tmp/src:/src -v /tmp/tests:/out \
+                   -w /work poldracklab/sdcflows:latest \
+                   pytest -v --junit-xml=/out/pytest.xml /src/sdcflows/sdcflows
 
       - store_artifacts:
           path: /tmp/tests
-
+      - store_artifacts:
+          path: /tmp/work
       - store_test_results:
           path: /tmp/tests
+      - save_cache:
+         key: workdir-v1-{{ .Branch }}-{{ .BuildNum }}
+         paths:
+            - /tmp/work
 
   deploy_docker:
     machine:

Dockerfile

@@ -19,6 +19,44 @@ RUN apt-get update && \
                     git && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
+# Installing freesurfer
+RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.1/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.1.tar.gz | tar zxv --no-same-owner -C /opt \
+    --exclude='freesurfer/diffusion' \
+    --exclude='freesurfer/docs' \
+    --exclude='freesurfer/fsfast' \
+    --exclude='freesurfer/lib/cuda' \
+    --exclude='freesurfer/lib/qt' \
+    --exclude='freesurfer/matlab' \
+    --exclude='freesurfer/mni/share/man' \
+    --exclude='freesurfer/subjects/fsaverage_sym' \
+    --exclude='freesurfer/subjects/fsaverage3' \
+    --exclude='freesurfer/subjects/fsaverage4' \
+    --exclude='freesurfer/subjects/cvs_avg35' \
+    --exclude='freesurfer/subjects/cvs_avg35_inMNI152' \
+    --exclude='freesurfer/subjects/bert' \
+    --exclude='freesurfer/subjects/lh.EC_average' \
+    --exclude='freesurfer/subjects/rh.EC_average' \
+    --exclude='freesurfer/subjects/sample-*.mgz' \
+    --exclude='freesurfer/subjects/V1_average' \
+    --exclude='freesurfer/trctrain'
+
+ENV FSL_DIR="/usr/share/fsl/5.0" \
+    OS="Linux" \
+    FS_OVERRIDE=0 \
+    FIX_VERTEX_AREA="" \
+    FSF_OUTPUT_FORMAT="nii.gz" \
+    FREESURFER_HOME="/opt/freesurfer"
+ENV SUBJECTS_DIR="$FREESURFER_HOME/subjects" \
+    FUNCTIONALS_DIR="$FREESURFER_HOME/sessions" \
+    MNI_DIR="$FREESURFER_HOME/mni" \
+    LOCAL_DIR="$FREESURFER_HOME/local" \
+    MINC_BIN_DIR="$FREESURFER_HOME/mni/bin" \
+    MINC_LIB_DIR="$FREESURFER_HOME/mni/lib" \
+    MNI_DATAPATH="$FREESURFER_HOME/mni/data"
+ENV PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \
+    MNI_PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \
+    PATH="$FREESURFER_HOME/bin:$FSFAST_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH"
+
 # Installing Neurodebian packages (FSL, AFNI, git)
 RUN curl -sSL "http://neuro.debian.net/lists/$( lsb_release -c | cut -f2 ).us-ca.full" >> /etc/apt/sources.list.d/neurodebian.sources.list && \
     apt-key add /usr/local/etc/neurodebian.gpg && \
@@ -32,8 +70,7 @@ RUN apt-get update && \
                     git-annex-standalone && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-ENV FSL_DIR="/usr/share/fsl/5.0" \
-    FSLDIR="/usr/share/fsl/5.0" \
+ENV FSLDIR="/usr/share/fsl/5.0" \
     FSLOUTPUTTYPE="NIFTI_GZ" \
     FSLMULTIFILEQUIT="TRUE" \
     POSSUMDIR="/usr/share/fsl/5.0" \

sdcflows/conftest.py

@@ -7,8 +7,9 @@
 
 test_data_env = os.getenv('TEST_DATA_HOME', str(Path.home() / 'sdcflows-tests'))
 test_output_dir = os.getenv('TEST_OUTPUT_DIR')
+test_workdir = os.getenv('TEST_WORK_DIR')
 
-layouts = {p.name: BIDSLayout(str(p), validate=False)
+layouts = {p.name: BIDSLayout(str(p), validate=False, derivatives=True)
            for p in Path(test_data_env).glob('*') if p.is_dir()}
 
 
@@ -28,6 +29,11 @@ def add_np(doctest_namespace):
         doctest_namespace[key] = Path(val.root)
 
 
+@pytest.fixture
+def workdir():
+    return None if test_workdir is None else Path(test_workdir)
+
+
 @pytest.fixture
 def output_path():
     return None if test_output_dir is None else Path(test_output_dir)