Merge pull request #305 from oesteban/fix/250-getters

[MNT] Deprecate getters module

Author: oesteban

.circleci/config.yml

@@ -66,13 +66,61 @@ jobs:
       #     paths:
       #       - cache/docker.tar.gz
 
+  get_data:
+    machine:
+      # Ubuntu 14.04 with Docker 17.10.0-ce
+      image: circleci/classic:201711-01
+    working_directory: /home/circleci/data
+    steps:
+      - restore_cache:
+          keys:
+            - data-v1-{{ epoch }}
+            - data-v1-
+      - run:
+          name: Get test data from ds000003
+          command: |
+            mkdir -p /tmp/data
+            if [[ ! -d /tmp/data/ds003_downsampled ]]; then
+              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
+                -O ds003_downsampled.tar.gz "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/57f328f6b83f6901ef94cf70"
+              tar xvzf ds003_downsampled.tar.gz -C /tmp/data/
+            else
+              echo "Dataset ds000003 was cached"
+            fi
+      - run:
+          name: Get BIDS test data stub
+          command: |
+            mkdir -p /tmp/data
+            if [[ ! -d /tmp/data/BIDS-examples-1-enh-ds054 ]]; then
+              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
+                -O BIDS-examples-1-enh-ds054.zip "http://github.com/chrisfilo/BIDS-examples-1/archive/enh/ds054.zip"
+              unzip BIDS-examples-1-enh-ds054.zip -d /tmp/data/
+            else
+              echo "BIDS stub was cached"
+            fi
+      - run:
+          name: Store FreeSurfer license file
+          command: |
+            mkdir -p /tmp/fslicense
+            printf "$FS_LICENSE_CONTENT" | base64 -d  >> /tmp/fslicense/license.txt
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - data
+            - fslicense
+      - save_cache:
+         key: data-v1-{{ epoch }}
+         paths:
+            - /tmp/data
+
+
   test_pytest:
     machine:
       image: circleci/classic:201711-01
     working_directory: /tmp/tests
     steps:
-      # - attach_workspace:
-      #     at: /tmp
+      - attach_workspace:
+          at: /tmp
       - restore_cache:
           keys:
             - docker-v1-{{ .Branch }}-{{ epoch }}
@@ -92,11 +140,6 @@ jobs:
               pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
               docker images
             fi
-      - run:
-          name: Store FreeSurfer license file
-          command: |
-            mkdir -p /tmp/fslicense
-            printf "$FS_LICENSE_CONTENT" | base64 -d  >> /tmp/fslicense/license.txt
       - run:
           name: Set PR number
           command: |
@@ -113,6 +156,7 @@ jobs:
             sudo setfacl -d -m group:ubuntu:rwx $PWD
             sudo setfacl -m group:ubuntu:rwx $PWD
             docker run -it --rm=false \
+              -e TEST_DATA_HOME=/data -v /tmp/data:/data \
               -v ${PWD}:/tmp niworkflows:py3 \
               pytest --junit-xml=/tmp/pytest.xml \
                      --cov niworkflows --cov-report xml:/tmp/unittests.xml \
@@ -132,6 +176,7 @@ jobs:
           command: |
             docker run -it --rm=false \
               -e SAVE_CIRCLE_ARTIFACTS="/tmp" \
+              -e TEST_DATA_HOME=/data -v /tmp/data:/data \
               -v /tmp/fslicense/license.txt:/opt/freesurfer/license.txt:ro \
               -v ${PWD}:/tmp niworkflows:py3 \
               pytest -n auto --junit-xml=/tmp/reportlets.xml \
@@ -174,18 +219,22 @@ workflows:
           filters:
             tags:
               only: /.*/
+      - get_data:
+          filters:
+            tags:
+              only: /.*/
 
       - test_pytest:
           requires:
             - build
+            - get_data
           filters:
             branches:
               ignore: /docs?\/.*/
             tags:
               only: /.*/
       - deploy:
           requires:
-            - build
             - test_pytest
           filters:
             branches:

Dockerfile

@@ -169,9 +169,7 @@ RUN echo "${VERSION}" > /src/niworkflows/niworkflows/VERSION && \
     pip install .[all] && \
     rm -rf ~/.cache/pip
 
-# Pre-install data
-RUN python -c 'from niworkflows.data.getters import get_ds003_downsampled; get_ds003_downsampled()'
-
+# Final settings
 WORKDIR /tmp
 ARG BUILD_DATE
 ARG VCS_REF

niworkflows/conftest.py

@@ -4,16 +4,17 @@
 import numpy
 import pytest
 
-from . import data
 from .utils.bids import collect_data
 
-data_dir = data.get_bids_examples(variant='BIDS-examples-1-enh-ds054')
+test_data_env = os.getenv('TEST_DATA_HOME',
+                          str(Path.home() / '.cache' / 'stanford-crn'))
+data_dir = Path(test_data_env) / 'BIDS-examples-1-enh-ds054'
 
 
 @pytest.fixture(autouse=True)
 def add_np(doctest_namespace):
     doctest_namespace['np'] = numpy
     doctest_namespace['os'] = os
     doctest_namespace['Path'] = Path
-    doctest_namespace['datadir'] = Path(data_dir)
+    doctest_namespace['datadir'] = data_dir
     doctest_namespace['bids_collect_data'] = collect_data

niworkflows/data/__init__.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
-# vi: set ft=python sts=4 ts=4 sw=4 et:
-""" Data fetchers module """
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-from .getters import (
-    get_brainweb_1mm_normal,
-    get_ds003_downsampled,
-    get_dataset,
-    get_template,
-    get_bids_examples,
-    TEMPLATE_ALIASES,
-)

niworkflows/data/getters.py

@@ -7,8 +7,12 @@
 """
 from __future__ import print_function, division, absolute_import, unicode_literals
 
+from warnings import warn
 from ..data.utils import fetch_file
 
+DEPRECATION_MSG = """\
+Module ``niworkflows.data.getters`` has been deprecated in 0.7.1, \
+and will be removed in 0.8."""
 
 OSF_PROJECT_URL = ('https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/')
 OSF_RESOURCES = {
@@ -54,6 +58,7 @@ def get_dataset(dataset_name, dataset_prefix=None, data_dir=None,
     :param str url: download URL of the dataset. Overwrite the default URL.
 
     """
+    warn(DEPRECATION_MSG)
     file_id, md5 = OSF_RESOURCES[dataset_name]
     if url is None:
         url = '{}/{}'.format(OSF_PROJECT_URL, file_id)
@@ -63,6 +68,7 @@ def get_dataset(dataset_name, dataset_prefix=None, data_dir=None,
 
 def get_template(template_name, data_dir=None, url=None, resume=True, verbose=1):
     """Download and load a template"""
+    warn(DEPRECATION_MSG)
     if template_name.startswith('tpl-'):
         template_name = template_name[4:]
 
@@ -80,6 +86,7 @@ def get_brainweb_1mm_normal(data_dir=None, url=None, resume=True, verbose=1):
     :param str url: download URL of the dataset. Overwrite the default URL.
 
     """
+    warn(DEPRECATION_MSG)
     return get_dataset('brainweb', data_dir=data_dir, url=url,
                        resume=resume, verbose=verbose)
 
@@ -92,13 +99,15 @@ def get_ds003_downsampled(data_dir=None, url=None, resume=True, verbose=1):
     :param str url: download URL of the dataset. Overwrite the default URL.
 
     """
+    warn(DEPRECATION_MSG)
     return get_dataset('ds003_downsampled', data_dir=data_dir, url=url,
                        resume=resume, verbose=verbose)
 
 
 def get_bids_examples(data_dir=None, url=None, resume=True, verbose=1,
                       variant='BIDS-examples-1-1.0.0-rc3u5'):
     """Download BIDS-examples-1"""
+    warn(DEPRECATION_MSG)
     variant = 'BIDS-examples-1-1.0.0-rc3u5' if variant not in BIDS_EXAMPLES else variant
     if url is None:
         url = BIDS_EXAMPLES[variant][0]

niworkflows/tests/conftest.py

@@ -3,15 +3,19 @@
 # vi: set ft=python sts=4 ts=4 sw=4 et:
 """ py.test configuration file """
 import os
+from pathlib import Path
 from tempfile import mkdtemp
 from datetime import datetime as dt
 import pytest
 from templateflow.api import get as get_template
-from niworkflows.data.getters import get_ds003_downsampled
 
 filepath = os.path.dirname(os.path.realpath(__file__))
 datadir = os.path.realpath(os.path.join(filepath, 'data'))
 
+test_data_env = os.getenv('TEST_DATA_HOME',
+                          str(Path.home() / '.cache' / 'stanford-crn'))
+data_dir = Path(test_data_env) / 'ds003_downsampled'
+
 
 def _run_interface_mock(objekt, runtime):
     runtime.returncode = 0
@@ -40,7 +44,7 @@ def reference_mask():
 
 @pytest.fixture
 def moving():
-    return os.path.join(get_ds003_downsampled(), 'sub-01/anat/sub-01_T1w.nii.gz')
+    return str(data_dir / 'sub-01/anat/sub-01_T1w.nii.gz')
 
 
 @pytest.fixture