rf/wip: removing boto3 requirements, removing some s3-related functions, updated travis.yml to see if it pushes to ghpages from travis

Author: akeshavan

.travis.yml

@@ -22,4 +22,4 @@ env:
   global:
   - GH_USER: nipy
   - GH_REPO: dmriprep
-  - secure: VmTfAChlLlC8WfK0QS9UzfTdI58so7YLkQdlTbB3WptqKIPDcGUSUjyB5ow9BO6nwupN0/zxaM3rneWJeJ7oEBQWGrbpdDkLWkCPB6htjECmUhfyWJccBH4ZnvVjDQH9Wl5SgNieJjGzI8QBLwnsvjzafH3VkP3SWFvqq4J3bGyDb97MUkWgDLdjy+3HsX6Ye7HCkYnLhi2BSwON/asQLd+Pr1FOv+P2tsJgwz9X4gkzws8gP5jGM4J5jxwYRUQKZKjgiEvDZbL0OqTvH/jxELKwcuIhDrqVeemZn5wWLRt7pvFUAzAwUliE0/NROUFi57RhU9NrwOZAo7ek3muM33AgBGodpkFJJ9vmsMJf5Diz71YN3xvMMD2vlfDByjcPOqYu/uM5o6sbuLZNDc+NExlsSSskMOnOBGz5e+Exo6c+X6Dxqx3rT6jmBPVTv7E9RiL1wwXF18fpCDrXwZb/mmxxbB0Bm3x/etIbi/mmbGlPo8K2GgOyzACPK3DHA6lRPd97qwU5cp/+7IWr7LWrVxAl6VS19/GIkVVlmBS3/7IJSg4ANgLAVZzXrcRcJTEY2xsF3TFJm840v6ArkFEogj84dGcxr1o9oLFLh2ZBSatoqSwdL8NezJruoOYDGzW37thp51iAHwNZ+Zfpv/PIcmWRrsBbhV+FA8KkFxmfIR0=
+  - secure: raeexu68fu/Hu4Ey0VkP3ljvAJut2X8xRAkTXO9nL6HlWQGt1pjYhXSxaLLZVQpgpDcDeOziUUCb4y7UALFHTkIrYrzp/Jq3Cn10tmsItEMAhH0cfzTuMr0Huhl/Zeiw67Qn16W7xpYEtV7zHBOYicQIFvzAOhNWDy/A/9c7XkKsEU47FqN/vxodCv+WMHQyvuM3ceaofccF+mVekAg7PztPrukzpwP+uishNfSeoHI3WeUDjX82bQZ3KE/oRHrB+YCB5rFg0wfekfsUnkyhS9HWF9CECM5mKWcPle0AZpjQXODrUIXk8dRSAprAzD1XGxRzmYG3nYPGx9a6gm6ofueHbLPery/0cLVYNX4IHBR2UpW6aA9VsgMBgG8LAdGiMqvvio2CZ+/tGdmIXx5twRCTGaJ4qJUdyyL881ChaUl8rBde7R8EtCBThKeV8Tnnun2PfLPt12qAc+EAc1r6a9zf/Mb8ahjTrZSU1heUn5irs44KvJk+QuftW9hF6hIvBfug7SHBIAdvyCd8TjMNShJNVPqQZU6NWSwTAGAj4Y/QsocvxYRoKHIhpyHbMZ+Lj+P27VfNYdkTe/viQcLEg8x4YYHCaKiYl4iwepEtyevtbtzUtb1Ba5clotgisw/PFao9q6GrWuy57eb/ZM8aKSHfb4JhN0swmi1t0uBOzP0=

dmriprep/__init__.py

@@ -10,21 +10,17 @@
 import logging
 import os
 
-from .dmriprep import upload_to_s3  # noqa
-from . import fetch_bids_s3  # noqa
-
-
 module_logger = logging.getLogger(__name__)
 
 # get the log level from environment variable
-if "AFQ_LOGLEVEL" in os.environ:
-    loglevel = os.environ['AFQ_LOGLEVEL']
+if "DMIRPREP_LOGLEVEL" in os.environ:
+    loglevel = os.environ['DMRIPREP_LOGLEVEL']
     module_logger.setLevel(getattr(logging, loglevel.upper()))
 else:
     module_logger.setLevel(logging.WARNING)
 
 # create a file handler
-logpath = os.path.join(os.path.expanduser('~'), '.afq', 'afq.log')
+logpath = os.path.join(os.path.expanduser('~'), '.dmriprep', 'dmriprep.log')
 
 # Create the config directory if it doesn't exist
 logdir = os.path.dirname(logpath)
@@ -49,8 +45,3 @@
 # add the handlers to the logger
 module_logger.addHandler(handler)
 module_logger.info('Started new dmriprep session')
-
-# Reduce verbosity of the boto logs
-logging.getLogger('boto').setLevel(logging.WARNING)
-logging.getLogger('boto3').setLevel(logging.WARNING)
-logging.getLogger('botocore').setLevel(logging.WARNING)

dmriprep/base.py

@@ -4,10 +4,6 @@
 
 from collections import namedtuple
 
-import boto3
-from botocore import UNSIGNED
-from botocore.client import Config
-
 
 # Define the different namedtuple return types
 InputFiles = namedtuple(
@@ -20,6 +16,3 @@
     'InputFilesWithSession',
     ['subject', 'site', 'session', 'files', 'file_type']
 )
-
-# Global s3 client to preserve anonymous config
-s3_client = boto3.client('s3', config=Config(signature_version=UNSIGNED))

dmriprep/dmriprep.py

@@ -7,91 +7,12 @@
 import os.path as op
 import subprocess
 
-import boto3
-import nibabel as nib
-
-from . import fetch_bids_s3 as fetch
-from .run_1 import run_dmriprep
+from .run import run_dmriprep
 
 
 mod_logger = logging.getLogger(__name__)
 
 
-def move_t1_to_freesurfer(t1_file):
-    """Move the T1 file back into the freesurfer directory.
-
-    This step is specific to the HBN dataset where the T1 files
-    are outside of the derivatives/sub-XXX/freesurfer directory.
-
-    Parameters
-    ----------
-    t1_file : string
-        Path to the T1 weighted nifty file
-    """
-    freesurfer_path = op.join(op.dirname(t1_file), 'freesurfer')
-
-    convert_cmd = 'mri_convert {in_:s} {out_:s}'.format(
-        in_=t1_file, out_=op.join(freesurfer_path, 'mri', 'orig.mgz')
-    )
-
-    fnull = open(os.devnull, 'w')
-    cmd = subprocess.call(convert_cmd.split(),
-                          stdout=fnull,
-                          stderr=subprocess.STDOUT)
-
-
-def upload_to_s3(output_files, outdir, bucket, prefix, site, session, subject):
-    """Upload output files to S3, using key format specified by input params
-
-    Parameters
-    ----------
-    output_files : list
-        Output files to transfer to S3. Assume that the user has passed in
-        relative paths that are appropriate to fill in after the 'dmriprep'
-        directory.
-
-    outdir : string
-        a path to the root of the data
-
-    bucket : string
-        Output S3 bucket
-
-    prefix : string
-        Output S3 prefix
-
-    site : string
-        Site ID, e.g. 'side-SI'
-
-    session : string
-        Session ID, e.g. 'sess-001'
-
-    subject : string
-        Subject ID, e.g. 'sub-ABCXYZ'
-
-    Returns
-    -------
-    list
-        S3 keys for each output file
-    """
-    s3 = boto3.client('s3')
-
-    def filename2s3key(filename):
-        return '/'.join([
-            prefix, site, subject, session,
-            'derivatives', 'dmriprep',
-            filename
-        ])
-
-    for file in output_files:
-        with open(op.abspath(op.join(outdir, file)), 'rb') as fp:
-            s3.put_object(
-                Bucket=bucket,
-                Body=fp,
-                Key=filename2s3key(file),
-            )
-
-    return [filename2s3key(f) for f in output_files]
-
 
 def pre_afq_individual(input_s3_keys, s3_prefix, out_bucket,
                        in_bucket='fcp-indi', workdir=op.abspath('.')):