Merge pull request #416 from mgxd/maint-deps

MAINT: Drop Python 2 support, update dcmstack + datalad

Author: mgxd

.travis.yml

@@ -1,10 +1,10 @@
 # vim ft=yaml
 language: python
 python:
-  - 2.7
   - 3.5
   - 3.6
   - 3.7
+  - 3.8
 
 cache:
   - apt

heudiconv/convert.py

@@ -72,8 +72,7 @@ def conversion_info(subject, outdir, info, filegroup, ses):
                 try:
                     files = filegroup[item]
                 except KeyError:
-                    PY3 = sys.version_info[0] >= 3
-                    files = filegroup[(str if PY3 else unicode)(item)]
+                    files = filegroup[str(item)]
                 outprefix = template.format(**parameters)
                 convert_info.append((op.join(outpath, outprefix),
                                     outtype, files))

heudiconv/external/dlad.py

@@ -10,7 +10,7 @@
 
 lgr = logging.getLogger(__name__)
 
-MIN_VERSION = '0.7'
+MIN_VERSION = '0.12.2'
 
 
 def prepare_datalad(studydir, outdir, sid, session, seqinfo, dicoms, bids):
@@ -34,36 +34,33 @@ def prepare_datalad(studydir, outdir, sid, session, seqinfo, dicoms, bids):
 def add_to_datalad(topdir, studydir, msg, bids):
     """Do all necessary preparations (if were not done before) and save
     """
-    from datalad.api import create
+    import datalad.api as dl
     from datalad.api import Dataset
     from datalad.support.annexrepo import AnnexRepo
     from datalad.support.external_versions import external_versions
     assert external_versions['datalad'] >= MIN_VERSION, (
-      "Need datalad >= {}".format(MIN_VERSION))  # add to reqs
+        "Need datalad >= {}".format(MIN_VERSION))  # add to reqs
 
-    create_kwargs = {}
-    if external_versions['datalad'] >= '0.10':
-        create_kwargs['fake_dates'] = True  # fake dates by default
 
     studyrelpath = op.relpath(studydir, topdir)
     assert not studyrelpath.startswith(op.pardir)  # so we are under
     # now we need to test and initiate a DataLad dataset all along the path
     curdir_ = topdir
     superds = None
-    subdirs = [''] + studyrelpath.split(op.sep)
+    subdirs = [''] + [d for d in studyrelpath.split(op.sep) if d != os.curdir]
     for isubdir, subdir in enumerate(subdirs):
         curdir_ = op.join(curdir_, subdir)
         ds = Dataset(curdir_)
         if not ds.is_installed():
             lgr.info("Initiating %s", ds)
             # would require annex > 20161018 for correct operation on annex v6
             # need to add .gitattributes first anyways
-            ds_ = create(curdir_, dataset=superds,
+            ds_ = dl.create(curdir_, dataset=superds,
                          force=True,
-                         no_annex=True,
+                         # initiate annex only at the bottom repository
+                         no_annex=isubdir<(len(subdirs)-1),
+                         fake_dates=True,
                          # shared_access='all',
-                         annex_version=6,
-                         **create_kwargs
                          )
             assert ds == ds_
         assert ds.is_installed()
@@ -93,17 +90,13 @@ def add_to_datalad(topdir, studydir, msg, bids):
     with open(gitattributes_path, 'wb') as f:
         f.write('\n'.join(known_attrs).encode('utf-8'))
 
-    # so for mortals it just looks like a regular directory!
-    if not ds.config.get('annex.thin'):
-        ds.config.add('annex.thin', 'true', where='local')
-    # initialize annex there if not yet initialized
-    AnnexRepo(ds.path, init=True)
+
     # ds might have memories of having ds.repo GitRepo
-    superds = None
-    del ds
-    ds = Dataset(studydir)
+    superds = Dataset(topdir)
+    assert op.realpath(ds.path) == op.realpath(studydir)
+    assert isinstance(ds.repo, AnnexRepo)
     # Add doesn't have all the options of save such as msg and supers
-    ds.add('.gitattributes', to_git=True, save=False)
+    ds.save(path=['.gitattributes'], message="Custom .gitattributes", to_git=True)
     dsh = dsh_path = None
     if op.lexists(op.join(ds.path, '.heudiconv')):
         dsh_path = op.join(ds.path, '.heudiconv')
@@ -120,7 +113,6 @@ def add_to_datalad(topdir, studydir, msg, bids):
             else:
                 dsh = ds.create(path='.heudiconv',
                                 force=True,
-                                **create_kwargs
                                 # shared_access='all'
                                 )
         # Since .heudiconv could contain sensitive information
@@ -146,7 +138,7 @@ def add_to_datalad(topdir, studydir, msg, bids):
     mark_sensitive(ds, '*/*/anat')  # within ses/subj
     if dsh_path:
         mark_sensitive(ds, '.heudiconv')  # entire .heudiconv!
-    ds.save(message=msg, recursive=True, super_datasets=True)
+    superds.save(path=ds.path, message=msg, recursive=True)
 
     assert not ds.repo.dirty
     # TODO:  they are still appearing as native annex symlinked beasts

heudiconv/info.py

@@ -12,22 +12,21 @@
     'Environment :: Console',
     'Intended Audience :: Science/Research',
     'License :: OSI Approved :: Apache Software License',
-    'Programming Language :: Python :: 2.7',
     'Programming Language :: Python :: 3.5',
     'Programming Language :: Python :: 3.6',
     'Programming Language :: Python :: 3.7',
     'Topic :: Scientific/Engineering'
 ]
 
-PYTHON_REQUIRES = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
+PYTHON_REQUIRES = ">=3.5"
 
 REQUIRES = [
     'nibabel',
     'pydicom',
     'nipype >=1.0.0; python_version > "3.0"',
     'nipype >=1.0.0,!=1.2.1,!=1.2.2; python_version == "2.7"',
     'pathlib',
-    'dcmstack>=0.7',
+    'dcmstack>=0.8',
     'etelemetry',
     'filelock>=3.0.12',
 ]
@@ -43,7 +42,7 @@
 EXTRA_REQUIRES = {
     'tests': TESTS_REQUIRES,
     'extras': [],  # Requires patched version ATM ['dcmstack'],
-    'datalad': ['datalad']
+    'datalad': ['datalad >=0.12.2']
 }
 
 # Flatten the lists

heudiconv/tests/test_main.py

@@ -35,8 +35,7 @@ def test_main_help(stdout):
     assert stdout.getvalue().startswith("usage: ")
 
 
-@patch('sys.stderr' if sys.version_info[:2] <= (3, 3) else
-       'sys.stdout', new_callable=StringIO)
+@patch('sys.stdout', new_callable=StringIO)
 def test_main_version(std):
     with pytest.raises(SystemExit):
         runner(['--version'])

heudiconv/tests/utils.py

@@ -52,9 +52,9 @@ def fetch_data(tmpdir, dataset, getpath=None):
     """
     from datalad import api
     targetdir = op.join(tmpdir, op.basename(dataset))
-    api.install(path=targetdir,
+    ds = api.install(path=targetdir,
                 source='http://datasets-tests.datalad.org/{}'.format(dataset))
 
     getdir = targetdir + (op.sep + getpath if getpath is not None else '')
-    api.get(getdir)
+    ds.get(getdir)
     return targetdir

heudiconv/utils.py

@@ -19,10 +19,7 @@
 import logging
 lgr = logging.getLogger(__name__)
 
-if sys.version_info[0] > 2:
-    from json.decoder import JSONDecodeError
-else:
-    JSONDecodeError = ValueError
+from json.decoder import JSONDecodeError
 
 
 seqinfo_fields = [
@@ -115,9 +112,7 @@ def anonymize_sid(sid, anon_sid_cmd):
     cmd = [anon_sid_cmd, sid]
     shell_return = check_output(cmd)
 
-    if all([sys.version_info[0] > 2,
-            isinstance(shell_return, bytes),
-            isinstance(sid, str)]):
+    if isinstance(shell_return, bytes) and isinstance(sid, str):
         anon_sid = shell_return.decode()
     else:
         anon_sid = shell_return
@@ -490,8 +485,6 @@ def create_tree(path, tree, archives_leading_dir=True):
             create_tree(full_name, load, archives_leading_dir=archives_leading_dir)
         else:
             with open(full_name, 'w') as f:
-                if sys.version_info[0] == 2 and not isinstance(load, str):
-                    load = load.encode('utf-8')
                 f.write(load)
         if executable:
             os.chmod(full_name, os.stat(full_name).st_mode | stat.S_IEXEC)