Merge branch 'master' of https://github.com/nipy/heudiconv into fix/group

Author: mgxd

heudiconv/bids.py

@@ -240,6 +240,27 @@ def add_participant_record(studydir, subject, age, sex):
             known_subjects = {l.split('\t')[0] for l in f.readlines()}
         if participant_id in known_subjects:
             return
+    else:
+        # Populate particpants.json (an optional file to describe column names in
+        # participant.tsv). This auto generation will make BIDS-validator happy.
+        participants_json = op.join(studydir, 'participants.json')
+        if not op.lexists(participants_json):
+            save_json(participants_json,
+                OrderedDict([
+                    ("participant_id", OrderedDict([
+                        ("Description", "Participant identifier")])),
+                    ("age", OrderedDict([
+                        ("Description", "Age in years (TODO - verify) as in the initial"
+                            " session, might not be correct for other sessions")])),
+                    ("sex", OrderedDict([
+                        ("Description", "self-rated by participant, M for male/F for "
+                            "female (TODO: verify)")])),
+                    ("group", OrderedDict([
+                        ("Description", "(TODO: adjust - by default everyone is in "
+                            "control group)")])),
+                ]),
+                sort_keys=False,
+                indent=2)
     # Add a new participant
     with open(participants_tsv, 'a') as f:
         f.write(
@@ -311,7 +332,8 @@ def save_scans_key(item, bids_files):
 
 def add_rows_to_scans_keys_file(fn, newrows):
     """
-    Add new rows to file fn for scans key filename
+    Add new rows to file fn for scans key filename and generate accompanying json
+    descriptor to make BIDS validator happy.
 
     Parameters
     ----------
@@ -334,6 +356,25 @@ def add_rows_to_scans_keys_file(fn, newrows):
         os.unlink(fn)
     else:
         fnames2info = newrows
+        # Populate _scans.json (an optional file to describe column names in
+        # _scans.tsv). This auto generation will make BIDS-validator happy.
+        scans_json = '.'.join(fn.split('.')[:-1] + ['json'])
+        if not op.lexists(scans_json):
+            save_json(scans_json,
+                OrderedDict([
+                    ("filename", OrderedDict([
+                        ("Description", "Name of the nifti file")])),
+                    ("acq_time", OrderedDict([
+                        ("LongName", "Acquisition time"),
+                        ("Description", "Acquisition time of the particular scan")])),
+                    ("operator", OrderedDict([
+                        ("Description", "Name of the operator")])),
+                    ("randstr", OrderedDict([
+                        ("LongName", "Random string"),
+                        ("Description", "md5 hash of UIDs")])),
+                ]),
+                sort_keys=False,
+                indent=2)
 
     header = ['filename', 'acq_time', 'operator', 'randstr']
     # prepare all the data rows
@@ -370,7 +411,7 @@ def get_formatted_scans_key_row(dcm_fn):
         time = dcm_data.ContentTime.split('.')[0]
         td = time + date
         acq_time = datetime.strptime(td, '%H%M%S%Y%m%d').isoformat()
-    except AttributeError as exc:
+    except (AttributeError, ValueError) as exc:
         lgr.warning("Failed to get date/time for the content: %s", str(exc))
         acq_time = None
     # add random string

heudiconv/cli/run.py

@@ -16,7 +16,7 @@
 import logging
 lgr = logging.getLogger(__name__)
 
-INIT_MSG = "Running {packname} version {version}".format
+INIT_MSG = "Running {packname} version {version} latest {latest}".format
 
 
 def is_interactive():
@@ -139,8 +139,11 @@ def get_parser():
                        '(can be compressed) are supported in addition to '
                        'directory. All matching tarballs for a subject are '
                        'extracted and their content processed in a single '
-                       'pass. Note that you might need to surround the value '
-                       'with quotes to avoid {...} being considered by shell')
+                       'pass. If multiple tarballs are found, each is '
+                       'assumed to be a separate session and the --ses '
+                       'argument is ignored. Note that you might need to '
+                       'surround the value with quotes to avoid {...} being '
+                       'considered by shell')
     group.add_argument('--files', nargs='*',
                        help='Files (tarballs, dicoms) or directories '
                        'containing files to process. Cannot be provided if '
@@ -243,8 +246,16 @@ def process_args(args):
 
     outdir = op.abspath(args.outdir)
 
+    import etelemetry
+    try:
+        latest = etelemetry.get_project("nipy/heudiconv")
+    except Exception as e:
+        lgr.warning("Could not check for version updates: ", e)
+        latest = {"version": 'Unknown'}
+
     lgr.info(INIT_MSG(packname=__packagename__,
-                      version=__version__))
+                      version=__version__,
+                      latest=latest["version"]))
 
     if args.command:
         process_extra_commands(outdir, args)

heudiconv/convert.py

@@ -519,11 +519,11 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam
         #   echo times for all bids_files and see if they are all the same or not.
 
         # Check for varying echo times
-        echo_times = set(
+        echo_times = sorted(list(set(
             load_json(b).get('EchoTime', None)
             for b in bids_files
             if b
-        )
+        )))
 
         is_multiecho = len(echo_times) > 1
 
@@ -575,11 +575,13 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam
             # For multi-echo sequences, we have to specify the echo number in
             # the file name:
             if bids_file and is_multiecho:
-                # Get the EchoNumber from json file info.  If not present, it's echo-1
-                echo_number = fileinfo.get('EchoNumber', 1)
-
+                # Get the EchoNumber from json file info.  If not present, use EchoTime
+                if 'EchoNumber' in fileinfo.keys():
+                    echo_number = fileinfo['EchoNumber']
+                else:
+                    echo_number = echo_times.index(fileinfo['EchoTime']) + 1
 
-                supported_multiecho = ['_bold', '_epi', '_sbref', '_T1w', '_PDT2']
+                supported_multiecho = ['_bold', '_phase', '_epi', '_sbref', '_T1w', '_PDT2']
                 # Now, decide where to insert it.
                 # Insert it **before** the following string(s), whichever appears first.
                 for imgtype in supported_multiecho:

heudiconv/info.py

@@ -8,12 +8,27 @@
 uses the dcmstack package and dcm2niix tool to convert DICOM directories or
 tarballs into collections of NIfTI files following pre-defined heuristic(s)."""
 
+CLASSIFIERS = [
+    'Environment :: Console',
+    'Intended Audience :: Science/Research',
+    'License :: OSI Approved :: Apache Software License',
+    'Programming Language :: Python :: 2.7',
+    'Programming Language :: Python :: 3.5',
+    'Programming Language :: Python :: 3.6',
+    'Programming Language :: Python :: 3.7',
+    'Topic :: Scientific/Engineering'
+]
+
+PYTHON_REQUIRES = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
+
 REQUIRES = [
     'nibabel',
     'pydicom',
-    'nipype>=1.0.0',
+    'nipype >=1.0.0; python_version > "3.0"',
+    'nipype >=1.0.0,!=1.2.1,!=1.2.2; python_version == "2.7"',
     'pathlib',
     'dcmstack>=0.7',
+    'etelemetry',
 ]
 
 TESTS_REQUIRES = [

heudiconv/tests/test_heuristics.py

@@ -165,7 +165,13 @@ def test_notop(tmpdir, bidsoptions):
     runner(args)
 
     assert op.exists(pjoin(tmppath, 'Halchenko/Yarik/950_bids_test4'))
-    for fname in ['CHANGES', 'dataset_description.json', 'participants.tsv', 'README']:
+    for fname in [
+        'CHANGES',
+        'dataset_description.json',
+        'participants.tsv',
+        'README',
+        'participants.json'
+    ]:
         if 'notop' in bidsoptions:
             assert not op.exists(pjoin(tmppath, 'Halchenko/Yarik/950_bids_test4', fname))
         else:

heudiconv/tests/test_main.py

@@ -22,6 +22,7 @@
 from os.path import join as opj
 from six.moves import StringIO
 import stat
+import os.path as op
 
 
 @patch('sys.stdout', new_callable=StringIO)
@@ -205,6 +206,7 @@ def _check_rows(fn, rows):
         assert dates == sorted(dates)
 
     _check_rows(fn, rows)
+    assert op.exists(opj(tmpdir.strpath, 'file.json'))
     # add a new one
     extra_rows = {
         'a_new_file.nii.gz': ['2016adsfasd23', '', 'fasadfasdf'],

heudiconv/tests/test_utils.py

@@ -1,10 +1,16 @@
+import json
 import os
 import os.path as op
+
 from heudiconv.utils import (
     get_known_heuristics_with_descriptions,
     get_heuristic_description,
     load_heuristic,
-    json_dumps_pretty)
+    json_dumps_pretty,
+    load_json,
+    create_tree,
+    save_json,
+    JSONDecodeError)
 
 import pytest
 from .utils import HEURISTICS_PATH
@@ -58,4 +64,24 @@ def test_json_dumps_pretty():
           'Mar  3 2017 10:46:13 by eja'
     # just the date which reveals the issue
     # tstr = 'Mar  3 2017 10:46:13 by eja'
-    assert pretty({'WipMemBlock': tstr}) == '{\n  "WipMemBlock": "%s"\n}' % tstr
+    assert pretty({'WipMemBlock': tstr}) == '{\n  "WipMemBlock": "%s"\n}' % tstr
+
+
+def test_load_json(tmp_path, caplog):
+    # test invalid json
+    ifname = 'invalid.json'
+    invalid_json_file = str(tmp_path / ifname)
+    create_tree(str(tmp_path), {ifname: u"I'm Jason Bourne"})
+
+    with pytest.raises(JSONDecodeError):
+        load_json(str(invalid_json_file))
+
+    assert ifname in caplog.text
+
+    # test valid json
+    vcontent = {"secret": "spy"}
+    vfname = "valid.json"
+    valid_json_file = str(tmp_path / vfname)
+    save_json(valid_json_file, vcontent)
+    
+    assert load_json(valid_json_file) == vcontent

heudiconv/utils.py

@@ -19,6 +19,11 @@
 import logging
 lgr = logging.getLogger(__name__)
 
+if sys.version_info[0] > 2:
+    from json.decoder import JSONDecodeError
+else:
+    JSONDecodeError = ValueError
+
 
 seqinfo_fields = [
     'total_files_till_now',  # 0
@@ -172,10 +177,15 @@ def load_json(filename):
     -------
     data : dict
     """
-    with open(filename, 'r') as fp:
-        data = json.load(fp)
-    return data
+    try:
+        with open(filename, 'r') as fp:
+            data = json.load(fp)
+    except JSONDecodeError:
+        lgr.error("{fname} is not a valid json file".format(fname=filename))
+        raise
 
+    return data
+    
 
 def assure_no_file_exists(path):
     """Check if file or symlink (git-annex?) exists, and if so -- remove"""

setup.py

@@ -47,11 +47,13 @@ def findsome(subdir, extensions):
         description=ldict['__description__'],
         long_description=ldict['__longdesc__'],
         license=ldict['__license__'],
+        classifiers=ldict['CLASSIFIERS'],
         packages=heudiconv_pkgs,
         entry_points={'console_scripts': [
             'heudiconv=heudiconv.cli.run:main',
             'heudiconv_monitor=heudiconv.cli.monitor:main',
         ]},
+        python_requires=ldict['PYTHON_REQUIRES'],
         install_requires=ldict['REQUIRES'],
         extras_require=ldict['EXTRA_REQUIRES'],
         package_data={