ref+tst: some refactoring, intial testing

Author: mgxd

heudiconv/convert.py

@@ -242,7 +242,7 @@ def convert(items, converter, scaninfo_suffix, custom_callable, with_prov,
         if not isinstance(outtypes, (list, tuple)):
             outtypes = (outtypes,)
 
-        prefix_dirname = op.dirname(prefix + '.ext')
+        prefix_dirname = op.dirname(prefix)
         outname_bids = prefix + '.json'
         bids_outfiles = []
         lgr.info('Converting %s (%d DICOMs) -> %s . '
@@ -442,8 +442,7 @@ def save_converted_files(res, item_dicoms, bids, outtype, prefix, outname_bids,
     """
     from nipype.interfaces.base import isdefined
 
-    prefix_dirname  = op.dirname(prefix + '.ext')
-    prefix_basename = op.basename(prefix)
+    prefix_dirname, prefix_basename = op.split(prefix)
 
     bids_outfiles = []
     res_files = res.outputs.converted_files
@@ -475,8 +474,8 @@ def save_converted_files(res, item_dicoms, bids, outtype, prefix, outname_bids,
         # Also copy BIDS files although they might need to
         # be merged/postprocessed later
         bids_files = sorted(res.outputs.bids
-                      if len(res.outputs.bids) == len(res_files)
-                      else [None] * len(res_files))
+                     if len(res.outputs.bids) == len(res_files)
+                     else [None] * len(res_files))
 
         ###   Do we have a multi-echo series?   ###
         #   Some Siemens sequences (e.g. CMRR's MB-EPI) set the label 'TE1',
@@ -488,23 +487,24 @@ def save_converted_files(res, item_dicoms, bids, outtype, prefix, outname_bids,
         #   series. To do that, the most straightforward way is to read the
         #   echo times for all bids_files and see if they are all the same or not.
 
-        # Get the echo times while not breaking non-BIDS compliance
-        echo_times = []
+        # Check for echotime information
+        echo_times = set()
+
         for bids_file in bids_files:
             if bids_file:
-                echo_times.append(load_json(bids_file).get('EchoTime'))
+                # check for varying EchoTimes
+                echot = load_json(bids_file).get('EchoTime', None)
+                if echot is not None:
+                    echo_times.add(echot)
 
         # To see if the echo times are the same, convert it to a set and see if
-        #   only one remains:
-        multiecho = False
-        if echo_times:
-            multiecho = len(set(echo_times)) == 1
+        # only one remains:
+        is_multiecho = len(echo_times) >= 1 if echo_times else False
 
         ### Loop through the bids_files, set the output name and save files
-
         for fl, suffix, bids_file in zip(res_files, suffixes, bids_files):
-            # load the json file info:
-            # TODO: time performance
+
+            # TODO: monitor conversion duration
             if bids_file:
                 fileinfo = load_json(bids_file)
 
@@ -515,7 +515,7 @@ def save_converted_files(res, item_dicoms, bids, outtype, prefix, outname_bids,
             # _sbref sequences reconstructing magnitude and phase generate
             # two NIfTI files IN THE SAME SERIES, so we cannot just add
             # the suffix, if we want to be bids compliant:
-            if (bids_file and (this_prefix_basename.endswith('_sbref'))):
+            if bids_file and this_prefix_basename.endswith('_sbref'):
                 # Check to see if it is magnitude or phase reconstruction:
                 if 'M' in fileinfo.get('ImageType'):
                     mag_or_phase = 'magnitude'
@@ -525,7 +525,7 @@ def save_converted_files(res, item_dicoms, bids, outtype, prefix, outname_bids,
                     mag_or_phase = suffix
 
                 # Insert reconstruction label
-                if not (("_rec-%s" % mag_or_phase) in this_prefix_basename):
+                if not ("_rec-%s" % mag_or_phase) in this_prefix_basename:
 
                     # If "_rec-" is specified, prepend the 'mag_or_phase' value.
                     if ('_rec-' in this_prefix_basename):
@@ -548,23 +548,23 @@ def save_converted_files(res, item_dicoms, bids, outtype, prefix, outname_bids,
             # (Note: it can be _sbref and multiecho, so don't use "elif"):
             # For multi-echo sequences, we have to specify the echo number in
             # the file name:
-            if bids and multiecho:
+            if bids_file and is_multiecho:
                 # Get the EchoNumber from json file info.  If not present, it's echo-1
                 echo_number = fileinfo.get('EchoNumber', 1)
 
 
-                supported_multiecho = ['_bold', '_sbref', '_T1w']  # epi?
+                supported_multiecho = ['_bold', '_epi', '_sbref', '_T1w']
                 # Now, decide where to insert it.
                 # Insert it **before** the following string(s), whichever appears first.
-                for imgtype in ['_bold', '_sbref', '_T1w']:
+                for imgtype in supported_multiecho:
                     if (imgtype in this_prefix_basename):
                         this_prefix_basename = this_prefix_basename.replace(
                             imgtype, "_echo-%d%s" % (echo_number, imgtype)
                         )
                         break
 
             # For Scout runs with multiple NIfTI images per run:
-            if (bids and ('scout' in this_prefix_basename.lower())):
+            if bids and 'scout' in this_prefix_basename.lower():
                 # in some cases (more than one slice slab), there are several
                 #   NIfTI images in the scout run, so distinguish them with "_acq-"
                 spt = this_prefix_basename.split('_acq-Scout', 1)
@@ -573,7 +573,7 @@ def save_converted_files(res, item_dicoms, bids, outtype, prefix, outname_bids,
             # Fallback option:
             # If we have failed to modify this_prefix_basename, because it didn't fall
             #   into any of the options above, just add the suffix at the end:
-            if ( this_prefix_basename == prefix_basename ):
+            if this_prefix_basename == prefix_basename:
                 this_prefix_basename += suffix
 
             # Finally, form the outname by stitching the directory and outtype:
@@ -586,6 +586,7 @@ def save_converted_files(res, item_dicoms, bids, outtype, prefix, outname_bids,
                 outname_bids_file = "%s.json" % (outname)
                 safe_copyfile(bids_file, outname_bids_file, overwrite)
                 bids_outfiles.append(outname_bids_file)
+
     # res_files is not a list
     else:
         outname = "{}.{}".format(prefix, outtype)

heudiconv/heuristics/bids-ME.py

@@ -0,0 +1,24 @@
+import os
+
+def create_key(template, outtype=('nii.gz',), annotation_classes=None):
+    if template is None or not template:
+        raise ValueError('Template must be a valid format string')
+    return template, outtype, annotation_classes
+
+def infotodict(seqinfo):
+    """Heuristic evaluator for determining which runs belong where
+
+    allowed template fields - follow python string module:
+
+    item: index within category
+    subject: participant id
+    seqitem: run number during scanning
+    subindex: sub index within group
+    """
+    bold = create_key('sub-{subject}/func/sub-{subject}_task-test_run-{item}_bold')
+
+    info = {bold: []}
+    for s in seqinfo:
+        if '_ME_' in s.series_description:
+            info[bold].append(s.series_id)
+    return info

heudiconv/tests/test_regression.py

@@ -1,6 +1,7 @@
 """Testing conversion with conversion saved on datalad"""
 import json
 from glob import glob
+import os.path as op
 
 import pytest
 
@@ -11,8 +12,8 @@
 except ImportError:
     have_datalad = False
 
-import heudiconv
 from heudiconv.cli.run import main as runner
+from heudiconv.utils import load_json
 # testing utilities
 from .utils import fetch_data, gen_heudiconv_args
 
@@ -24,12 +25,19 @@
 def test_conversion(tmpdir, subject, heuristic, anon_cmd):
     tmpdir.chdir()
     try:
-        datadir = fetch_data(tmpdir.strpath, subject)
+        datadir = fetch_data(tmpdir.strpath,
+                             "dbic/QA",  # path from datalad database root
+                             getpath=op.join('sourcedata', subject))
     except IncompleteResultsError as exc:
         pytest.skip("Failed to fetch test data: %s" % str(exc))
     outdir = tmpdir.mkdir('out').strpath
 
-    args = gen_heudiconv_args(datadir, outdir, subject, heuristic, anon_cmd)
+    args = gen_heudiconv_args(datadir,
+                              outdir,
+                              subject,
+                              heuristic,
+                              anon_cmd,
+                              template=op.join('sourcedata/{subject}/*/*/*.tgz'))
     runner(args) # run conversion
 
     # verify functionals were converted
@@ -38,8 +46,38 @@ def test_conversion(tmpdir, subject, heuristic, anon_cmd):
 
     # compare some json metadata
     json_ = '{}/task-rest_acq-24mm64sl1000tr32te600dyn_bold.json'.format
-    orig, conv = (json.load(open(json_(datadir))),
-                  json.load(open(json_(outdir))))
+    orig, conv = (load_json(json_(datadir)),
+                  load_json(json_(outdir)))
     keys = ['EchoTime', 'MagneticFieldStrength', 'Manufacturer', 'SliceTiming']
     for key in keys:
         assert orig[key] == conv[key]
+
+@pytest.mark.skipif(not have_datalad, reason="no datalad")
+def test_multiecho(tmpdir, subject='MEEPI', heuristic='bids-ME.py'):
+    tmpdir.chdir()
+    try:
+        datadir = fetch_data(tmpdir.strpath, "dicoms/velasco/MEEPI")
+    except IncompleteResultsError as exc:
+        pytest.skip("Failed to fetch test data: %s" % str(exc))
+
+    outdir = tmpdir.mkdir('out').strpath
+    args = gen_heudiconv_args(datadir, outdir, subject, heuristic)
+    runner(args) # run conversion
+
+    # check if we have echo functionals
+    echoes = glob(op.join('out', 'sub-' + subject, 'func', '*echo*nii.gz'))
+    assert len(echoes) == 3
+
+    # check EchoTime of each functional
+    # ET1 < ET2 < ET3
+    prev_echo = 0
+    for echo in sorted(echoes):
+        _json = echo.replace('.nii.gz', '.json')
+        assert _json
+        echotime = load_json(_json).get('EchoTime', None)
+        assert echotime > prev_echo
+        prev_echo = echotime
+
+    events = glob(op.join('out', 'sub-' + subject, 'func', '*events.tsv'))
+    for event in events:
+        assert 'echo-' not in event

heudiconv/tests/utils.py

@@ -5,14 +5,24 @@
 TESTS_DATA_PATH = op.join(op.dirname(__file__), 'data')
 
 
-def gen_heudiconv_args(datadir, outdir, subject, heuristic_file, anon_cmd=None, xargs=None):
+def gen_heudiconv_args(datadir, outdir, subject, heuristic_file,
+                       anon_cmd=None, template=None, xargs=None):
     heuristic = op.realpath(op.join(HEURISTICS_PATH, heuristic_file))
-    args = ["-d", op.join(datadir, 'sourcedata/{subject}/*/*/*.tgz'),
+
+    if template:
+        # use --dicom_dir_template
+        args = ["-d", op.join(datadir, template)]
+    else:
+        args = ["--files", datadir]
+
+    args.extend([
             "-c", "dcm2niix",
             "-o", outdir,
             "-s", subject,
             "-f", heuristic,
-            "--bids",]
+            "--bids",
+            "--minmeta",]
+            )
     if anon_cmd:
         args += ["--anon-cmd", op.join(op.dirname(__file__), anon_cmd), "-a", outdir]
     if xargs:
@@ -21,10 +31,30 @@ def gen_heudiconv_args(datadir, outdir, subject, heuristic_file, anon_cmd=None,
     return args
 
 
-def fetch_data(tmpdir, subject):
-    """Fetches some test dicoms using datalad"""
+def fetch_data(tmpdir, dataset, getpath=None):
+    """
+    Utility function to interface with datalad database.
+    Performs datalad `install` and datalad `get` operations.
+
+    Parameters
+    ----------
+    tmpdir : str
+        directory to temporarily store data
+    dataset : str
+        dataset path from `http://datasets-tests.datalad.org`
+    getpath : str [optional]
+        exclusive path to get
+
+    Returns
+    -------
+    targetdir : str
+        directory with installed dataset
+    """
     from datalad import api
-    targetdir = op.join(tmpdir, 'QA')
-    api.install(path=targetdir, source='http://datasets-tests.datalad.org/dbic/QA')
-    api.get('{}/sourcedata/{}'.format(targetdir, subject))
+    targetdir = op.join(tmpdir, op.basename(dataset))
+    api.install(path=targetdir,
+                source='http://datasets-tests.datalad.org/{}'.format(dataset))
+
+    getdir = targetdir + (op.sep + getpath if getpath is not None else '')
+    api.get(getdir)
     return targetdir