Merge tag 'v0.5.1' into debian

Heudiconv v0.5.1

* tag 'v0.5.1': (35 commits)
  rel: proper release day
  Fix links to heuristcs in README.md
  doc+enh: update docker dcm2niix, add entry to changelog
  doc: changelog + version bump
  fix: do not load phasediff until exists
  BF: workaround for elderly buggy python 2.7 exec
  BF: for datalad 0.10.0  - check/populate .gitattributes correctly
  BF+ENH: set dist-restrictions metadata correctly
  Implement renaming suggested by @yarikoptic
  Provide SeriesInstanceUID in SeqInfo
  enh: update docker build with latest dcm2niix release
  tst: add check for running with no heuristic
  fix: syntax
  fix: simply regenerate conversion table if heurisitic has changed
  fix: various small fixes
  fix multiple conversion #196
  BF: Don't create a sourcedata-dir just for a README; Check lexists for the actual file instead
  BF: Do not overwrite existing files with BIDS templates
  DOC: Point to James' youtube video
  ENH: print pip version right before installing dev-requirements
  ...

Author: yarikoptic

.travis.yml

@@ -25,6 +25,7 @@ before_install:
   - pip install --upgrade virtualenv
   - virtualenv --python=python venv
   - source venv/bin/activate
+  - pip --version # check again since seems that python_requires='>=3.5' in secretstorage is not in effect
   - python --version # just to check
   - pip install -r dev-requirements.txt
   - pip install requests # below installs pyld but that assumes we have requests already

CHANGELOG.md

@@ -4,6 +4,33 @@ All notable changes to this project will be documented (for humans) in this file
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [0.5.1] - 2018-07-05
+Bugfix release
+
+### Added
+- Video tutorial / updated slides
+- Helper to set metadata restrictions correctly
+- Usage is now shown when run without arguments
+- New fields to Seqinfo
+  - series_uid
+- Reproin heuristic support for xnat
+### Changed
+- Dockerfile updated to use `dcm2niix v1.0.20180622`
+- Conversion table will be regenerated if heurisic has changed
+- Do not touch existing BIDS files
+  - events.tsv
+  - task JSON
+### Fixed
+- Python 2.7.8 and older installation
+- Support for updated packages
+  - `Datalad` 0.10
+  - `pydicom` 1.0.2
+- Later versions of `pydicom` are prioritized first
+- JSON pretty print should not remove spaces
+- Phasediff fieldmaps behavior
+  - ensure phasediff exists
+  - support for single magnitude acquisitions
+
 ## [0.5] - 2018-03-01
 The first release after major refactoring:
 

Dockerfile

@@ -5,7 +5,7 @@
 # pull request on our GitHub repository:
 #     https://github.com/kaczmarj/neurodocker
 #
-# Timestamp: 2018-02-02 16:32:55
+# Timestamp: 2018-06-29 18:14:08
 
 FROM debian:stretch
 
@@ -32,27 +32,48 @@ RUN apt-get update -qq && apt-get install -yq --no-install-recommends  \
     && chmod -R 777 /neurodocker && chmod a+s /neurodocker
 ENTRYPOINT ["/neurodocker/startup.sh"]
 
-RUN apt-get update -qq \
-    && apt-get install -y -q --no-install-recommends git \
-                                                     gcc \
-                                                     pigz \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
 #------------------------
-# Install dcm2niix v1.0.20171215
+# Install dcm2niix v1.0.20180622
 #------------------------
 WORKDIR /tmp
 RUN deps='cmake g++ gcc git make pigz zlib1g-dev' \
     && apt-get update -qq && apt-get install -yq --no-install-recommends $deps \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
     && mkdir dcm2niix \
-    && curl -sSL https://github.com/rordenlab/dcm2niix/tarball/v1.0.20171215 | tar xz -C dcm2niix --strip-components 1 \
+    && curl -sSL https://github.com/rordenlab/dcm2niix/tarball/v1.0.20180622 | tar xz -C dcm2niix --strip-components 1 \
     && mkdir dcm2niix/build && cd dcm2niix/build \
     && cmake .. && make \
     && make install \
-    && rm -rf /tmp/*
+    && rm -rf /tmp/* \
+    && apt-get purge -y --auto-remove $deps
+
+#--------------------------------------------------
+# Add NeuroDebian repository
+# Please note that some packages downloaded through
+# NeuroDebian may have restrictive licenses.
+#--------------------------------------------------
+RUN apt-get update -qq && apt-get install -yq --no-install-recommends dirmngr gnupg \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+    && curl -sSL http://neuro.debian.net/lists/stretch.us-nh.full \
+    > /etc/apt/sources.list.d/neurodebian.sources.list \
+    && curl -sSL https://dl.dropbox.com/s/zxs209o955q6vkg/neurodebian.gpg \
+    | apt-key add - \
+    && (apt-key adv --refresh-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 || true) \
+    && apt-get update
+
+# Install NeuroDebian packages
+RUN apt-get update -qq && apt-get install -yq --no-install-recommends git-annex-standalone \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN apt-get update -qq \
+    && apt-get install -y -q --no-install-recommends git \
+                                                     gcc \
+                                                     pigz \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 COPY [".", "/src/heudiconv"]
 
@@ -83,26 +104,6 @@ RUN conda create -y -q --name neuro python=2 \
     && sync \
     && sed -i '$isource activate neuro' $ND_ENTRYPOINT
 
-#--------------------------------------------------
-# Add NeuroDebian repository
-# Please note that some packages downloaded through
-# NeuroDebian may have restrictive licenses.
-#--------------------------------------------------
-RUN apt-get update -qq && apt-get install -yq --no-install-recommends dirmngr gnupg \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-    && curl -sSL http://neuro.debian.net/lists/stretch.us-nh.full \
-    > /etc/apt/sources.list.d/neurodebian.sources.list \
-    && curl -sSL https://dl.dropbox.com/s/zxs209o955q6vkg/neurodebian.gpg \
-    | apt-key add - \
-    && (apt-key adv --refresh-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 || true) \
-    && apt-get update
-
-# Install NeuroDebian packages
-RUN apt-get update -qq && apt-get install -yq --no-install-recommends git-annex-standalone \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
 ENTRYPOINT ["/neurodocker/startup.sh", "heudiconv"]
 
 #--------------------------------------
@@ -117,6 +118,20 @@ RUN echo '{ \
     \n      "debian:stretch" \
     \n    ], \
     \n    [ \
+    \n      "dcm2niix", \
+    \n      { \
+    \n        "version": "v1.0.20180622" \
+    \n      } \
+    \n    ], \
+    \n    [ \
+    \n      "neurodebian", \
+    \n      { \
+    \n        "os_codename": "stretch", \
+    \n        "download_server": "usa-nh", \
+    \n        "pkgs": "git-annex-standalone" \
+    \n      } \
+    \n    ], \
+    \n    [ \
     \n      "install", \
     \n      [ \
     \n        "git", \
@@ -125,12 +140,6 @@ RUN echo '{ \
     \n      ] \
     \n    ], \
     \n    [ \
-    \n      "dcm2niix", \
-    \n      { \
-    \n        "version": "v1.0.20171215" \
-    \n      } \
-    \n    ], \
-    \n    [ \
     \n      "copy", \
     \n      [ \
     \n        ".", \
@@ -142,23 +151,15 @@ RUN echo '{ \
     \n      { \
     \n        "env_name": "neuro", \
     \n        "conda_install": "python=2 traits=4.6.0", \
-    \n        "pip_install": "https://github.com/moloney/dcmstack/tarball/master /src/heudiconv[all]", \
-    \n        "activate": true \
-    \n      } \
-    \n    ], \
-    \n    [ \
-    \n      "neurodebian", \
-    \n      { \
-    \n        "os_codename": "stretch", \
-    \n        "download_server": "usa-nh", \
-    \n        "pkgs": "git-annex-standalone" \
+    \n        "activate": true, \
+    \n        "pip_install": "https://github.com/moloney/dcmstack/tarball/master /src/heudiconv[all]" \
     \n      } \
     \n    ], \
     \n    [ \
     \n      "entrypoint", \
     \n      "/neurodocker/startup.sh heudiconv" \
     \n    ] \
     \n  ], \
-    \n  "generation_timestamp": "2018-02-02 16:32:55", \
+    \n  "generation_timestamp": "2018-06-29 18:14:08", \
     \n  "neurodocker_version": "0.3.2" \
     \n}' > /neurodocker/neurodocker_specs.json

README.md

@@ -23,9 +23,9 @@ HeuDiConv operates using a heuristic, which provides information on
 how your files should be converted. A number of example heuristics are
 provided to address various use-cases
 
-- the [cmrr_heuristic](heuristics/cmrr_heuristic.py) provides an
+- the [cmrr_heuristic](heudiconv/heuristics/cmrr_heuristic.py) provides an
   example for a conversion to [BIDS]
-- the [reproin](heuristics/reproin.py) could be used to establish
+- the [reproin](heudiconv/heuristics/reproin.py) could be used to establish
   a complete imaging center wide automation to convert all acquired
   data to [BIDS] following a simple naming
   [convention](https://goo.gl/o0YASC) for studies and sequences
@@ -36,7 +36,7 @@ provided to address various use-cases
 
 Released versions of HeuDiConv are available from PyPI so you could
 just `pip install heudiconv[all]` for the most complete installation, 
-and it would require manual installation ony
+and it would require manual installation only
 of the [dcm2niix](https://github.com/rordenlab/dcm2niix/).  On
 Debian-based systems we recommend to use
 [NeuroDebian](http://neuro.debian.net) providing
@@ -64,14 +64,19 @@ manager appropriate for your OS.
 ## Tutorial with example conversion to BIDS format using Docker
 Please read this tutorial to understand how heudiconv works in practice.
 
-[Slides here](http://nipy.org/workshops/2017-03-boston/lectures/bids-heudiconv/#1)
+[Slides here](http://nipy.org/heudiconv/#1)
 
 To generate lean BIDS output, consider using both the `-b` and the `--minmeta` flags 
 to your heudiconv command. The `-b` flag generates a json file with BIDS keys, while
 the `--minmeta` flag restricts the json file to only BIDS keys. Without `--minmeta`,
 the json file and the associated Nifti file contains DICOM metadata extracted using
 dicomstack.
 
+### Other tutorials
+
+- YouTube:
+    - ["Heudiconv Example"](https://www.youtube.com/watch?v=O1kZAuR7E00) by [James Kent](https://github.com/jdkent)
+
 ## How it works (in some more detail)
 
 Call `heudiconv` like this:

heudiconv/bids.py

@@ -29,7 +29,7 @@ def populate_bids_templates(path, defaults={}):
 
     lgr.info("Populating template files under %s", path)
     descriptor = op.join(path, 'dataset_description.json')
-    if not op.exists(descriptor):
+    if not op.lexists(descriptor):
         save_json(descriptor,
               OrderedDict([
                   ('Name', "TODO: name of the dataset"),
@@ -89,18 +89,22 @@ def populate_bids_templates(path, defaults={}):
         suf = '_bold.json'
         assert fpath.endswith(suf)
         events_file = fpath[:-len(suf)] + '_events.tsv'
-        lgr.debug("Generating %s", events_file)
-        with open(events_file, 'w') as f:
-            f.write("onset\tduration\ttrial_type\tresponse_time\tstim_file\tTODO -- fill in rows and add more tab-separated columns if desired")
+        # do not touch any existing thing, it may be precious
+        if not op.lexists(events_file):
+            lgr.debug("Generating %s", events_file)
+            with open(events_file, 'w') as f:
+                f.write("onset\tduration\ttrial_type\tresponse_time\tstim_file\tTODO -- fill in rows and add more tab-separated columns if desired")
     # extract tasks files stubs
     for task_acq, fields in tasks.items():
         task_file = op.join(path, task_acq + '_bold.json')
-        lgr.debug("Generating %s", task_file)
-        fields["TaskName"] = ("TODO: full task name for %s" %
-                              task_acq.split('_')[0].split('-')[1])
-        fields["CogAtlasID"] = "TODO"
-        with open(task_file, 'w') as f:
-            f.write(json_dumps_pretty(fields, indent=2, sort_keys=True))
+        # do not touch any existing thing, it may be precious
+        if not op.lexists(task_file):
+            lgr.debug("Generating %s", task_file)
+            fields["TaskName"] = ("TODO: full task name for %s" %
+                                  task_acq.split('_')[0].split('-')[1])
+            fields["CogAtlasID"] = "TODO"
+            with open(task_file, 'w') as f:
+                f.write(json_dumps_pretty(fields, indent=2, sort_keys=True))
 
 
 def tuneup_bids_json_files(json_files):
@@ -133,8 +137,9 @@ def tuneup_bids_json_files(json_files):
         json_basename = '_'.join(jsonfile.split('_')[:-1])
         # if we got by now all needed .json files -- we can fix them up
         # unfortunately order of "items" is not guaranteed atm
-        if len(glob(json_basename + '*.json')) == 3:
-            json_phasediffname = json_basename + '_phasediff.json'
+        json_phasediffname = json_basename + '_phasediff.json'
+        json_mag = json_basename + '_magnitude*.json'
+        if op.exists(json_phasediffname) and len(glob(json_mag)) >= 1:
             json_ = load_json(json_phasediffname)
             # TODO: we might want to reorder them since ATM
             # the one for shorter TE is the 2nd one!
@@ -293,10 +298,14 @@ def get_formatted_scans_key_row(item):
                                             force=True))
     # we need to store filenames and acquisition times
     # parse date and time and get it into isoformat
-    date = mw.dcm_data.ContentDate
-    time = mw.dcm_data.ContentTime.split('.')[0]
-    td = time + date
-    acq_time = datetime.strptime(td, '%H%M%S%Y%m%d').isoformat()
+    try:
+        date = mw.dcm_data.ContentDate
+        time = mw.dcm_data.ContentTime.split('.')[0]
+        td = time + date
+        acq_time = datetime.strptime(td, '%H%M%S%Y%m%d').isoformat()
+    except AttributeError as exc:
+        lgr.warning("Failed to get date/time for the content: %s", str(exc))
+        acq_time = None
     # add random string
     randstr = ''.join(map(chr, sample(k=8, population=range(33, 127))))
     try:
@@ -326,6 +335,10 @@ def convert_sid_bids(subject_id):
     """
     cleaner = lambda y: ''.join([x for x in y if x.isalnum()])
     sid = cleaner(subject_id)
+    if not sid:
+        raise ValueError(
+            "Subject ID became empty after cleanup.  Please provide manually "
+            "a suitable alphanumeric subject ID")
     lgr.warning('{0} contained nonalphanumeric character(s), subject '
                 'ID was cleaned to be {1}'.format(subject_id, sid))
     return sid, subject_id

heudiconv/cli/run.py

@@ -99,6 +99,11 @@ def process_extra_commands(outdir, args):
 def main(argv=None):
     parser = get_parser()
     args = parser.parse_args(argv)
+    # exit if nothing to be done
+    if not args.files and not args.dicom_dir_template and not args.command:
+        lgr.warning("Nothing to be done - displaying usage help")
+        parser.print_help()
+        sys.exit(1)
     # To be done asap so anything random is deterministic
     if args.random_seed is not None:
         import random
@@ -237,6 +242,9 @@ def process_args(args):
     #
     # Load heuristic -- better do it asap to make sure it loads correctly
     #
+    if not args.heuristic:
+        raise RuntimeError("No heuristic specified - add to arguments and rerun")
+
     heuristic = load_heuristic(args.heuristic)
 
     study_sessions = get_study_sessions(args.dicom_dir_template, args.files,