Merge tag 'v0.4' into debian

A "usable" release!

Since v0.3 a number of fixes (permissions settings, handling of datalad
support, etc).

TODO: We need to establishg a ChangeLog ;)

* tag 'v0.4': (221 commits)
  Time for a new "snapshot release"
  ENH: it is ok not to test in the tests code that test is skipped if no datalad
  ENH: skip the test relying on datalad if no datalad is available
  ENH: make use of set_readonly in the code
  BF: set_readonly helper to set or reset file to/from readonly mode
  Fix: #93 (Missing PerformingPhysicianName entry)
  Sort _scans entries first by date and then by filename
  BF: open non part 10 compliant DICOMs
  fix: compatible with mac install
  add instructions for generating lean bids output
  Update README.md
  Update Dockerfile
  fix: add support for pigz
  enh: update to latest dcm2niix
  fix: minmeta added to convert and assertion removed to enable numeric comparison
  BF: do not try to produce scans files if layout is not fully BIDS yet
  more of abnoxious globs for validator to skip
  ENH: allow to use with existing datalad datasets which do not have .heudiconv/ as a submodule
  BF(TST): adjusted to using ContentTime + removed not needed () around assert statement
  BF+RF: save_scans_key should get a single item to be applied to all related bids files, use ContentTime not SeriesTime
  ...

Author: yarikoptic

.coveragerc

@@ -0,0 +1,4 @@
+[run]
+include = tests/*
+          bin/*
+          setup.py

.gitignore

@@ -1 +1,4 @@
 *.pyc
+.cache/
+.coverage
+*.egg-info/

.travis.yml

@@ -2,9 +2,9 @@
 language: python
 python:
   - 2.7
-  - 3.3
   - 3.4
   - 3.5
+  - 3.6
 
 cache:
   - apt
@@ -17,17 +17,25 @@ env:
 
 before_install:
   # The ultimate one-liner setup for NeuroDebian repository
-  # which might be needed later for tools
-  #- bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh)
-  #- travis_retry sudo apt-get update -qq
-  # for now even remove requirements.txt since dependencies aren't avail
-  - echo '' > requirements.txt
+  - bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh)
+  - travis_retry sudo apt-get update -qq
+  - travis_retry sudo apt-get install git-annex-standalone dcm2niix
+  # Install in our own virtualenv
+  - python -m pip install --upgrade pip
+  - pip install --upgrade virtualenv
+  - virtualenv --python=python venv
+  - source venv/bin/activate
+  - python --version # just to check
   - pip install -r dev-requirements.txt
-  - pip install codecov
+  - pip install datalad
+  - pip install codecov pytest
 
+install:
+  - git config --global user.email "[email protected]"
+  - git config --global user.name "Travis Almighty"
 
 script:
-  - nosetests -s -v --with-doctest --doctest-tests --with-cov --cover-package . --logging-level=INFO tests
+  - coverage run `which py.test` -s -v tests heuristics
 
 after_success:
   - codecov

Dockerfile

@@ -1,11 +1,36 @@
+
 FROM continuumio/miniconda
 
 MAINTAINER <[email protected]>
 
-RUN apt-get update && apt-get upgrade -y && apt-get install -y g++ && apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y
-RUN cd /tmp && git clone https://github.com/neurolabusc/dcm2niix.git && cd dcm2niix/console/ && git checkout e262240bb27e8f4c9440d6b1a97dfd98ef0f9939 && g++ -O3 -I. main_console.cpp nii_dicom.cpp jpg_0XC3.cpp ujpeg.cpp nifti1_io_core.cpp nii_ortho.cpp nii_dicom_batch.cpp  -o dcm2niix -DmyDisableOpenJPEG -DmyDisableJasper && cp dcm2niix /usr/local/bin/ 
-RUN conda install -y -c conda-forge nipype && pip install https://github.com/moloney/dcmstack/archive/c12d27d2c802d75a33ad70110124500a83e851ee.zip && pip install https://github.com/nipy/nipype/archive/dd1ed4f0d5735c69c1743f29875acf09d23a62e0.zip
-RUN curl -O https://raw.githubusercontent.com/nipy/heudiconv/master/bin/heudiconv && chmod +x heudiconv && cp heudiconv /usr/local/bin/
-RUN curl -O https://raw.githubusercontent.com/nipy/heudiconv/master/heuristics/convertall.py && chmod +x convertall.py
+RUN apt-get update && apt-get upgrade -y && \
+    apt-get install -y g++ pkg-config make && \
+    apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y
+RUN (wget -O- http://neuro.debian.net/lists/jessie.us-nh.full | tee /etc/apt/sources.list.d/neurodebian.sources.list) && \
+    apt-key adv --recv-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 && \
+    apt-get update -qq && apt-get install -y git-annex-standalone && \
+    apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y
+RUN conda install -y -c conda-forge nipype && \
+    conda install cmake && \
+    pip install https://github.com/moloney/dcmstack/archive/c12d27d2c802d75a33ad70110124500a83e851ee.zip && \
+    pip install datalad && \
+    conda clean -tipsy && rm -rf ~/.pip/
+RUN apt-get update && apt-get upgrade -y && \
+    apt-get install -y pigz && \
+    apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y && \
+    cd /tmp && git clone https://github.com/neurolabusc/dcm2niix.git && \
+    cd dcm2niix && \
+    git checkout 6ba27b9befcbae925209664bb8acbb00e266114a && \
+    mkdir build && cd build && cmake -DBATCH_VERSION=ON .. && \
+    make && make install && \
+    cd / && rm -rf /tmp/dcm2niix
+
+COPY bin/heudiconv /usr/local/bin/heudiconv
+RUN chmod +x /usr/local/bin/heudiconv
+RUN mkdir /heuristics
+COPY heuristics/convertall.py /heuristics
+RUN chmod +x /heuristics/convertall.py
+RUN git config --global user.email "[email protected]" && \
+    git config --global user.name "Docker Almighty"
 
 ENTRYPOINT ["/usr/local/bin/heudiconv"]

Makefile

@@ -6,9 +6,8 @@ install:
 	mkdir -p $(DESTDIR)$(PREFIX)/share/heudiconv/heuristics
 	mkdir -p $(DESTDIR)$(PREFIX)/share/doc/heudiconv/examples/heuristics
 	mkdir -p $(DESTDIR)$(PREFIX)/bin
-	install -t $(DESTDIR)$(PREFIX)/bin bin/heudiconv
-	install -m 644 -t $(DESTDIR)$(PREFIX)/share/heudiconv/heuristics heuristics/*
-	install -m 644 -t $(DESTDIR)$(PREFIX)/share/doc/heudiconv/examples/heuristics/  examples/heuristics/*
+	install bin/heudiconv $(DESTDIR)$(PREFIX)/bin
+	install -m 644 heuristics/* $(DESTDIR)$(PREFIX)/share/heudiconv/heuristics
 
 uninstall:
 	rm -f $(DESTDIR)$(PREFIX)/bin/heudiconv

README.md

@@ -10,7 +10,7 @@ structured directory layouts.
 - it's faster than parsesdicomdir or mri_convert if you use dcm2niix option
 - it tracks the provenance of the conversion from DICOM to NIfTI in W3C
   PROV format
-- the cmrr_heuristic example shows a conversion to [BIDS](http://bids.neuroimaging.io) 
+- the cmrr_heuristic example shows a conversion to [BIDS](http://bids.neuroimaging.io)
   layout structure
 
 ## Install
@@ -33,18 +33,30 @@ as long as the following dependencies are in your path you can use the script
 - nibabel
 - dcm2niix
 
+## Tutorial with example conversion to BIDS format using Docker
+Please read this tutorial to understand how heudiconv works in practice.
+
+[Slides here](http://nipy.org/workshops/2017-03-boston/lectures/bids-heudiconv/#1)
+
+To generate lean BIDS output, consider using both the `-b` and the `--minmeta` flags 
+to your heudiconv command. The `-b` flag generates a json file with BIDS keys, while
+the `--minmeta` flag restricts the json file to only BIDS keys. Without `--minmeta`,
+the json file and the associated Nifti file contains DICOM metadata extracted using
+dicomstack.
+
 ## How it works (in some more detail)
 
 Call `heudiconv` like this:
 
-    heudiconv -d '%s*.tar*' -s xx05 -f ~/myheuristics/convertall.py
+    heudiconv -d '{subject}*.tar*' -s xx05 -f ~/myheuristics/convertall.py
 
-where `-d '%s*tar*'` is an expression used to find DICOM files (`%s` expands to
-a subject ID so that the expression will match any `.tar` files, compressed
-or not that start with the subject ID in their name). `-s od05` specifies a
-subject ID for the conversion (this could be a list of multiple IDs), and
-`-f ~/myheuristics/convertall.py` identifies a heuristic implementation for this
-conversion (see below) for details.
+where `-d '{subject}*tar*'` is an expression used to find DICOM files
+(`{subject}` expands to a subject ID so that the expression will match any
+`.tar` files, compressed or not that start with the subject ID in their name).
+An additional flag for session (`{session}`) can be included in the expression
+as well. `-s od05` specifies a subject ID for the conversion (this could be a
+list of multiple IDs), and `-f ~/myheuristics/convertall.py` identifies a
+heuristic implementation for this conversion (see below) for details.
 
 This call will locate the DICOMs (in any number of matching tarballs), extract
 them to a temporary directory, search for any DICOM series it can find, and
@@ -70,65 +82,82 @@ soon you'll be able to:
 ## The heuristic file
 
 The heuristic file controls how information about the dicoms is used to convert
-to a file system layout (e.g., BIDS). This is a python file that must have the 
+to a file system layout (e.g., BIDS). This is a python file that must have the
 function `infotodict`, which takes a single argument `seqinfo`.  
 
 ### `seqinfo` and the `s` variable
 
-Each item in `seqinfo` contains the following ordered elements.
+`seqinfo` is a list of namedtuple objects, each containing the following fields:
+
+* total_files_till_now
+* example_dcm_file
+* series_id
+* dcm_dir_name
+* unspecified2
+* unspecified3
+* dim1
+* dim2
+* dim3
+* dim4
+* TR
+* TE
+* protocol_name
+* is_motion_corrected
+* is_derived
+* patient_id
+* study_description
+* referring_physician_name
+* series_description
+* image_type
 
 ```
-[total_files_till_now, example_dcm_file, series_number, unspecified, unspecified, 
-unspecified, dim1, dim2, dim3,  dim4, TR, TE, SeriesDescription, MotionCorrectedOrNot]
-
 128     125000-1-1.dcm  1       -       -       
 -       160     160     128     1       0.00315 1.37    AAHScout        False
 ```
 
 ### The dictionary returned by `infotodict`
- 
+
 This dictionary contains as keys a 3-tuple `(template, a tuple of output types,
  annotation classes)`.
 
 template - how the file should be relative to the base directory
-tuple of output types - what format of output should be created - nii.gz, dicom, 
+tuple of output types - what format of output should be created - nii.gz, dicom,
  etc.,.
 annotation classes - unused
 
 ```
-Example: ('func/sub-{subject}_task-face_run-{item:02d}_acq-PA_bold', ('nii.gz', 
+Example: ('func/sub-{subject}_task-face_run-{item:02d}_acq-PA_bold', ('nii.gz',
         'dicom'), None)
 ```
 
 A few fields are defined by default and can be used in the template:
 
-- item: index within category 
-- subject: participant id 
+- item: index within category
+- subject: participant id
 - seqitem: run number during scanning
 - subindex: sub index within group
-- session: session info for multi-session studies and when session has been 
+- session: session info for multi-session studies and when session has been
   defined as a parameter for heudiconv
 
-Additional variables may be added and can be returned in the value of the 
+Additional variables may be added and can be returned in the value of the
 dictionary returned from the function.
 
-`info[some_3-tuple] = [12, 14, 16]` would assign dicom sequence groups 12, 14 
+`info[some_3-tuple] = [12, 14, 16]` would assign dicom sequence groups 12, 14
 and 16 to be converted using the template specified in `some_3-tuple`.
 
-if the template contained a non-sanctioned variable, it would have to be 
+if the template contained a non-sanctioned variable, it would have to be
 provided in the values for that key.
 
 ```
-some_3_tuple = ('func/sub-{subject}_task-face_run-{item:02d}_acq-{acq}_bold', ('nii.gz', 
+some_3_tuple = ('func/sub-{subject}_task-face_run-{item:02d}_acq-{acq}_bold', ('nii.gz',
         'dicom'), None)
 ```
 
-In the above example `{acq}` is not a standard variable. In this case, values 
+In the above example `{acq}` is not a standard variable. In this case, values
 for this variable needs to be added.
 
 ```
 info[some_3-tuple] = [{'item': 12, 'acq': 'AP'},
                       {'item': 14, 'acq': 'AP'},
                       {'item': 16, 'acq': 'PA'}]
 ```
-