Merge branch 'master' of https://github.com/nipy/heudiconv into multiecho

Author: mgxd

CHANGELOG.md

@@ -4,6 +4,23 @@ All notable changes to this project will be documented (for humans) in this file
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [0.5.2] - Date
+
+TODO Summary
+
+### Added
+### Changed
+
+- Reproin heuristic: `__dup` indices would now be assigned incrementally
+  individually per each sequence, so there is a chance to properly treat
+  associate for multi-file (e.g. `fmap`) sequences
+
+### Deprecated
+### Fixed
+### Removed
+### Security
+
+
 ## [0.5.1] - 2018-07-05
 Bugfix release
 

Dockerfile

@@ -1,142 +1,131 @@
-# Generated by Neurodocker v0.3.2.
-#
+# Generated by Neurodocker version 0.4.1-28-g83dbc15
+# Timestamp: 2018-11-01 22:00:14 UTC
+# 
 # Thank you for using Neurodocker. If you discover any issues
 # or ways to improve this software, please submit an issue or
 # pull request on our GitHub repository:
+# 
 #     https://github.com/kaczmarj/neurodocker
-#
-# Timestamp: 2018-06-29 18:14:08
 
-FROM debian:stretch
+FROM neurodebian:stretch
 
-ARG DEBIAN_FRONTEND=noninteractive
+ARG DEBIAN_FRONTEND="noninteractive"
 
-#----------------------------------------------------------
-# Install common dependencies and create default entrypoint
-#----------------------------------------------------------
 ENV LANG="en_US.UTF-8" \
-    LC_ALL="C.UTF-8" \
+    LC_ALL="en_US.UTF-8" \
     ND_ENTRYPOINT="/neurodocker/startup.sh"
-RUN apt-get update -qq && apt-get install -yq --no-install-recommends  \
-    	apt-utils bzip2 ca-certificates curl locales unzip \
+RUN export ND_ENTRYPOINT="/neurodocker/startup.sh" \
+    && apt-get update -qq \
+    && apt-get install -y -q --no-install-recommends \
+           apt-utils \
+           bzip2 \
+           ca-certificates \
+           curl \
+           locales \
+           unzip \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-    && localedef --force --inputfile=en_US --charmap=UTF-8 C.UTF-8 \
+    && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \
+    && dpkg-reconfigure --frontend=noninteractive locales \
+    && update-locale LANG="en_US.UTF-8" \
     && chmod 777 /opt && chmod a+s /opt \
     && mkdir -p /neurodocker \
     && if [ ! -f "$ND_ENTRYPOINT" ]; then \
-         echo '#!/usr/bin/env bash' >> $ND_ENTRYPOINT \
-         && echo 'set +x' >> $ND_ENTRYPOINT \
-         && echo 'if [ -z "$*" ]; then /usr/bin/env bash; else $*; fi' >> $ND_ENTRYPOINT; \
-       fi \
+         echo '#!/usr/bin/env bash' >> "$ND_ENTRYPOINT" \
+    &&   echo 'set -e' >> "$ND_ENTRYPOINT" \
+    &&   echo 'if [ -n "$1" ]; then "$@"; else /usr/bin/env bash; fi' >> "$ND_ENTRYPOINT"; \
+    fi \
     && chmod -R 777 /neurodocker && chmod a+s /neurodocker
+
 ENTRYPOINT ["/neurodocker/startup.sh"]
 
-#------------------------
-# Install dcm2niix v1.0.20180622
-#------------------------
-WORKDIR /tmp
-RUN deps='cmake g++ gcc git make pigz zlib1g-dev' \
-    && apt-get update -qq && apt-get install -yq --no-install-recommends $deps \
+ENV PATH="/opt/dcm2niix-v1.0.20180622/bin:$PATH"
+RUN apt-get update -qq \
+    && apt-get install -y -q --no-install-recommends \
+           cmake \
+           g++ \
+           gcc \
+           git \
+           make \
+           pigz \
+           zlib1g-dev \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-    && mkdir dcm2niix \
-    && curl -sSL https://github.com/rordenlab/dcm2niix/tarball/v1.0.20180622 | tar xz -C dcm2niix --strip-components 1 \
-    && mkdir dcm2niix/build && cd dcm2niix/build \
-    && cmake .. && make \
+    && git clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix \
+    && cd /tmp/dcm2niix \
+    && git fetch --tags \
+    && git checkout v1.0.20180622 \
+    && mkdir /tmp/dcm2niix/build \
+    && cd /tmp/dcm2niix/build \
+    && cmake  -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20180622 .. \
+    && make \
     && make install \
-    && rm -rf /tmp/* \
-    && apt-get purge -y --auto-remove $deps
-
-#--------------------------------------------------
-# Add NeuroDebian repository
-# Please note that some packages downloaded through
-# NeuroDebian may have restrictive licenses.
-#--------------------------------------------------
-RUN apt-get update -qq && apt-get install -yq --no-install-recommends dirmngr gnupg \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-    && curl -sSL http://neuro.debian.net/lists/stretch.us-nh.full \
-    > /etc/apt/sources.list.d/neurodebian.sources.list \
-    && curl -sSL https://dl.dropbox.com/s/zxs209o955q6vkg/neurodebian.gpg \
-    | apt-key add - \
-    && (apt-key adv --refresh-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 || true) \
-    && apt-get update
-
-# Install NeuroDebian packages
-RUN apt-get update -qq && apt-get install -yq --no-install-recommends git-annex-standalone \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    && rm -rf /tmp/dcm2niix
 
 RUN apt-get update -qq \
-    && apt-get install -y -q --no-install-recommends git \
-                                                     gcc \
-                                                     pigz \
+    && apt-get install -y -q --no-install-recommends \
+           git \
+           gcc \
+           pigz \
+           liblzma-dev \
+           libc-dev \
+           git-annex-standalone \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 COPY [".", "/src/heudiconv"]
 
-#------------------
-# Install Miniconda
-#------------------
-ENV CONDA_DIR=/opt/conda \
-    PATH=/opt/conda/bin:$PATH
-RUN echo "Downloading Miniconda installer ..." \
-    && miniconda_installer=/tmp/miniconda.sh \
-    && curl -sSL --retry 5 -o $miniconda_installer https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-    && /bin/bash $miniconda_installer -b -p $CONDA_DIR \
-    && rm -f $miniconda_installer \
+ENV CONDA_DIR="/opt/miniconda-latest" \
+    PATH="/opt/miniconda-latest/bin:$PATH"
+RUN export PATH="/opt/miniconda-latest/bin:$PATH" \
+    && echo "Downloading Miniconda installer ..." \
+    && conda_installer="/tmp/miniconda.sh" \
+    && curl -fsSL --retry 5 -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+    && bash "$conda_installer" -b -p /opt/miniconda-latest \
+    && rm -f "$conda_installer" \
+    && conda update -yq -nbase conda \
     && conda config --system --prepend channels conda-forge \
     && conda config --system --set auto_update_conda false \
     && conda config --system --set show_channel_urls true \
-    && conda clean -tipsy && sync
-
-#-------------------------
-# Create conda environment
-#-------------------------
-RUN conda create -y -q --name neuro python=2 \
-                                    traits=4.6.0 \
     && sync && conda clean -tipsy && sync \
-    && /bin/bash -c "source activate neuro \
-      && pip install -q --no-cache-dir https://github.com/moloney/dcmstack/tarball/master \
-                                       /src/heudiconv[all]" \
-    && sync \
-    && sed -i '$isource activate neuro' $ND_ENTRYPOINT
+    && conda install -y -q --name base \
+           python=3.6 \
+           traits>=4.6.0 \
+           scipy \
+           numpy \
+           nomkl \
+    && sync && conda clean -tipsy && sync \
+    && bash -c "source activate base \
+    &&   pip install --no-cache-dir --editable \
+             /src/heudiconv[all]" \
+    && rm -rf ~/.cache/pip/* \
+    && sync
 
-ENTRYPOINT ["/neurodocker/startup.sh", "heudiconv"]
+ENTRYPOINT ["heudiconv"]
 
-#--------------------------------------
-# Save container specifications to JSON
-#--------------------------------------
 RUN echo '{ \
     \n  "pkg_manager": "apt", \
-    \n  "check_urls": true, \
     \n  "instructions": [ \
     \n    [ \
     \n      "base", \
-    \n      "debian:stretch" \
+    \n      "neurodebian:stretch" \
     \n    ], \
     \n    [ \
     \n      "dcm2niix", \
     \n      { \
-    \n        "version": "v1.0.20180622" \
-    \n      } \
-    \n    ], \
-    \n    [ \
-    \n      "neurodebian", \
-    \n      { \
-    \n        "os_codename": "stretch", \
-    \n        "download_server": "usa-nh", \
-    \n        "pkgs": "git-annex-standalone" \
+    \n        "version": "v1.0.20180622", \
+    \n        "method": "source" \
     \n      } \
     \n    ], \
     \n    [ \
     \n      "install", \
     \n      [ \
     \n        "git", \
     \n        "gcc", \
-    \n        "pigz" \
+    \n        "pigz", \
+    \n        "liblzma-dev", \
+    \n        "libc-dev", \
+    \n        "git-annex-standalone" \
     \n      ] \
     \n    ], \
     \n    [ \
@@ -149,17 +138,23 @@ RUN echo '{ \
     \n    [ \
     \n      "miniconda", \
     \n      { \
-    \n        "env_name": "neuro", \
-    \n        "conda_install": "python=2 traits=4.6.0", \
-    \n        "activate": true, \
-    \n        "pip_install": "https://github.com/moloney/dcmstack/tarball/master /src/heudiconv[all]" \
+    \n        "use_env": "base", \
+    \n        "conda_install": [ \
+    \n          "python=3.6", \
+    \n          "traits>=4.6.0", \
+    \n          "scipy", \
+    \n          "numpy", \
+    \n          "nomkl" \
+    \n        ], \
+    \n        "pip_install": [ \
+    \n          "/src/heudiconv[all]" \
+    \n        ], \
+    \n        "pip_opts": "--editable" \
     \n      } \
     \n    ], \
     \n    [ \
     \n      "entrypoint", \
-    \n      "/neurodocker/startup.sh heudiconv" \
+    \n      "heudiconv" \
     \n    ] \
-    \n  ], \
-    \n  "generation_timestamp": "2018-06-29 18:14:08", \
-    \n  "neurodocker_version": "0.3.2" \
+    \n  ] \
     \n}' > /neurodocker/neurodocker_specs.json

heudiconv/bids.py

@@ -1,5 +1,6 @@
 """Handle BIDS specific operations"""
 
+import hashlib
 import os
 import os.path as op
 import logging
@@ -217,7 +218,7 @@ def save_scans_key(item, bids_files):
     """
     Parameters
     ----------
-    items:
+    item:
     bids_files: str or list
 
     Returns
@@ -235,7 +236,7 @@ def save_scans_key(item, bids_files):
         # get filenames
         f_name = '/'.join(bids_file.split('/')[-2:])
         f_name = f_name.replace('json', 'nii.gz')
-        rows[f_name] = get_formatted_scans_key_row(item)
+        rows[f_name] = get_formatted_scans_key_row(item[-1][0])
         subj_, ses_ = find_subj_ses(f_name)
         if not subj_:
             lgr.warning(
@@ -300,7 +301,7 @@ def add_rows_to_scans_keys_file(fn, newrows):
         writer.writerows([header] + data_rows_sorted)
 
 
-def get_formatted_scans_key_row(item):
+def get_formatted_scans_key_row(dcm_fn):
     """
     Parameters
     ----------
@@ -312,25 +313,27 @@ def get_formatted_scans_key_row(item):
         [ISO acquisition time, performing physician name, random string]
 
     """
-    dcm_fn = item[-1][0]
-    from heudiconv.external.dcmstack import ds
-    mw = ds.wrapper_from_data(dcm.read_file(dcm_fn,
-                                            stop_before_pixels=True,
-                                            force=True))
+    dcm_data = dcm.read_file(dcm_fn, stop_before_pixels=True, force=True)
     # we need to store filenames and acquisition times
     # parse date and time and get it into isoformat
     try:
-        date = mw.dcm_data.ContentDate
-        time = mw.dcm_data.ContentTime.split('.')[0]
+        date = dcm_data.ContentDate
+        time = dcm_data.ContentTime.split('.')[0]
         td = time + date
         acq_time = datetime.strptime(td, '%H%M%S%Y%m%d').isoformat()
     except AttributeError as exc:
         lgr.warning("Failed to get date/time for the content: %s", str(exc))
         acq_time = None
     # add random string
-    randstr = ''.join(map(chr, sample(k=8, population=range(33, 127))))
+    # But let's make it reproducible by using all UIDs
+    # (might change across versions?)
+    randcontent = u''.join(
+        [getattr(dcm_data, f) or '' for f in sorted(dir(dcm_data))
+         if f.endswith('UID')]
+    )
+    randstr = hashlib.md5(randcontent.encode()).hexdigest()[:8]
     try:
-        perfphys = mw.dcm_data.PerformingPhysicianName
+        perfphys = dcm_data.PerformingPhysicianName
     except AttributeError:
         perfphys = ''
     row = [acq_time, perfphys, randstr]