Merge pull request #695 from nipreps/bp/missing_ok

FIX: Add unlink() with missing_ok for Python 3.7

Author: effigies

Dockerfile

@@ -2,7 +2,7 @@
 #
 # MIT License
 #
-# Copyright (c) 2021 The NiPreps Developers
+# Copyright (c) 2022 The NiPreps Developers
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -22,29 +22,10 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# Use Ubuntu 20.04 LTS
-FROM ubuntu:focal-20210416
+FROM nipreps/miniconda:py38_1.4.2
 
-# Prepare environment
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-                    apt-utils \
-                    autoconf \
-                    build-essential \
-                    bzip2 \
-                    ca-certificates \
-                    curl \
-                    git \
-                    libtool \
-                    lsb-release \
-                    pkg-config \
-                    unzip \
-                    xvfb && \
-    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-ENV DEBIAN_FRONTEND="noninteractive" \
-    LANG="en_US.UTF-8" \
-    LC_ALL="en_US.UTF-8"
+ARG DEBIAN_FRONTEND=noninteractive
+ENV LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:${CONDA_PATH}/lib"
 
 # Installing freesurfer
 RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.1/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.1.tar.gz | tar zxv --no-same-owner -C /opt \
@@ -69,7 +50,7 @@ RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.1/frees
     --exclude='freesurfer/trctrain'
 
 # Simulate SetUpFreeSurfer.sh
-ENV FSL_DIR="/opt/fsl-5.0.11" \
+ENV FSL_DIR="/opt/fsl" \
     OS="Linux" \
     FS_OVERRIDE=0 \
     FIX_VERTEX_AREA="" \
@@ -86,86 +67,98 @@ ENV PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \
     MNI_PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \
     PATH="$FREESURFER_HOME/bin:$FSFAST_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH"
 
-# FSL 5.0.11 (neurodocker build)
-RUN apt-get update -qq \
-    && apt-get install -y -q --no-install-recommends \
-           bc \
-           dc \
-           file \
-           libfontconfig1 \
-           libfreetype6 \
-           libgl1-mesa-dev \
-           libgl1-mesa-dri \
-           libglu1-mesa-dev \
-           libgomp1 \
-           libice6 \
-           libxcursor1 \
-           libxft2 \
-           libxinerama1 \
-           libxrandr2 \
-           libxrender1 \
-           libxt6 \
-           sudo \
-           wget \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* \
-    && echo "Downloading FSL ..." \
-    && mkdir -p /opt/fsl-5.0.11 \
-    && curl -fsSL --retry 5 https://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-5.0.11-centos6_64.tar.gz \
-    | tar -xz -C /opt/fsl-5.0.11 --strip-components 1 \
-    --exclude "fsl/config" \
-    --exclude "fsl/data/atlases" \
-    --exclude "fsl/data/first" \
-    --exclude "fsl/data/mist" \
-    --exclude "fsl/data/possum" \
-    --exclude "fsl/data/standard/bianca" \
-    --exclude "fsl/data/standard/tissuepriors" \
-    --exclude "fsl/doc" \
-    --exclude "fsl/etc/default_flobs.flobs" \
-    --exclude "fsl/etc/fslconf" \
-    --exclude "fsl/etc/js" \
-    --exclude "fsl/etc/luts" \
-    --exclude "fsl/etc/matlab" \
-    --exclude "fsl/extras" \
-    --exclude "fsl/include" \
-    --exclude "fsl/python" \
-    --exclude "fsl/refdoc" \
-    --exclude "fsl/src" \
-    --exclude "fsl/tcl" \
-    --exclude "fsl/bin/FSLeyes" \
-    && find /opt/fsl-5.0.11/bin -type f -not \( \
-        -name "applywarp" -or \
-        -name "bet" -or \
-        -name "bet2" -or \
-        -name "convert_xfm" -or \
-        -name "fast" -or \
-        -name "flirt" -or \
-        -name "fsl_regfilt" -or \
-        -name "fslhd" -or \
-        -name "fslinfo" -or \
-        -name "fslmaths" -or \
-        -name "fslmerge" -or \
-        -name "fslroi" -or \
-        -name "fslsplit" -or \
-        -name "fslstats" -or \
-        -name "imtest" -or \
-        -name "mcflirt" -or \
-        -name "melodic" -or \
-        -name "prelude" -or \
-        -name "remove_ext" -or \
-        -name "susan" -or \
-        -name "topup" -or \
-        -name "zeropad" \) -delete \
-    && find /opt/fsl-5.0.11/data/standard -type f -not -name "MNI152_T1_2mm_brain.nii.gz" -delete
-ENV FSLDIR="/opt/fsl-5.0.11" \
-    PATH="/opt/fsl-5.0.11/bin:$PATH" \
+# Install AFNI latest (neurodocker build)
+ENV AFNI_DIR="/opt/afni"
+RUN echo "Downloading AFNI ..." \
+    && mkdir -p ${AFNI_DIR} \
+    && curl -fsSL --retry 5 https://afni.nimh.nih.gov/pub/dist/tgz/linux_openmp_64.tgz \
+       | tar -xz -C ${AFNI_DIR} --strip-components 1 \
+    # Keep only what we use
+    && find ${AFNI_DIR} -type f -not \( \
+        -name "3dTshift" -or \
+        -name "3dUnifize" -or \
+        -name "3dAutomask" -or \
+        -name "3dvolreg" \) -delete
+
+ENV PATH="${AFNI_DIR}:$PATH" \
+    AFNI_IMSAVE_WARNINGS="NO" \
+    AFNI_MODELPATH="${AFNI_DIR}/models" \
+    AFNI_TTATLAS_DATASET="${AFNI_DIR}/atlases" \
+    AFNI_PLUGINPATH="${AFNI_DIR}/plugins"
+
+# Install AFNI's dependencies
+RUN ${CONDA_PATH}/bin/conda install -c conda-forge -c anaconda \
+                            gsl                                \
+                            xorg-libxp                         \
+                            scipy=1.8                          \
+    && ${CONDA_PATH}/bin/conda install -c sssdgc png \
+    && sync \
+    && ${CONDA_PATH}/bin/conda clean -afy; sync \
+    && rm -rf ~/.conda ~/.cache/pip/*; sync \
+    && ln -s ${CONDA_PATH}/lib/libgsl.so.25 /usr/lib/x86_64-linux-gnu/libgsl.so.19 \
+    && ln -s ${CONDA_PATH}/lib/libgsl.so.25 /usr/lib/x86_64-linux-gnu/libgsl.so.0 \
+    && ldconfig
+
+RUN apt-get update \
+ && apt-get install -y -q --no-install-recommends     \
+                    libcurl4-openssl-dev              \
+                    libgdal-dev                       \
+                    libgfortran-8-dev                 \
+                    libgfortran4                      \
+                    libglw1-mesa                      \
+                    libgomp1                          \
+                    libjpeg62                         \
+                    libnode-dev                       \
+                    libssl-dev                        \
+                    libudunits2-dev                   \
+                    libxm4                            \
+                    libxml2-dev                       \
+                    netpbm                            \
+                    tcsh                              \
+                    xfonts-base                       \
+ && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+ && ldconfig
+
+# Installing ANTs 2.3.4 (NeuroDocker build)
+ENV ANTSPATH="/opt/ants"
+WORKDIR $ANTSPATH
+RUN curl -sSL "https://dl.dropbox.com/s/gwf51ykkk5bifyj/ants-Linux-centos6_x86_64-v2.3.4.tar.gz" \
+    | tar -xzC $ANTSPATH --strip-components 1
+ENV PATH="$ANTSPATH:$PATH"
+
+# FSL 5.0.11
+RUN curl -sSL https://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-5.0.11-centos6_64.tar.gz | tar zxv --no-same-owner -C /opt \
+    --exclude='fsl/doc' \
+    --exclude='fsl/refdoc' \
+    --exclude='fsl/python/oxford_asl' \
+    --exclude='fsl/data/possum' \
+    --exclude='fsl/data/first' \
+    --exclude='fsl/data/mist' \
+    --exclude='fsl/data/atlases' \
+    --exclude='fsl/data/xtract_data' \
+    --exclude='fsl/extras/doc' \
+    --exclude='fsl/extras/man' \
+    --exclude='fsl/extras/src' \
+    --exclude='fsl/src' \
+    --exclude='fsl/tcl'
+
+ENV FSLDIR="/opt/fsl" \
+    PATH="/opt/fsl/bin:$PATH" \
     FSLOUTPUTTYPE="NIFTI_GZ" \
     FSLMULTIFILEQUIT="TRUE" \
+    FSLTCLSH="/opt/fsl/bin/fsltclsh" \
+    FSLWISH="/opt/fsl/bin/fslwish" \
     FSLLOCKDIR="" \
     FSLMACHINELIST="" \
     FSLREMOTECALL="" \
     FSLGECUDAQ="cuda.q" \
-    LD_LIBRARY_PATH="/opt/fsl-5.0.11/lib:$LD_LIBRARY_PATH"
+    POSSUMDIR="/opt/fsl" \
+    LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/fsl"
+
+# Unless otherwise specified each process should only use one thread - nipype
+# will handle parallelization
+ENV MKL_NUM_THREADS=1 \
+    OMP_NUM_THREADS=1
 
 # Convert3D (neurodocker build)
 RUN echo "Downloading Convert3D ..." \
@@ -178,88 +171,11 @@ RUN echo "Downloading Convert3D ..." \
 ENV C3DPATH="/opt/convert3d-1.0.0" \
     PATH="/opt/convert3d-1.0.0/bin:$PATH"
 
-# AFNI latest (neurodocker build)
-RUN apt-get update -qq \
-    && apt-get install -y -q --no-install-recommends \
-           apt-utils \
-           ed \
-           gsl-bin \
-           libglib2.0-0 \
-           libglu1-mesa-dev \
-           libglw1-mesa \
-           libgomp1 \
-           libjpeg62 \
-           libxm4 \
-           netpbm \
-           tcsh \
-           xfonts-base \
-           xvfb \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* \
-    && curl -sSL --retry 5 -o /tmp/multiarch.deb http://archive.ubuntu.com/ubuntu/pool/main/g/glibc/multiarch-support_2.27-3ubuntu1.2_amd64.deb \
-    && dpkg -i /tmp/multiarch.deb \
-    && rm /tmp/multiarch.deb \
-    && curl -sSL --retry 5 -o /tmp/libxp6.deb http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb \
-    && dpkg -i /tmp/libxp6.deb \
-    && rm /tmp/libxp6.deb \
-    && curl -sSL --retry 5 -o /tmp/libpng.deb http://snapshot.debian.org/archive/debian-security/20160113T213056Z/pool/updates/main/libp/libpng/libpng12-0_1.2.49-1%2Bdeb7u2_amd64.deb \
-    && dpkg -i /tmp/libpng.deb \
-    && rm /tmp/libpng.deb \
-    && apt-get install -f \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* \
-    && gsl2_path="$(find / -name 'libgsl.so.19' || printf '')" \
-    && if [ -n "$gsl2_path" ]; then \
-         ln -sfv "$gsl2_path" "$(dirname $gsl2_path)/libgsl.so.0"; \
-    fi \
-    && ldconfig \
-    && echo "Downloading AFNI ..." \
-    && mkdir -p /opt/afni-latest \
-    && curl -fsSL --retry 5 https://afni.nimh.nih.gov/pub/dist/tgz/linux_openmp_64.tgz \
-    | tar -xz -C /opt/afni-latest --strip-components 1 \
-    --exclude "linux_openmp_64/*.gz" \
-    --exclude "linux_openmp_64/funstuff" \
-    --exclude "linux_openmp_64/shiny" \
-    --exclude "linux_openmp_64/afnipy" \
-    --exclude "linux_openmp_64/lib/RetroTS" \
-    --exclude "linux_openmp_64/meica.libs" \
-    # Keep only what we use
-    && find /opt/afni-latest -type f -not \( \
-        -name "3dTshift" -or \
-        -name "3dUnifize" -or \
-        -name "3dAutomask" -or \
-        -name "3dvolreg" \) -delete
-
-ENV PATH="/opt/afni-latest:$PATH" \
-    AFNI_IMSAVE_WARNINGS="NO" \
-    AFNI_PLUGINPATH="/opt/afni-latest"
-
-# Installing ANTs 2.3.3 (NeuroDocker build)
-# Note: the URL says 2.3.4 but it is actually 2.3.3
-ENV ANTSPATH="/opt/ants" \
-    PATH="/opt/ants:$PATH"
-WORKDIR $ANTSPATH
-RUN curl -sSL "https://dl.dropbox.com/s/gwf51ykkk5bifyj/ants-Linux-centos6_x86_64-v2.3.4.tar.gz" \
-    | tar -xzC $ANTSPATH --strip-components 1
-
-COPY --from=nipreps/miniconda:py38_1.3.2 /opt/conda /opt/conda
-
-RUN ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
-    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
-    echo "conda activate base" >> ~/.bashrc
-
-# Set CPATH for packages relying on compiled libs (e.g. indexed_gzip)
-ENV PATH="/opt/conda/bin:$PATH" \
-    CPATH="/opt/conda/include:$CPATH" \
-    LD_LIBRARY_PATH="/opt/conda/lib:$LD_LIBRARY_PATH" \
-    LANG="C.UTF-8" \
-    LC_ALL="C.UTF-8" \
-    PYTHONNOUSERSITE=1
-
 # Create a shared $HOME directory
 RUN useradd -m -s /bin/bash -G users niworkflows
 WORKDIR /home/niworkflows
-ENV HOME="/home/niworkflows"
+ENV HOME="/home/niworkflows" \
+    LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH"
 
 # Unless otherwise specified each process should only use one thread - nipype
 # will handle parallelization

niworkflows/interfaces/bids.py

@@ -50,7 +50,7 @@
 from templateflow.api import templates as _get_template_list
 from ..utils.bids import _init_layout, relative_to_root
 from ..utils.images import set_consumables, unsafe_write_nifti_header_and_data
-from ..utils.misc import splitext as _splitext, _copy_any
+from ..utils.misc import splitext as _splitext, _copy_any, unlink
 
 regz = re.compile(r"\.gz$")
 _pybids_spec = loads(Path(_pkgres("niworkflows", "data/nipreps.json")).read_text())
@@ -669,7 +669,7 @@ def _run_interface(self, runtime):
                         new_header.set_data_dtype(data_dtype)
                 del nii
 
-            out_file.unlink(missing_ok=True)
+            unlink(out_file, missing_ok=True)
             if new_data is new_header is None:
                 _copy_any(orig_file, str(out_file))
             else:
@@ -710,11 +710,11 @@ def _run_interface(self, runtime):
                             legacy_metadata[key] = self._metadata.pop(key)
                     if legacy_metadata:
                         sidecar = out_file.parent / f"{_splitext(str(out_file))[0]}.json"
-                        sidecar.unlink(missing_ok=True)
+                        unlink(sidecar, missing_ok=True)
                         sidecar.write_text(dumps(legacy_metadata, sort_keys=True, indent=2))
                 # The future: the extension is the first . and everything after
                 sidecar = out_file.parent / f"{out_file.name.split('.', 1)[0]}.json"
-                sidecar.unlink(missing_ok=True)
+                unlink(sidecar, missing_ok=True)
                 sidecar.write_text(dumps(self._metadata, sort_keys=True, indent=2))
                 self._results["out_meta"] = str(sidecar)
         return runtime

niworkflows/utils/misc.py

@@ -21,6 +21,7 @@
 #     https://www.nipreps.org/community/licensing/
 #
 """Miscellaneous utilities."""
+import os
 
 
 __all__ = [
@@ -333,5 +334,15 @@ def check_valid_fs_license():
     return proc.returncode == 0 and "ERROR:" not in proc.stdout.decode()
 
 
+def unlink(pathlike, missing_ok=False):
+    """Backport of Path.unlink from Python 3.8+ with missing_ok keyword"""
+    # PY37 hack; drop when python_requires >= 3.8
+    try:
+        os.unlink(pathlike)
+    except FileNotFoundError:
+        if not missing_ok:
+            raise
+
+
 if __name__ == "__main__":
     pass

setup.cfg

@@ -27,6 +27,7 @@ python_requires = >= 3.7
 install_requires =
     attrs
     jinja2
+    markupsafe ~= 2.0.1  # jinja2 imports deprecated function removed in 2.1
     matplotlib >= 3.4.2
     nibabel ~= 3.0
     nilearn >= 0.5.2