Merge pull request #1916 from FCP-INDI/upgrade/python

⬆️ Upgrade dependencies for 1.8.6

Author: e-kenneally

.circleci/main.yml

@@ -1,5 +1,6 @@
 --- 
 version: 2.1
+# Singularity started failing to set up on Circle circa May 2023, so those tests are currently disabled
 
 parameters:
   branch:
@@ -73,7 +74,7 @@ commands:
       version:
         description: "Version of Singularity to clone"
         type: string
-        default: v3.7.2
+        default: v3.8.7
     steps:
       - install-singularity-requirements
       - restore_cache:
@@ -84,7 +85,7 @@ commands:
           command: |
             if [[ ! -d singularity-<< parameters.version >> ]]
             then
-              git clone -b << parameters.version >> https://github.com/sylabs/singularity singularity-<< parameters.version >>
+              git clone -b << parameters.version >> https://github.com/apptainer/singularity singularity-<< parameters.version >>
             fi
       - save_cache:
           key: singularity-<< parameters.version >>
@@ -94,16 +95,29 @@ commands:
          name: "Setting up Singularity"
          command: |
            cd singularity-<< parameters.version >>
-           ./mconfig
+           ./mconfig -p /usr/local
            make -C builddir
            sudo make -C builddir install
            cd ..
   install-singularity-requirements:
     steps:
       - run:
           name: "Installing Singularity requirements"
+          # https://github.com/apptainer/apptainer/blob/release-1.1/INSTALL.md
           command: |
-           sudo apt-get update && sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.1 libarchive-dev libgpgme11-dev libseccomp-dev -y
+            sudo apt-get update -y && \
+            sudo apt-get install -y \
+              build-essential \
+              libseccomp-dev \
+              pkg-config \
+              uidmap \
+              squashfs-tools \
+              squashfuse \
+              fuse2fs \
+              fuse-overlayfs \
+              fakeroot \
+              cryptsetup \
+              curl wget git
   run-pytest-docker:
     steps:
       - create-docker-test-container:
@@ -115,17 +129,18 @@ commands:
     steps:
       - restore_cache:
           keys:
-            - pip-ci-requirements-{{ checksum "dev/circleci_data/requirements.txt" }}-3.8.5
+            - pip-ci-requirements-{{ checksum "dev/circleci_data/requirements.txt" }}-3.10.10
       - run:
           name: "Installing CI dependencies"
           command: |
-            pyenv global 3.8.5
+            yes n | pyenv install 3.10.10 || true
+            pyenv global 3.10.10
             pip install --upgrade pip
             pip install -r dev/circleci_data/requirements.txt
       - save_cache:
-          key: pip-ci-requirements-{{ checksum "dev/circleci_data/requirements.txt" }}-3.8.5
+          key: pip-ci-requirements-{{ checksum "dev/circleci_data/requirements.txt" }}-3.10.10
           paths: 
-            - /opt/circleci/.pyenv/versions/3.8.5
+            - /opt/circleci/.pyenv/versions/3.10.10
   set-up-variant:
     parameters:
       variant:
@@ -162,21 +177,21 @@ commands:
 jobs:
   combine-coverage:
     machine:
-      image: ubuntu-2004:202010-01
+      image: ubuntu-2004:2023.04.2
     steps:
       - checkout
       - restore_cache:
           key: coverage-docker--{{ .Revision }}
-      - restore_cache:
-          key: coverage-singularity--{{ .Revision }}
+      # - restore_cache:
+      #     key: coverage-singularity--{{ .Revision }}
       - restore_cache:
           key: coverage-docker-ABCD-HCP-{{ .Revision }}
-      - restore_cache:
-          key: coverage-singularity-ABCD-HCP-{{ .Revision }}
+      # - restore_cache:
+      #     key: coverage-singularity-ABCD-HCP-{{ .Revision }}
       - restore_cache:
           key: coverage-docker-lite-{{ .Revision }}
-      - restore_cache:
-          key: coverage-singularity-lite-{{ .Revision }}
+      # - restore_cache:
+      #     key: coverage-singularity-lite-{{ .Revision }}
       - set-python-version
       - combine-coverage
       - store_artifacts:
@@ -187,7 +202,7 @@ jobs:
         type: string
         default: ""
     machine:
-      image: ubuntu-2004:202010-01
+      image: ubuntu-2004:2023.04.2
     resource_class: medium
     steps:
       - checkout
@@ -209,7 +224,7 @@ jobs:
         type: string
         default: ""
     machine:
-      image: ubuntu-2004:202010-01
+      image: ubuntu-2004:2023.04.2
     resource_class: large
     steps:
       - checkout
@@ -231,7 +246,7 @@ jobs:
         type: string
         default: ""
     machine:
-      image: ubuntu-2004:202010-01
+      image: ubuntu-2004:2023.04.2
     resource_class: large
     steps:
       - checkout
@@ -262,11 +277,11 @@ workflows:
           name: "Combine coverage"
           requires:
             - "Test in Docker"
-            - "Test in Singularity"
+            # - "Test in Singularity"
             - "Test lite variant in Docker"
-            - "Test lite variant in Singularity"
+            # - "Test lite variant in Singularity"
             - "Test ABCD-HCP variant in Docker"
-            - "Test ABCD-HCP variant in Singularity"
+            # - "Test ABCD-HCP variant in Singularity"
             - "Test fMRIPrep-LTS variant in Docker"
       - push-branch-to-docker-hub:
           filters:
@@ -292,11 +307,11 @@ workflows:
           variant: fMRIPrep-LTS
       - pytest-docker:
           name: "Test lite variant in Docker"
-      - pytest-singularity:
-          name: "Test in Singularity"
-      - pytest-singularity:
-          name: "Test ABCD-HCP variant in Singularity"
-          variant: ABCD-HCP
-      - pytest-singularity:
-          name: "Test lite variant in Singularity"
-          variant: lite
+      # - pytest-singularity:
+      #     name: "Test in Singularity"
+      # - pytest-singularity:
+      #     name: "Test ABCD-HCP variant in Singularity"
+      #     variant: ABCD-HCP
+      # - pytest-singularity:
+      #     name: "Test lite variant in Singularity"
+      #     variant: lite

.dockerignore

@@ -1,8 +1,8 @@
-!.github/scripts
 bids-examples
 .circleci
 cpac_runs
 .env*
 .git
 .github
+!.github/scripts
 *.tar.gz

.github/Dockerfiles/AFNI.23.0.07-bionic.Dockerfile

@@ -0,0 +1,60 @@
+FROM ghcr.io/fcp-indi/c-pac/ubuntu:python3.10-bionic-non-free as AFNI
+USER root
+
+# install AFNI
+COPY dev/docker_data/required_afni_pkgs.txt /opt/required_afni_pkgs.txt
+COPY dev/docker_data/checksum/AFNI.23.0.07.sha384 /tmp/AFNI.23.0.07.sha384
+RUN if [ -f /usr/lib/x86_64-linux-gnu/mesa/libGL.so.1.2.0 ]; then \
+        ln -svf /usr/lib/x86_64-linux-gnu/mesa/libGL.so.1.2.0 /usr/lib/x86_64-linux-gnu/libGL.so.1; \
+    fi \
+    && libs_path=/usr/lib/x86_64-linux-gnu \
+    && if [ -f $libs_path/libgsl.so.23 ]; then \
+        ln -svf $libs_path/libgsl.so.23 $libs_path/libgsl.so.19 \
+    &&  ln -svf $libs_path/libgsl.so.23 $libs_path/libgsl.so.0; \
+    elif [ -f $libs_path/libgsl.so.23.0.0 ]; then \
+       ln -svf $libs_path/libgsl.so.23.0.0 $libs_path/libgsl.so.19 \
+    &&  ln -svf $libs_path/libgsl.so.23.0.0 $libs_path/libgsl.so.0; \
+    elif [ -f $libs_path/libgsl.so ]; then \
+        ln -svf $libs_path/libgsl.so $libs_path/libgsl.so.0; \
+    fi \
+    && LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH \
+    && export LD_LIBRARY_PATH \
+    && apt-get update && apt-get install -y libglw1-mesa-dev \
+    && AFNI_VERSION="23.0.07" \
+    && curl -LOJ https://github.com/afni/afni/archive/AFNI_${AFNI_VERSION}.tar.gz \
+    && sha384sum --check /tmp/AFNI.23.0.07.sha384 \
+    && mkdir /opt/afni \
+    && tar -xvf afni-AFNI_${AFNI_VERSION}.tar.gz -C /opt/afni --strip-components 1 \
+    && rm -rf afni-AFNI_${AFNI_VERSION}.tar.gz \
+    && cd /opt/afni/src \
+    && sed '/^INSTALLDIR =/c INSTALLDIR = /opt/afni' Makefile.linux_ubuntu_16_64 > Makefile \
+    && make vastness && make cleanest \
+    && cd /opt/afni \
+    # filter down to required packages
+    ls > full_ls \
+    && sed 's/linux_openmp_64\///g' /opt/required_afni_pkgs.txt | sort > required_ls \
+    && comm -2 -3 full_ls required_ls | xargs rm -rf full_ls required_ls \
+    && apt-get remove -y libglw1-mesa-dev \
+    && ldconfig
+
+# set up AFNI
+ENV PATH=/opt/afni:$PATH
+
+ENTRYPOINT ["/bin/bash"]
+
+# Link libraries for Singularity images
+RUN ldconfig
+
+RUN apt-get clean \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+FROM scratch
+LABEL org.opencontainers.image.description "NOT INTENDED FOR USE OTHER THAN AS A STAGE IMAGE IN A MULTI-STAGE BUILD \
+AFNI 21.1.00 (Domitian) stage"
+LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC
+COPY --from=AFNI /lib/x86_64-linux-gnu/ld* /lib/x86_64-linux-gnu/
+COPY --from=AFNI /lib/x86_64-linux-gnu/lib*so* /lib/x86_64-linux-gnu/
+COPY --from=AFNI /lib64/ld* /lib64/
+COPY --from=AFNI /opt/afni/ /opt/afni/
+COPY --from=AFNI /usr/lib/x86_64-linux-gnu/lib*so* /usr/lib/x86_64-linux-gnu/

.github/Dockerfiles/ANTs.2.4.3.Python3.10-bionic.Dockerfile

@@ -0,0 +1,21 @@
+FROM ghcr.io/fcp-indi/c-pac/ubuntu:python3.10-bionic-non-free as ANTs
+
+USER root
+COPY dev/docker_data/checksum/ANTs.2.4.3.sha384 /tmp/checksum.sha384
+RUN curl -sL https://github.com/ANTsX/ANTs/releases/download/v2.4.3/ants-2.4.3-ubuntu-18.04-X64-gcc.zip -o /tmp/ANTs.zip \
+  && curl -sL https://s3-eu-west-1.amazonaws.com/pfigshare-u-files/3133832/Oasis.zip -o /tmp/Oasis.zip \
+  && sha384sum --check /tmp/checksum.sha384 \
+  && unzip /tmp/ANTs.zip -d /tmp \
+  && mkdir /usr/lib/ants \
+  && mv /tmp/ants-2.4.3/* /usr/lib/ants \
+  && mkdir /ants_template \
+  && unzip /tmp/Oasis.zip -d /tmp \
+  && mv /tmp/MICCAI2012-Multi-Atlas-Challenge-Data /ants_template/oasis
+
+# Only keep what we need
+FROM scratch
+LABEL org.opencontainers.image.description "NOT INTENDED FOR USE OTHER THAN AS A STAGE IMAGE IN A MULTI-STAGE BUILD \
+ANTs 2.4.3 stage"
+LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC
+COPY --from=ANTs /usr/lib/ants/ /usr/lib/ants/
+COPY --from=ANTs /ants_template/ /ants_template/

.github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile

@@ -17,11 +17,12 @@ RUN rm -Rf /code/docker_data/Dockerfiles && \
 ENTRYPOINT ["/code/run-with-freesurfer.sh"]
 
 # Link libraries for Singularity images
-RUN ldconfig
-
-RUN apt-get clean && \
-    apt-get autoremove -y && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN ldconfig \
+    && apt-get clean \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+    && chmod 777 / \
+    && chmod 777 $(ls / | grep -v sys | grep -v proc)
 
 # set user
 # USER c-pac_user

.github/Dockerfiles/C-PAC.develop-bionic.Dockerfile

@@ -17,9 +17,10 @@ RUN rm -Rf /code/docker_data/Dockerfiles && \
 ENTRYPOINT ["/code/run-with-freesurfer.sh"]
 
 # link libraries & clean up
-RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    ldconfig && \
-    chmod 777 $(ls / | grep -v sys | grep -v proc)
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+    && ldconfig \
+    && chmod 777 / \
+    && chmod 777 $(ls / | grep -v sys | grep -v proc)
 
 # set user
 # USER c-pac_user

.github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile

@@ -17,13 +17,14 @@ RUN rm -Rf /code/docker_data/Dockerfiles && \
 ENTRYPOINT ["/code/run-with-freesurfer.sh"]
 
 # link libraries & clean up
-RUN sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
-    locale-gen && \
-    apt-get clean && \
-    apt-get autoremove -y && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    ldconfig && \
-    chmod 777 $(ls / | grep -v sys | grep -v proc)
+RUN sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \
+    && locale-gen \
+    && apt-get clean \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+    && ldconfig \
+    && chmod 777 / \
+    && chmod 777 $(ls / | grep -v sys | grep -v proc)
 
 # set user
 # USER c-pac_user

.github/Dockerfiles/C-PAC.develop-lite-bionic.Dockerfile

@@ -1,13 +1,26 @@
-FROM ghcr.io/fcp-indi/c-pac:latest-bionic
+FROM ghcr.io/fcp-indi/c-pac/stage-base:lite-v1.8.6.dev1
 LABEL org.opencontainers.image.description "Full C-PAC image without FreeSurfer"
 LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC
 USER root
+
+# install C-PAC
+COPY dev/circleci_data/pipe-test_ci.yml /cpac_resources/pipe-test_ci.yml
+COPY . /code
+RUN pip install -e /code
+# set up runscript
+COPY dev/docker_data /code/docker_data
+RUN rm -Rf /code/docker_data/Dockerfiles && \
+    mv /code/docker_data/* /code && \
+    rm -Rf /code/docker_data && \
+    chmod +x /code/run.py && \
+    rm -Rf /code/run-with-freesurfer.sh
 ENTRYPOINT ["/code/run.py"]
 
-# remove FreeSurfer, link libraries & clean up
-RUN rm -rf /usr/lib/freesurfer/ /code/run-with-freesurfer.sh /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    ln -svf /usr/lib/x86_64-linux-gnu/libgsl.so.23 /usr/lib/x86_64-linux-gnu/libgsl.so.0 && ldconfig && \
-    chmod 777 $(ls / | grep -v sys | grep -v proc)
+# link libraries & clean up
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+    && ldconfig \
+    && chmod 777 / \
+    && chmod 777 $(ls / | grep -v sys | grep -v proc)
 
 # set user
 # USER c-pac_user

.github/Dockerfiles/FSL.5.0.10-bionic.Dockerfile

@@ -1,4 +1,20 @@
-FROM ghcr.io/shnizzedy/c-pac/fsl:neurodebian-bionic as FSL-Neurodebian
+# Copyright (C) 2021-2023  C-PAC Developers
+
+# This file is part of C-PAC.
+
+# C-PAC is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+
+# C-PAC is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+# License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
+FROM ghcr.io/fcp-indi/c-pac/fsl:data as data
 FROM ghcr.io/fcp-indi/c-pac/ubuntu:bionic-non-free AS FSL
 
 USER root
@@ -14,7 +30,6 @@ ENV FSLDIR=/usr/share/fsl/5.0 \
     PATH=/usr/lib/fsl/5.0:$PATH \
     TZ=America/New_York
 
-
 # Installing and setting up FSL
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
      echo $TZ > /etc/timezone && \
@@ -45,4 +60,6 @@ COPY --from=FSL /usr/bin/wish /usr/bin/wish
 COPY --from=FSL /usr/share/fsl/ /usr/share/fsl/
 COPY --from=FSL /usr/lib/ /usr/lib/
 COPY --from=FSL /lib/x86_64-linux-gnu/lib*so* /lib/x86_64-linux-gnu/
-COPY --from=FSL-Neurodebian /usr/share/fsl/5.0/data/standard/tissuepriors/*mm /usr/share/fsl/5.0/data/standard/tissuepriors/
+# install C-PAC resources into FSL
+COPY --from=data /fsl_data/standard /usr/share/fsl/5.0/data/standard
+COPY --from=data /fsl_data/atlases /usr/share/fsl/5.0/data/atlases