🔀 Merge develop into fix/distcorelif

Author: shnizzedy

.github/Dockerfiles/Ubuntu.bionic-non-free.Dockerfile

@@ -107,9 +107,9 @@ RUN groupadd -r c-pac && \
       libxp-dev && \
     add-apt-repository --remove --yes ppa:zeehio/libxp && \
     apt-get update && \
-    curl -sO https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.2-Linux-x86_64.sh && \
-    bash Miniconda3-py37_4.8.2-Linux-x86_64.sh -b -p /usr/local/miniconda && \
-    rm Miniconda3-py37_4.8.2-Linux-x86_64.sh && chmod -R 777 /usr/local/miniconda && \
+    curl -sO https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh && \
+    bash Miniconda3-py37_4.12.0-Linux-x86_64.sh -b -p /usr/local/miniconda && \
+    rm Miniconda3-py37_4.12.0-Linux-x86_64.sh && chmod -R 777 /usr/local/miniconda && \
     ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
     echo $TZ > /etc/timezone && \
     sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \

.github/Dockerfiles/Ubuntu.xenial-20200114.Dockerfile

@@ -120,9 +120,9 @@ RUN apt-get update && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Installing and setting up miniconda
-RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \
-    bash Miniconda3-4.5.11-Linux-x86_64.sh -b -p /usr/local/miniconda && \
-    rm Miniconda3-4.5.11-Linux-x86_64.sh && chmod -R 777 /usr/local/miniconda
+RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh && \
+    bash Miniconda3-py37_4.12.0-Linux-x86_64.sh -b -p /usr/local/miniconda && \
+    rm Miniconda3-py37_4.12.0-Linux-x86_64.sh && chmod -R 777 /usr/local/miniconda
 
 # Set CPATH for packages relying on compiled libs (e.g. indexed_gzip)
 ENV PATH="/usr/local/miniconda/bin:$PATH" \

.github/scripts/get_package_id.py

@@ -11,18 +11,18 @@
 
 def get_packages(owner, tag, api_token=None):
     """Function to collect GHCR packages for a given owner & tag
-    
+
     Parameters
     ----------
     owner : str
         username or org name
-    
+
     tag : str
         image tag (part left of the colon)
 
     api_token : str or None
         GitHub API personal access token with read.packages permission
-    
+
     Returns
     -------
     list
@@ -32,11 +32,11 @@ def get_packages(owner, tag, api_token=None):
 
     def fetch(url):
         """Method to make API call and return response, given a URL
-        
+
         Parameters
         ----------
         url : str
-        
+
         Returns
         -------
         dict or list
@@ -59,34 +59,36 @@ def fetch(url):
                 '$VERSION_TAG $GITHUB_TOKEN`'
             ]))
         return response
-    packages = fetch(
+    _packages = fetch(
         f'https://api.github.com/orgs/{owner}/packages/container/'
         f'{tag}/versions')
-    if packages.get('message', 'Not Found') == 'Not Found':
-        packages = fetch(
-            f'https://api.github.com/users/{owner}/packages/container/'
-            f'{tag}/versions')
+    packages = []
+    for _package in _packages:
+        if _package.get('message', 'Not Found') == 'Not Found':
+            packages += fetch(
+                f'https://api.github.com/users/{owner}/packages/container/'
+                f'{tag}/versions')
     return packages
 
 
 def id_from_tag(owner, image, tag, api_token=None):
     """Function to return a package ID given an image version tag
-    
+
     Parameters
     ----------
     owner : str
         GitHub username or org name
-    
+
     image : str
         Image tag (the part to the left of the colon)
-    
+
     tag : str
         Image version tag (the part to the right of the colon)
 
     api_token: str or None
         GitHub API personal access token with read.packages permission
     """
-    packages = get_packages(owner, image)
+    packages = get_packages(owner, image, api_token)
     versions = [image['id'] for image in packages if tag in image.get(
         'metadata', {}
     ).get('container', {}).get('tags', [])]

.github/workflows/build_stages.yml

@@ -66,7 +66,7 @@ jobs:
           DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}/$TAG" | tr '[:upper:]' '[:lower:]')
           echo DOCKER_TAG=$DOCKER_TAG >> $GITHUB_ENV
           docker manifest inspect $DOCKER_TAG >/dev/null
-          echo "::set-output name=not_yet_exists::$?"
+          echo "not_yet_exists=$?" >> $GITHUB_OUTPUT
       - name: Set up Docker Buildx
         uses: docker/[email protected]
         if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile)
@@ -156,7 +156,7 @@ jobs:
           DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}/$TAG" | tr '[:upper:]' '[:lower:]')
           echo DOCKER_TAG=$DOCKER_TAG >> $GITHUB_ENV
           docker manifest inspect $DOCKER_TAG >/dev/null
-          echo "::set-output name=not_yet_exists::$?"
+          echo "not_yet_exists=$?" >> $GITHUB_OUTPUT
         if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile)
       - name: Prep Dockerfiles for forked repository
         if: github.repository_owner != 'FCP-INDI' && contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile)
@@ -230,7 +230,7 @@ jobs:
         DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}/$TAG" | tr '[:upper:]' '[:lower:]')
         echo DOCKER_TAG=$DOCKER_TAG >> $GITHUB_ENV
         docker manifest inspect $DOCKER_TAG >/dev/null
-        echo "::set-output name=not_yet_exists::$?"
+        echo "not_yet_exists=$?" >> $GITHUB_OUTPUT
       id: docker_tag
     - name: Clear up some space on runner
       if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1

.github/workflows/check_updated_version.yml renamed to .github/workflows/check_updated_preconfigs.yml

@@ -1,19 +1,23 @@
-name: Check updated version
+name: Check updated preconfigs
 
 on:
   pull_request_target:
     types: [edited, opened, reopened, ready_for_review, synchronize]
   push:
 
 jobs:
-  check-updated-version:
-    name: Check if version updated
+  check-updated-preconfigs:
+    name: Check if preconfigs need updated
     runs-on: ubuntu-latest
     steps:
       - name: Check out C-PAC
         uses: actions/checkout@v2
         with:
           fetch-depth: 2
+      - uses: actions/setup-python@v4 
+        with:
+          python-version: '3.9'
+          cache: 'pip'
       - name: Check if version updated
         run: |
           git config --global user.email "[email protected]"
@@ -37,6 +41,14 @@ jobs:
             [skip ci]"
             git push origin HEAD:${GITHUB_BRANCH} || true
           fi
+          export PYTHONPATH=$PWD
+          pip install nipype numpy matplotlib pandas pathvalidate pytz pyyaml voluptuous yamlordereddictloader
+          python ./CPAC/utils/configuration/yaml_template.py
+          if [[ ! -z $(git diff origin/${GITHUB_BRANCH}) ]]
+          then
+            git add CPAC/resources/configs
+            git commit -m ":bulb: Update comments based on default preconfig"
+          fi
           COMMIT_MESSAGE=$(git log -1 --pretty=%B)
           if [[ ! "$COMMIT_MESSAGE" == *"Update version to"* ]]
           then
@@ -48,14 +60,14 @@ jobs:
             git add version CPAC/resources/configs
             if [[ ! -z $(git diff origin/${GITHUB_BRANCH}) ]]
             then
-              git commit -m ":bookmark: Update version to ${VERSION} ($COMMIT_MESSAGE)"
+              git commit -m ":bookmark: Update version to ${VERSION} ($COMMIT_MESSAGE)" || true
               git push origin HEAD:${GITHUB_BRANCH} || true
             fi
             cd ..
           fi
 
   trigger-next-workflow:
-    needs: check-updated-version
+    needs: check-updated-preconfigs
     runs-on: ubuntu-latest
     steps:
       - name: Set up check

CHANGELOG.md

@@ -34,8 +34,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added crash messages from during and before graph building to logs
 - Added data-config-specific hash string to C-PAC-generated config files
 - Updated `rbc-options` preconfig to use `fmriprep-options` preprocessing
+- Changed minimized pipeline base from `default` to `blank`
 
 ### Fixed
+- Fixed [a bug](https://github.com/FCP-INDI/C-PAC/issues/1779) in which generated pipeline configs were not 100% accurate. The only affected configurable option discovered in testing was seed-based correlation analysis always reverting to the default configuration.
 - Fixed [bug](https://github.com/FCP-INDI/C-PAC/issues/1795) that was causing `cpac run` to fail when passing a manual random seed via `--random_seed`.
 - Replaces ``DwellTime`` with ``EffectiveEchoSpacing`` for FSL usage of the term
 - Fixed an issue that was causing some epi field maps to not be ingressed if the BIDS tags were not in the correct order.

CPAC/alff/alff.py

@@ -246,8 +246,7 @@ def alff_falff(wf, cfg, strat_pool, pipe_num, opt=None):
      "switch": ["run"],
      "option_key": "None",
      "option_val": "None",
-     "inputs": [["desc-denoisedNofilt_bold", "desc-brain_bold", 
-                 "desc-preproc_bold", "bold"],
+     "inputs": ["desc-denoisedNofilt_bold",
                 "space-bold_desc-brain_mask"],
      "outputs": ["alff",
                  "falff"]}
@@ -262,8 +261,7 @@ def alff_falff(wf, cfg, strat_pool, pipe_num, opt=None):
     alff.get_node('hp_input').iterables = ('hp', alff.inputs.hp_input.hp)
     alff.get_node('lp_input').iterables = ('lp', alff.inputs.lp_input.lp)
 
-    node, out = strat_pool.get_data(["desc-denoisedNofilt_bold", "desc-brain_bold",
-                                     "desc-preproc_bold", "bold"])
+    node, out = strat_pool.get_data("desc-denoisedNofilt_bold")
     wf.connect(node, out, alff, 'inputspec.rest_res')
 
     node, out = strat_pool.get_data('space-bold_desc-brain_mask')