🔀 Merge develop into fix/minimal-schema

Author: shnizzedy

.github/Dockerfiles/Ubuntu.bionic-non-free.Dockerfile

@@ -107,9 +107,9 @@ RUN groupadd -r c-pac && \
       libxp-dev && \
     add-apt-repository --remove --yes ppa:zeehio/libxp && \
     apt-get update && \
-    curl -sO https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.2-Linux-x86_64.sh && \
-    bash Miniconda3-py37_4.8.2-Linux-x86_64.sh -b -p /usr/local/miniconda && \
-    rm Miniconda3-py37_4.8.2-Linux-x86_64.sh && chmod -R 777 /usr/local/miniconda && \
+    curl -sO https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh && \
+    bash Miniconda3-py37_4.12.0-Linux-x86_64.sh -b -p /usr/local/miniconda && \
+    rm Miniconda3-py37_4.12.0-Linux-x86_64.sh && chmod -R 777 /usr/local/miniconda && \
     ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
     echo $TZ > /etc/timezone && \
     sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \

.github/Dockerfiles/Ubuntu.xenial-20200114.Dockerfile

@@ -120,9 +120,9 @@ RUN apt-get update && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Installing and setting up miniconda
-RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \
-    bash Miniconda3-4.5.11-Linux-x86_64.sh -b -p /usr/local/miniconda && \
-    rm Miniconda3-4.5.11-Linux-x86_64.sh && chmod -R 777 /usr/local/miniconda
+RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh && \
+    bash Miniconda3-py37_4.12.0-Linux-x86_64.sh -b -p /usr/local/miniconda && \
+    rm Miniconda3-py37_4.12.0-Linux-x86_64.sh && chmod -R 777 /usr/local/miniconda
 
 # Set CPATH for packages relying on compiled libs (e.g. indexed_gzip)
 ENV PATH="/usr/local/miniconda/bin:$PATH" \

.github/scripts/get_package_id.py

@@ -11,18 +11,18 @@
 
 def get_packages(owner, tag, api_token=None):
     """Function to collect GHCR packages for a given owner & tag
-    
+
     Parameters
     ----------
     owner : str
         username or org name
-    
+
     tag : str
         image tag (part left of the colon)
 
     api_token : str or None
         GitHub API personal access token with read.packages permission
-    
+
     Returns
     -------
     list
@@ -32,11 +32,11 @@ def get_packages(owner, tag, api_token=None):
 
     def fetch(url):
         """Method to make API call and return response, given a URL
-        
+
         Parameters
         ----------
         url : str
-        
+
         Returns
         -------
         dict or list
@@ -59,34 +59,36 @@ def fetch(url):
                 '$VERSION_TAG $GITHUB_TOKEN`'
             ]))
         return response
-    packages = fetch(
+    _packages = fetch(
         f'https://api.github.com/orgs/{owner}/packages/container/'
         f'{tag}/versions')
-    if packages.get('message', 'Not Found') == 'Not Found':
-        packages = fetch(
-            f'https://api.github.com/users/{owner}/packages/container/'
-            f'{tag}/versions')
+    packages = []
+    for _package in _packages:
+        if _package.get('message', 'Not Found') == 'Not Found':
+            packages += fetch(
+                f'https://api.github.com/users/{owner}/packages/container/'
+                f'{tag}/versions')
     return packages
 
 
 def id_from_tag(owner, image, tag, api_token=None):
     """Function to return a package ID given an image version tag
-    
+
     Parameters
     ----------
     owner : str
         GitHub username or org name
-    
+
     image : str
         Image tag (the part to the left of the colon)
-    
+
     tag : str
         Image version tag (the part to the right of the colon)
 
     api_token: str or None
         GitHub API personal access token with read.packages permission
     """
-    packages = get_packages(owner, image)
+    packages = get_packages(owner, image, api_token)
     versions = [image['id'] for image in packages if tag in image.get(
         'metadata', {}
     ).get('container', {}).get('tags', [])]

.github/workflows/build_stages.yml

@@ -66,7 +66,7 @@ jobs:
           DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}/$TAG" | tr '[:upper:]' '[:lower:]')
           echo DOCKER_TAG=$DOCKER_TAG >> $GITHUB_ENV
           docker manifest inspect $DOCKER_TAG >/dev/null
-          echo "::set-output name=not_yet_exists::$?"
+          echo "not_yet_exists=$?" >> $GITHUB_OUTPUT
       - name: Set up Docker Buildx
         uses: docker/[email protected]
         if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile)
@@ -156,7 +156,7 @@ jobs:
           DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}/$TAG" | tr '[:upper:]' '[:lower:]')
           echo DOCKER_TAG=$DOCKER_TAG >> $GITHUB_ENV
           docker manifest inspect $DOCKER_TAG >/dev/null
-          echo "::set-output name=not_yet_exists::$?"
+          echo "not_yet_exists=$?" >> $GITHUB_OUTPUT
         if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile)
       - name: Prep Dockerfiles for forked repository
         if: github.repository_owner != 'FCP-INDI' && contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile)
@@ -230,7 +230,7 @@ jobs:
         DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}/$TAG" | tr '[:upper:]' '[:lower:]')
         echo DOCKER_TAG=$DOCKER_TAG >> $GITHUB_ENV
         docker manifest inspect $DOCKER_TAG >/dev/null
-        echo "::set-output name=not_yet_exists::$?"
+        echo "not_yet_exists=$?" >> $GITHUB_OUTPUT
       id: docker_tag
     - name: Clear up some space on runner
       if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1

CHANGELOG.md

@@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added the ability to ingress TotalReadoutTime from epi field map meta-data from the JSON sidecars.
 - Added the ability to use TotalReadoutTime of epi field maps in the calculation of FSL topup distortion correction.
 - Difference method (``-``) for ``CPAC.utils.configuration.Configuration`` instances
+- Calculate reho and alff when timeseries in template space
 
 ### Changed
 - Added a level of depth to `working` directories to match `log` and `output` directory structure

CPAC/alff/alff.py

@@ -246,7 +246,7 @@ def alff_falff(wf, cfg, strat_pool, pipe_num, opt=None):
      "switch": ["run"],
      "option_key": "None",
      "option_val": "None",
-     "inputs": [["desc-cleanedNofilt_bold", "desc-brain_bold", 
+     "inputs": [["desc-denoisedNofilt_bold", "desc-brain_bold", 
                  "desc-preproc_bold", "bold"],
                 "space-bold_desc-brain_mask"],
      "outputs": ["alff",
@@ -262,7 +262,7 @@ def alff_falff(wf, cfg, strat_pool, pipe_num, opt=None):
     alff.get_node('hp_input').iterables = ('hp', alff.inputs.hp_input.hp)
     alff.get_node('lp_input').iterables = ('lp', alff.inputs.lp_input.lp)
 
-    node, out = strat_pool.get_data(["desc-cleanedNofilt_bold", "desc-brain_bold",
+    node, out = strat_pool.get_data(["desc-denoisedNofilt_bold", "desc-brain_bold",
                                      "desc-preproc_bold", "bold"])
     wf.connect(node, out, alff, 'inputspec.rest_res')
 
@@ -275,3 +275,38 @@ def alff_falff(wf, cfg, strat_pool, pipe_num, opt=None):
     }
 
     return (wf, outputs)
+
+def alff_falff_space_template(wf, cfg, strat_pool, pipe_num, opt=None):
+    '''
+    {"name": "alff_falff_space_template",
+     "config": ["amplitude_low_frequency_fluctuation"],
+     "switch": ["run"],
+     "option_key": "None",
+     "option_val": "None",
+     "inputs": [["space-template_desc-denoisedNofilt_bold"],
+                "space-template_desc-bold_mask"],
+     "outputs": ["space-template_alff",
+                 "space-template_falff"]}
+    '''
+
+    alff = create_alff(f'alff_falff_{pipe_num}')
+
+    alff.inputs.hp_input.hp = \
+        cfg.amplitude_low_frequency_fluctuation['highpass_cutoff']
+    alff.inputs.lp_input.lp = \
+        cfg.amplitude_low_frequency_fluctuation['lowpass_cutoff']
+    alff.get_node('hp_input').iterables = ('hp', alff.inputs.hp_input.hp)
+    alff.get_node('lp_input').iterables = ('lp', alff.inputs.lp_input.lp)
+
+    node, out = strat_pool.get_data(["space-template_desc-denoisedNofilt_bold"])
+    wf.connect(node, out, alff, 'inputspec.rest_res')
+
+    node, out = strat_pool.get_data("space-template_desc-bold_mask")
+    wf.connect(node, out, alff, 'inputspec.rest_mask')
+
+    outputs = {
+        'space-template_alff': (alff, 'outputspec.alff_img'),
+        'space-template_falff': (alff, 'outputspec.falff_img')
+    }
+
+    return (wf, outputs)

CPAC/info.py

@@ -177,6 +177,7 @@ def get_cpac_gitversion():
     "future",
     "INDI-Tools",
     "lockfile==0.12.2",
+    "joblib==1.0.1",
     "matplotlib==3.1.3",
     "networkx==2.4",
     "nibabel==2.3.3",

CPAC/nuisance/nuisance.py

@@ -2532,7 +2532,6 @@ def nuisance_regression(wf, cfg, strat_pool, pipe_num, opt, space):
         outputs = {
             desc_keys[0]: (nuis, 'outputspec.residual_file_path'),
             desc_keys[1]: (nuis, 'outputspec.residual_file_path'),
-            desc_keys[2]: (nuis, 'outputspec.residual_file_path')
         }
 
     return (wf, outputs)
@@ -2587,13 +2586,10 @@ def nuisance_regression_template(wf, cfg, strat_pool, pipe_num, opt=None):
                  "dvars"),
                 "TR"],
      "outputs": {"space-template_desc-preproc_bold": {
-        "Description": "Preprocessed BOLD image that was nuisance-"
+        "Description": "Preprocessed BOLD image that was nusiance-"
                        "regressed in template space"},
                  "space-template_desc-cleaned_bold": {
-        "Description": "Preprocessed BOLD image that was nuisance-"
-                       "regressed in template space"},
-                 "space-template_desc-denoisedNofilt_bold": {
-        "Description": "Preprocessed BOLD image that was nuisance-"
+        "Description": "Preprocessed BOLD image that was nusiance-"
                        "regressed in template space"},
                  "regressors": {
         "Description": "Regressors that were applied in template space"}}}

CPAC/pipeline/cpac_pipeline.py

@@ -102,6 +102,7 @@
     warp_bold_mean_to_T1template,
     warp_bold_mask_to_T1template,
     warp_deriv_mask_to_T1template,
+    warp_denoiseNofilt_to_T1template,
     warp_timeseries_to_EPItemplate,
     warp_bold_mean_to_EPItemplate,
     warp_bold_mask_to_EPItemplate,
@@ -183,8 +184,8 @@
     multiple_regression
 )
 
-from CPAC.alff.alff import alff_falff
-from CPAC.reho.reho import reho
+from CPAC.alff.alff import alff_falff, alff_falff_space_template
+from CPAC.reho.reho import reho, reho_space_template
 
 from CPAC.vmhc.vmhc import (
     smooth_func_vmhc,
@@ -1282,6 +1283,10 @@ def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None,
     if not rpool.check_rpool('space-template_desc-bold_mask'):
         pipeline_blocks += [warp_bold_mask_to_T1template,
                             warp_deriv_mask_to_T1template]
+        
+    target_space_alff = cfg.amplitude_low_frequency_fluctuation['target_space']
+    if 'Template' in target_space_alff and not rpool.check_rpool('space-template_desc-denoisedNofilt_bold'):
+        pipeline_blocks += [warp_denoiseNofilt_to_T1template]
 
     template = cfg.registration_workflows['functional_registration']['func_registration_to_template']['target_template']['using']
 
@@ -1324,6 +1329,7 @@ def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None,
     tse_atlases, sca_atlases = gather_extraction_maps(cfg)
     cfg.timeseries_extraction['tse_atlases'] = tse_atlases
     cfg.seed_based_correlation_analysis['sca_atlases'] = sca_atlases
+    target_space_reho = cfg.regional_homogeneity['target_space']
 
     if not rpool.check_rpool('desc-Mean_timeseries') and \
                     'Avg' in tse_atlases:
@@ -1349,11 +1355,21 @@ def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None,
                     'MultReg' in sca_atlases:
         pipeline_blocks += [multiple_regression]
 
-    if not rpool.check_rpool('alff'):
-        pipeline_blocks += [alff_falff]
-
-    if not rpool.check_rpool('reho'):
-        pipeline_blocks += [reho]
+    if 'Native' in target_space_alff:
+        if not rpool.check_rpool('alff'):
+            pipeline_blocks += [alff_falff]
+
+    if 'Template' in target_space_alff:
+        if not rpool.check_rpool('space-template_alff'):
+            pipeline_blocks += [alff_falff_space_template]
+
+    if 'Native' in target_space_reho:
+        if not rpool.check_rpool('reho'):
+            pipeline_blocks += [reho]
+            
+    if 'Template' in target_space_reho:
+        if not rpool.check_rpool('space-template_reho'):
+            pipeline_blocks += [reho_space_template]
 
     if not rpool.check_rpool('vmhc'):
         pipeline_blocks += [smooth_func_vmhc,

CPAC/pipeline/engine.py

@@ -88,8 +88,10 @@ def __init__(self, rpool=None, name=None, cfg=None, pipe_list=None):
 
             self.run_smoothing = 'smoothed' in cfg.post_processing[
                 'spatial_smoothing']['output']
+            self.smoothing_bool = cfg.post_processing['spatial_smoothing']['run']
             self.run_zscoring = 'z-scored' in cfg.post_processing[
                 'z-scoring']['output']
+            self.zscoring_bool = cfg.post_processing['z-scoring']['run']
             self.fwhm = cfg.post_processing['spatial_smoothing']['fwhm']
             self.smooth_opts = cfg.post_processing['spatial_smoothing'][
                 'smoothing_method']
@@ -697,7 +699,7 @@ def post_process(self, wf, label, connection, json_info, pipe_idx, pipe_x,
                     mask_idx = self.generate_prov_string(mask_prov)[1]
                     break
 
-        if self.run_smoothing:
+        if self.smoothing_bool:
             if label in Outputs.to_smooth:
                 for smooth_opt in self.smooth_opts:
 
@@ -736,7 +738,7 @@ def post_process(self, wf, label, connection, json_info, pipe_idx, pipe_x,
                                   pipe_idx, f'spatial_smoothing_{smooth_opt}',
                                   fork=True)
 
-        if self.run_zscoring:            
+        if self.zscoring_bool:            
             for label_con_tpl in post_labels:
                 label = label_con_tpl[0]
                 connection = (label_con_tpl[1], label_con_tpl[2])