Merge branch 'develop' into upgrade/python

Author: nx10

.github/READTHIS.md

@@ -0,0 +1,123 @@
+<!-- Don't rename to '.github/README.md' or it will override the root readme. -->
+
+# C-PAC/.github README
+
+This directory contains Dockerfiles, scripts, requirements files, and workflow configurations for [this repository's GitHub Actions](https://github.com/FCP-INDI/C-PAC/actions).
+
+When updating these files, please update this README as necessary.
+
+[The `workflows` directory](./workflows) contains the configurations for the Actions themselves. The other directories support these configs.
+
+```mermaid
+flowchart TD
+    subgraph Dockerfiles
+      base[base-.*]
+
+      cpacdockerfiles["C-PAC.develop-(?!lite).*"]
+
+      cpaclitedockerfile[C-PAC.develop-lite.*]
+
+      stagedockerfiles[AFNI.*\nANTs.*\nc3d.*\nconnectome-workbench.*\nFSL.*\nICA-AROMA.*\nmsm.*]
+
+      ubuntudockerfiles[Ubuntu.*]
+    end
+    subgraph scripts
+      get_package_id.py
+      get_pr_base_shas
+      local_ghcr
+    end
+    subgraph smoke_test_participant.yml
+      smoke_test_human
+      smoke_test_nhp
+      smoke_test_rodent
+    end
+    subgraph stage_requirements
+      .*.txt
+    end
+    subgraph workflows
+      subgraph build_stages.yml
+
+        ubuntu[[Ubnutu]]-->stages[[stages]]-->build-base[[build-base]]
+
+      end
+      subgraph build_C-PAC.yml
+        bCPAC[[C-PAC]]
+      end
+      subgraph build_and_test.yml
+        Circle_tests[[Circle_tests]]
+      
+        C-PAC[[C-PAC]]-->bCPAC
+        C-PAC-->Circle_tests
+        C-PAC-->smoke-tests-participant
+        C-PAC-->C-PAC-lite
+
+        C-PAC-lite[[C-PAC-lite]]-->bCPAC
+        C-PAC-lite-->Circle_tests
+        C-PAC-lite-->smoke-tests-participant
+
+        C-PAC-ABCD-HCP[[C-PAC-ABCD-HCP]]-->bCPAC
+        C-PAC-ABCD-HCP-->Circle_tests
+        C-PAC-ABCD-HCP-->smoke-tests-participant
+
+        C-PAC-fMRIPrep[[C-PAC-fMRIPrep-LTS]]-->bCPAC
+        C-PAC-fMRIPrep-->Circle_tests
+        C-PAC-fMRIPrep-->smoke-tests-participant
+
+        smoke-tests-participant[[smoke-tests-participant]]
+      end
+
+      check_updated_preconfigs.yml-->build_stages.yml
+
+      delete_images.yml
+    end
+
+    subgraph yaml_template[CPAC/utils/configuration/yaml_template.py]
+
+      update_all_preconfigs[[update_all_preconfigs]]
+    end
+
+    base<-->build-base
+
+    Circle_tests-->CircleCI((Run tests on Circle CI))
+
+    build_stages.yml-->build_and_test.yml
+
+    check_updated_preconfigs.yml<-->get_pr_base_shas
+    check_updated_preconfigs.yml-->update_all_preconfigs
+
+    cpacdockerfiles<-->C-PAC
+    cpacdockerfiles<-->C-PAC-ABCD-HCP
+    cpacdockerfiles<-->C-PAC-fMRIPrep
+
+    cpaclitedockerfile<-->C-PAC-lite
+
+    delete>delete branch]-->delete_images.yml
+
+    delete_images.yml<-->get_package_id.py
+
+    build-base<-->local_ghcr
+    bCPAC<-->local_ghcr
+    stages<-->local_ghcr
+
+    push>git push]-->check_updated_preconfigs.yml
+
+    smoke-tests-participant-->smoke_test_human
+    smoke-tests-participant-->smoke_test_nhp
+    smoke-tests-participant-->smoke_test_rodent
+
+    stagedockerfiles<-->stages
+
+    build-base<-->.*.txt
+
+    ubuntudockerfiles<-->ubuntu
+```
+
+In this [Mermaid flowchart](https://mermaid.js.org/syntax/flowchart.html), these shapes are used:
+
+concept | shape
+---|---
+directory | `subgraph`
+file | rectangle (`[]`) or `subgraph`
+workflow job | subprocess (`[[]]`)
+trigger action | asymetric (`>]`)
+external API | circle (`(())`)

.github/workflows/deploy_to_Docker_Hub.yml

@@ -32,6 +32,7 @@ jobs:
           TAG1=release-${{ github.ref_name }}$VARIANT
           DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
           echo "original tag: ${DOCKER_TAG}:${TAG1}"
+          docker pull ${DOCKER_TAG}:${TAG1}
           docker tag ${DOCKER_TAG}:${TAG1} fcpindi/c-pac:${TAG1}
           docker push fcpindi/c-pac:${TAG1}
           if [[ ${{ github.ref_name }} =~ ${TAG_PATTERN} ]]

.github/workflows/smoke_test_participant.yml

@@ -122,14 +122,6 @@ jobs:
           - ''
         participant:
           - 032102 032106 032164 032167 032130 032128 2215 2312 032191 032195
-        include:
-          # requires T2w
-          - preconfig: monkey-ABCD
-            variant: ''
-            participant: 032102 032106 032130 032128 032191 032195
-          - preconfig: monkey-ABCD
-            variant: ABCD-HCP
-            participant: 032102 032106 032130 032128 032191 032195
     steps:
       - name: Get C-PAC
         run: |

CHANGELOG.md

@@ -88,6 +88,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added NodeBlock information to `pypeline.log` when verbose debugging is on
 - Added the ability to ingress FreeSurfer data into CPAC
 - Added the ability to toggle FreeSurfer derived masks for brain extraction
+- Added an optional volume center to FD-J calculation
+- Added new preconfig `abcd-prep`, which performs minimal preprocessing on the T1w data in preparation for Freesurfer Recon-All
 
 ### Changed
 

CPAC/anat_preproc/anat_preproc.py

@@ -16,7 +16,8 @@
     mri_convert, \
     wb_command, \
     fslmaths_command, \
-    VolumeRemoveIslands
+    VolumeRemoveIslands, \
+    normalize_wmparc
 from CPAC.pipeline.engine import flatten_list
 from CPAC.utils.docs import docstring_parameter
 from CPAC.utils.interfaces.fsl import Merge as fslMerge
@@ -914,16 +915,44 @@ def freesurfer_abcd_brain_connector(wf, cfg, strat_pool, pipe_num, opt):
     ### ABCD harmonization - anatomical brain mask generation ###
     # Ref: https://github.com/DCAN-Labs/DCAN-HCP/blob/master/PostFreeSurfer/PostFreeSurferPipeline.sh#L151-L156
 
+    
     wmparc_to_nifti = pe.Node(util.Function(input_names=['in_file',
                                                          'reslice_like',
                                                          'args'],
                                             output_names=['out_file'],
                                             function=mri_convert),
                               name=f'wmparc_to_nifti_{pipe_num}')
-    wmparc_to_nifti.inputs.args = '-rt nearest'
+    
+    # Register wmparc file if ingressing FreeSurfer data
+    if strat_pool.check_rpool('pipeline-fs_xfm'):
+
+        wmparc_to_native = pe.Node(util.Function(input_names=['source_file',
+                                                            'target_file',
+                                                            'xfm',
+                                                            'out_file'],
+                                                output_names=['transformed_file'],
+                                                function=normalize_wmparc),
+                                        name=f'wmparc_to_native_{pipe_num}')
+        
+        wmparc_to_native.inputs.out_file = 'wmparc_warped.mgz'
+
+        node, out = strat_pool.get_data('pipeline-fs_wmparc')
+        wf.connect(node, out, wmparc_to_native, 'source_file')
+
+        node, out = strat_pool.get_data('pipeline-fs_raw-average')
+        wf.connect(node, out, wmparc_to_native, 'target_file')
 
-    node, out = strat_pool.get_data('pipeline-fs_wmparc')
-    wf.connect(node, out, wmparc_to_nifti, 'in_file')
+        node, out = strat_pool.get_data('pipeline-fs_xfm')
+        wf.connect(node, out, wmparc_to_native, 'xfm')
+
+        wf.connect(wmparc_to_native, 'transformed_file', wmparc_to_nifti, 'in_file')
+    
+    else:
+        
+        node, out = strat_pool.get_data('pipeline-fs_wmparc')
+        wf.connect(node, out, wmparc_to_nifti, 'in_file')
+
+    wmparc_to_nifti.inputs.args = '-rt nearest'
 
     node, out = strat_pool.get_data('desc-preproc_T1w')
     wf.connect(node, out, wmparc_to_nifti, 'reslice_like')
@@ -1244,21 +1273,24 @@ def acpc_align_head(wf, cfg, strat_pool, pipe_num, opt=None):
                 ["anatomical_preproc", "run"]],
      "option_key": "None",
      "option_val": "None",
-     "inputs": ["desc-head_T1w",
-                "desc-preproc_T1w",
+     "inputs": ["desc-preproc_T1w",
+                "desc-head_T1w",
                 "T1w-ACPC-template"],
      "outputs": ["desc-head_T1w",
                  "desc-preproc_T1w",
                  "from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm"]}
     '''
-
+    
+    # NOTE: desc-preproc_T1w and desc-head_T1w are both needed in this 
+    # nodeblock despite the list called in get_data() below
+    
     acpc_align = acpc_alignment(config=cfg,
                                 acpc_target=cfg.anatomical_preproc[
                                     'acpc_alignment']['acpc_target'],
                                 mask=False,
                                 wf_name=f'acpc_align_{pipe_num}')
 
-    node, out = strat_pool.get_data(['desc-head_T1w', 'desc-preproc_T1w'])
+    node, out = strat_pool.get_data(['desc-preproc_T1w','desc-head_T1w'])
     wf.connect(node, out, acpc_align, 'inputspec.anat_leaf')
 
     node, out = strat_pool.get_data('T1w-ACPC-template')
@@ -1772,6 +1804,8 @@ def brain_mask_freesurfer_abcd(wf, cfg, strat_pool, pipe_num, opt=None):
      "option_val": "FreeSurfer-ABCD",
      "inputs": ["desc-preproc_T1w",
                 "pipeline-fs_wmparc",
+                "pipeline-fs_raw-average",
+                "pipeline-fs_xfm",
                 "freesurfer-subject-dir"],
      "outputs": ["space-T1w_desc-brain_mask"]}
     '''
@@ -1813,6 +1847,8 @@ def brain_mask_acpc_freesurfer_abcd(wf, cfg, strat_pool, pipe_num, opt=None):
      "option_val": "FreeSurfer-ABCD",
      "inputs": ["desc-preproc_T1w",
                 "pipeline-fs_wmparc",
+                "pipeline-fs_raw-average",
+                "pipeline-fs_xfm",
                 "freesurfer-subject-dir"],
      "outputs": ["space-T1w_desc-acpcbrain_mask"]}
     '''
@@ -2533,10 +2569,10 @@ def brain_extraction_temp_T2(wf, cfg, strat_pool, pipe_num, opt=None):
 def freesurfer_abcd_preproc(wf, cfg, strat_pool, pipe_num, opt=None):
     '''
     {"name": "freesurfer_abcd_preproc",
-     "config": ["anatomical_preproc", "brain_extraction"],
-     "switch": "None",
-     "option_key": "using",
-     "option_val": "FreeSurfer-ABCD",
+     "config": ["surface_analysis", "abcd_prefreesurfer_prep"],
+     "switch": ["run"],
+     "option_key": "None",
+     "option_val": "None",
      "inputs": ["desc-preproc_T1w",
                 "T1w-template",
                 "T1w-brain-template-mask",
@@ -2545,16 +2581,17 @@ def freesurfer_abcd_preproc(wf, cfg, strat_pool, pipe_num, opt=None):
                 "freesurfer-subject-dir"],
      "outputs": ["desc-restore_T1w",
                  "desc-restore-brain_T1w",
+                 "desc-ABCDpreproc_T1w",
                  "pipeline-fs_desc-fast_biasfield",
                  "pipeline-fs_hemi-L_desc-surface_curv",
-                 "pipeline-fs_hemi-R_desc-surface_curv", 
+                 "pipeline-fs_hemi-R_desc-surface_curv",
                  "pipeline-fs_hemi-L_desc-surfaceMesh_pial",
                  "pipeline-fs_hemi-R_desc-surfaceMesh_pial",
                  "pipeline-fs_hemi-L_desc-surfaceMesh_smoothwm",
                  "pipeline-fs_hemi-R_desc-surfaceMesh_smoothwm",
                  "pipeline-fs_hemi-L_desc-surfaceMesh_sphere",
                  "pipeline-fs_hemi-R_desc-surfaceMesh_sphere",
-                 "pipeline-fs_hemi-L_desc-surfaceMap_sulc", 
+                 "pipeline-fs_hemi-L_desc-surfaceMap_sulc",
                  "pipeline-fs_hemi-R_desc-surfaceMap_sulc",
                  "pipeline-fs_hemi-L_desc-surfaceMap_thickness",
                  "pipeline-fs_hemi-R_desc-surfaceMap_thickness",
@@ -2565,7 +2602,6 @@ def freesurfer_abcd_preproc(wf, cfg, strat_pool, pipe_num, opt=None):
                  "pipeline-fs_wmparc",
                  "freesurfer-subject-dir"]}
     '''
-
     # fnirt-based brain extraction
     brain_extraction = fnirt_based_brain_extraction(config=cfg,
                                                     wf_name=f'fnirt_based_brain_extraction_{pipe_num}')
@@ -2657,52 +2693,15 @@ def freesurfer_abcd_preproc(wf, cfg, strat_pool, pipe_num, opt=None):
     wf.connect(average_brain, 'out_stat',
                normalize_head, 'number')
 
-    if strat_pool.check_rpool('freesurfer-subject-dir'):
-        outputs = {
+    outputs = {
             'desc-restore_T1w': (fast_correction, 'outputspec.anat_restore'),
             'desc-restore-brain_T1w': (fast_correction,
                                        'outputspec.anat_brain_restore'),
-            'pipeline-fs_desc-fast_biasfield': (fast_correction, 'outputspec.bias_field')}
-        return (wf, outputs)
-
-    else:
-        ### recon-all -all step ###
-        reconall = pe.Node(interface=freesurfer.ReconAll(),
-                           name=f'anat_freesurfer_{pipe_num}',
-                           mem_gb=2.7)
-
-        sub_dir = cfg.pipeline_setup['working_directory']['path']
-        freesurfer_subject_dir = os.path.join(sub_dir,
-                                              'cpac_'+cfg['subject_id'],
-                                              f'anat_preproc_freesurfer_{pipe_num}',
-                                              'anat_freesurfer')
-
-        # create the directory for FreeSurfer node
-        if not os.path.exists(freesurfer_subject_dir):
-            os.makedirs(freesurfer_subject_dir)
-
-        reconall.inputs.directive = 'all'
-        reconall.inputs.subjects_dir = freesurfer_subject_dir
-        reconall.inputs.openmp = cfg.pipeline_setup['system_config']['num_OMP_threads']
-
-        wf.connect(normalize_head, 'out_file',
-                   reconall, 'T1_files')
-
-        wf, hemisphere_outputs = freesurfer_hemispheres(wf, reconall, pipe_num)
-
-        outputs = {
-        'desc-restore_T1w': (fast_correction, 'outputspec.anat_restore'),
-        'desc-restore-brain_T1w': (fast_correction,
-                                   'outputspec.anat_brain_restore'),
-        'pipeline-fs_desc-fast_biasfield': (fast_correction, 'outputspec.bias_field'),
-        'pipeline-fs_wmparc': (reconall, 'wmparc'),
-        'freesurfer-subject-dir': (reconall, 'subjects_dir'),
-        **hemisphere_outputs
-    }
-
+            'pipeline-fs_desc-fast_biasfield': (fast_correction, 'outputspec.bias_field'),
+            'desc-ABCDpreproc_T1w': (normalize_head, 'out_file')
+            }
     return (wf, outputs)
 
-
 # we're grabbing the postproc outputs and appending them to
 # the reconall outputs
 @docstring_parameter(postproc_outputs=str(flatten_list(freesurfer_abcd_preproc, 'outputs')
@@ -2715,7 +2714,8 @@ def freesurfer_reconall(wf, cfg, strat_pool, pipe_num, opt=None):
       "switch": ["run_reconall"],
       "option_key": "None",
       "option_val": "None",
-      "inputs": ["desc-preproc_T1w"],
+      "inputs": [["desc-ABCDpreproc_T1w",
+                 "desc-preproc_T1w"]],
       "outputs": ["freesurfer-subject-dir",
                   "pipeline-fs_raw-average",
                   "pipeline-fs_subcortical-seg",
@@ -2748,7 +2748,7 @@ def freesurfer_reconall(wf, cfg, strat_pool, pipe_num, opt=None):
         reconall.inputs.args = cfg.surface_analysis['freesurfer'][
             'reconall_args']
 
-    node, out = strat_pool.get_data("desc-preproc_T1w")
+    node, out = strat_pool.get_data(["desc-ABCDpreproc_T1w","desc-preproc_T1w"])
     wf.connect(node, out, reconall, 'T1_files')
 
     wf, hemisphere_outputs = freesurfer_hemispheres(wf, reconall, pipe_num)
@@ -2983,9 +2983,9 @@ def fast_bias_field_correction(config=None, wf_name='fast_bias_field_correction'
 def correct_restore_brain_intensity_abcd(wf, cfg, strat_pool, pipe_num, opt=None):
     '''
     {"name": "correct_restore_brain_intensity_abcd",
-     "config": ["anatomical_preproc", "brain_extraction"],
-     "switch": "None",
-     "option_key": "using",
+     "config": "None",
+     "switch": ["anatomical_preproc", "brain_extraction", "run"],
+     "option_key": ["anatomical_preproc", "brain_extraction", "using"],
      "option_val": "FreeSurfer-ABCD",
      "inputs": [("desc-preproc_T1w",
                  "desc-n4_T1w",