Merge branch 'develop' into verbose_debugging_update

Author: sgiavasis

CHANGELOG.md

@@ -28,6 +28,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added abililty to fork on motion filter
 - Added [`sdcflows`](https://www.nipreps.org/sdcflows/2.0/) to CPAC requirements
 - Added NodeBlock information to `pypeline.log` when verbose debugging is on
+- Added the ability to ingress FreeSurfer data into CPAC
+- Added the ability to toggle FreeSurfer derived masks for brain extraction
 
 ### Changed
 - Freesurfer output directory ingress moved to the data configuration YAML

CPAC/anat_preproc/anat_preproc.py

@@ -174,22 +174,22 @@ def split_hemi_interface():
     for label in splits:
         wf.connect(reconall, label, splits[label], 'multi_file')
     outputs = {
-        'hemi-L_desc-surface_curv': (splits['curv'], 'lh'),
-        'hemi-R_desc-surface_curv': (splits['curv'], 'rh'),
-        'hemi-L_desc-surfaceMesh_pial': (splits['pial'], 'lh'),
-        'hemi-R_desc-surfaceMesh_pial': (splits['pial'], 'rh'),
-        'hemi-L_desc-surfaceMesh_smoothwm': (splits['smoothwm'], 'lh'),
-        'hemi-R_desc-surfaceMesh_smoothwm': (splits['smoothwm'], 'rh'),
-        'hemi-L_desc-surfaceMesh_sphere': (splits['sphere'], 'lh'),
-        'hemi-R_desc-surfaceMesh_sphere': (splits['sphere'], 'rh'),
-        'hemi-L_desc-surfaceMap_sulc': (splits['sulc'], 'lh'),
-        'hemi-R_desc-surfaceMap_sulc': (splits['sulc'], 'rh'),
-        'hemi-L_desc-surfaceMap_thickness': (splits['thickness'], 'lh'),
-        'hemi-R_desc-surfaceMap_thickness': (splits['thickness'], 'rh'),
-        'hemi-L_desc-surfaceMap_volume': (splits['volume'], 'lh'),
-        'hemi-R_desc-surfaceMap_volume': (splits['volume'], 'rh'),
-        'hemi-L_desc-surfaceMesh_white': (splits['white'], 'lh'),
-        'hemi-R_desc-surfaceMesh_white': (splits['white'], 'rh')}
+        'pipeline-fs_hemi-L_desc-surface_curv': (splits['curv'], 'lh'),
+        'pipeline-fs_hemi-R_desc-surface_curv': (splits['curv'], 'rh'),
+        'pipeline-fs_hemi-L_desc-surfaceMesh_pial': (splits['pial'], 'lh'),
+        'pipeline-fs_hemi-R_desc-surfaceMesh_pial': (splits['pial'], 'rh'),
+        'pipeline-fs_hemi-L_desc-surfaceMesh_smoothwm': (splits['smoothwm'], 'lh'),
+        'pipeline-fs_hemi-R_desc-surfaceMesh_smoothwm': (splits['smoothwm'], 'rh'),
+        'pipeline-fs_hemi-L_desc-surfaceMesh_sphere': (splits['sphere'], 'lh'),
+        'pipeline-fs_hemi-R_desc-surfaceMesh_sphere': (splits['sphere'], 'rh'),
+        'pipeline-fs_hemi-L_desc-surfaceMap_sulc': (splits['sulc'], 'lh'),
+        'pipeline-fs_hemi-R_desc-surfaceMap_sulc': (splits['sulc'], 'rh'),
+        'pipeline-fs_hemi-L_desc-surfaceMap_thickness': (splits['thickness'], 'lh'),
+        'pipeline-fs_hemi-R_desc-surfaceMap_thickness': (splits['thickness'], 'rh'),
+        'pipeline-fs_hemi-L_desc-surfaceMap_volume': (splits['volume'], 'lh'),
+        'pipeline-fs_hemi-R_desc-surfaceMap_volume': (splits['volume'], 'rh'),
+        'pipeline-fs_hemi-L_desc-surfaceMesh_white': (splits['white'], 'lh'),
+        'pipeline-fs_hemi-R_desc-surfaceMesh_white': (splits['white'], 'rh')}
 
     return wf, outputs
 

CPAC/anat_preproc/utils.py

@@ -23,6 +23,7 @@
 import copy
 import faulthandler
 
+from logging import getLogger
 from time import strftime
 
 import nipype
@@ -41,7 +42,6 @@
 from CPAC.pipeline.engine import NodeBlock, initiate_rpool
 from CPAC.anat_preproc.anat_preproc import (
     freesurfer_reconall,
-    freesurfer_postproc,
     freesurfer_abcd_preproc,
     anatomical_init,
     acpc_align_head,
@@ -81,7 +81,7 @@
     brain_mask_T2,
     brain_mask_acpc_T2,
     brain_extraction_temp_T2,
-    brain_extraction_T2
+    brain_extraction_T2,
 )
 
 from CPAC.registration.registration import (
@@ -211,10 +211,10 @@
 
 from CPAC.qc.pipeline import create_qc_workflow
 from CPAC.qc.xcp import qc_xcp
+
 from CPAC.utils.monitoring import log_nodes_cb, log_nodes_initial, \
                                   LOGTAIL, set_up_logger, \
                                   WARNING_FREESURFER_OFF_WITH_DATA
-from CPAC.utils.monitoring.custom_logging import getLogger
 from CPAC.utils.monitoring.draw_gantt_chart import resource_report
 from CPAC.utils.utils import (
     check_config_resources,
@@ -865,40 +865,27 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None):
         ]
         pipeline_blocks += anat_init_blocks
 
-    if rpool.check_rpool('freesurfer-subject-dir'):
-        pipeline_blocks += [freesurfer_postproc]
-    else:
+    if not rpool.check_rpool('freesurfer-subject-dir'):
         pipeline_blocks += [freesurfer_reconall]  # includes postproc
 
     if not rpool.check_rpool('desc-preproc_T1w'):
 
         # brain masking for ACPC alignment
         if cfg.anatomical_preproc['acpc_alignment']['acpc_target'] == 'brain':
-            if rpool.check_rpool('space-T1w_desc-brain_mask') or \
-                    cfg.surface_analysis['freesurfer']['run_reconall']:
-                acpc_blocks = [
-                    brain_extraction_temp,
-                    acpc_align_brain_with_mask
-                    # outputs space-T1w_desc-brain_mask for later - keep the mask (the user provided)
-                ]
-                acpc_blocks.append(
-                    [brain_mask_acpc_freesurfer_fsl_tight,
-                     brain_mask_acpc_freesurfer_fsl_loose]
-                )
-            else:
                 acpc_blocks = [
                     [brain_mask_acpc_afni,
                      brain_mask_acpc_fsl,
                      brain_mask_acpc_niworkflows_ants,
                      brain_mask_acpc_unet,
-                     brain_mask_acpc_freesurfer_abcd],
-                    #  brain_mask_acpc_freesurfer
-                    # we don't want these masks to be used later
+                     brain_mask_acpc_freesurfer_abcd,
+                     brain_mask_acpc_freesurfer,
+                     brain_mask_acpc_freesurfer_fsl_tight,
+                     brain_mask_acpc_freesurfer_fsl_loose],
+                    acpc_align_brain_with_mask,
                     brain_extraction_temp,
                     acpc_align_brain
                 ]
-        elif cfg.anatomical_preproc['acpc_alignment'][
-            'acpc_target'] == 'whole-head':
+        elif cfg.anatomical_preproc['acpc_alignment']['acpc_target'] == 'whole-head':
             if (rpool.check_rpool('space-T1w_desc-brain_mask') and \
                 cfg.anatomical_preproc['acpc_alignment']['align_brain_mask']) or \
                     cfg.surface_analysis['freesurfer']['run_reconall']:
@@ -910,6 +897,7 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None):
                 acpc_blocks = [
                     acpc_align_head  # does not output nor generate a mask
                 ]
+        
 
         anat_preproc_blocks = [
             (non_local_means, ('T1w', ['desc-preproc_T1w',
@@ -927,18 +915,18 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None):
         pipeline_blocks += [freesurfer_abcd_preproc]
 
     # Anatomical T1 brain masking
-    if not rpool.check_rpool('space-T1w_desc-brain_mask') or \
-        cfg.surface_analysis['freesurfer']['run_reconall']:
-        anat_brain_mask_blocks = [
-            [brain_mask_afni,
-             brain_mask_fsl,
-             brain_mask_niworkflows_ants,
-             brain_mask_unet,
-             brain_mask_freesurfer_abcd,
-             brain_mask_freesurfer_fsl_tight,
-             brain_mask_freesurfer_fsl_loose]
-        ]
-        pipeline_blocks += anat_brain_mask_blocks
+
+    anat_brain_mask_blocks = [
+        [brain_mask_afni,
+        brain_mask_fsl,
+        brain_mask_niworkflows_ants,
+        brain_mask_unet,
+        brain_mask_freesurfer_abcd,
+        brain_mask_freesurfer,
+        brain_mask_freesurfer_fsl_tight,
+        brain_mask_freesurfer_fsl_loose]
+    ]
+    pipeline_blocks += anat_brain_mask_blocks
 
     # T2w Anatomical Preprocessing
     if rpool.check_rpool('T2w'):
@@ -1030,7 +1018,6 @@ def build_T1w_registration_stack(rpool, cfg, pipeline_blocks=None):
             warp_T1mask_to_template
         ]
 
-
     if not rpool.check_rpool('desc-restore-brain_T1w'):
         reg_blocks.append(correct_restore_brain_intensity_abcd)
 
@@ -1362,6 +1349,7 @@ def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None,
 	    apply_func_warp['EPI'] = (_r_w_f_r['coregistration']['run'] and _r_w_f_r['func_registration_to_template']['run_EPI'])
     else:
         apply_func_warp['EPI'] = (_r_w_f_r['func_registration_to_template']['run_EPI'])
+    
     del _r_w_f_r
 
     template_funcs = [
@@ -1393,6 +1381,7 @@ def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None,
 
     # PostFreeSurfer and fMRISurface
     if not rpool.check_rpool('space-fsLR_den-32k_bold.dtseries'):
+        
         pipeline_blocks += [surface_postproc]
 
     # Extractions and Derivatives

CPAC/pipeline/cpac_pipeline.py

@@ -17,6 +17,7 @@
 import ast
 import copy
 from itertools import chain
+import logging
 import os
 import re
 from types import FunctionType
@@ -54,7 +55,8 @@
 
 from CPAC.resources.templates.lookup_table import lookup_identifier
 
-logger = getLogger('nipype.workflow')
+logger = logging.getLogger('nipype.workflow')
+verbose_logger = logging.getLogger('engine')
 
 
 class ResourcePool:
@@ -437,6 +439,7 @@ def flatten_prov(self, prov):
             return flat_prov
 
     def get_strats(self, resources, debug=False):
+
         # TODO: NOTE: NOT COMPATIBLE WITH SUB-RPOOL/STRAT_POOLS
         # TODO: (and it doesn't have to be)
 
@@ -445,7 +448,6 @@ def get_strats(self, resources, debug=False):
         linked_resources = []
         resource_list = []
         if debug:
-            verbose_logger = getLogger('engine')
             verbose_logger.debug('\nresources: %s', resources)
         for resource in resources:
             # grab the linked-input tuples
@@ -469,7 +471,6 @@ def get_strats(self, resources, debug=False):
         variant_pool = {}
         len_inputs = len(resource_list)
         if debug:
-            verbose_logger = getLogger('engine')
             verbose_logger.debug('linked_resources: %s',
                                  linked_resources)
             verbose_logger.debug('resource_list: %s', resource_list)
@@ -482,8 +483,7 @@ def get_strats(self, resources, debug=False):
                 continue
             sub_pool = []
             if debug:
-                verbose_logger = getLogger('engine')
-                verbose_logger.debug('%s len(rp_dct): %s\n', resource, len(rp_dct))
+                verbose_logger.debug('len(rp_dct): %s\n', len(rp_dct))
             for strat in rp_dct.keys():
                 json_info = self.get_json(fetched_resource, strat)
                 cpac_prov = json_info['CpacProvenance']
@@ -496,6 +496,7 @@ def get_strats(self, resources, debug=False):
                             variant_pool[fetched_resource] += val
                             variant_pool[fetched_resource].append(
                                 f'NO-{val[0]}')
+
             if debug:
                 verbose_logger.debug('%s sub_pool: %s\n', resource, sub_pool)
             total_pool.append(sub_pool)
@@ -1014,6 +1015,7 @@ def gather_pipes(self, wf, cfg, all=False, add_incl=None, add_excl=None):
 
                 unique_id = out_dct['unique_id']
                 resource_idx = resource
+
                 if isinstance(num_variant, int):
                     if True in cfg['functional_preproc',
                                    'motion_estimates_and_correction',
@@ -1131,7 +1133,7 @@ def gather_pipes(self, wf, cfg, all=False, add_incl=None, add_excl=None):
                 nii_name.inputs.keep_ext = True
                 wf.connect(id_string, 'out_filename',
                            nii_name, 'format_string')
-
+                
                 node, out = self.rpool[resource][pipe_idx]['data']
                 try:
                     wf.connect(node, out, nii_name, 'in_file')
@@ -1439,7 +1441,6 @@ def connect_block(self, wf, cfg, rpool):
                             node_name = f'{node_name}_{opt["Name"]}'
 
                         if debug:
-                            verbose_logger = getLogger('engine')
                             verbose_logger.debug('\n=======================')
                             verbose_logger.debug('Node name: %s', node_name)
                             prov_dct = \
@@ -1684,7 +1685,9 @@ def ingress_raw_anat_data(wf, rpool, cfg, data_paths, unique_id, part_id,
         rpool.set_data('T2w', anat_flow_T2, 'outputspec.anat', {},
                     "", "anat_ingress")
 
-    if 'freesurfer_dir' in data_paths['anat']:
+    ingress_fs = cfg.surface_analysis['freesurfer']['ingress_reconall']
+    
+    if 'freesurfer_dir' in data_paths['anat'] and ingress_fs:
         anat['freesurfer_dir'] = data_paths['anat']['freesurfer_dir']
 
         fs_ingress = create_general_datasource('gather_freesurfer_dir')
@@ -1697,27 +1700,27 @@ def ingress_raw_anat_data(wf, rpool, cfg, data_paths, unique_id, part_id,
                        {}, "", "freesurfer_config_ingress")
 
         recon_outs = {
-            'raw-average': 'mri/rawavg.mgz',
-            'subcortical-seg': 'mri/aseg.mgz',
-            'brainmask': 'mri/brainmask.mgz',
-            'wmparc': 'mri/wmparc.mgz',
-            'T1': 'mri/T1.mgz',
-            'hemi-L_desc-surface_curv': 'surf/lh.curv',
-            'hemi-R_desc-surface_curv': 'surf/rh.curv',
-            'hemi-L_desc-surfaceMesh_pial': 'surf/lh.pial',
-            'hemi-R_desc-surfaceMesh_pial': 'surf/rh.pial',
-            'hemi-L_desc-surfaceMesh_smoothwm': 'surf/lh.smoothwm',
-            'hemi-R_desc-surfaceMesh_smoothwm': 'surf/rh.smoothwm',
-            'hemi-L_desc-surfaceMesh_sphere': 'surf/lh.sphere',
-            'hemi-R_desc-surfaceMesh_sphere': 'surf/rh.sphere',
-            'hemi-L_desc-surfaceMap_sulc': 'surf/lh.sulc',
-            'hemi-R_desc-surfaceMap_sulc': 'surf/rh.sulc',
-            'hemi-L_desc-surfaceMap_thickness': 'surf/lh.thickness',
-            'hemi-R_desc-surfaceMap_thickness': 'surf/rh.thickness',
-            'hemi-L_desc-surfaceMap_volume': 'surf/lh.volume',
-            'hemi-R_desc-surfaceMap_volume': 'surf/rh.volume',
-            'hemi-L_desc-surfaceMesh_white': 'surf/lh.white',
-            'hemi-R_desc-surfaceMesh_white': 'surf/rh.white',
+            'pipeline-fs_raw-average': 'mri/rawavg.mgz',
+            'pipeline-fs_subcortical-seg': 'mri/aseg.mgz',
+            'pipeline-fs_brainmask': 'mri/brainmask.mgz',
+            'pipeline-fs_wmparc': 'mri/wmparc.mgz',
+            'pipeline-fs_T1': 'mri/T1.mgz',
+            'pipeline-fs_hemi-L_desc-surface_curv': 'surf/lh.curv',
+            'pipeline-fs_hemi-R_desc-surface_curv': 'surf/rh.curv',
+            'pipeline-fs_hemi-L_desc-surfaceMesh_pial': 'surf/lh.pial',
+            'pipeline-fs_hemi-R_desc-surfaceMesh_pial': 'surf/rh.pial',
+            'pipeline-fs_hemi-L_desc-surfaceMesh_smoothwm': 'surf/lh.smoothwm',
+            'pipeline-fs_hemi-R_desc-surfaceMesh_smoothwm': 'surf/rh.smoothwm',
+            'pipeline-fs_hemi-L_desc-surfaceMesh_sphere': 'surf/lh.sphere',
+            'pipeline-fs_hemi-R_desc-surfaceMesh_sphere': 'surf/rh.sphere',
+            'pipeline-fs_hemi-L_desc-surfaceMap_sulc': 'surf/lh.sulc',
+            'pipeline-fs_hemi-R_desc-surfaceMap_sulc': 'surf/rh.sulc',
+            'pipeline-fs_hemi-L_desc-surfaceMap_thickness': 'surf/lh.thickness',
+            'pipeline-fs_hemi-R_desc-surfaceMap_thickness': 'surf/rh.thickness',
+            'pipeline-fs_hemi-L_desc-surfaceMap_volume': 'surf/lh.volume',
+            'pipeline-fs_hemi-R_desc-surfaceMap_volume': 'surf/rh.volume',
+            'pipeline-fs_hemi-L_desc-surfaceMesh_white': 'surf/lh.white',
+            'pipeline-fs_hemi-R_desc-surfaceMesh_white': 'surf/rh.white',
         }
 
         for key, outfile in recon_outs.items():
@@ -1772,7 +1775,6 @@ def ingress_raw_func_data(wf, rpool, cfg, data_paths, unique_id, part_id,
         # pylint: disable=protected-access
         wf._local_func_scans = local_func_scans
         if cfg.pipeline_setup['Debugging']['verbose']:
-            verbose_logger = getLogger('engine')
             verbose_logger.debug('local_func_scans: %s', local_func_scans)
     del local_func_scans
 

CPAC/pipeline/engine.py

@@ -75,7 +75,7 @@ def str_to_bool1_1(x):  # pylint: disable=invalid-name
     'brain_extraction': {
         'using': ['3dSkullStrip', 'BET', 'UNet', 'niworkflows-ants',
                   'FreeSurfer-BET-Tight', 'FreeSurfer-BET-Loose',
-                  'FreeSurfer-ABCD']
+                  'FreeSurfer-ABCD', 'FreeSurfer-Brainmask']
     },
     'centrality': {
        'method_options': ['degree_centrality', 'eigenvector_centrality',
@@ -437,6 +437,7 @@ def sanitize(filename):
                 'BiasFieldSmoothingSigma': Maybe(int),
             },),
         ),
+
         'acpc_alignment': Required(
             # require 'T1w_brain_ACPC_template' and
             # 'T2w_brain_ACPC_template' if 'acpc_target' is 'brain'
@@ -712,7 +713,9 @@ def sanitize(filename):
     'surface_analysis': {
         'freesurfer': {
             'run_reconall': bool1_1,
-            'reconall_args': Maybe(str)
+            'reconall_args': Maybe(str),
+            # 'generate_masks': bool1_1,
+            'ingress_reconall': bool1_1,
         },
         'post_freesurfer': {
             'run': bool1_1,