Merge pull request #1630 from FCP-INDI/feature/surface_convenience

Feature/surface convenience

Author: sgiavasis

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added changelog
 - Added CHD8 mouse template (`/cpac_templates/chd8_functional_template_noise_mask_ag.nii.gz`) 
 - Added commandline flags `--T1w_label` and `--bold_label`
+- Added the ability to ingress an entire FreeSurfer output directory to bypass surface analysis if already completed elsewhere
 - Added AFNI and Nilearn implementations of Pearson and partial correlation matrices
 
 ### Changed
@@ -34,6 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Improved memory management for multi-core node allocation.
 - Fixed [bug](https://github.com/FCP-INDI/C-PAC/issues/1548) where `--participant_label [A B C]` would skip first and last labels (`A` and `C`).
 - Stripped the ABI tag note (which was preventing the library from loading dynamically on some host operating systems) from `libQt5Core.so.5` in the ABCD-HCP variant image.
+- Fixed an issue blocking non-C-PAC output data from being read in without sidecar meta-data.
 
 ## [1.8.1] - 2021-09-17
 

CPAC/pipeline/cpac_pipeline.py

@@ -791,7 +791,8 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None):
         ]
         pipeline_blocks += anat_init_blocks
 
-    pipeline_blocks += [freesurfer_preproc]
+    if not rpool.check_rpool('freesurfer-subject-dir'):
+        pipeline_blocks += [freesurfer_preproc]
 
     if not rpool.check_rpool('desc-preproc_T1w'):
 
@@ -806,7 +807,7 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None):
                 ]
                 acpc_blocks.append(
                     [brain_mask_acpc_freesurfer_fsl_tight,
-                    brain_mask_acpc_freesurfer_fsl_loose]
+                     brain_mask_acpc_freesurfer_fsl_loose]
                 )
             else:
                 acpc_blocks = [
@@ -846,8 +847,9 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None):
             anat_blocks = anat_preproc_blocks + acpc_blocks
 
         pipeline_blocks += anat_blocks
-
-        pipeline_blocks += [freesurfer_abcd_preproc]
+    
+        if not rpool.check_rpool('freesurfer-subject-dir'):
+            pipeline_blocks += [freesurfer_abcd_preproc]
 
     # Anatomical T1 brain masking
     if not rpool.check_rpool('space-T1w_desc-brain_mask') or \
@@ -948,9 +950,11 @@ def build_T1w_registration_stack(rpool, cfg, pipeline_blocks=None):
     if not rpool.check_rpool('from-T1w_to-template_mode-image_xfm'):
         reg_blocks = [
             [register_ANTs_anat_to_template, register_FSL_anat_to_template],
-             overwrite_transform_anat_to_template,
-             correct_restore_brain_intensity_abcd # ABCD-options pipeline
+             overwrite_transform_anat_to_template
         ]
+        
+    if not rpool.check_rpool('desc-restore-brain_T1w'):
+        reg_blocks.append(correct_restore_brain_intensity_abcd)
 
     if cfg.voxel_mirrored_homotopic_connectivity['run']:
         if not rpool.check_rpool('from-T1w_to-symtemplate_mode-image_xfm'):
@@ -971,7 +975,6 @@ def build_segmentation_stack(rpool, cfg, pipeline_blocks=None):
         seg_blocks = [
             [tissue_seg_fsl_fast,
              tissue_seg_ants_prior]
-             #tissue_seg_freesurfer
         ]
         if 'T1_Template' in cfg.segmentation['tissue_segmentation'][
             'Template_Based']['template_for_segmentation']:
@@ -1076,10 +1079,7 @@ def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None,
     pipeline_blocks = build_segmentation_stack(rpool, cfg, pipeline_blocks)
 
     # Functional Preprocessing, including motion correction and BOLD masking
-    if cfg.functional_preproc['run'] and \
-            (not rpool.check_rpool('desc-brain_bold') or
-             not rpool.check_rpool('space-bold_desc-brain_mask') or
-             not rpool.check_rpool('movement-parameters')):
+    if cfg.functional_preproc['run']:
         func_init_blocks = [
             func_scaling,
             func_truncate
@@ -1089,14 +1089,20 @@ def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None,
             func_slice_time,
             func_reorient
         ]
+        
+        if not rpool.check_rpool('desc-mean_bold'):
+            func_preproc_blocks.append(func_mean)
+        
+        func_mask_blocks = []
+        if not rpool.check_rpool('space-bold_desc-brain_mask'):
+            func_mask_blocks = [
+                [bold_mask_afni, bold_mask_fsl, bold_mask_fsl_afni,
+                 bold_mask_anatomical_refined, bold_mask_anatomical_based,
+                 bold_mask_anatomical_resampled, bold_mask_ccs],
+                bold_masking]
+            
         func_prep_blocks = [
-            [bold_mask_afni, bold_mask_fsl, bold_mask_fsl_afni,
-             bold_mask_anatomical_refined, bold_mask_anatomical_based,
-             bold_mask_anatomical_resampled,
-             bold_mask_ccs],
-            bold_masking,
             calc_motion_stats,
-            func_mean,
             func_normalize
         ]
 
@@ -1114,24 +1120,31 @@ def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None,
                 distcor_blocks = [distcor_blocks]
             func_prep_blocks += distcor_blocks
 
-        if cfg['functional_preproc']['motion_estimates_and_correction'][
-            'motion_estimates']['calculate_motion_first']:
-            func_motion_blocks = [
-                get_motion_ref,
-                func_motion_estimates,
-                motion_estimate_filter
-            ]
-            func_blocks = func_init_blocks + func_motion_blocks + \
-                          func_preproc_blocks + [func_motion_correct_only] + \
-                          func_prep_blocks
+        func_motion_blocks = []
+        if not rpool.check_rpool('movement-parameters'):
+            if cfg['functional_preproc']['motion_estimates_and_correction'][
+                'motion_estimates']['calculate_motion_first']:
+                func_motion_blocks = [
+                    get_motion_ref,
+                    func_motion_estimates,
+                    motion_estimate_filter
+                ]
+                func_blocks = func_init_blocks + func_motion_blocks + \
+                              func_preproc_blocks + [func_motion_correct_only] + \
+                              func_mask_blocks + func_prep_blocks
+            else:
+                func_motion_blocks = [
+                    get_motion_ref,
+                    func_motion_correct,
+                    motion_estimate_filter
+                ]
+                func_blocks = func_init_blocks + func_preproc_blocks + \
+                              func_motion_blocks + func_mask_blocks + \
+                              func_prep_blocks
         else:
-            func_motion_blocks = [
-                get_motion_ref,
-                func_motion_correct,
-                motion_estimate_filter
-            ]
             func_blocks = func_init_blocks + func_preproc_blocks + \
-                          func_motion_blocks + func_prep_blocks
+                          func_motion_blocks + func_mask_blocks + \
+                          func_prep_blocks
 
         pipeline_blocks += func_blocks
 

CPAC/pipeline/engine.py

@@ -1466,6 +1466,12 @@ def ingress_raw_func_data(wf, rpool, cfg, data_paths, unique_id, part_id,
 def ingress_output_dir(cfg, rpool, unique_id, creds_path=None):
 
     out_dir = cfg.pipeline_setup['output_directory']['path']
+    
+    if not os.path.isdir(out_dir):
+        print(f"\nOutput directory {out_dir} does not exist yet, "
+              "initializing.")
+        os.makedirs(out_dir)
+    
     source = False
 
     if cfg.pipeline_setup['output_directory']['pull_source_once']:
@@ -1520,7 +1526,7 @@ def ingress_output_dir(cfg, rpool, unique_id, creds_path=None):
     cpac_dir_anat = os.path.join(cpac_dir, 'anat')
     cpac_dir_func = os.path.join(cpac_dir, 'func')
 
-    exts = ['.nii', '.gz', '.mat', '.1D', '.txt', '.csv', '.rms']
+    exts = ['.nii', '.gz', '.mat', '.1D', '.txt', '.csv', '.rms', '.mgz']
 
     all_output_dir = []
     if os.path.isdir(cpac_dir_anat):
@@ -1562,10 +1568,6 @@ def ingress_output_dir(cfg, rpool, unique_id, creds_path=None):
 
         unique_data_label = str(data_label)
 
-        #if 'sub-' in data_label or 'ses-' in data_label:
-        #    raise Exception('\n\n[!] Possibly wrong participant or '
-        #                    'session in this directory?\n\nDirectory: '
-        #                    f'{cpac_dir_anat}\nFilepath: {filepath}\n\n')
         suffix = data_label.split('_')[-1]
         desc_val = None
         for tag in data_label.split('_'):
@@ -1578,16 +1580,21 @@ def ingress_output_dir(cfg, rpool, unique_id, creds_path=None):
         jsonpath = f"{jsonpath}.json"
 
         if not os.path.exists(jsonpath):
-            print(f'\n\n[!] No JSON found for file {filepath}.\nCreating '
-                  f'{jsonpath}..\n\n')
+            print(f'\n\n[!] No JSON found for file {filepath}.')
+            if not source:
+                print(f'Creating {jsonpath}..\n\n')
+            else:
+                print('Creating meta-data for the data..\n\n')
             json_info = {
+                'CpacProvenance': [f'{data_label}:Non-C-PAC Origin'],
                 'Description': 'This data was generated elsewhere and '
                                'supplied by the user into this C-PAC run\'s '
                                'output directory. This JSON file was '
                                'automatically generated by C-PAC because a '
                                'JSON file was not supplied with the data.'
             }
-            write_output_json(json_info, jsonpath)
+            if not source:
+                write_output_json(json_info, jsonpath)
         else:        
             json_info = read_json(jsonpath)
 
@@ -1664,7 +1671,7 @@ def ingress_pipeconfig_paths(cfg, rpool, unique_id, creds_path=None):
             res_keys = [x.lstrip() for x in resolution.split(',')]
             tag = res_keys[-1]
 
-        json_info = {} 
+        json_info = {}
 
         if '$FSLDIR' in val:
             val = val.replace('$FSLDIR', cfg.pipeline_setup[
@@ -1722,7 +1729,20 @@ def ingress_pipeconfig_paths(cfg, rpool, unique_id, creds_path=None):
                 )
                 rpool.set_data(key, config_ingress, 'outputspec.data', json_info,
                                "", f"{key}_config_ingress")
-            
+
+    # Freesurfer directory, not a template, so not in cpac_templates.tsv
+    if cfg.surface_analysis['freesurfer']['freesurfer_dir']:
+        fs_ingress = create_general_datasource(f'gather_freesurfer_dir')
+        fs_ingress.inputs.inputnode.set(
+                    unique_id=unique_id,
+                    data=cfg.surface_analysis['freesurfer']['freesurfer_dir'],
+                    creds_path=creds_path,
+                    dl_dir=cfg.pipeline_setup['working_directory']['path']
+        )
+        rpool.set_data("freesurfer-subject-dir", fs_ingress, 'outputspec.data', 
+                       json_info, "", f"freesurfer_config_ingress")
+
+
     # templates, resampling from config
     '''
     template_keys = [

CPAC/pipeline/schema.py

@@ -687,6 +687,7 @@ def _changes_1_8_0_to_1_8_1(config_dict):
         'freesurfer': {
             'run': bool,
             'reconall_args': Maybe(str),
+            'freesurfer_dir': Maybe(str)
         },
         'post_freesurfer': {
             'run': bool,

CPAC/registration/registration.py

@@ -2429,10 +2429,10 @@ def overwrite_transform_anat_to_template(wf, cfg, strat_pool, pipe_num, opt=None
                  ["desc-restore_T1w", "desc-preproc_T1w", "desc-reorient_T1w", "T1w"],
                  ["desc-preproc_T1w", "desc-reorient_T1w", "T1w"],
                  "space-T1w_desc-brain_mask",
+                 "T1w-template",
                  "from-T1w_to-template_mode-image_xfm",
                  "from-template_to-T1w_mode-image_xfm",
-                 "space-template_desc-brain_T1w"),
-                "T1w-template",],
+                 "space-template_desc-brain_T1w")],
      "outputs": ["space-template_desc-brain_T1w",
                  "space-template_desc-head_T1w",
                  "space-template_desc-T1w_mask",

CPAC/resources/cpac_outputs.tsv

@@ -139,6 +139,7 @@ rh-pial-surface-mesh	surface-derived		anat
 raw-average	surface-derived		anat						
 lh-smoothed-surface-mesh	surface-derived		anat						
 rh-smoothed-surface-mesh	surface-derived		anat						
+space-fsLR_den-32k_bold-dtseries	surface-derived		anat						
 lh-spherical-surface-mesh	surface-derived		anat						Yes
 rh-spherical-surface-mesh	surface-derived		anat						Yes
 lh-sulcal-depth-surface-maps	surface-derived		anat						Yes
@@ -147,10 +148,13 @@ lh-surface-curvature	surface-derived		anat
 rh-surface-curvature	surface-derived		anat						
 lh-white-matter-surface-mesh	surface-derived		anat						Yes
 rh-white-matter-surface-mesh	surface-derived		anat						Yes
+wmparc	surface-derived		anat						Yes
 space-symtemplate_desc-brain_T1w	T1w	symmetric template	anat	NIfTI					
 desc-brain_T1w	T1w	T1w	anat	NIfTI					
 desc-preproc_T1w	T1w	T1w	anat	NIfTI					
 desc-reorient_T1w	T1w	T1w	anat	NIfTI					
+desc-restore_T1w	T1w	T1w	anat	NIfTI					
+desc-restore-brain_T1w	T1w	T1w	anat	NIfTI					
 space-template_desc-brain_T1w	T1w	template	anat	NIfTI					
 desc-Mean_timeseries	timeseries		func	1D					
 desc-MeanSCA_timeseries	timeseries		func	1D					

CPAC/surface/surf_preproc.py

@@ -116,7 +116,7 @@ def surface_connector(wf, cfg, strat_pool, pipe_num, opt):
     wf.connect(node, out, surf, 'scout_bold')
 
     outputs = {
-        'space-fsLR_den-32k_bold.dtseries': (surf, 'out_file')
+        'space-fsLR_den-32k_bold-dtseries': (surf, 'out_file')
     }
 
     return wf, outputs
@@ -136,9 +136,9 @@ def surface_preproc(wf, cfg, strat_pool, pipe_num, opt=None):
                 "from-template_to-T1w_mode-image_xfm",
                 "space-template_desc-brain_bold",
                 "space-template_desc-scout_bold"],
-     "outputs": ["space-fsLR_den-32k_bold.dtseries"]}
+     "outputs": ["space-fsLR_den-32k_bold-dtseries"]}
     '''
 
     wf, outputs = surface_connector(wf, cfg, strat_pool, pipe_num, opt)
 
-    return (wf, outputs)
+    return (wf, outputs)

CPAC/utils/bids_utils.py

@@ -30,8 +30,8 @@ def bids_decode_fname(file_path, dbg=False, raise_error=True):
 
     if len(sub) > 1:
         print("Odd that there is more than one subject directory" +
-               "in (%s), does the filename conform to" % file_path +
-               " BIDS format?")
+              "in (%s), does the filename conform to" % file_path +
+              " BIDS format?")
     if sub:
         sub_ndx = file_path_vals.index(sub[0])
         if sub_ndx > 0 and file_path_vals[sub_ndx - 1]:

dev/docker_data/default_pipeline.yml

@@ -162,6 +162,9 @@ surface_analysis:
 
     # Add extra arguments to recon-all command
     reconall_args: None
+    
+    # (Optional) Provide an already-existing FreeSurfer output directory to ingress already-computed surfaces
+    freesurfer_dir: None
 
   # Run ABCD-HCP post FreeSurfer and fMRISurface pipeline
   post_freesurfer: