FCP-INDI
diff --git a/‎.pylintrc‎
Lines changed: 3 additions & 1 deletion b/‎.pylintrc‎
Lines changed: 3 additions & 1 deletion
diff --git a/‎CHANGELOG.md‎
Lines changed: 10 additions & 1 deletion b/‎CHANGELOG.md‎
Lines changed: 10 additions & 1 deletion
diff --git a/‎CPAC/alff/alff.py‎
Lines changed: 3 additions & 3 deletions b/‎CPAC/alff/alff.py‎
Lines changed: 3 additions & 3 deletions
diff --git a/‎CPAC/cwas/cwas.py‎
Lines changed: 34 additions & 14 deletions b/‎CPAC/cwas/cwas.py‎
Lines changed: 34 additions & 14 deletions
diff --git a/‎CPAC/cwas/mdmr.py‎
Lines changed: 5 additions & 5 deletions b/‎CPAC/cwas/mdmr.py‎
Lines changed: 5 additions & 5 deletions
diff --git a/‎CPAC/cwas/pipeline.py‎
Lines changed: 24 additions & 7 deletions b/‎CPAC/cwas/pipeline.py‎
Lines changed: 24 additions & 7 deletions
@@ -291,7 +291,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=
+generated-members=CPAC.utils.configuration.configuration.Configuration.*
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).
@@ -435,6 +435,7 @@ good-names=c,
            ex,
            nb,
            Run,
+           TR,
            v,
            wf,
            _,
@@ -443,6 +444,7 @@ good-names=c,
 # they will always be accepted
 good-names-rgxs=^_version_(extra|m[a-n]{2}[or]{2})$,  # version parts in info.py
                 .*EPI.*,
+                .*TE.*,
                 .*T1.*,
                 .*T2.*
 
 
@@ -14,22 +14,31 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [unreleased]
 
 ### Added
 - Added the ability to downsample to 10K or 2K resolution for freesurfer runs
+- Added the ability to ingress TotalReadoutTime from epi field map meta-data from the JSON sidecars.
+- Added the ability to use TotalReadoutTime of epi field maps in the calculation of FSL topup distortion correction.
 - Added ability to set minimum quality measure thresholds to all registration steps
 - Difference method (``-``) for ``CPAC.utils.configuration.Configuration`` instances
 
 ### Changed
+- Added a level of depth to `working` directories to match `log` and `output` directory structure
+- Renamed participant-pipeline-level `output` directory prefix to `pipeline_` to match `log` and `working` paths
 - Changed the 1mm atlases chosen in the rbc-options preconfig to the 2mm versions
 - For Nilearn-generated correlation matrices, diagonals are now set to all `1`s (were all `0`s)
 - Added ability to apply nusiance correction to template-space BOLD images
 - Removed ability to run single-step-resampling on motion-corrected BOLD data
 - Moved default pipeline config into directory with other preconfigs
 - Added crash messages from during and before graph building to logs
+- Added data-config-specific hash string to C-PAC-generated config files
+- Updated `rbc-options` preconfig to use `fmriprep-options` preprocessing
 
 ### Fixed
+- Fixed [bug](https://github.com/FCP-INDI/C-PAC/issues/1795) that was causing `cpac run` to fail when passing a manual random seed via `--random_seed`.
+- Replaces ``DwellTime`` with ``EffectiveEchoSpacing`` for FSL usage of the term
+- Fixed an issue that was causing some epi field maps to not be ingressed if the BIDS tags were not in the correct order.
 
 ## [v1.8.4] - 2022-06-27
 
 
@@ -246,8 +246,8 @@ def alff_falff(wf, cfg, strat_pool, pipe_num, opt=None):
      "switch": ["run"],
      "option_key": "None",
      "option_val": "None",
-     "inputs": [["desc-cleaned_bold", "desc-brain_bold", "desc-preproc_bold",
-                 "bold"],
+     "inputs": [["desc-cleanedNofilt_bold", "desc-brain_bold", 
+                 "desc-preproc_bold", "bold"],
                 "space-bold_desc-brain_mask"],
      "outputs": ["alff",
                  "falff"]}
@@ -262,7 +262,7 @@ def alff_falff(wf, cfg, strat_pool, pipe_num, opt=None):
     alff.get_node('hp_input').iterables = ('hp', alff.inputs.hp_input.hp)
     alff.get_node('lp_input').iterables = ('lp', alff.inputs.lp_input.lp)
 
-    node, out = strat_pool.get_data(["desc-cleaned_bold", "desc-brain_bold",
+    node, out = strat_pool.get_data(["desc-cleanedNofilt_bold", "desc-brain_bold",
                                      "desc-preproc_bold", "bold"])
     wf.connect(node, out, alff, 'inputspec.rest_res')
 
 
@@ -3,6 +3,9 @@
 import nibabel as nb
 import numpy as np
 import pandas as pd
+import scipy.stats
+from scipy.stats import t
+from numpy import inf
 
 from CPAC.cwas.mdmr import mdmr
 from CPAC.utils import correlation
@@ -35,7 +38,6 @@ def joint_mask(subjects, mask_file=None):
         mask_file = os.path.join(os.getcwd(), 'joint_mask.nii.gz')
         create_merged_copefile(files, cope_file)
         create_merge_mask(cope_file, mask_file)
-
     return mask_file
 
 
@@ -48,7 +50,6 @@ def calc_mdmrs(D, regressor, cols, permutations):
 def calc_subdists(subjects_data, voxel_range):
     subjects, voxels, _ = subjects_data.shape
     D = np.zeros((len(voxel_range), subjects, subjects))
-
     for i, v in enumerate(voxel_range):
         profiles = np.zeros((subjects, voxels))
         for si in range(subjects):
@@ -67,6 +68,12 @@ def calc_cwas(subjects_data, regressor, regressor_selected_cols, permutations, v
         D, regressor, regressor_selected_cols, permutations)
     return F_set, p_set
 
+def pval_to_zval(p_set, permu):
+    inv_pval = 1 - p_set
+    zvals = t.ppf(inv_pval, (len(p_set) - 1))
+    zvals[zvals == -inf] = permu / (permu + 1)
+    zvals[zvals == inf] = permu / (permu + 1)
+    return zvals
 
 def nifti_cwas(subjects, mask_file, regressor_file, participant_column,
                columns_string, permutations, voxel_range):
@@ -130,7 +137,7 @@ def nifti_cwas(subjects, mask_file, regressor_file, participant_column,
         for sub_id in subject_ids:
             if str(sub_id).lstrip('0') == str(pheno_sub_id):
                 regressor_data.at[index, participant_column] = str(sub_id)
-
+    
     regressor_data.index = regressor_data[participant_column]
 
     # Keep only data from specific subjects
@@ -144,33 +151,27 @@ def nifti_cwas(subjects, mask_file, regressor_file, participant_column,
     if len(regressor_selected_cols) == 0:
         regressor_selected_cols = [i for i, c in enumerate(regressor_cols)]
     regressor_selected_cols = np.array(regressor_selected_cols)
-
     # Remove participant id column from the dataframe and convert it to a numpy matrix
     regressor = ordered_regressor_data \
         .drop(columns=[participant_column]) \
         .reset_index(drop=True) \
         .values \
         .astype(np.float64)
-
     if len(regressor.shape) == 1:
         regressor = regressor[:, np.newaxis]
     elif len(regressor.shape) != 2:
         raise ValueError('Bad regressor shape: %s' % str(regressor.shape))
-
     if len(subject_files) != regressor.shape[0]:
         raise ValueError('Number of subjects does not match regressor size')
-
-    mask = nb.load(mask_file).get_data().astype('bool')
+    mask = nb.load(mask_file).get_fdata().astype('bool')
     mask_indices = np.where(mask)
-
     subjects_data = np.array([
-        nb.load(subject_file).get_data().astype('float64')[mask_indices].T
+        nb.load(subject_file).get_fdata().astype('float64')[mask_indices]
         for subject_file in subject_files
     ])
 
     F_set, p_set = calc_cwas(subjects_data, regressor, regressor_selected_cols,
                              permutations, voxel_range)
-
     cwd = os.getcwd()
     F_file = os.path.join(cwd, 'pseudo_F.npy')
     p_file = os.path.join(cwd, 'significance_p.npy')
@@ -182,7 +183,7 @@ def nifti_cwas(subjects, mask_file, regressor_file, participant_column,
 
 
 def create_cwas_batches(mask_file, batches):
-    mask = nb.load(mask_file).get_data().astype('bool')
+    mask = nb.load(mask_file).get_fdata().astype('bool')
     voxels = mask.sum(dtype=int)
     return np.array_split(np.arange(voxels), batches)
 
@@ -198,7 +199,7 @@ def volumize(mask_image, data):
     )
 
 
-def merge_cwas_batches(cwas_batches, mask_file):
+def merge_cwas_batches(cwas_batches, mask_file, z_score, permutations):
     _, _, voxel_range = zip(*cwas_batches)
     voxels = np.array(np.concatenate(voxel_range))
 
@@ -211,18 +212,37 @@ def merge_cwas_batches(cwas_batches, mask_file):
         p_set[voxel_range] = np.load(p_file)
 
     log_p_set = -np.log10(p_set)
+    one_p_set = 1 - p_set
 
     F_vol = volumize(mask_image, F_set)
     p_vol = volumize(mask_image, p_set)
     log_p_vol = volumize(mask_image, log_p_set)
+    one_p_vol = volumize(mask_image, one_p_set)
 
     cwd = os.getcwd()
     F_file = os.path.join(cwd, 'pseudo_F_volume.nii.gz')
     p_file = os.path.join(cwd, 'p_significance_volume.nii.gz')
     log_p_file = os.path.join(cwd, 'neglog_p_significance_volume.nii.gz')
+    one_p_file = os.path.join(cwd, 'one_minus_p_values.nii.gz')
 
     F_vol.to_filename(F_file)
     p_vol.to_filename(p_file)
     log_p_vol.to_filename(log_p_file)
+    one_p_vol.to_filename(one_p_file)
+
+    if 1 in z_score:
+        zvals = pval_to_zval(p_set, permutations)
+        z_file = zstat_image(zvals, mask_file)
+    
+    return F_file, p_file, log_p_file, one_p_file, z_file
+    
+def zstat_image(zvals, mask_file):
+    mask_image = nb.load(mask_file)
 
-    return F_file, p_file, log_p_file
+    z_vol = volumize(mask_image, zvals)
+
+    cwd = os.getcwd()
+    z_file = os.path.join(cwd, 'zstat.nii.gz')
+ 
+    z_vol.to_filename(z_file)
+    return z_file
@@ -83,15 +83,15 @@ def mdmr(D, X, columns, permutations):
         Gs[:, di] = gower(D[di]).flatten()
 
     X1 = np.hstack((np.ones((subjects, 1)), X))
-    columns = columns.copy() + 1
+    columns = columns.copy() #removed a +1
 
     regressors = X1.shape[1]
-    
-    permutation_indexes = np.zeros((permutations + 1, subjects), dtype=np.int)
+
+    permutation_indexes = np.zeros((permutations, subjects), dtype=np.int)
     permutation_indexes[0, :] = range(subjects)
-    for i in range(1, permutations + 1):
+    for i in range(1, permutations):
         permutation_indexes[i,:] = np.random.permutation(subjects)
-        
+    
     H2perms = gen_h2_perms(X1, columns, permutation_indexes)
     IHperms = gen_ih_perms(X1, columns, permutation_indexes)
 
 
@@ -2,6 +2,7 @@
 import os
 from CPAC.pipeline import nipype_pipeline_engine as pe
 import nipype.interfaces.utility as util
+from nipype import config
 
 from CPAC.utils.interfaces.function import Function
 
@@ -11,6 +12,7 @@
     create_cwas_batches,
     merge_cwas_batches,
     nifti_cwas,
+    zstat_image,
 )
 
 
@@ -54,6 +56,8 @@ def create_cwas(name='cwas', working_dir=None, crash_dir=None):
             Pseudo F values of CWAS
         outputspec.p_map : string (nifti file)
             Significance p values calculated from permutation tests
+        outputspec.z_map : string (nifti file)
+            Significance p values converted to z-scores 
             
     CWAS Procedure:
     
@@ -83,10 +87,9 @@ def create_cwas(name='cwas', working_dir=None, crash_dir=None):
     
     References
     ----------
-    .. [1] Shehzad Z, Kelly C, Reiss PT, Emerson JW, McMahon K, Copland DA, Castellanos FX, Milham MP. An Analytic Framework for Connectome-Wide Association Studies. Under Review.
+    .. [1] Shehzad Z, Kelly C, Reiss PT, Cameron Craddock R, Emerson JW, McMahon K, Copland DA, Castellanos FX, Milham MP. A multivariate distance-based analytic framework for connectome-wide association studies. Neuroimage. 2014 Jun;93 Pt 1(0 1):74-94. doi: 10.1016/j.neuroimage.2014.02.024. Epub 2014 Feb 28. PMID: 24583255; PMCID: PMC4138049.
     
     """
-
     if not working_dir:
         working_dir = os.path.join(os.getcwd(), 'MDMR_work_dir')
     if not crash_dir:
@@ -95,20 +98,24 @@ def create_cwas(name='cwas', working_dir=None, crash_dir=None):
     workflow = pe.Workflow(name=name)
     workflow.base_dir = working_dir
     workflow.config['execution'] = {'hash_method': 'timestamp',
-                                    'crashdump_dir': os.path.abspath(crash_dir)}
+                                    'crashdump_dir': os.path.abspath(crash_dir), 
+                                    'crashfile_format': 'txt'}
 
     inputspec = pe.Node(util.IdentityInterface(fields=['roi',
                                                        'subjects',
                                                        'regressor',
                                                        'participant_column',
                                                        'columns',
                                                        'permutations',
-                                                       'parallel_nodes']),
+                                                       'parallel_nodes',
+                                                       'z_score']),
                         name='inputspec')
 
     outputspec = pe.Node(util.IdentityInterface(fields=['F_map',
                                                         'p_map',
-                                                        'neglog_p_map']),
+                                                        'neglog_p_map',
+                                                        'one_p_map',
+                                                        'z_map']),
                          name='outputspec')
 
     ccb = pe.Node(Function(input_names=['mask_file',
@@ -139,10 +146,14 @@ def create_cwas(name='cwas', working_dir=None, crash_dir=None):
                     name='joint_mask')
 
     mcwasb = pe.Node(Function(input_names=['cwas_batches',
-                                           'mask_file'],
+                                           'mask_file',
+                                           'z_score',
+                                           'permutations'],
                               output_names=['F_file',
                                             'p_file',
-                                            'neglog_p_file'],
+                                            'neglog_p_file',
+                                            'one_p_file',
+                                            'z_file'],
                               function=merge_cwas_batches,
                               as_module=True),
                      name='cwas_volumes')
@@ -181,9 +192,15 @@ def create_cwas(name='cwas', working_dir=None, crash_dir=None):
                      mcwasb, 'cwas_batches')
     workflow.connect(jmask, 'joint_mask',
                      mcwasb, 'mask_file')
+    workflow.connect(inputspec, 'z_score',
+                     mcwasb, 'z_score')
+    workflow.connect(inputspec, 'permutations',
+                     mcwasb, 'permutations')
 
     workflow.connect(mcwasb, 'F_file', outputspec, 'F_map')
     workflow.connect(mcwasb, 'p_file', outputspec, 'p_map')
     workflow.connect(mcwasb, 'neglog_p_file', outputspec, 'neglog_p_map')
+    workflow.connect(mcwasb, 'one_p_file', outputspec, 'one_p_map')
+    workflow.connect(mcwasb, 'z_file', outputspec, 'z_map')
 
     return workflow