diff --git a/nibabies/cli/parser.py b/nibabies/cli/parser.py index 8bb1d201..56c865eb 100644 --- a/nibabies/cli/parser.py +++ b/nibabies/cli/parser.py @@ -619,11 +619,12 @@ def _str_none(val): '--output-layout', action='store', default='bids', - choices=('bids', 'legacy'), + choices=('bids', 'legacy', 'multiverse'), help='Organization of outputs. bids (default) places NiBabies derivatives ' 'directly in the output directory, and defaults to placing FreeSurfer ' 'derivatives in /sourcedata/freesurfer. legacy creates derivative ' - 'datasets as subdirectories of outputs.', + 'datasets as subdirectories of outputs. multiverse appends the version and a hash ' + 'of parameters used to the output folder - the hash is also applied to the output files.', ) g_other.add_argument( '-w', @@ -834,27 +835,62 @@ def parse_args(args=None, namespace=None): applied.""" ) + config.workflow.skull_strip_template = config.workflow.skull_strip_template[0] + bids_dir = config.execution.bids_dir output_dir = config.execution.output_dir work_dir = config.execution.work_dir version = config.environment.version output_layout = config.execution.output_layout + config.execution.parameters_hash = config.hash_config(config.get()) - if config.execution.fs_subjects_dir is None: - if output_layout == 'bids': - config.execution.fs_subjects_dir = output_dir / 'sourcedata' / 'freesurfer' - elif output_layout == 'legacy': - config.execution.fs_subjects_dir = output_dir / 'freesurfer' + # Multiverse behaves as a cross between bids and legacy if config.execution.nibabies_dir is None: - if output_layout == 'bids': - config.execution.nibabies_dir = output_dir - elif output_layout == 'legacy': - config.execution.nibabies_dir = output_dir / 'nibabies' + match output_layout: + case 'bids': + config.execution.nibabies_dir = output_dir + case 'legacy': + config.execution.nibabies_dir = output_dir / 'nibabies' + case 'multiverse': + config.loggers.cli.warning( + 'Multiverse output selected - assigning output directory based on version' + ' and configuration hash.' + ) + config.execution.nibabies_dir = ( + output_dir + / f'nibabies-{version.split("+", 1)[0]}-{config.execution.parameters_hash}' + ) + case _: + config.loggers.cli.warning('Unknown output layout %s', output_layout) + pass + + nibabies_dir = config.execution.nibabies_dir + + if config.execution.fs_subjects_dir is None: + match output_layout: + case 'bids': + config.execution.fs_subjects_dir = output_dir / 'sourcedata' / 'freesurfer' + case 'legacy': + config.execution.fs_subjects_dir = output_dir / 'freesurfer' + case 'multiverse': + config.execution.fs_subjects_dir = ( + nibabies_dir / 'sourcedata' / f'freesurfer-{config.execution.parameters_hash}' + ) + case _: + pass + if config.workflow.surface_recon_method == 'mcribs': - if output_layout == 'bids': - config.execution.mcribs_dir = output_dir / 'sourcedata' / 'mcribs' - elif output_layout == 'legacy': - config.execution.mcribs_dir = output_dir / 'mcribs' + match output_layout: + case 'bids': + config.execution.mcribs_dir = output_dir / 'sourcedata' / 'mcribs' + case 'legacy': + config.execution.mcribs_dir = output_dir / 'mcribs' + case 'multiverse': + config.execution.mcribs_dir = ( + nibabies_dir / 'sourcedata' / f'mcribs-{config.execution.parameters_hash}' + ) + case _: + pass # Ensure the directory is created config.execution.mcribs_dir.mkdir(exist_ok=True, parents=True) @@ -909,7 +945,6 @@ def parse_args(args=None, namespace=None): participant_ids=config.execution.participant_label, session_ids=config.execution.session_id, ) - config.workflow.skull_strip_template = config.workflow.skull_strip_template[0] # finally, write config to file config_file = config.execution.work_dir / config.execution.run_uuid / 'config.toml' diff --git a/nibabies/cli/run.py b/nibabies/cli/run.py index fc1c8a36..b577b13f 100755 --- a/nibabies/cli/run.py +++ b/nibabies/cli/run.py @@ -144,16 +144,20 @@ def main(): finally: from ..reports.core import generate_reports + add_hash = config.execution.output_layout == 'multiverse' + # Generate reports phase generate_reports( config.execution.unique_labels, config.execution.nibabies_dir, config.execution.run_uuid, + config_hash=config.execution.parameters_hash if add_hash else None, ) write_derivative_description( config.execution.bids_dir, config.execution.nibabies_dir, config.execution.dataset_links, + config.execution.parameters_hash, ) write_bidsignore(config.execution.nibabies_dir) diff --git a/nibabies/config.py b/nibabies/config.py index cab424cd..c4a4c91e 100644 --- a/nibabies/config.py +++ b/nibabies/config.py @@ -89,6 +89,7 @@ """ import os +import typing as ty from multiprocessing import set_start_method from templateflow.conf import TF_LAYOUT @@ -409,6 +410,8 @@ class execution(_Config): output_spaces = None """List of (non)standard spaces designated (with the ``--output-spaces`` flag of the command line) as spatial references for outputs.""" + parameters_hash = None + """Unique hash of the current configuration parameters.""" reference_anat = None """Force usage of this anatomical scan as the structural reference.""" reports_only = False @@ -792,15 +795,79 @@ def _process_initializer(cwd, omp_nthreads): os.environ['OMP_NUM_THREADS'] = f'{omp_nthreads}' -def dismiss_echo(entities: list | None = None): +def dismiss_entities(entities: list | None = None) -> list: """Set entities to dismiss in a DerivativesDataSink.""" from niworkflows.utils.connections import listify - entities = entities or [] + entities = set(entities or []) echo_idx = execution.echo_idx if echo_idx is None or len(listify(echo_idx)) > 2: - entities.append('echo') - return entities + entities.add('echo') + output_layout = execution.output_layout + if output_layout != 'multiverse': + entities.add('hash') + return list(entities) + + +DEFAULT_DISMISS_ENTITIES = dismiss_entities() + +DEFAULT_CONFIG_HASH_FIELDS = { + 'execution': [ + 'sloppy', + 'echo_idx', + 'reference_anat', + ], + 'workflow': [ + 'surface_recon_method', + 'bold2anat_dof', + 'bold2anat_init', + 'dummy_scans', + 'fd_radius', + 'fmap_bspline', + 'fmap_demean', + 'force_syn', + 'hmc_bold_frame', + 'longitudinal', + 'medial_surface_nan', + 'multi_step_reg', + 'norm_csf', + 'project_goodvoxels', + 'regressors_dvars_th', + 'regressors_fd_th', + 'skull_strip_fixed_seed', + 'skull_strip_template', + 'skull_strip_anat', + 'slice_time_ref', + 'surface_recon_method', + 'use_bbr', + 'use_syn_sdc', + 'me_t2s_fit_method', + ], +} + + +def hash_config( + conf: dict[str, ty.Any], + *, + fields_required: dict[str, list[str]] = DEFAULT_CONFIG_HASH_FIELDS, + version: str = None, + digest_size: int = 4, +) -> str: + """ + Generate a unique BLAKE2b hash of configuration attributes. + + By default, uses a preselected list of workflow-altering parameters. + """ + import json + from hashlib import blake2b + + if version is None: + from nibabies import __version__ as version + data = {} + for level, fields in fields_required.items(): + for f in fields: + data[f] = conf[level].get(f, None) -DEFAULT_DISMISS_ENTITIES = dismiss_echo() + datab = json.dumps(data, sort_keys=True).encode() + return blake2b(datab, digest_size=digest_size).hexdigest() diff --git a/nibabies/reports/core.py b/nibabies/reports/core.py index ed7ccdd2..9331b64f 100644 --- a/nibabies/reports/core.py +++ b/nibabies/reports/core.py @@ -10,9 +10,9 @@ def run_reports( subject, run_uuid, session=None, - out_filename=None, + bootstrap_file=None, + out_filename='report.html', reportlets_dir=None, - packagename=None, ): """ Run the reports. @@ -22,8 +22,9 @@ def run_reports( run_uuid, subject=subject, session=session, - bootstrap_file=load_data.readable('reports-spec.yml'), + bootstrap_file=load_data('reports-spec.yml'), reportlets_dir=reportlets_dir, + out_filename=out_filename, ).generate_report() @@ -31,8 +32,10 @@ def generate_reports( sub_ses_list, output_dir, run_uuid, + *, work_dir=None, - packagename=None, + bootstrap_file=None, + config_hash=None, ): """Execute run_reports on a list of subjects.""" reportlets_dir = None @@ -41,14 +44,23 @@ def generate_reports( report_errors = [] for subject, session in sub_ses_list: + # Determine the output filename + html_report = f'sub-{subject}' + if session is not None: + html_report += f'_ses-{session}' + if config_hash is not None: + html_report += f'_{config_hash}' + html_report += '.html' + report_errors.append( run_reports( output_dir, subject, run_uuid, - session=session, - packagename=packagename, + bootstrap_file=bootstrap_file, reportlets_dir=reportlets_dir, + out_filename=html_report, + session=session, ) ) diff --git a/nibabies/tests/test_config.py b/nibabies/tests/test_config.py index f585e912..1f4da13e 100644 --- a/nibabies/tests/test_config.py +++ b/nibabies/tests/test_config.py @@ -146,3 +146,19 @@ def _load_spaces(age): # Conditional based on workflow necessities spaces = init_workflow_spaces(init_execution_spaces(), age) return spaces + + +def test_hash_config(): + # This may change with changes to config defaults / new attributes! + expected = 'cfee5aaf' + assert config.hash_config(config.get()) == expected + _reset_config() + + config.execution.log_level = 5 # non-vital attributes do not matter + assert config.hash_config(config.get()) == expected + _reset_config() + + # but altering a vital attribute will create a new hash + config.workflow.surface_recon_method = 'mcribs' + assert config.hash_config(config.get()) != expected + _reset_config() diff --git a/nibabies/utils/bids.py b/nibabies/utils/bids.py index ff330330..8fc01aa9 100644 --- a/nibabies/utils/bids.py +++ b/nibabies/utils/bids.py @@ -40,7 +40,7 @@ def write_bidsignore(deriv_dir): ignore_file.write_text('\n'.join(bids_ignore) + '\n') -def write_derivative_description(bids_dir, deriv_dir, dataset_links=None): +def write_derivative_description(bids_dir, deriv_dir, dataset_links=None, config_hash=None): from nibabies import __version__ DOWNLOAD_URL = f'https://github.com/nipreps/nibabies/archive/{__version__}.tar.gz' @@ -56,6 +56,7 @@ def write_derivative_description(bids_dir, deriv_dir, dataset_links=None): 'Name': 'NiBabies', 'Version': __version__, 'CodeURL': DOWNLOAD_URL, + 'ConfigurationHash': config_hash, } ], 'HowToAcknowledge': 'TODO', diff --git a/nibabies/utils/derivatives.py b/nibabies/utils/derivatives.py index 6191a1d5..d8c395df 100644 --- a/nibabies/utils/derivatives.py +++ b/nibabies/utils/derivatives.py @@ -136,6 +136,7 @@ def copy_derivatives( modality: str, subject_id: str, session_id: str | None = None, + config_hash: str | None = None, ) -> None: """ Creates a copy of any found derivatives into output directory. @@ -154,8 +155,19 @@ def copy_derivatives( if not isinstance(deriv, str): continue deriv = Path(deriv) - - shutil.copy2(deriv, outpath / deriv.name) - json = deriv.parent / (deriv.name.split('.')[0] + '.json') + outname = deriv.name + + if config_hash: + ents = outname.split('_') + if any(ent.startswith('hash-') for ent in ents): + # Avoid adding another hash + pass + else: + idx = 2 if ents[1].startswith('ses-') else 1 + ents.insert(idx, f'hash-{config_hash}') + outname = '_'.join(ents) + + shutil.copy2(deriv, outpath / outname) + json = deriv.parent / (outname.split('.')[0] + '.json') if json.exists(): shutil.copy2(json, outpath / json.name) diff --git a/nibabies/workflows/base.py b/nibabies/workflows/base.py index fea9c574..f62f53ce 100644 --- a/nibabies/workflows/base.py +++ b/nibabies/workflows/base.py @@ -330,6 +330,9 @@ def init_single_subject_wf( modality='anat', subject_id=f'sub-{subject_id}', session_id=f'ses-{session_id}' if session_id else None, + config_hash=config.execution.parameters_hash + if config.execution.output_layout == 'multiverse' + else None, ) # Determine some session level options here, as we should have @@ -368,7 +371,6 @@ def init_single_subject_wf( ) anat = reference_anat.lower() # To be used for workflow connections - LOGGER.info( 'Collected the following data for %s:\nRaw:\n%s\n\nDerivatives:\n\n%s\n', f'sub-{subject_id}' if not session_id else f'sub-{subject_id}_ses-{session_id}', @@ -739,6 +741,9 @@ def init_single_subject_wf( modality='func', subject_id=f'sub-{subject_id}', session_id=f'ses-{session_id}' if session_id else None, + config_hash=config.execution.parameters_hash + if config.execution.output_layout == 'multiverse' + else None, ) bold_wf = init_bold_wf( @@ -837,6 +842,10 @@ def clean_datasinks(workflow: pe.Workflow) -> pe.Workflow: for node in workflow.list_node_names(): if node.split('.')[-1].startswith('ds_'): workflow.get_node(node).interface.out_path_base = '' + workflow.get_node(node).interface.inputs.base_directory = config.execution.nibabies_dir + + if config.execution.output_layout == 'multiverse': + workflow.get_node(node).interface.inputs.hash = config.execution.parameters_hash return workflow diff --git a/nibabies/workflows/bold/outputs.py b/nibabies/workflows/bold/outputs.py index 24cdd4a5..8d051f02 100644 --- a/nibabies/workflows/bold/outputs.py +++ b/nibabies/workflows/bold/outputs.py @@ -32,7 +32,7 @@ from nibabies import config from nibabies._types import Anatomical -from nibabies.config import DEFAULT_DISMISS_ENTITIES, DEFAULT_MEMORY_MIN_GB, dismiss_echo +from nibabies.config import DEFAULT_DISMISS_ENTITIES, DEFAULT_MEMORY_MIN_GB, dismiss_entities from nibabies.interfaces import DerivativesDataSink from nibabies.interfaces.bids import BIDSURI @@ -487,7 +487,7 @@ def init_ds_registration_wf( mode='image', suffix='xfm', extension='.txt', - dismiss_entities=dismiss_echo(['part']), + dismiss_entities=dismiss_entities(['part']), **{'from': source, 'to': dest}, ), name='ds_xform', diff --git a/pyproject.toml b/pyproject.toml index 03295de6..5414fe0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,8 @@ dependencies = [ "nireports >= 23.2.0", "nitime", "nitransforms >= 24.1.1", - "niworkflows >= 1.13.1", + #"niworkflows >= 1.13.1", + "niworkflows @ git+https://github.com/nipreps/niworkflows.git@master", "numpy >= 1.21.0", "packaging", "pandas < 3",