ENH: Adopt config module

Close #87

Author: oesteban

dmriprep/__init__.py

@@ -1,23 +1,14 @@
 """Top-level package for dmriprep."""
-import warnings as _warnings
 from .__about__ import (
     __version__,
     __copyright__,
     __credits__,
     __packagename__,
 )
 
-
 __all__ = [
     '__version__',
     '__copyright__',
     '__credits__',
     '__packagename__',
 ]
-
-# cmp is not used by dmriprep, so ignore nipype-generated warnings
-_warnings.filterwarnings('ignore', r'cmp not installed')
-_warnings.filterwarnings('ignore', r'This has not been fully tested. Please report any failures.')
-_warnings.filterwarnings('ignore', r"can't resolve package from __spec__ or __package__")
-_warnings.simplefilter('ignore', DeprecationWarning)
-_warnings.simplefilter('ignore', ResourceWarning)

dmriprep/cli/parser.py

@@ -0,0 +1,116 @@
+"""Test parser."""
+from packaging.version import Version
+import pytest
+from ..parser import _build_parser
+from .. import version as _version
+from ... import config
+
+MIN_ARGS = ['data/', 'out/', 'participant']
+
+
+@pytest.mark.parametrize('args,code', [
+    ([], 2),
+    (MIN_ARGS, 2),  # bids_dir does not exist
+    (MIN_ARGS + ['--fs-license-file'], 2),
+    (MIN_ARGS + ['--fs-license-file', 'fslicense.txt'], 2),
+])
+def test_parser_errors(args, code):
+    """Check behavior of the parser."""
+    with pytest.raises(SystemExit) as error:
+        _build_parser().parse_args(args)
+
+    assert error.value.code == code
+
+
+@pytest.mark.parametrize('args', [
+    MIN_ARGS,
+    MIN_ARGS + ['--fs-license-file'],
+])
+def test_parser_valid(tmp_path, args):
+    """Check valid arguments."""
+    datapath = (tmp_path / 'data')
+    datapath.mkdir(exist_ok=True)
+    args[0] = str(datapath)
+
+    if '--fs-license-file' in args:
+        _fs_file = tmp_path / 'license.txt'
+        _fs_file.write_text('')
+        args.insert(args.index('--fs-license-file') + 1,
+                    str(_fs_file.absolute()))
+
+    opts = _build_parser().parse_args(args)
+
+    assert opts.bids_dir == datapath
+
+
+@pytest.mark.parametrize('argval,gb', [
+    ('1G', 1),
+    ('1GB', 1),
+    ('1000', 1),    # Default units are MB
+    ('32000', 32),  # Default units are MB
+    ('4000', 4),    # Default units are MB
+    ('1000M', 1),
+    ('1000MB', 1),
+    ('1T', 1000),
+    ('1TB', 1000),
+    ('%dK' % 1e6, 1),
+    ('%dKB' % 1e6, 1),
+    ('%dB' % 1e9, 1),
+])
+def test_memory_arg(tmp_path, argval, gb):
+    """Check the correct parsing of the memory argument."""
+    datapath = (tmp_path / 'data')
+    datapath.mkdir(exist_ok=True)
+    _fs_file = tmp_path / 'license.txt'
+    _fs_file.write_text('')
+
+    args = MIN_ARGS + ['--fs-license-file', str(_fs_file)] \
+        + ['--mem', argval]
+    opts = _build_parser().parse_args(args)
+
+    assert opts.memory_gb == gb
+
+
+@pytest.mark.parametrize('current,latest', [
+    ('1.0.0', '1.3.2'),
+    ('1.3.2', '1.3.2')
+])
+def test_get_parser_update(monkeypatch, capsys, current, latest):
+    """Make sure the out-of-date banner is shown."""
+    expectation = Version(current) < Version(latest)
+
+    def _mock_check_latest(*args, **kwargs):
+        return Version(latest)
+
+    monkeypatch.setattr(config.environment, 'version', current)
+    monkeypatch.setattr(_version, 'check_latest', _mock_check_latest)
+
+    _build_parser()
+    captured = capsys.readouterr().err
+
+    msg = """\
+You are using dMRIPrep-%s, and a newer version of dMRIPrep is available: %s.
+Please check out our documentation about how and when to upgrade:
+https://dmriprep.readthedocs.io/en/latest/faq.html#upgrading""" % (current, latest)
+
+    assert (msg in captured) is expectation
+
+
+@pytest.mark.parametrize('flagged', [
+    (True, None),
+    (True, 'random reason'),
+    (False, None),
+])
+def test_get_parser_blacklist(monkeypatch, capsys, flagged):
+    """Make sure the blacklisting banner is shown."""
+    def _mock_is_bl(*args, **kwargs):
+        return flagged
+
+    monkeypatch.setattr(_version, 'is_flagged', _mock_is_bl)
+
+    _build_parser()
+    captured = capsys.readouterr().err
+
+    assert ('FLAGGED' in captured) is flagged[0]
+    if flagged[0]:
+        assert ((flagged[1] or 'reason: unknown') in captured)

dmriprep/cli/run.py

@@ -0,0 +1,160 @@
+"""
+The workflow builder factory method.
+
+All the checks and the construction of the workflow are done
+inside this function that has pickleable inputs and output
+dictionary (``retval``) to allow isolation using a
+``multiprocessing.Process`` that allows dmriprep to enforce
+a hard-limited memory-scope.
+
+"""
+
+
+def build_workflow(config_file, retval):
+    """Create the Nipype Workflow that supports the whole execution graph."""
+    from niworkflows.utils.bids import collect_participants, check_pipeline_version
+    from niworkflows.reports import generate_reports
+    from .. import config
+    from ..utils.misc import check_deps
+    from ..workflows.base import init_dmriprep_wf
+
+    config.load(config_file)
+    build_log = config.loggers.workflow
+
+    output_dir = config.execution.output_dir
+    version = config.environment.version
+
+    retval['return_code'] = 1
+    retval['workflow'] = None
+
+    # warn if older results exist: check for dataset_description.json in output folder
+    msg = check_pipeline_version(
+        version, output_dir / 'dmriprep' / 'dataset_description.json'
+    )
+    if msg is not None:
+        build_log.warning(msg)
+
+    # Please note this is the input folder's dataset_description.json
+    dset_desc_path = config.execution.bids_dir / 'dataset_description.json'
+    if dset_desc_path.exists():
+        from hashlib import sha256
+        desc_content = dset_desc_path.read_bytes()
+        config.execution.bids_description_hash = sha256(desc_content).hexdigest()
+
+    # First check that bids_dir looks like a BIDS folder
+    subject_list = collect_participants(
+        config.execution.layout,
+        participant_label=config.execution.participant_label
+    )
+
+    # Called with reports only
+    if config.execution.reports_only:
+        from pkg_resources import resource_filename as pkgrf
+
+        build_log.log(25, 'Running --reports-only on participants %s', ', '.join(subject_list))
+        retval['return_code'] = generate_reports(
+            subject_list,
+            config.execution.output_dir,
+            config.execution.work_dir,
+            config.execution.run_uuid,
+            config=pkgrf('dmriprep', 'config/reports-spec.yml'),
+            packagename='dmriprep')
+        return retval
+
+    # Build main workflow
+    INIT_MSG = """
+    Running dMRIPREP version {version}:
+      * BIDS dataset path: {bids_dir}.
+      * Participant list: {subject_list}.
+      * Run identifier: {uuid}.
+      * Output spaces: {spaces}.
+    """.format
+    build_log.log(25, INIT_MSG(
+        version=config.environment.version,
+        bids_dir=config.execution.bids_dir,
+        subject_list=subject_list,
+        uuid=config.execution.run_uuid,
+        spaces=config.execution.output_spaces)
+    )
+
+    retval['workflow'] = init_dmriprep_wf()
+
+    # Check workflow for missing commands
+    missing = check_deps(retval['workflow'])
+    if missing:
+        build_log.critical(
+            "Cannot run dMRIPrep. Missing dependencies:%s",
+            '\n\t* %s'.join(["{} (Interface: {})".format(cmd, iface)
+                             for iface, cmd in missing])
+        )
+        retval['return_code'] = 127  # 127 == command not found.
+        return retval
+
+    config.to_filename(config_file)
+    build_log.info(
+        "dMRIPrep workflow graph with %d nodes built successfully.",
+        len(retval['workflow']._get_all_nodes())
+    )
+    retval['return_code'] = 0
+    return retval
+
+
+def build_boilerplate(config_file, workflow):
+    """Write boilerplate in an isolated process."""
+    from .. import config
+
+    config.load(config_file)
+    logs_path = config.execution.output_dir / 'dmriprep' / 'logs'
+    boilerplate = workflow.visit_desc()
+    citation_files = {
+        ext: logs_path / ('CITATION.%s' % ext)
+        for ext in ('bib', 'tex', 'md', 'html')
+    }
+
+    if boilerplate:
+        # To please git-annex users and also to guarantee consistency
+        # among different renderings of the same file, first remove any
+        # existing one
+        for citation_file in citation_files.values():
+            try:
+                citation_file.unlink()
+            except FileNotFoundError:
+                pass
+
+    citation_files['md'].write_text(boilerplate)
+
+    if not config.execution.md_only_boilerplate and citation_files['md'].exists():
+        from subprocess import check_call, CalledProcessError, TimeoutExpired
+        from pkg_resources import resource_filename as pkgrf
+        from shutil import copyfile
+        # Generate HTML file resolving citations
+        cmd = ['pandoc', '-s', '--bibliography',
+               pkgrf('dmriprep', 'data/boilerplate.bib'),
+               '--filter', 'pandoc-citeproc',
+               '--metadata', 'pagetitle="dMRIPrep citation boilerplate"',
+               str(citation_files['md']),
+               '-o', str(citation_files['html'])]
+
+        config.loggers.cli.info(
+            'Generating an HTML version of the citation boilerplate...')
+        try:
+            check_call(cmd, timeout=10)
+        except (FileNotFoundError, CalledProcessError, TimeoutExpired):
+            config.loggers.cli.warning(
+                'Could not generate CITATION.html file:\n%s', ' '.join(cmd))
+
+        # Generate LaTex file resolving citations
+        cmd = ['pandoc', '-s', '--bibliography',
+               pkgrf('dmriprep', 'data/boilerplate.bib'),
+               '--natbib', str(citation_files['md']),
+               '-o', str(citation_files['tex'])]
+        config.loggers.cli.info(
+            'Generating a LaTeX version of the citation boilerplate...')
+        try:
+            check_call(cmd, timeout=10)
+        except (FileNotFoundError, CalledProcessError, TimeoutExpired):
+            config.loggers.cli.warning(
+                'Could not generate CITATION.tex file:\n%s', ' '.join(cmd))
+        else:
+            copyfile(pkgrf('dmriprep', 'data/boilerplate.bib'),
+                     citation_files['bib'])