diff --git a/.circleci/version.py b/.circleci/version.py index 8bbac435c3..cbaaacb31b 100644 --- a/.circleci/version.py +++ b/.circleci/version.py @@ -1,4 +1,5 @@ """Get sdcflows version.""" + import sdcflows -print(sdcflows.__version__, end="", file=open("/tmp/.docker-version.txt", "w")) +print(sdcflows.__version__, end='', file=open('/tmp/.docker-version.txt', 'w')) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..fe21fefa26 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,6 @@ +# Thu May 15 09:29:29 2025 -0400 - markiewicz@stanford.edu - run: ruff format [ignore-rev] +d97ae316c0bdf71084a0732760ceed5221033fc2 +# Thu May 15 09:26:57 2025 -0400 - markiewicz@stanford.edu - run: ruff check --fix [ignore-rev] +cacd409cce71f2e530ad1bd1e8b79f397cab81d4 +# Thu May 15 09:22:00 2025 -0400 - markiewicz@stanford.edu - run: pre-commit run --all [ignore-rev] +2b40bee0d6b2c88628760a8b1513851a439411e8 diff --git a/.git_archival.txt b/.git_archival.txt index b1a286bbb6..8fb235d704 100644 --- a/.git_archival.txt +++ b/.git_archival.txt @@ -1,4 +1,4 @@ node: $Format:%H$ node-date: $Format:%cI$ describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ -ref-names: $Format:%D$ \ No newline at end of file +ref-names: $Format:%D$ diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index bd8b7c2b0c..a3e3eae35a 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -17,11 +17,12 @@ permissions: contents: read jobs: - flake8: + style: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: pipx run flake8-pyproject sdcflows/ + - run: pipx run ruff check sdcflows/ + - run: pipx run ruff format --diff sdcflows/ # codespell: # runs-on: ubuntu-latest diff --git a/.maint/update_authors.py b/.maint/update_authors.py index 0ca78d4993..48fe34eed0 100644 --- a/.maint/update_authors.py +++ b/.maint/update_authors.py @@ -8,9 +8,11 @@ # ] # /// """Update and sort the creators list of the zenodo record.""" + +import json import sys from pathlib import Path -import json + import click from fuzzywuzzy import fuzz, process @@ -43,21 +45,18 @@ def read_md_table(md_text): keys = None retval = [] for line in md_text.splitlines(): - if line.strip().startswith("| --- |"): - keys = ( - k.replace("*", "").strip() - for k in prev.split("|") - ) + if line.strip().startswith('| --- |'): + keys = (k.replace('*', '').strip() for k in prev.split('|')) keys = [k.lower() for k in keys if k] continue elif not keys: prev = line continue - if not line or not line.strip().startswith("|"): + if not line or not line.strip().startswith('|'): break - values = [v.strip() or None for v in line.split("|")][1:-1] + values = [v.strip() or None for v in line.split('|')][1:-1] retval.append({k: v for k, v in zip(keys, values) if v}) return retval @@ -66,21 +65,15 @@ def read_md_table(md_text): def sort_contributors(entries, git_lines, exclude=None, last=None): """Return a list of author dictionaries, ordered by contribution.""" last = last or [] - sorted_authors = sorted(entries, key=lambda i: i["name"]) + sorted_authors = sorted(entries, key=lambda i: i['name']) - first_last = [ - " ".join(val["name"].split(",")[::-1]).strip() for val in sorted_authors - ] - first_last_excl = [ - " ".join(val["name"].split(",")[::-1]).strip() for val in exclude or [] - ] + first_last = [' '.join(val['name'].split(',')[::-1]).strip() for val in sorted_authors] + first_last_excl = [' '.join(val['name'].split(',')[::-1]).strip() for val in exclude or []] unmatched = [] author_matches = [] for ele in git_lines: - matches = process.extract( - ele, first_last, scorer=fuzz.token_sort_ratio, limit=2 - ) + matches = process.extract(ele, first_last, scorer=fuzz.token_sort_ratio, limit=2) # matches is a list [('First match', % Match), ('Second match', % Match)] if matches[0][1] > 80: val = sorted_authors[first_last.index(matches[0][0])] @@ -93,7 +86,7 @@ def sort_contributors(entries, git_lines, exclude=None, last=None): if val not in author_matches: author_matches.append(val) - names = {" ".join(val["name"].split(",")[::-1]).strip() for val in author_matches} + names = {' '.join(val['name'].split(',')[::-1]).strip() for val in author_matches} for missing_name in first_last: if missing_name not in names: missing = sorted_authors[first_last.index(missing_name)] @@ -101,7 +94,7 @@ def sort_contributors(entries, git_lines, exclude=None, last=None): position_matches = [] for i, item in enumerate(author_matches): - pos = item.pop("position", None) + pos = item.pop('position', None) if pos is not None: position_matches.append((i, int(pos))) @@ -113,7 +106,7 @@ def sort_contributors(entries, git_lines, exclude=None, last=None): return author_matches, unmatched -def get_git_lines(fname="line-contributors.txt"): +def get_git_lines(fname='line-contributors.txt'): """Run git-line-summary.""" import shutil import subprocess as sp @@ -122,17 +115,17 @@ def get_git_lines(fname="line-contributors.txt"): lines = [] if contrib_file.exists(): - print("WARNING: Reusing existing line-contributors.txt file.", file=sys.stderr) + print('WARNING: Reusing existing line-contributors.txt file.', file=sys.stderr) lines = contrib_file.read_text().splitlines() - cmd = [shutil.which("git-line-summary")] + cmd = [shutil.which('git-line-summary')] if cmd == [None]: - cmd = [shutil.which("git-summary"), "--line"] + cmd = [shutil.which('git-summary'), '--line'] if not lines and cmd[0]: - print(f"Running {' '.join(cmd)!r} on repo") + print(f'Running {" ".join(cmd)!r} on repo') lines = sp.check_output(cmd).decode().splitlines() - lines = [l for l in lines if "Not Committed Yet" not in l] - contrib_file.write_text("\n".join(lines)) + lines = [line for line in lines if 'Not Committed Yet' not in line] + contrib_file.write_text('\n'.join(lines)) if not lines: raise RuntimeError( @@ -142,15 +135,15 @@ def get_git_lines(fname="line-contributors.txt"): git-line-summary not found, please install git-extras. """ * (cmd[0] is None) ) - return [" ".join(line.strip().split()[1:-1]) for line in lines if "%" in line] + return [' '.join(line.strip().split()[1:-1]) for line in lines if '%' in line] def _namelast(inlist): retval = [] for i in inlist: - i["name"] = (f"{i.pop('name', '')} {i.pop('lastname', '')}").strip() - if not i["name"]: - i["name"] = i.get("handle", "") + i['name'] = (f'{i.pop("name", "")} {i.pop("lastname", "")}').strip() + if not i['name']: + i['name'] = i.get('handle', '') retval.append(i) return retval @@ -162,12 +155,13 @@ def cli(): @cli.command() -@click.option("-z", "--zenodo-file", type=click.Path(exists=True), default=".zenodo.json") -@click.option("-m", "--maintainers", type=click.Path(exists=True), default=".maint/MAINTAINERS.md") -@click.option("-c", "--contributors", type=click.Path(exists=True), - default=".maint/CONTRIBUTORS.md") -@click.option("--pi", type=click.Path(exists=True), default=".maint/PIs.md") -@click.option("-f", "--former-file", type=click.Path(exists=True), default=".maint/FORMER.md") +@click.option('-z', '--zenodo-file', type=click.Path(exists=True), default='.zenodo.json') +@click.option('-m', '--maintainers', type=click.Path(exists=True), default='.maint/MAINTAINERS.md') +@click.option( + '-c', '--contributors', type=click.Path(exists=True), default='.maint/CONTRIBUTORS.md' +) +@click.option('--pi', type=click.Path(exists=True), default='.maint/PIs.md') +@click.option('-f', '--former-file', type=click.Path(exists=True), default='.maint/FORMER.md') def zenodo( zenodo_file, maintainers, @@ -188,65 +182,53 @@ def zenodo( ) zen_contributors, miss_contributors = sort_contributors( - _namelast(read_md_table(Path(contributors).read_text())), - data, - exclude=former + _namelast(read_md_table(Path(contributors).read_text())), data, exclude=former ) zen_pi = _namelast(reversed(read_md_table(Path(pi).read_text()))) - zenodo["creators"] = zen_creators - zenodo["contributors"] = zen_contributors + [ - pi for pi in zen_pi if pi not in zen_contributors - ] - creator_names = { - c["name"] for c in zenodo["creators"] - if c["name"] != "" - } - - zenodo["contributors"] = [ - c for c in zenodo["contributors"] - if c["name"] not in creator_names - ] + zenodo['creators'] = zen_creators + zenodo['contributors'] = zen_contributors + [pi for pi in zen_pi if pi not in zen_contributors] + creator_names = {c['name'] for c in zenodo['creators'] if c['name'] != ''} + + zenodo['contributors'] = [c for c in zenodo['contributors'] if c['name'] not in creator_names] misses = set(miss_creators).intersection(miss_contributors) if misses: print( - "Some people made commits, but are missing in .maint/ " - f"files: {', '.join(misses)}", + f'Some people made commits, but are missing in .maint/ files: {", ".join(misses)}', file=sys.stderr, ) # Remove position - for creator in zenodo["creators"]: - creator.pop("position", None) - creator.pop("handle", None) - if "affiliation" not in creator: - creator["affiliation"] = "Unknown affiliation" - elif isinstance(creator["affiliation"], list): - creator["affiliation"] = creator["affiliation"][0] - - for creator in zenodo["contributors"]: - creator.pop("handle", None) - creator["type"] = "Researcher" - creator.pop("position", None) - - if "affiliation" not in creator: - creator["affiliation"] = "Unknown affiliation" - elif isinstance(creator["affiliation"], list): - creator["affiliation"] = creator["affiliation"][0] - - Path(zenodo_file).write_text( - "%s\n" % json.dumps(zenodo, indent=2, ensure_ascii=False) - ) + for creator in zenodo['creators']: + creator.pop('position', None) + creator.pop('handle', None) + if 'affiliation' not in creator: + creator['affiliation'] = 'Unknown affiliation' + elif isinstance(creator['affiliation'], list): + creator['affiliation'] = creator['affiliation'][0] + + for creator in zenodo['contributors']: + creator.pop('handle', None) + creator['type'] = 'Researcher' + creator.pop('position', None) + + if 'affiliation' not in creator: + creator['affiliation'] = 'Unknown affiliation' + elif isinstance(creator['affiliation'], list): + creator['affiliation'] = creator['affiliation'][0] + + Path(zenodo_file).write_text('%s\n' % json.dumps(zenodo, indent=2, ensure_ascii=False)) @cli.command() -@click.option("-m", "--maintainers", type=click.Path(exists=True), default=".maint/MAINTAINERS.md") -@click.option("-c", "--contributors", type=click.Path(exists=True), - default=".maint/CONTRIBUTORS.md") -@click.option("--pi", type=click.Path(exists=True), default=".maint/PIs.md") -@click.option("-f", "--former-file", type=click.Path(exists=True), default=".maint/FORMER.md") +@click.option('-m', '--maintainers', type=click.Path(exists=True), default='.maint/MAINTAINERS.md') +@click.option( + '-c', '--contributors', type=click.Path(exists=True), default='.maint/CONTRIBUTORS.md' +) +@click.option('--pi', type=click.Path(exists=True), default='.maint/PIs.md') +@click.option('-f', '--former-file', type=click.Path(exists=True), default='.maint/FORMER.md') def publication( maintainers, contributors, @@ -254,9 +236,8 @@ def publication( former_file, ): """Generate the list of authors and affiliations for papers.""" - members = ( - _namelast(read_md_table(Path(maintainers).read_text())) - + _namelast(read_md_table(Path(contributors).read_text())) + members = _namelast(read_md_table(Path(maintainers).read_text())) + _namelast( + read_md_table(Path(contributors).read_text()) ) former_names = _namelast(read_md_table(Path(former_file).read_text())) @@ -267,11 +248,8 @@ def publication( ) pi_hits = _namelast(reversed(read_md_table(Path(pi).read_text()))) - pi_names = [pi["name"] for pi in pi_hits] - hits = [ - hit for hit in hits - if hit["name"] not in pi_names - ] + pi_hits + pi_names = [pi['name'] for pi in pi_hits] + hits = [hit for hit in hits if hit['name'] not in pi_names] + pi_hits def _aslist(value): if isinstance(value, (list, tuple)): @@ -281,16 +259,16 @@ def _aslist(value): # Remove position affiliations = [] for item in hits: - item.pop("position", None) - for a in _aslist(item.get("affiliation", "Unaffiliated")): + item.pop('position', None) + for a in _aslist(item.get('affiliation', 'Unaffiliated')): if a not in affiliations: affiliations.append(a) aff_indexes = [ - ", ".join( + ', '.join( [ - "%d" % (affiliations.index(a) + 1) - for a in _aslist(author.get("affiliation", "Unaffiliated")) + '%d' % (affiliations.index(a) + 1) + for a in _aslist(author.get('affiliation', 'Unaffiliated')) ] ) for author in hits @@ -298,30 +276,22 @@ def _aslist(value): if misses: print( - "Some people made commits, but are missing in .maint/ " - f"files: {', '.join(misses)}", + f'Some people made commits, but are missing in .maint/ files: {", ".join(misses)}', file=sys.stderr, ) - print("Authors (%d):" % len(hits)) + print('Authors (%d):' % len(hits)) print( - "%s." - % "; ".join( - [ - "%s \\ :sup:`%s`\\ " % (i["name"], idx) - for i, idx in zip(hits, aff_indexes) - ] - ) + '%s.' + % '; '.join(['%s \\ :sup:`%s`\\ ' % (i['name'], idx) for i, idx in zip(hits, aff_indexes)]) ) print( - "\n\nAffiliations:\n%s" - % "\n".join( - ["{0: >2}. {1}".format(i + 1, a) for i, a in enumerate(affiliations)] - ) + '\n\nAffiliations:\n%s' + % '\n'.join(['{0: >2}. {1}'.format(i + 1, a) for i, a in enumerate(affiliations)]) ) -if __name__ == "__main__": +if __name__ == '__main__': """ Install entry-point """ cli() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 17c213359a..ce9679c707 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,9 @@ exclude: ".*/data/.*" repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: check-added-large-files - - id: check-docstring-first - id: check-merge-conflict - id: check-json - id: check-toml diff --git a/docs/conf.py b/docs/conf.py index 517302b16a..49053b911e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -6,8 +6,10 @@ http://www.sphinx-doc.org/en/master/config """ + from packaging.version import Version -from sdcflows import __version__, __copyright__, __packagename__ + +from sdcflows import __copyright__, __packagename__, __version__ # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, @@ -18,7 +20,7 @@ # -- Project information ----------------------------------------------------- project = __packagename__ copyright = __copyright__ -author = "The SDCflows Developers" +author = 'The SDCflows Developers' # The short X.Y version version = Version(__version__).public @@ -27,57 +29,57 @@ # -- General configuration --------------------------------------------------- extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.coverage", - "sphinx.ext.doctest", - "sphinx.ext.githubpages", - "sphinx.ext.ifconfig", - "sphinx.ext.intersphinx", - "sphinx.ext.mathjax", - "sphinx.ext.viewcode", - "sphinxarg.ext", - "sphinxcontrib.apidoc", - "nbsphinx", - "nipype.sphinxext.apidoc", - "nipype.sphinxext.plot_workflow", + 'sphinx.ext.autodoc', + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', + 'sphinx.ext.githubpages', + 'sphinx.ext.ifconfig', + 'sphinx.ext.intersphinx', + 'sphinx.ext.mathjax', + 'sphinx.ext.viewcode', + 'sphinxarg.ext', + 'sphinxcontrib.apidoc', + 'nbsphinx', + 'nipype.sphinxext.apidoc', + 'nipype.sphinxext.plot_workflow', ] autodoc_mock_imports = [ - "matplotlib", - "migas", - "nilearn", - "nipy", - "nitime", - "nireports", - "pandas", - "seaborn", - "skimage", - "svgutils", - "transforms3d", + 'matplotlib', + 'migas', + 'nilearn', + 'nipy', + 'nitime', + 'nireports', + 'pandas', + 'seaborn', + 'skimage', + 'svgutils', + 'transforms3d', ] # Accept custom section names to be parsed for numpy-style docstrings # of parameters. napoleon_use_param = False napoleon_custom_sections = [ - ("Inputs", "Parameters"), - ("Outputs", "Parameters"), - ("Attributes", "Parameters"), - ("Mandatory Inputs", "Parameters"), - ("Optional Inputs", "Parameters"), + ('Inputs', 'Parameters'), + ('Outputs', 'Parameters'), + ('Attributes', 'Parameters'), + ('Mandatory Inputs', 'Parameters'), + ('Optional Inputs', 'Parameters'), ] # Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] +templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = ".rst" +source_suffix = '.rst' # The master toctree document. -master_doc = "index" +master_doc = 'index' numfig = True @@ -86,16 +88,16 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = "en" +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [ - "_build", - "Thumbs.db", - ".DS_Store", - "api/sdcflows.rst", + '_build', + 'Thumbs.db', + '.DS_Store', + 'api/sdcflows.rst', ] # The name of the Pygments (syntax highlighting) style to use. @@ -107,7 +109,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "furo" +html_theme = 'furo' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -118,12 +120,12 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +html_static_path = ['_static'] html_js_files = [ - "js/version-switch.js", + 'js/version-switch.js', ] html_css_files = [ - "css/version-switch.css", + 'css/version-switch.css', ] # Custom sidebar templates, must be a dictionary that maps document names @@ -140,7 +142,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = "sdcflowsdoc" +htmlhelp_basename = 'sdcflowsdoc' # -- Options for LaTeX output ------------------------------------------------ @@ -166,10 +168,10 @@ latex_documents = [ ( master_doc, - "sdcflows.tex", - "SDCFlows Documentation", - "The SDCFlows Developers", - "manual", + 'sdcflows.tex', + 'SDCFlows Documentation', + 'The SDCFlows Developers', + 'manual', ), ] @@ -178,7 +180,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_doc, "sdcflows", "SDCFlows Documentation", [author], 1)] +man_pages = [(master_doc, 'sdcflows', 'SDCFlows Documentation', [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -189,12 +191,12 @@ texinfo_documents = [ ( master_doc, - "sdcflows", - "SDCFlows Documentation", + 'sdcflows', + 'SDCFlows Documentation', author, - "SDCFlows", - "One line description of project.", - "Miscellaneous", + 'SDCFlows', + 'One line description of project.', + 'Miscellaneous', ), ] @@ -214,31 +216,31 @@ # epub_uid = '' # A list of files that should not be packed into the epub file. -epub_exclude_files = ["search.html"] +epub_exclude_files = ['search.html'] # -- Extension configuration ------------------------------------------------- -apidoc_module_dir = "../sdcflows" -apidoc_output_dir = "api" -apidoc_excluded_paths = ["conftest.py", "*/tests/*", "tests/*"] +apidoc_module_dir = '../sdcflows' +apidoc_output_dir = 'api' +apidoc_excluded_paths = ['conftest.py', '*/tests/*', 'tests/*'] apidoc_separate_modules = True -apidoc_extra_args = ["--module-first", "-d 1", "-T"] +apidoc_extra_args = ['--module-first', '-d 1', '-T'] # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - "python": ("https://docs.python.org/3/", None), - "numpy": ("https://numpy.org/doc/stable/", None), - "scipy": ("https://docs.scipy.org/doc/scipy/", None), - "matplotlib": ("https://matplotlib.org/stable", None), - "bids": ("https://bids-standard.github.io/pybids/", None), - "nibabel": ("https://nipy.org/nibabel/", None), - "nipype": ("https://nipype.readthedocs.io/en/latest/", None), - "niworkflows": ("https://www.nipreps.org/niworkflows/", None), - "smriprep": ("https://www.nipreps.org/smriprep/", None), - "templateflow": ("https://www.templateflow.org/python-client", None), + 'python': ('https://docs.python.org/3/', None), + 'numpy': ('https://numpy.org/doc/stable/', None), + 'scipy': ('https://docs.scipy.org/doc/scipy/', None), + 'matplotlib': ('https://matplotlib.org/stable', None), + 'bids': ('https://bids-standard.github.io/pybids/', None), + 'nibabel': ('https://nipy.org/nibabel/', None), + 'nipype': ('https://nipype.readthedocs.io/en/latest/', None), + 'niworkflows': ('https://www.nipreps.org/niworkflows/', None), + 'smriprep': ('https://www.nipreps.org/smriprep/', None), + 'templateflow': ('https://www.templateflow.org/python-client', None), } # -- Options for versioning extension ---------------------------------------- diff --git a/docs/examples.rst b/docs/examples.rst index 4efb78ab02..af6f58b215 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -5,4 +5,4 @@ Examples :maxdepth: 2 :caption: Contents: - notebooks/SDC - Theory and physics \ No newline at end of file + notebooks/SDC - Theory and physics diff --git a/docs/notebooks/SDC - Theory and physics.ipynb b/docs/notebooks/SDC - Theory and physics.ipynb index 1a4b17e697..d8fa57fdac 100644 --- a/docs/notebooks/SDC - Theory and physics.ipynb +++ b/docs/notebooks/SDC - Theory and physics.ipynb @@ -31,11 +31,11 @@ "from matplotlib import pyplot as plt\n", "from mpl_toolkits.axes_grid1.inset_locator import inset_axes\n", "\n", - "plt.rcParams[\"figure.figsize\"] = (12, 9)\n", - "plt.rcParams[\"xtick.bottom\"] = False\n", - "plt.rcParams[\"xtick.labelbottom\"] = False\n", - "plt.rcParams[\"ytick.left\"] = False\n", - "plt.rcParams[\"ytick.labelleft\"] = False" + "plt.rcParams['figure.figsize'] = (12, 9)\n", + "plt.rcParams['xtick.bottom'] = False\n", + "plt.rcParams['xtick.labelbottom'] = False\n", + "plt.rcParams['ytick.left'] = False\n", + "plt.rcParams['ytick.labelleft'] = False" ] }, { @@ -54,9 +54,10 @@ "outputs": [], "source": [ "from itertools import product\n", + "\n", + "import nibabel as nb\n", "import numpy as np\n", "from scipy import ndimage as ndi\n", - "import nibabel as nb\n", "from templateflow.api import get" ] }, @@ -71,44 +72,45 @@ " samples_x = np.arange(6, brain_slice.shape[1] - 3, step=12).astype(int)\n", " samples_y = np.arange(6, brain_slice.shape[0] - 3, step=12).astype(int)\n", " return zip(*product(samples_x, samples_y))\n", - " \n", "\n", - "def plot_brain(brain_slice, brain_cmap=\"RdPu_r\", grid=False, voxel_centers_c=None):\n", + "\n", + "def plot_brain(brain_slice, brain_cmap='RdPu_r', grid=False, voxel_centers_c=None):\n", " fig, ax = plt.subplots()\n", "\n", " # Plot image\n", - " ax.imshow(brain_slice, cmap=brain_cmap, origin=\"lower\");\n", + " ax.imshow(brain_slice, cmap=brain_cmap, origin='lower')\n", "\n", " # Generate focus axes\n", " axins = inset_axes(\n", " ax,\n", - " width=\"200%\",\n", - " height=\"100%\",\n", - " bbox_to_anchor=(1, .6, .5, .4),\n", + " width='200%',\n", + " height='100%',\n", + " bbox_to_anchor=(1, 0.6, 0.5, 0.4),\n", " bbox_transform=ax.transAxes,\n", " loc=2,\n", " )\n", - " axins.set_aspect(\"auto\")\n", + " axins.set_aspect('auto')\n", "\n", " # sub region of the original image\n", " x1, x2 = (np.array((0, 48)) + (z_s.shape[1] - 1) * 0.5).astype(int)\n", " y1, y2 = np.round(np.array((-15, 15)) + (z_s.shape[0] - 1) * 0.70).astype(int)\n", "\n", - " axins.imshow(brain_slice[y1:y2, x1:x2], extent=(x1, x2, y1, y2), cmap=brain_cmap, origin=\"lower\");\n", + " axins.imshow(\n", + " brain_slice[y1:y2, x1:x2], extent=(x1, x2, y1, y2), cmap=brain_cmap, origin='lower'\n", + " )\n", " axins.set_xlim(x1, x2)\n", " axins.set_ylim(y1, y2)\n", " axins.set_xticklabels([])\n", " axins.set_yticklabels([])\n", "\n", - " ax.indicate_inset_zoom(axins, edgecolor=\"black\", alpha=1, linewidth=1.5);\n", + " ax.indicate_inset_zoom(axins, edgecolor='black', alpha=1, linewidth=1.5)\n", "\n", - " \n", " if grid:\n", " params = {}\n", " if voxel_centers_c is not None:\n", - " params[\"norm\"] = mpl.colors.CenteredNorm()\n", - " params[\"c\"] = voxel_centers_c\n", - " params[\"cmap\"] = \"seismic\"\n", + " params['norm'] = mpl.colors.CenteredNorm()\n", + " params['c'] = voxel_centers_c\n", + " params['cmap'] = 'seismic'\n", " elif voxel_centers_c is None:\n", " params['c'] = 'black'\n", "\n", @@ -121,23 +123,24 @@ " e_j = np.arange(0, z_slice.shape[0], step=12).astype(int)\n", "\n", " # Plot grid\n", - " ax.plot([e_i[1:-1]] * len(e_j), e_j, c='k', lw=1, alpha=0.3);\n", - " ax.plot(e_i, [e_j[1:-1]] * len(e_i), c='k', lw=1, alpha=0.3);\n", - " axins.plot([e_i[1:-1]] * len(e_j), e_j, c='k', lw=1, alpha=0.3);\n", - " axins.plot(e_i, [e_j[1:-1]] * len(e_i), c='k', lw=1, alpha=0.3);\n", - " \n", + " ax.plot([e_i[1:-1]] * len(e_j), e_j, c='k', lw=1, alpha=0.3)\n", + " ax.plot(e_i, [e_j[1:-1]] * len(e_i), c='k', lw=1, alpha=0.3)\n", + " axins.plot([e_i[1:-1]] * len(e_j), e_j, c='k', lw=1, alpha=0.3)\n", + " axins.plot(e_i, [e_j[1:-1]] * len(e_i), c='k', lw=1, alpha=0.3)\n", + "\n", " return fig, ax, axins\n", "\n", + "\n", "def axis_cbar(ax):\n", " cbar = inset_axes(\n", " ax,\n", - " width=\"100%\",\n", - " height=\"10%\",\n", + " width='100%',\n", + " height='10%',\n", " bbox_to_anchor=(0, -0.15, 1, 0.5),\n", " bbox_transform=ax.transAxes,\n", " loc='lower center',\n", " )\n", - " cbar.set_aspect(\"auto\")\n", + " cbar.set_aspect('auto')\n", " return cbar" ] }, @@ -158,7 +161,9 @@ "metadata": {}, "outputs": [], "source": [ - "data = np.asanyarray(nb.load(get(\"MNI152NLin2009cAsym\", desc=\"brain\", resolution=1, suffix=\"T1w\")).dataobj);" + "data = np.asanyarray(\n", + " nb.load(get('MNI152NLin2009cAsym', desc='brain', resolution=1, suffix='T1w')).dataobj\n", + ");" ] }, { @@ -168,7 +173,7 @@ "metadata": {}, "outputs": [], "source": [ - "z_slice = np.swapaxes(data[..., 90], 0, 1).astype(\"float32\")\n", + "z_slice = np.swapaxes(data[..., 90], 0, 1).astype('float32')\n", "z_s = z_slice.copy()\n", "z_slice[z_slice == 0] = np.nan" ] @@ -273,7 +278,7 @@ "metadata": {}, "outputs": [], "source": [ - "field = nb.load(\"fieldmap.nii.gz\").get_fdata(dtype=\"float32\")" + "field = nb.load('fieldmap.nii.gz').get_fdata(dtype='float32')" ] }, { @@ -295,8 +300,8 @@ ], "source": [ "fig, ax, axins = plot_brain(z_slice, grid=True)\n", - "ax.imshow(field, cmap=\"seismic\", norm=mpl.colors.CenteredNorm(), origin=\"lower\", alpha=0.4);\n", - "axins.imshow(field, cmap=\"seismic\", norm=mpl.colors.CenteredNorm(), origin=\"lower\", alpha=0.4);" + "ax.imshow(field, cmap='seismic', norm=mpl.colors.CenteredNorm(), origin='lower', alpha=0.4)\n", + "axins.imshow(field, cmap='seismic', norm=mpl.colors.CenteredNorm(), origin='lower', alpha=0.4);" ] }, { @@ -336,15 +341,14 @@ } ], "source": [ - "fig, ax, axins = plot_brain(np.ones_like(z_slice) * np.nan, grid=True, voxel_centers_c=sampled_field)\n", - "im = ax.imshow(field, cmap=\"seismic\", norm=mpl.colors.CenteredNorm(), origin=\"lower\", alpha=0.3);\n", - "axins.imshow(field, cmap=\"seismic\", norm=mpl.colors.CenteredNorm(), origin=\"lower\", alpha=0.3);\n", + "fig, ax, axins = plot_brain(\n", + " np.ones_like(z_slice) * np.nan, grid=True, voxel_centers_c=sampled_field\n", + ")\n", + "im = ax.imshow(field, cmap='seismic', norm=mpl.colors.CenteredNorm(), origin='lower', alpha=0.3)\n", + "axins.imshow(field, cmap='seismic', norm=mpl.colors.CenteredNorm(), origin='lower', alpha=0.3)\n", "\n", "fig.colorbar(\n", - " im,\n", - " cax=axis_cbar(axins),\n", - " orientation='horizontal',\n", - " ticks=[-field.max(), 0, field.max()]\n", + " im, cax=axis_cbar(axins), orientation='horizontal', ticks=[-field.max(), 0, field.max()]\n", ").set_ticklabels(\n", " [r'$\\Delta B_0 < 0$', r'$\\Delta B_0 = 0$', r'$\\Delta B_0 > 0$'],\n", " fontsize=16,\n", @@ -378,14 +382,11 @@ ], "source": [ "fig, ax, axins = plot_brain(z_slice, grid=True, voxel_centers_c=sampled_field)\n", - "im = ax.imshow(field, cmap=\"seismic\", norm=mpl.colors.CenteredNorm(), origin=\"lower\", alpha=0.3);\n", - "axins.imshow(field, cmap=\"seismic\", norm=mpl.colors.CenteredNorm(), origin=\"lower\", alpha=0.3);\n", + "im = ax.imshow(field, cmap='seismic', norm=mpl.colors.CenteredNorm(), origin='lower', alpha=0.3)\n", + "axins.imshow(field, cmap='seismic', norm=mpl.colors.CenteredNorm(), origin='lower', alpha=0.3)\n", "\n", "fig.colorbar(\n", - " im,\n", - " cax=axis_cbar(axins),\n", - " orientation='horizontal',\n", - " ticks=[-field.max(), 0, field.max()]\n", + " im, cax=axis_cbar(axins), orientation='horizontal', ticks=[-field.max(), 0, field.max()]\n", ").set_ticklabels(\n", " [r'$\\Delta B_0 < 0$', r'$\\Delta B_0 = 0$', r'$\\Delta B_0 > 0$'],\n", " fontsize=16,\n", @@ -415,7 +416,7 @@ "metadata": {}, "outputs": [], "source": [ - "trt = - 0.15 # in seconds" + "trt = -0.15 # in seconds" ] }, { @@ -465,7 +466,7 @@ " scale_units='xy',\n", " scale=1,\n", ")\n", - "im = ax.imshow(field, cmap=\"seismic\", norm=mpl.colors.CenteredNorm(), origin=\"lower\", alpha=0.3);\n", + "im = ax.imshow(field, cmap='seismic', norm=mpl.colors.CenteredNorm(), origin='lower', alpha=0.3)\n", "\n", "# Inset plotting\n", "q = axins.quiver(\n", @@ -480,14 +481,14 @@ " width=0.008,\n", " scale=1,\n", ")\n", - "axins.imshow(field, cmap=\"seismic\", norm=mpl.colors.CenteredNorm(), origin=\"lower\", alpha=0.3);\n", + "axins.imshow(field, cmap='seismic', norm=mpl.colors.CenteredNorm(), origin='lower', alpha=0.3)\n", "\n", "fig.colorbar(\n", " im,\n", " cax=axis_cbar(axins),\n", " orientation='horizontal',\n", " ticks=[-field.max(), 0, field.max()],\n", - ").set_ticklabels([r'$s_{min}$', '0.0' , r'$s_{max}$'], fontsize=16)" + ").set_ticklabels([r'$s_{min}$', '0.0', r'$s_{max}$'], fontsize=16)" ] }, { @@ -517,26 +518,44 @@ "\n", "# Plot main view\n", "ax.scatter(np.array(x_coord)[is_in_fov], np.array(y_coord)[is_in_fov], s=10, c='k')\n", - "ax.scatter(np.array(x_coord)[is_in_fov], y_coord_moved[is_in_fov], c=c, s=10, cmap=\"seismic\", vmin=-1, vmax=1)\n", - "ax.plot([e_i[1:-1]] * len(e_j), e_j, c='k', lw=1, alpha=0.3);\n", - "im = ax.imshow(field, cmap=\"seismic\", norm=mpl.colors.CenteredNorm(), origin=\"lower\", alpha=0.3);\n", + "ax.scatter(\n", + " np.array(x_coord)[is_in_fov],\n", + " y_coord_moved[is_in_fov],\n", + " c=c,\n", + " s=10,\n", + " cmap='seismic',\n", + " vmin=-1,\n", + " vmax=1,\n", + ")\n", + "ax.plot([e_i[1:-1]] * len(e_j), e_j, c='k', lw=1, alpha=0.3)\n", + "im = ax.imshow(field, cmap='seismic', norm=mpl.colors.CenteredNorm(), origin='lower', alpha=0.3)\n", "\n", "# Plot inset\n", - "axins.scatter(np.array(x_coord)[is_in_fov], np.array(y_coord)[is_in_fov], s=80, c=None, fc='none', ec='k')\n", - "axins.scatter(np.array(x_coord)[is_in_fov], y_coord_moved[is_in_fov], s=80, c=c, cmap='seismic', vmin=-1, vmax=1)\n", + "axins.scatter(\n", + " np.array(x_coord)[is_in_fov], np.array(y_coord)[is_in_fov], s=80, c=None, fc='none', ec='k'\n", + ")\n", + "axins.scatter(\n", + " np.array(x_coord)[is_in_fov],\n", + " y_coord_moved[is_in_fov],\n", + " s=80,\n", + " c=c,\n", + " cmap='seismic',\n", + " vmin=-1,\n", + " vmax=1,\n", + ")\n", "axins.quiver(\n", " np.array(x_coord)[is_in_fov],\n", " y_coord_moved[is_in_fov],\n", " u,\n", " -v,\n", - " color=\"black\",\n", + " color='black',\n", " angles='xy',\n", " scale_units='xy',\n", " width=0.008,\n", " scale=1,\n", ")\n", - "axins.plot([e_i[1:-1]] * len(e_j), e_j, c='k', lw=1, alpha=0.3);\n", - "axins.imshow(field, cmap=\"seismic\", norm=mpl.colors.CenteredNorm(), origin=\"lower\", alpha=0.3);\n", + "axins.plot([e_i[1:-1]] * len(e_j), e_j, c='k', lw=1, alpha=0.3)\n", + "axins.imshow(field, cmap='seismic', norm=mpl.colors.CenteredNorm(), origin='lower', alpha=0.3)\n", "\n", "# Calculate warping\n", "j_axis = np.arange(z_slice.shape[1], dtype=int)\n", @@ -544,8 +563,8 @@ " warped_edges = i + trt * field[i, :]\n", " warped_edges[warped_edges <= 0] = np.nan\n", " warped_edges[warped_edges >= field.shape[0] - 1] = np.nan\n", - " ax.plot(j_axis, warped_edges, c='k', lw=1, alpha=0.5);\n", - " axins.plot(j_axis, warped_edges, c='k', lw=1, alpha=0.5);\n", + " ax.plot(j_axis, warped_edges, c='k', lw=1, alpha=0.5)\n", + " axins.plot(j_axis, warped_edges, c='k', lw=1, alpha=0.5)\n", "\n", "fig.colorbar(\n", " im,\n", @@ -573,7 +592,7 @@ "outputs": [], "source": [ "all_indexes = tuple([np.arange(s) for s in z_slice.shape])\n", - "all_ndindex = np.array(np.meshgrid(*all_indexes, indexing=\"ij\")).reshape(2, -1)" + "all_ndindex = np.array(np.meshgrid(*all_indexes, indexing='ij')).reshape(2, -1)" ] }, { @@ -583,7 +602,7 @@ "metadata": {}, "outputs": [], "source": [ - "all_ndindex_warped = all_ndindex.astype(\"float32\")\n", + "all_ndindex_warped = all_ndindex.astype('float32')\n", "all_ndindex_warped[0, :] += trt * field.reshape(-1)" ] }, @@ -658,9 +677,9 @@ } ], "source": [ - "fig, ax, axins = plot_brain(warped_brain, grid=False, brain_cmap='gray');\n", - "im = ax.imshow(z_slice, cmap=\"RdPu_r\", origin=\"lower\", alpha=0.5);\n", - "axins.imshow(z_slice, cmap=\"RdPu_r\", origin=\"lower\", alpha=0.5);" + "fig, ax, axins = plot_brain(warped_brain, grid=False, brain_cmap='gray')\n", + "im = ax.imshow(z_slice, cmap='RdPu_r', origin='lower', alpha=0.5)\n", + "axins.imshow(z_slice, cmap='RdPu_r', origin='lower', alpha=0.5);" ] }, { @@ -691,7 +710,7 @@ } ], "source": [ - "all_ndindex_warped = all_ndindex.astype(\"float32\")\n", + "all_ndindex_warped = all_ndindex.astype('float32')\n", "all_ndindex_warped[0, :] -= trt * field.reshape(-1)\n", "\n", "warped_brain = ndi.map_coordinates(\n", @@ -699,9 +718,9 @@ " all_ndindex_warped,\n", ").reshape(z_slice.shape)\n", "\n", - "fig, ax, axins = plot_brain(warped_brain, grid=False, brain_cmap='gray');\n", - "im = ax.imshow(z_slice, cmap=\"RdPu_r\", origin=\"lower\", alpha=0.5);\n", - "axins.imshow(z_slice, cmap=\"RdPu_r\", origin=\"lower\", alpha=0.5);" + "fig, ax, axins = plot_brain(warped_brain, grid=False, brain_cmap='gray')\n", + "im = ax.imshow(z_slice, cmap='RdPu_r', origin='lower', alpha=0.5)\n", + "axins.imshow(z_slice, cmap='RdPu_r', origin='lower', alpha=0.5);" ] } ], diff --git a/docs/tools/apigen.py b/docs/tools/apigen.py index 716fd488f0..f602f45fa8 100644 --- a/docs/tools/apigen.py +++ b/docs/tools/apigen.py @@ -21,7 +21,6 @@ import os import re from inspect import getmodule - from types import BuiltinFunctionType, FunctionType # suppress print statements (warnings for empty files) @@ -29,21 +28,21 @@ class ApiDocWriter(object): - """ Class for automatic detection and parsing of API docs + """Class for automatic detection and parsing of API docs to Sphinx-parsable reST format""" # only separating first two levels - rst_section_levels = ["*", "=", "-", "~", "^"] + rst_section_levels = ['*', '=', '-', '~', '^'] def __init__( self, package_name, - rst_extension=".txt", + rst_extension='.txt', package_skip_patterns=None, module_skip_patterns=None, other_defines=True, ): - """ Initialize package for parsing + r"""Initialize package for parsing Parameters ---------- @@ -73,9 +72,9 @@ def __init__( particular module but not defined there. """ if package_skip_patterns is None: - package_skip_patterns = ["\\.tests$"] + package_skip_patterns = ['\\.tests$'] if module_skip_patterns is None: - module_skip_patterns = ["\\.setup$", "\\._"] + module_skip_patterns = ['\\.setup$', '\\._'] self.package_name = package_name self.rst_extension = rst_extension self.package_skip_patterns = package_skip_patterns @@ -86,7 +85,7 @@ def get_package_name(self): return self._package_name def set_package_name(self, package_name): - """ Set package_name + """Set package_name >>> docwriter = ApiDocWriter('sphinx') >>> import sphinx @@ -103,20 +102,18 @@ def set_package_name(self, package_name): self.root_path = root_module.__path__[-1] self.written_modules = None - package_name = property( - get_package_name, set_package_name, None, "get/set package_name" - ) + package_name = property(get_package_name, set_package_name, None, 'get/set package_name') def _import(self, name): - """ Import namespace package """ + """Import namespace package""" mod = __import__(name) - components = name.split(".") + components = name.split('.') for comp in components[1:]: mod = getattr(mod, comp) return mod def _get_object_name(self, line): - """ Get second token in line + """Get second token in line >>> docwriter = ApiDocWriter('sphinx') >>> docwriter._get_object_name(" def func(): ") 'func' @@ -125,13 +122,13 @@ def _get_object_name(self, line): >>> docwriter._get_object_name(" class Klass: ") 'Klass' """ - name = line.split()[1].split("(")[0].strip() + name = line.split()[1].split('(')[0].strip() # in case we have classes which are not derived from object # ie. old style classes - return name.rstrip(":") + return name.rstrip(':') def _uri2path(self, uri): - """ Convert uri to absolute filepath + """Convert uri to absolute filepath Parameters ---------- @@ -159,36 +156,36 @@ def _uri2path(self, uri): """ if uri == self.package_name: - return os.path.join(self.root_path, "__init__.py") - path = uri.replace(self.package_name + ".", "") - path = path.replace(".", os.path.sep) + return os.path.join(self.root_path, '__init__.py') + path = uri.replace(self.package_name + '.', '') + path = path.replace('.', os.path.sep) path = os.path.join(self.root_path, path) # XXX maybe check for extensions as well? - if os.path.exists(path + ".py"): # file - path += ".py" - elif os.path.exists(os.path.join(path, "__init__.py")): - path = os.path.join(path, "__init__.py") + if os.path.exists(path + '.py'): # file + path += '.py' + elif os.path.exists(os.path.join(path, '__init__.py')): + path = os.path.join(path, '__init__.py') else: return None return path def _path2uri(self, dirpath): - """ Convert directory path to uri """ - package_dir = self.package_name.replace(".", os.path.sep) + """Convert directory path to uri""" + package_dir = self.package_name.replace('.', os.path.sep) relpath = dirpath.replace(self.root_path, package_dir) if relpath.startswith(os.path.sep): relpath = relpath[1:] - return relpath.replace(os.path.sep, ".") + return relpath.replace(os.path.sep, '.') def _parse_module(self, uri): - """ Parse module defined in *uri* """ + """Parse module defined in *uri*""" filename = self._uri2path(uri) if filename is None: - print(filename, "erk") + print(filename, 'erk') # nothing that we could handle here. return ([], []) - f = open(filename, "rt") + f = open(filename, 'rt') functions, classes = self._parse_lines(f) f.close() return functions, classes @@ -211,7 +208,7 @@ def _parse_module_with_import(self, uri): """ mod = __import__(uri, fromlist=[uri]) # find all public objects in the module. - obj_strs = [obj for obj in dir(mod) if not obj.startswith("_")] + obj_strs = [obj for obj in dir(mod) if not obj.startswith('_')] functions = [] classes = [] for obj_str in obj_strs: @@ -224,7 +221,7 @@ def _parse_module_with_import(self, uri): continue # figure out if obj is a function or class if ( - hasattr(obj, "func_name") + hasattr(obj, 'func_name') or isinstance(obj, BuiltinFunctionType) or isinstance(obj, FunctionType) ): @@ -239,19 +236,19 @@ def _parse_module_with_import(self, uri): return functions, classes def _parse_lines(self, linesource): - """ Parse lines of text for functions and classes """ + """Parse lines of text for functions and classes""" functions = [] classes = [] for line in linesource: - if line.startswith("def ") and line.count("("): + if line.startswith('def ') and line.count('('): # exclude private stuff name = self._get_object_name(line) - if not name.startswith("_"): + if not name.startswith('_'): functions.append(name) - elif line.startswith("class "): + elif line.startswith('class '): # exclude private stuff name = self._get_object_name(line) - if not name.startswith("_"): + if not name.startswith('_'): classes.append(name) else: pass @@ -277,59 +274,53 @@ def generate_api_doc(self, uri): # get the names of all classes and functions functions, classes = self._parse_module_with_import(uri) if not len(functions) and not len(classes) and DEBUG: - print("WARNING: Empty -", uri) # dbg + print('WARNING: Empty -', uri) # dbg # Make a shorter version of the uri that omits the package name for # titles - uri_short = re.sub(r"^%s\." % self.package_name, "", uri) + uri_short = re.sub(r'^%s\.' % self.package_name, '', uri) - head = ".. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n" - body = "" + head = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n' + body = '' # Set the chapter title to read 'module' for all modules except for the # main packages - if "." in uri_short: - title = "Module: :mod:`" + uri_short + "`" - head += title + "\n" + self.rst_section_levels[2] * len(title) + if '.' in uri_short: + title = 'Module: :mod:`' + uri_short + '`' + head += title + '\n' + self.rst_section_levels[2] * len(title) else: - title = ":mod:`" + uri_short + "`" - head += title + "\n" + self.rst_section_levels[1] * len(title) + title = ':mod:`' + uri_short + '`' + head += title + '\n' + self.rst_section_levels[1] * len(title) - head += "\n.. automodule:: " + uri + "\n" - head += "\n.. currentmodule:: " + uri + "\n" - body += "\n.. currentmodule:: " + uri + "\n\n" + head += '\n.. automodule:: ' + uri + '\n' + head += '\n.. currentmodule:: ' + uri + '\n' + body += '\n.. currentmodule:: ' + uri + '\n\n' for c in classes: - body += ( - "\n:class:`" - + c - + "`\n" - + self.rst_section_levels[3] * (len(c) + 9) - + "\n\n" - ) - body += "\n.. autoclass:: " + c + "\n" + body += '\n:class:`' + c + '`\n' + self.rst_section_levels[3] * (len(c) + 9) + '\n\n' + body += '\n.. autoclass:: ' + c + '\n' # must NOT exclude from index to keep cross-refs working body += ( - " :members:\n" - " :undoc-members:\n" - " :show-inheritance:\n" - "\n" - " .. automethod:: __init__\n\n" + ' :members:\n' + ' :undoc-members:\n' + ' :show-inheritance:\n' + '\n' + ' .. automethod:: __init__\n\n' ) - head += ".. autosummary::\n\n" + head += '.. autosummary::\n\n' for f in classes + functions: - head += " " + f + "\n" - head += "\n" + head += ' ' + f + '\n' + head += '\n' for f in functions: # must NOT exclude from index to keep cross-refs working - body += f + "\n" - body += self.rst_section_levels[3] * len(f) + "\n" - body += "\n.. autofunction:: " + f + "\n\n" + body += f + '\n' + body += self.rst_section_levels[3] * len(f) + '\n' + body += '\n.. autofunction:: ' + f + '\n\n' return head, body def _survives_exclude(self, matchstr, match_type): - """ Returns True if *matchstr* does not match patterns + """Returns True if *matchstr* does not match patterns ``self.package_name`` removed from front of string if present @@ -349,9 +340,9 @@ def _survives_exclude(self, matchstr, match_type): >>> dw._survives_exclude('sphinx.badmod', 'module') False """ - if match_type == "module": + if match_type == 'module': patterns = self.module_skip_patterns - elif match_type == "package": + elif match_type == 'package': patterns = self.package_skip_patterns else: raise ValueError('Cannot interpret match type "%s"' % match_type) @@ -370,7 +361,7 @@ def _survives_exclude(self, matchstr, match_type): return True def discover_modules(self): - """ Return module sequence discovered from ``self.package_name`` + r"""Return module sequence discovered from ``self.package_name`` Parameters @@ -403,27 +394,23 @@ def discover_modules(self): # dipy does not import a whole bunch of modules we'll # include those here as well (the *.py filenames). filenames = [ - f[:-3] - for f in filenames - if f.endswith(".py") and not f.startswith("__init__") + f[:-3] for f in filenames if f.endswith('.py') and not f.startswith('__init__') ] for filename in filenames: - package_uri = "/".join((dirpath, filename)) + package_uri = '/'.join((dirpath, filename)) for subpkg_name in dirnames + filenames: - package_uri = ".".join((root_uri, subpkg_name)) + package_uri = '.'.join((root_uri, subpkg_name)) package_path = self._uri2path(package_uri) - if package_path and self._survives_exclude(package_uri, "package"): + if package_path and self._survives_exclude(package_uri, 'package'): modules.append(package_uri) return sorted(modules) def write_modules_api(self, modules, outdir): # upper-level modules - main_module = modules[0].split(".")[0] ulms = [ - ".".join(m.split(".")[:2]) if m.count(".") >= 1 else m.split(".")[0] - for m in modules + '.'.join(m.split('.')[:2]) if m.count('.') >= 1 else m.split('.')[0] for m in modules ] from collections import OrderedDict @@ -439,12 +426,12 @@ def write_modules_api(self, modules, outdir): written_modules = [] for ulm, mods in module_by_ulm.items(): - print("Generating docs for %s:" % ulm) + print('Generating docs for %s:' % ulm) document_head = [] document_body = [] for m in mods: - print(" -> " + m) + print(' -> ' + m) head, body = self.generate_api_doc(m) document_head.append(head) @@ -452,7 +439,7 @@ def write_modules_api(self, modules, outdir): out_module = ulm + self.rst_extension outfile = os.path.join(outdir, out_module) - fileobj = open(outfile, "wt") + fileobj = open(outfile, 'wt') fileobj.writelines(document_head + document_body) fileobj.close() @@ -483,7 +470,7 @@ def write_api_docs(self, outdir): modules = self.discover_modules() self.write_modules_api(modules, outdir) - def write_index(self, outdir, froot="gen", relative_to=None): + def write_index(self, outdir, froot='gen', relative_to=None): """Make a reST API index file from written files Parameters @@ -502,22 +489,22 @@ def write_index(self, outdir, froot="gen", relative_to=None): leave path as it is. """ if self.written_modules is None: - raise ValueError("No modules written") + raise ValueError('No modules written') # Get full filename path path = os.path.join(outdir, froot + self.rst_extension) # Path written into index is relative to rootpath if relative_to is not None: - relpath = (outdir + os.path.sep).replace(relative_to + os.path.sep, "") + relpath = (outdir + os.path.sep).replace(relative_to + os.path.sep, '') else: relpath = outdir - idx = open(path, "wt") + idx = open(path, 'wt') w = idx.write - w(".. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n") + w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n') - title = "API Reference" - w(title + "\n") - w("=" * len(title) + "\n\n") - w(".. toctree::\n\n") + title = 'API Reference' + w(title + '\n') + w('=' * len(title) + '\n\n') + w('.. toctree::\n\n') for f in self.written_modules: - w(" %s\n" % os.path.join(relpath, f)) + w(' %s\n' % os.path.join(relpath, f)) idx.close() diff --git a/docs/tools/buildmodref.py b/docs/tools/buildmodref.py index 769c696bf1..e0a7c41059 100755 --- a/docs/tools/buildmodref.py +++ b/docs/tools/buildmodref.py @@ -1,23 +1,22 @@ #!/usr/bin/env python -"""Script to auto-generate API docs. -""" -from __future__ import print_function, division +"""Script to auto-generate API docs.""" + +from __future__ import division, print_function # stdlib imports import sys -import re - -# local imports -from apigen import ApiDocWriter # version comparison from distutils.version import LooseVersion as V +# local imports +from apigen import ApiDocWriter + # ***************************************************************************** def abort(error): - print("*WARNING* API documentation not generated: %s" % error) + print('*WARNING* API documentation not generated: %s' % error) exit() @@ -28,7 +27,7 @@ def writeapi(package, outdir, source_version, other_defines=True): try: __import__(package) except ImportError: - abort("Can not import " + package) + abort('Can not import ' + package) module = sys.modules[package] @@ -39,21 +38,21 @@ def writeapi(package, outdir, source_version, other_defines=True): installed_version = V(module.__version__) if source_version != installed_version: - abort("Installed version does not match source version") + abort('Installed version does not match source version') - docwriter = ApiDocWriter(package, rst_extension=".rst", other_defines=other_defines) + docwriter = ApiDocWriter(package, rst_extension='.rst', other_defines=other_defines) docwriter.package_skip_patterns += [ - r"\.%s$" % package, - r".*test.*$", - r"\.version.*$", + r'\.%s$' % package, + r'.*test.*$', + r'\.version.*$', ] docwriter.write_api_docs(outdir) - docwriter.write_index(outdir, "index", relative_to=outdir) - print("%d files written" % len(docwriter.written_modules)) + docwriter.write_index(outdir, 'index', relative_to=outdir) + print('%d files written' % len(docwriter.written_modules)) -if __name__ == "__main__": +if __name__ == '__main__': package = sys.argv[1] outdir = sys.argv[2] try: @@ -61,6 +60,6 @@ def writeapi(package, outdir, source_version, other_defines=True): except IndexError: other_defines = True else: - other_defines = other_defines in ("True", "true", "1") + other_defines = other_defines in ('True', 'true', '1') writeapi(package, outdir, other_defines=other_defines) diff --git a/pyproject.toml b/pyproject.toml index 3fab6b2b79..040fab4974 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,10 +65,8 @@ mem = [ ] dev = [ - "black", "pre-commit", - "isort", - "flake8-pyproject", + "ruff", ] test = [ @@ -117,39 +115,9 @@ version-file = "sdcflows/_version.py" # Developer tool configurations # +# Disable black, use ruff below [tool.black] -line-length = 99 -target-version = ['py39'] -skip-string-normalization = true -exclude = ''' -# Directories -/( - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | venv - | _build - | build - | dist -)/ -''' - -[tool.isort] -profile = 'black' -skip_gitignore = true - -[tool.flake8] -max-line-length = "99" -doctests = "False" -exclude = "*build/" -ignore = ["W503", "E203"] -per-file-ignores = [ - "**/__init__.py : F401", - "docs/conf.py : E265", -] +exclude = "*" [tool.pytest.ini_options] minversion = "6" @@ -206,3 +174,30 @@ skip = """ ignore = [ "W002", # Test data contains duplicates ] + +[tool.ruff] +line-length = 99 + +[tool.ruff.lint] +select = [ + "F", + "E", + "W", + "I", +] +ignore = [ + "E203", + "B019", + "SIM108", + "C901", +] + +[tool.ruff.lint.flake8-quotes] +inline-quotes = "single" + +[tool.ruff.lint.extend-per-file-ignores] +"*/__init__.py" = ["F401"] +"docs/conf.py" = ["E265"] + +[tool.ruff.format] +quote-style = "single" diff --git a/sdcflows/__init__.py b/sdcflows/__init__.py index e0cb64387f..07c3833a49 100644 --- a/sdcflows/__init__.py +++ b/sdcflows/__init__.py @@ -1,13 +1,15 @@ """SDCflows - :abbr:`SDC (susceptibility distortion correction)` by DUMMIES, for dummies.""" -__packagename__ = "sdcflows" -__copyright__ = "2022, The NiPreps developers" + +__packagename__ = 'sdcflows' +__copyright__ = '2022, The NiPreps developers' try: from ._version import __version__ except ModuleNotFoundError: - from importlib.metadata import version, PackageNotFoundError + from importlib.metadata import PackageNotFoundError, version + try: __version__ = version(__packagename__) except PackageNotFoundError: - __version__ = "0+unknown" + __version__ = '0+unknown' del version del PackageNotFoundError diff --git a/sdcflows/_warnings.py b/sdcflows/_warnings.py index e9708230ac..15d0d1fe4f 100644 --- a/sdcflows/_warnings.py +++ b/sdcflows/_warnings.py @@ -25,10 +25,11 @@ # The original file this work derives from is found at: # https://github.com/nipreps/mriqc/blob/8ceadba8669cc2a86119a97b9311ab968f11c6eb/mriqc/_warnings.py """Manipulate Python warnings.""" + import logging import warnings -_wlog = logging.getLogger("py.warnings") +_wlog = logging.getLogger('py.warnings') _wlog.addHandler(logging.NullHandler()) @@ -36,9 +37,9 @@ def _warn(message, category=None, stacklevel=1, source=None): """Redefine the warning function.""" if category is not None: category = type(category).__name__ - category = category.replace("type", "WARNING") + category = category.replace('type', 'WARNING') - logging.getLogger("py.warnings").warning(f"{category or 'WARNING'}: {message}") + logging.getLogger('py.warnings').warning(f'{category or "WARNING"}: {message}') def _showwarning(message, category, filename, lineno, file=None, line=None): diff --git a/sdcflows/cli/main.py b/sdcflows/cli/main.py index 0ba9b4e076..d96751dd2c 100644 --- a/sdcflows/cli/main.py +++ b/sdcflows/cli/main.py @@ -25,11 +25,12 @@ def main(argv=None): """Entry point for SDCFlows' CLI.""" + import atexit import gc import os import sys from tempfile import mktemp - import atexit + from sdcflows import config from sdcflows.cli.parser import parse_args @@ -42,25 +43,24 @@ def main(argv=None): from niworkflows.utils.debug import setup_exceptionhook setup_exceptionhook() - config.nipype.plugin = "Linear" + config.nipype.plugin = 'Linear' # CRITICAL Save the config to a file. This is necessary because the execution graph # is built as a separate process to keep the memory footprint low. The most # straightforward way to communicate with the child process is via the filesystem. # The config file name needs to be unique, otherwise multiple sdcflows instances # will create write conflicts. - config_file = mktemp( - dir=config.execution.work_dir, prefix=".sdcflows.", suffix=".toml" - ) + config_file = mktemp(dir=config.execution.work_dir, prefix='.sdcflows.', suffix='.toml') config.to_filename(config_file) config.file_path = config_file exitcode = 0 - if config.workflow.analysis_level != ["participant"]: + if config.workflow.analysis_level != ['participant']: raise ValueError("Analysis level can only be 'participant'") if config.execution.dry_run: # --dry-run: pretty print results from niworkflows.utils.bids import collect_participants + from sdcflows.utils.wrangler import find_estimators subjects = collect_participants( @@ -76,31 +76,31 @@ def main(argv=None): logger=config.loggers.cli, ) - print(f"Estimation for <{config.execution.bids_dir}> complete. Found:") + print(f'Estimation for <{config.execution.bids_dir}> complete. Found:') for subject, estimators in estimators_record.items(): - print(f"\tsub-{subject}") + print(f'\tsub-{subject}') if not estimators: - print("\t\tNo estimators found") + print('\t\tNo estimators found') continue for estimator in estimators: - print(f"\t\t{estimator}") + print(f'\t\t{estimator}') for fl in estimator.sources: - fl_relpath = fl.path.relative_to(config.execution.bids_dir / f"sub-{subject}") - pe_dir = fl.metadata.get("PhaseEncodingDirection") - print(f"\t\t\t{pe_dir}\t{fl_relpath}") + fl_relpath = fl.path.relative_to(config.execution.bids_dir / f'sub-{subject}') + pe_dir = fl.metadata.get('PhaseEncodingDirection') + print(f'\t\t\t{pe_dir}\t{fl_relpath}') sys.exit(exitcode) # Initialize process pool if multiprocessing _pool = None - if config.nipype.plugin in ("MultiProc", "LegacyMultiProc"): - from contextlib import suppress + if config.nipype.plugin in ('MultiProc', 'LegacyMultiProc'): import multiprocessing as mp from concurrent.futures import ProcessPoolExecutor + from contextlib import suppress - os.environ["OMP_NUM_THREADS"] = "1" + os.environ['OMP_NUM_THREADS'] = '1' with suppress(RuntimeError): - mp.set_start_method("fork") + mp.set_start_method('fork') gc.collect() _pool = ProcessPoolExecutor( @@ -127,8 +127,8 @@ def main(argv=None): p.start() p.join() - sdcflows_wf = retval.get("workflow", None) - exitcode = p.exitcode or retval.get("return_code", 0) + sdcflows_wf = retval.get('workflow', None) + exitcode = p.exitcode or retval.get('return_code', 0) # CRITICAL Load the config from the file. This is necessary because the ``build_workflow`` # function executed constrained in a process may change the config (and thus the global @@ -151,12 +151,12 @@ def main(argv=None): config.loggers.init() # Resource management options - if config.nipype.plugin in ("MultiProc", "LegacyMultiProc") and ( + if config.nipype.plugin in ('MultiProc', 'LegacyMultiProc') and ( 1 < config.nipype.nprocs < config.nipype.omp_nthreads ): config.loggers.cli.warning( - "Per-process threads (--omp-nthreads=%d) exceed total " - "threads (--nthreads/--n_cpus=%d)", + 'Per-process threads (--omp-nthreads=%d) exceed total ' + 'threads (--nthreads/--n_cpus=%d)', config.nipype.omp_nthreads, config.nipype.nprocs, ) @@ -165,7 +165,7 @@ def main(argv=None): sys.exit(os.EX_SOFTWARE) if sdcflows_wf and config.execution.write_graph: - sdcflows_wf.write_graph(graph2use="colored", format="svg", simple_form=True) + sdcflows_wf.write_graph(graph2use='colored', format='svg', simple_form=True) # Clean up master process before running workflow, which may create forks gc.collect() @@ -175,13 +175,11 @@ def main(argv=None): from niworkflows.engine.plugin import MultiProcPlugin _plugin = { - "plugin": MultiProcPlugin( - pool=_pool, plugin_args=config.nipype.plugin_args - ), + 'plugin': MultiProcPlugin(pool=_pool, plugin_args=config.nipype.plugin_args), } sdcflows_wf.run(**_plugin) - config.loggers.cli.log(25, "Finished all workload") + config.loggers.cli.log(25, 'Finished all workload') -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/sdcflows/cli/parser.py b/sdcflows/cli/parser.py index 63570fd717..184f375717 100644 --- a/sdcflows/cli/parser.py +++ b/sdcflows/cli/parser.py @@ -21,6 +21,7 @@ # https://www.nipreps.org/community/licensing/ # """Standalone command line executable for estimation of fieldmaps.""" + import re from argparse import Action, ArgumentDefaultsHelpFormatter, ArgumentParser from functools import partial @@ -46,22 +47,19 @@ def _parse_participant_labels(value): """ return sorted( - set( - re.sub(r"^sub-", "", item.strip()) - for item in re.split(r"\s+", f"{value}".strip()) - ) + set(re.sub(r'^sub-', '', item.strip()) for item in re.split(r'\s+', f'{value}'.strip())) ) def _parser(): class ParticipantLabelAction(Action): def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, _parse_participant_labels(" ".join(values))) + setattr(namespace, self.dest, _parse_participant_labels(' '.join(values))) def _path_exists(path, parser): """Ensure a given path exists.""" if path is None or not Path(path).exists(): - raise parser.error(f"Path does not exist: <{path}>.") + raise parser.error(f'Path does not exist: <{path}>.') return Path(path).expanduser().absolute() def _min_one(value, parser): @@ -72,10 +70,10 @@ def _min_one(value, parser): return value def _to_gb(value): - scale = {"G": 1, "T": 10**3, "M": 1e-3, "K": 1e-6, "B": 1e-9} - digits = "".join([c for c in value if c.isdigit()]) + scale = {'G': 1, 'T': 10**3, 'M': 1e-3, 'K': 1e-6, 'B': 1e-9} + digits = ''.join([c for c in value if c.isdigit()]) n_digits = len(digits) - units = value[n_digits:] or "G" + units = value[n_digits:] or 'G' return int(digits) * scale[units[0]] def _bids_filter(value): @@ -95,88 +93,88 @@ def _bids_filter(value): PositiveInt = partial(_min_one, parser=parser) parser.add_argument( - "bids_dir", - action="store", + 'bids_dir', + action='store', type=PathExists, - help="The root folder of a BIDS valid dataset (sub-XXXXX folders should " - "be found at the top level in this folder).", + help='The root folder of a BIDS valid dataset (sub-XXXXX folders should ' + 'be found at the top level in this folder).', ) parser.add_argument( - "output_dir", - action="store", + 'output_dir', + action='store', type=Path, - help="The directory where the output files " - "should be stored. If you are running group level analysis " - "this folder should be prepopulated with the results of the " - "participant level analysis.", + help='The directory where the output files ' + 'should be stored. If you are running group level analysis ' + 'this folder should be prepopulated with the results of the ' + 'participant level analysis.', ) parser.add_argument( - "analysis_level", - action="store", - nargs="+", - help="Level of the analysis that will be performed. " - "Multiple participant level analyses can be run independently " - "(in parallel) using the same output_dir.", - choices=["participant", "group"], + 'analysis_level', + action='store', + nargs='+', + help='Level of the analysis that will be performed. ' + 'Multiple participant level analyses can be run independently ' + '(in parallel) using the same output_dir.', + choices=['participant', 'group'], ) # optional arguments parser.add_argument( - "--version", action="version", version=f"SDCFlows {config.environment.version}" + '--version', action='version', version=f'SDCFlows {config.environment.version}' ) parser.add_argument( - "-v", - "--verbose", - dest="verbose_count", - action="count", + '-v', + '--verbose', + dest='verbose_count', + action='count', default=0, - help="Increases log verbosity for each occurrence, debug level is -vvv.", + help='Increases log verbosity for each occurrence, debug level is -vvv.', ) # main options - g_bids = parser.add_argument_group("Options related to BIDS") + g_bids = parser.add_argument_group('Options related to BIDS') g_bids.add_argument( - "--participant-label", + '--participant-label', action=ParticipantLabelAction, - nargs="+", - help="A space delimited list of participant identifiers or a single " - "identifier (the sub- prefix can be removed).", + nargs='+', + help='A space delimited list of participant identifiers or a single ' + 'identifier (the sub- prefix can be removed).', ) g_bids.add_argument( - "--session-label", - action="store", - nargs="*", + '--session-label', + action='store', + nargs='*', type=str, - help="Filter input dataset by session label.", + help='Filter input dataset by session label.', ) g_bids.add_argument( - "--bids-filter-file", - action="store", + '--bids-filter-file', + action='store', type=Path, - metavar="PATH", - help="a JSON file describing custom BIDS input filter using pybids " - "{:{:,...},...} " - "(https://github.com/bids-standard/pybids/blob/master/bids/layout/config/bids.json)", + metavar='PATH', + help='a JSON file describing custom BIDS input filter using pybids ' + '{:{:,...},...} ' + '(https://github.com/bids-standard/pybids/blob/master/bids/layout/config/bids.json)', ) g_bids.add_argument( - "--bids-database-dir", - metavar="PATH", + '--bids-database-dir', + metavar='PATH', type=PathExists, - help="Path to an existing PyBIDS database folder, for faster indexing " - "(especially useful for large datasets).", + help='Path to an existing PyBIDS database folder, for faster indexing ' + '(especially useful for large datasets).', ) g_bids.add_argument( - "--bids-database-wipe", - action="store_true", + '--bids-database-wipe', + action='store_true', default=False, - help="Wipe out previously existing BIDS indexing caches, forcing re-indexing.", + help='Wipe out previously existing BIDS indexing caches, forcing re-indexing.', ) # General performance - g_perfm = parser.add_argument_group("Options to handle performance") + g_perfm = parser.add_argument_group('Options to handle performance') g_perfm.add_argument( - "--nprocs", - action="store", + '--nprocs', + action='store', type=PositiveInt, help="""\ Maximum number of simultaneously running parallel processes executed by *SDCFlows* \ @@ -192,8 +190,8 @@ def _bids_filter(value): not be what you want in, e.g., shared systems like a HPC cluster.""", ) g_perfm.add_argument( - "--omp-nthreads", - action="store", + '--omp-nthreads', + action='store', type=PositiveInt, help="""\ Maximum number of threads that multi-threaded processes executed by *SDCFlows* \ @@ -202,69 +200,69 @@ def _bids_filter(value): not be what you want in, e.g., shared systems like a HPC cluster.""", ) g_perfm.add_argument( - "--mem-gb", - dest="memory_gb", - action="store", + '--mem-gb', + dest='memory_gb', + action='store', type=_to_gb, - help="Upper bound memory limit for SDCFlows processes.", + help='Upper bound memory limit for SDCFlows processes.', ) g_perfm.add_argument( - "--debug", - action="store_true", + '--debug', + action='store_true', default=False, - help="Enable changes to processing to aid in debugging", + help='Enable changes to processing to aid in debugging', ) g_perfm.add_argument( - "--pdb", - dest="pdb", - action="store_true", + '--pdb', + dest='pdb', + action='store_true', default=False, - help="Open Python debugger (pdb) on exceptions.", + help='Open Python debugger (pdb) on exceptions.', ) g_perfm.add_argument( - "--sloppy", - action="store_true", + '--sloppy', + action='store_true', default=False, - help="Use sloppy mode for minimal footprint.", + help='Use sloppy mode for minimal footprint.', ) # Control instruments - g_outputs = parser.add_argument_group("Instrumental options") + g_outputs = parser.add_argument_group('Instrumental options') g_outputs.add_argument( - "-w", - "--work-dir", - action="store", + '-w', + '--work-dir', + action='store', type=Path, - default=Path("work").absolute(), - help="Path where intermediate results should be stored.", + default=Path('work').absolute(), + help='Path where intermediate results should be stored.', ) g_outputs.add_argument( - "-n", - "--dry-run", - action="store_true", + '-n', + '--dry-run', + action='store_true', default=False, - help="only find estimable fieldmaps (that is, estimation is not triggered)", + help='only find estimable fieldmaps (that is, estimation is not triggered)', ) g_outputs.add_argument( - "--no-fmapless", - action="store_false", - dest="fmapless", + '--no-fmapless', + action='store_false', + dest='fmapless', default=True, - help="Allow fieldmap-less estimation", + help='Allow fieldmap-less estimation', ) g_outputs.add_argument( - "--use-plugin", - action="store", + '--use-plugin', + action='store', default=None, type=Path, - help="Nipype plugin configuration file.", + help='Nipype plugin configuration file.', ) g_outputs.add_argument( - "--notrack", - action="store_true", - help="Opt-out of sending tracking information of this run to the NiPreps developers. " - "This information helps to improve SDCFlows and provides an indicator of " - "real world usage for obtaining funding.", + '--notrack', + action='store_true', + help='Opt-out of sending tracking information of this run to the NiPreps developers. ' + 'This information helps to improve SDCFlows and provides an indicator of ' + 'real world usage for obtaining funding.', ) return parser @@ -272,8 +270,8 @@ def _bids_filter(value): def parse_args(args=None, namespace=None): """Parse args and run further checks on the command line.""" - from logging import DEBUG from json import loads + from logging import DEBUG parser = _parser() opts = parser.parse_args(args, namespace) @@ -286,13 +284,11 @@ def parse_args(args=None, namespace=None): with open(opts.use_plugin) as f: plugin_settings = loadyml(f) - _plugin = plugin_settings.get("plugin") + _plugin = plugin_settings.get('plugin') if _plugin: config.nipype.plugin = _plugin - config.nipype.plugin_args = plugin_settings.get("plugin_args", {}) - config.nipype.nprocs = config.nipype.plugin_args.get( - "nprocs", config.nipype.nprocs - ) + config.nipype.plugin_args = plugin_settings.get('plugin_args', {}) + config.nipype.nprocs = config.nipype.plugin_args.get('nprocs', config.nipype.nprocs) # Load BIDS filters if opts.bids_filter_file: @@ -306,21 +302,21 @@ def parse_args(args=None, namespace=None): # Ensure input and output folders are not the same if output_dir == bids_dir: parser.error( - "The selected output folder is the same as the input BIDS folder. " - "Please modify the output path (suggestion: %s)." + 'The selected output folder is the same as the input BIDS folder. ' + 'Please modify the output path (suggestion: %s).' % bids_dir - / "derivatives" - / ("sdcflows_%s" % version.split("+")[0]) + / 'derivatives' + / ('sdcflows_%s' % version.split('+')[0]) ) if bids_dir in work_dir.parents: parser.error( - "The selected working directory is a subdirectory of the input BIDS folder. " - "Please modify the output path." + 'The selected working directory is a subdirectory of the input BIDS folder. ' + 'Please modify the output path.' ) # Setup directories - config.execution.log_dir = output_dir / "logs" + config.execution.log_dir = output_dir / 'logs' # Check and create output and working directories config.execution.log_dir.mkdir(exist_ok=True, parents=True) output_dir.mkdir(exist_ok=True, parents=True) @@ -335,8 +331,8 @@ def parse_args(args=None, namespace=None): missing_subjects = selected_label - set(participant_label) if missing_subjects: parser.error( - "One or more participant labels were not found in the BIDS directory: " - f"{', '.join(missing_subjects)}." + 'One or more participant labels were not found in the BIDS directory: ' + f'{", ".join(missing_subjects)}.' ) participant_label = selected_label diff --git a/sdcflows/cli/tests/test_find_estimators.py b/sdcflows/cli/tests/test_find_estimators.py index e5cd1432cc..a1cc4da98c 100644 --- a/sdcflows/cli/tests/test_find_estimators.py +++ b/sdcflows/cli/tests/test_find_estimators.py @@ -21,7 +21,9 @@ # https://www.nipreps.org/community/licensing/ # """Check the CLI.""" + from importlib import reload + import pytest from niworkflows.utils.testing import generate_bids_skeleton @@ -40,95 +42,95 @@ """ intendedfor_config = { - "01": [ - {"anat": [{"suffix": "T1w", "metadata": {"EchoTime": 1}}]}, + '01': [ + {'anat': [{'suffix': 'T1w', 'metadata': {'EchoTime': 1}}]}, { - "fmap": [ + 'fmap': [ { - "dir": "AP", - "suffix": "epi", - "metadata": { - "TotalReadoutTime": 0.1, - "PhaseEncodingDirection": "j-", - "IntendedFor": "func/sub-01_task-rest_bold.nii.gz", + 'dir': 'AP', + 'suffix': 'epi', + 'metadata': { + 'TotalReadoutTime': 0.1, + 'PhaseEncodingDirection': 'j-', + 'IntendedFor': 'func/sub-01_task-rest_bold.nii.gz', }, }, { - "dir": "PA", - "suffix": "epi", - "metadata": { - "TotalReadoutTime": 0.1, - "PhaseEncodingDirection": "j", - "IntendedFor": "func/sub-01_task-rest_bold.nii.gz", + 'dir': 'PA', + 'suffix': 'epi', + 'metadata': { + 'TotalReadoutTime': 0.1, + 'PhaseEncodingDirection': 'j', + 'IntendedFor': 'func/sub-01_task-rest_bold.nii.gz', }, }, ] }, { - "func": [ + 'func': [ { - "task": "rest", - "suffix": "bold", - "metadata": { - "RepetitionTime": 0.8, - "PhaseEncodingDirection": "j", + 'task': 'rest', + 'suffix': 'bold', + 'metadata': { + 'RepetitionTime': 0.8, + 'PhaseEncodingDirection': 'j', }, } ] }, ], - "02": [{"anat": [{"suffix": "T1w", "metadata": {"EchoTime": 1}}]}], + '02': [{'anat': [{'suffix': 'T1w', 'metadata': {'EchoTime': 1}}]}], } b0field_config = { - "01": [ - {"anat": [{"suffix": "T1w", "metadata": {"EchoTime": 1}}]}, + '01': [ + {'anat': [{'suffix': 'T1w', 'metadata': {'EchoTime': 1}}]}, { - "fmap": [ + 'fmap': [ { - "dir": "AP", - "suffix": "epi", - "metadata": { - "B0FieldIdentifier": "pepolar", - "TotalReadoutTime": 0.1, - "PhaseEncodingDirection": "j-", + 'dir': 'AP', + 'suffix': 'epi', + 'metadata': { + 'B0FieldIdentifier': 'pepolar', + 'TotalReadoutTime': 0.1, + 'PhaseEncodingDirection': 'j-', }, }, { - "dir": "PA", - "suffix": "epi", - "metadata": { - "B0FieldIdentifier": "pepolar", - "TotalReadoutTime": 0.1, - "PhaseEncodingDirection": "j", + 'dir': 'PA', + 'suffix': 'epi', + 'metadata': { + 'B0FieldIdentifier': 'pepolar', + 'TotalReadoutTime': 0.1, + 'PhaseEncodingDirection': 'j', }, }, ] }, { - "func": [ + 'func': [ { - "task": "rest", - "suffix": "bold", - "metadata": { - "B0FieldSource": "pepolar", - "RepetitionTime": 0.8, - "PhaseEncodingDirection": "j", + 'task': 'rest', + 'suffix': 'bold', + 'metadata': { + 'B0FieldSource': 'pepolar', + 'RepetitionTime': 0.8, + 'PhaseEncodingDirection': 'j', }, } ] }, ], - "02": [{"anat": [{"suffix": "T1w", "metadata": {"EchoTime": 1}}]}], + '02': [{'anat': [{'suffix': 'T1w', 'metadata': {'EchoTime': 1}}]}], } @pytest.mark.parametrize( - "test_id,config,estimator_id", + 'test_id,config,estimator_id', [ - ("intendedfor", intendedfor_config, "auto_00000"), - ("b0field", b0field_config, "pepolar"), + ('intendedfor', intendedfor_config, 'auto_00000'), + ('b0field', b0field_config, 'pepolar'), ], ) def test_cli_finder_wrapper(tmp_path, capsys, test_id, config, estimator_id): @@ -141,7 +143,7 @@ def test_cli_finder_wrapper(tmp_path, capsys, test_id, config, estimator_id): path = (tmp_path / test_id).absolute() generate_bids_skeleton(path, config) with pytest.raises(SystemExit) as wrapped_exit: - cli_finder_wrapper([str(path), str(tmp_path / "out"), "participant", "--dry-run"]) + cli_finder_wrapper([str(path), str(tmp_path / 'out'), 'participant', '--dry-run']) assert wrapped_exit.value.code == 0 output = OUTPUT.format(path=path, estimator_id=estimator_id) diff --git a/sdcflows/cli/workflow.py b/sdcflows/cli/workflow.py index c97b4ef696..ab0ac72342 100644 --- a/sdcflows/cli/workflow.py +++ b/sdcflows/cli/workflow.py @@ -26,8 +26,9 @@ def build_workflow(config_file, retval): """Create the Nipype Workflow that supports the whole execution graph.""" import os + # We do not need OMP > 1 for workflow creation - os.environ["OMP_NUM_THREADS"] = "1" + os.environ['OMP_NUM_THREADS'] = '1' from sdcflows import config from sdcflows.workflows.fit.base import init_sdcflows_wf @@ -38,15 +39,18 @@ def build_workflow(config_file, retval): # Make sure loggers are started config.loggers.init() - config.loggers.cli.log(25, f"""\ + config.loggers.cli.log( + 25, + f"""\ Running SDCFlows {config.environment.version}: * BIDS dataset path: {config.execution.bids_dir}. * Output folder: {config.execution.output_dir}. * Analysis levels: {config.workflow.analysis_level}. -""") +""", + ) - retval["return_code"] = 1 - retval["workflow"] = None - retval["workflow"] = init_sdcflows_wf() - retval["return_code"] = int(retval["workflow"] is None) + retval['return_code'] = 1 + retval['workflow'] = None + retval['workflow'] = init_sdcflows_wf() + retval['return_code'] = int(retval['workflow'] is None) return retval diff --git a/sdcflows/config.py b/sdcflows/config.py index 2bcaedd757..e2e851184d 100644 --- a/sdcflows/config.py +++ b/sdcflows/config.py @@ -110,6 +110,7 @@ :py:class:`~bids.layout.BIDSLayout`, etc.) """ + import os import sys from pathlib import Path @@ -123,53 +124,51 @@ from importlib_metadata import version as get_version # Ignore annoying warnings -from sdcflows._warnings import logging from sdcflows import __version__ +from sdcflows._warnings import logging _pre_exec_env = dict(os.environ) # Reduce numpy's vms by limiting OMP_NUM_THREADS -_default_omp_threads = int(os.getenv("OMP_NUM_THREADS", os.cpu_count())) +_default_omp_threads = int(os.getenv('OMP_NUM_THREADS', os.cpu_count())) # Disable NiPype etelemetry always -_disable_et = bool( - os.getenv("NO_ET") is not None or os.getenv("NIPYPE_NO_ET") is not None -) -os.environ["NIPYPE_NO_ET"] = "1" -os.environ["NO_ET"] = "1" +_disable_et = bool(os.getenv('NO_ET') is not None or os.getenv('NIPYPE_NO_ET') is not None) +os.environ['NIPYPE_NO_ET'] = '1' +os.environ['NO_ET'] = '1' -if not hasattr(sys, "_is_pytest_session"): +if not hasattr(sys, '_is_pytest_session'): sys._is_pytest_session = False # Trick to avoid sklearn's FutureWarnings # Disable all warnings in main and children processes only on production versions if not any( ( - "+" in __version__, - __version__.endswith(".dirty"), - os.getenv("SDCFLOWS_DEV", "0").lower() in ("1", "on", "true", "y", "yes"), + '+' in __version__, + __version__.endswith('.dirty'), + os.getenv('SDCFLOWS_DEV', '0').lower() in ('1', 'on', 'true', 'y', 'yes'), ) ): - os.environ["PYTHONWARNINGS"] = "ignore" + os.environ['PYTHONWARNINGS'] = 'ignore' -logging.addLevelName(25, "IMPORTANT") # Add a new level between INFO and WARNING -logging.addLevelName(15, "VERBOSE") # Add a new level between INFO and DEBUG +logging.addLevelName(25, 'IMPORTANT') # Add a new level between INFO and WARNING +logging.addLevelName(15, 'VERBOSE') # Add a new level between INFO and DEBUG DEFAULT_MEMORY_MIN_GB = 0.01 _exec_env = os.name _docker_ver = None # special variable set in the container -if os.getenv("IS_DOCKER_8395080871"): - _exec_env = "singularity" - _cgroup = Path("/proc/1/cgroup") - if _cgroup.exists() and "docker" in _cgroup.read_text(): - _docker_ver = os.getenv("DOCKER_VERSION_8395080871") - _exec_env = "docker" +if os.getenv('IS_DOCKER_8395080871'): + _exec_env = 'singularity' + _cgroup = Path('/proc/1/cgroup') + if _cgroup.exists() and 'docker' in _cgroup.read_text(): + _docker_ver = os.getenv('DOCKER_VERSION_8395080871') + _exec_env = 'docker' del _cgroup _templateflow_home = Path( os.getenv( - "TEMPLATEFLOW_HOME", - os.path.join(os.getenv("HOME"), ".cache", "templateflow"), + 'TEMPLATEFLOW_HOME', + os.path.join(os.getenv('HOME'), '.cache', 'templateflow'), ) ) @@ -180,39 +179,34 @@ except Exception: _free_mem_at_start = None -_oc_limit = "n/a" -_oc_policy = "n/a" +_oc_limit = 'n/a' +_oc_policy = 'n/a' try: # Memory policy may have a large effect on types of errors experienced - _proc_oc_path = Path("/proc/sys/vm/overcommit_memory") + _proc_oc_path = Path('/proc/sys/vm/overcommit_memory') if _proc_oc_path.exists(): - _oc_policy = {"0": "heuristic", "1": "always", "2": "never"}.get( - _proc_oc_path.read_text().strip(), "unknown" + _oc_policy = {'0': 'heuristic', '1': 'always', '2': 'never'}.get( + _proc_oc_path.read_text().strip(), 'unknown' ) - if _oc_policy != "never": - _proc_oc_kbytes = Path("/proc/sys/vm/overcommit_kbytes") + if _oc_policy != 'never': + _proc_oc_kbytes = Path('/proc/sys/vm/overcommit_kbytes') if _proc_oc_kbytes.exists(): _oc_limit = _proc_oc_kbytes.read_text().strip() - if ( - _oc_limit in ("0", "n/a") - and Path("/proc/sys/vm/overcommit_ratio").exists() - ): - _oc_limit = "{}%".format( - Path("/proc/sys/vm/overcommit_ratio").read_text().strip() - ) + if _oc_limit in ('0', 'n/a') and Path('/proc/sys/vm/overcommit_ratio').exists(): + _oc_limit = '{}%'.format(Path('/proc/sys/vm/overcommit_ratio').read_text().strip()) except Exception: pass _memory_gb = None try: - if "linux" in sys.platform: - with open("/proc/meminfo", "r") as f_in: + if 'linux' in sys.platform: + with open('/proc/meminfo', 'r') as f_in: _meminfo_lines = f_in.readlines() - _mem_total_line = [line for line in _meminfo_lines if "MemTotal" in line][0] + _mem_total_line = [line for line in _meminfo_lines if 'MemTotal' in line][0] _mem_total = float(_mem_total_line.split()[1]) _memory_gb = _mem_total / (1024.0**2) - elif "darwin" in sys.platform: - _mem_str = os.popen("sysctl hw.memsize").read().strip().split(" ")[-1] + elif 'darwin' in sys.platform: + _mem_str = os.popen('sysctl hw.memsize').read().strip().split(' ')[-1] _memory_gb = float(_mem_str) / (1024.0**3) except Exception: pass @@ -230,7 +224,7 @@ class _Config: def __init__(self): """Avert instantiation.""" - raise RuntimeError("Configuration type is not instantiable.") + raise RuntimeError('Configuration type is not instantiable.') @classmethod def load(cls, settings, init=True): @@ -255,7 +249,7 @@ def get(cls): """Return defined settings.""" out = {} for k, v in cls.__dict__.items(): - if k.startswith("_") or v is None: + if k.startswith('_') or v is None: continue if callable(getattr(cls, k)): continue @@ -291,9 +285,9 @@ class environment(_Config): """Linux's kernel virtual memory overcommit policy.""" overcommit_limit = _oc_limit """Linux's kernel virtual memory overcommit limits.""" - nipype_version = get_version("nipype") + nipype_version = get_version('nipype') """Nipype's current version.""" - templateflow_version = get_version("templateflow") + templateflow_version = get_version('templateflow') """The TemplateFlow client version installed.""" total_memory = _memory_gb """Total memory available, in GB.""" @@ -306,7 +300,7 @@ class environment(_Config): class nipype(_Config): """Nipype settings.""" - crashfile_format = "txt" + crashfile_format = 'txt' """The file format for crashfiles, either text or pickle.""" get_linked_libs = False """Run NiPype's tool to enlist linked libraries for every interface.""" @@ -318,11 +312,11 @@ class nipype(_Config): """Number of processes (compute tasks) that can be run in parallel (multiprocessing only).""" omp_nthreads = _default_omp_threads """Number of CPUs a single process can access for multithreaded execution.""" - plugin = "MultiProc" + plugin = 'MultiProc' """NiPype's execution plugin.""" plugin_args = { - "maxtasksperchild": 1, - "raise_insufficient": False, + 'maxtasksperchild': 1, + 'raise_insufficient': False, } """Settings for NiPype's execution plugin.""" remove_node_directories = False @@ -336,13 +330,13 @@ class nipype(_Config): def get_plugin(cls): """Format a dictionary for Nipype consumption.""" out = { - "plugin": cls.plugin, - "plugin_args": cls.plugin_args, + 'plugin': cls.plugin, + 'plugin_args': cls.plugin_args, } - if cls.plugin in ("MultiProc", "LegacyMultiProc"): - out["plugin_args"]["n_procs"] = int(cls.nprocs) + if cls.plugin in ('MultiProc', 'LegacyMultiProc'): + out['plugin_args']['n_procs'] = int(cls.nprocs) if cls.memory_gb: - out["plugin_args"]["memory_gb"] = float(cls.memory_gb) + out['plugin_args']['memory_gb'] = float(cls.memory_gb) return out @classmethod @@ -353,11 +347,11 @@ def init(cls): # Nipype config (logs and execution) ncfg.update_config( { - "execution": { - "crashdump_dir": str(execution.log_dir), - "crashfile_format": cls.crashfile_format, - "get_linked_libs": cls.get_linked_libs, - "stop_on_first_crash": cls.stop_on_first_crash, + 'execution': { + 'crashdump_dir': str(execution.log_dir), + 'crashfile_format': cls.crashfile_format, + 'get_linked_libs': cls.get_linked_libs, + 'stop_on_first_crash': cls.stop_on_first_crash, } } ) @@ -398,7 +392,7 @@ class execution(_Config): """List of participant identifiers that are to be preprocessed.""" pdb = False """Drop into PDB when exceptions are encountered.""" - run_uuid = "{}_{}".format(strftime("%Y%m%d-%H%M%S"), uuid4()) + run_uuid = '{}_{}'.format(strftime('%Y%m%d-%H%M%S'), uuid4()) """Unique identifier of this particular run.""" session_label = None """Filter input dataset by session identifier.""" @@ -406,7 +400,7 @@ class execution(_Config): """Run in sloppy mode (meaning, suboptimal parameters that minimize run-time).""" templateflow_home = _templateflow_home """The root folder of the TemplateFlow client.""" - work_dir = Path("work").absolute() + work_dir = Path('work').absolute() """Path to a working directory where intermediate results will be available.""" write_graph = False """Write out the computational graph corresponding to the planned preprocessing.""" @@ -414,13 +408,13 @@ class execution(_Config): _layout = None _paths = ( - "bids_dir", - "bids_database_dir", - "layout", - "log_dir", - "output_dir", - "templateflow_home", - "work_dir", + 'bids_dir', + 'bids_database_dir', + 'layout', + 'log_dir', + 'output_dir', + 'templateflow_home', + 'work_dir', ) @classmethod @@ -432,25 +426,24 @@ def init(cls): if cls._layout is None: import re - from bids.layout.index import BIDSLayoutIndexer + from bids.layout import BIDSLayout + from bids.layout.index import BIDSLayoutIndexer ignore_paths = [ # Ignore folders at the top if they don't start with /sub-