added in example specs from mriqc and reworked test_package_complete test

Author: tclose

conftest.py

@@ -17,11 +17,6 @@ def gen_test_conftest():
     return PKG_DIR / "scripts" / "pkg_gen" / "resources" / "conftest.py"
 
 
-@pytest.fixture(params=[str(p.stem) for p in EXAMPLE_WORKFLOWS_DIR.glob("*.yaml")])
-def workflow_spec_file(request):
-    return (EXAMPLE_WORKFLOWS_DIR / request.param).with_suffix(".yaml")
-
-
 @pytest.fixture
 def work_dir():
     work_dir = tempfile.mkdtemp()

example-specs/workflow/mriqc/interfaces/add_provenance.yaml

@@ -15,11 +15,11 @@ inputs:
   rename:
   # dict[str, str] - fields to rename in the Pydra interface
   types:
-  # dict[str, type] - override inferred types (use "mime-like" string for file-format types,
-  # e.g. 'medimage/nifti-gz'). For most fields the type will be correctly inferred
-  # from the nipype interface, but you may want to be more specific, particularly
-  # for file types, where specifying the format also specifies the file that will be
-  # passed to the field in the automatically generated unittests.
+    # dict[str, type] - override inferred types (use "mime-like" string for file-format types,
+    # e.g. 'medimage/nifti-gz'). For most fields the type will be correctly inferred
+    # from the nipype interface, but you may want to be more specific, particularly
+    # for file types, where specifying the format also specifies the file that will be
+    # passed to the field in the automatically generated unittests.
     air_msk: generic/file
     # type=file|default=<undefined>: air mask file
     in_file: generic/file
@@ -43,39 +43,43 @@ outputs:
   # for file types, where specifying the format also specifies the file that will be
   # passed to the field in the automatically generated unittests.
   callables:
-  # dict[str, str] - names of methods/callable classes defined in the adjacent `*_callables.py`
-  # to set to the `callable` attribute of output fields
+    # dict[str, str] - names of methods/callable classes defined in the adjacent `*_callables.py`
+    # to set to the `callable` attribute of output fields
     out_prov: out_prov_callable
-    # type=dict: 
+    # type=dict:
   templates:
   # dict[str, str] - `output_file_template` values to be provided to output fields
   requirements:
   # dict[str, list[str]] - input fields that are required to be provided for the output field to be present
 tests:
-- inputs:
-  # dict[str, str] - values to provide to inputs fields in the task initialisation
-  # (if not specified, will try to choose a sensible value)
-    in_file:
-    # type=file|default=<undefined>: input file
-    air_msk:
-    # type=file|default=<undefined>: air mask file
-    rot_msk:
-    # type=file|default=<undefined>: rotation mask file
-    modality:
-    # type=str|default='': provenance type
-  imports:
-  # list[nipype2pydra.task.base.explicitimport] - list import statements required by the test, with each list item
-  # consisting of 'module', 'name', and optionally 'alias' keys
-  expected_outputs:
-  # dict[str, str] - expected values for selected outputs, noting that tests will typically
-  # be terminated before they complete for time-saving reasons, and therefore
-  # these values will be ignored, when running in CI
-  timeout: 10
-  # int - the value to set for the timeout in the generated test, 
-  # after which the test will be considered to have been initialised 
-  # successfully. Set to 0 to disable the timeout (warning, this could
-  # lead to the unittests taking a very long time to complete)
-  xfail: true
-  # bool - whether the unittest is expected to fail or not. Set to false
-  # when you are satisfied with the edits you have made to this file
+  - inputs:
+      # dict[str, str] - values to provide to inputs fields in the task initialisation
+      # (if not specified, will try to choose a sensible value)
+      in_file:
+      # type=file|default=<undefined>: input file
+      air_msk:
+      # type=file|default=<undefined>: air mask file
+      rot_msk:
+      # type=file|default=<undefined>: rotation mask file
+      modality:
+      # type=str|default='': provenance type
+    imports:
+    # list[nipype2pydra.task.base.explicitimport] - list import statements required by the test, with each list item
+    # consisting of 'module', 'name', and optionally 'alias' keys
+    expected_outputs:
+    # dict[str, str] - expected values for selected outputs, noting that tests will typically
+    # be terminated before they complete for time-saving reasons, and therefore
+    # these values will be ignored, when running in CI
+    timeout: 10
+    # int - the value to set for the timeout in the generated test,
+    # after which the test will be considered to have been initialised
+    # successfully. Set to 0 to disable the timeout (warning, this could
+    # lead to the unittests taking a very long time to complete)
+    xfail: true
+    # bool - whether the unittest is expected to fail or not. Set to false
+    # when you are satisfied with the edits you have made to this file
 doctests: []
+find_replace:
+  - ["config.environment.version", "'<version-not-captured>'"]
+  - ["config.execution.debug", "False"]
+  - ["config.workflow.fd_thres,", "0.2,  # <configuration>.fd_thres"]

example-specs/workflow/mriqc/interfaces/conform_image.yaml

@@ -5,13 +5,13 @@
 #
 # Docs
 # ----
-# 
+#
 #     Conforms an input image.
-# 
+#
 #     List of nifti datatypes:
-# 
+#
 #     .. note: Original Analyze 7.5 types
-# 
+#
 #           DT_NONE                    0
 #           DT_UNKNOWN                 0     / what it says, dude           /
 #           DT_BINARY                  1     / binary (1 bit/voxel)         /
@@ -23,19 +23,19 @@
 #           DT_DOUBLE                 64     / double (64 bits/voxel)       /
 #           DT_RGB                   128     / RGB triple (24 bits/voxel)   /
 #           DT_ALL                   255     / not very useful (?)          /
-# 
+#
 #     .. note: Added names for the same data types
-# 
+#
 #           DT_UINT8                   2
 #           DT_INT16                   4
 #           DT_INT32                   8
 #           DT_FLOAT32                16
 #           DT_COMPLEX64              32
 #           DT_FLOAT64                64
 #           DT_RGB24                 128
-# 
+#
 #     .. note: New codes for NIfTI
-# 
+#
 #           DT_INT8                  256     / signed char (8 bits)         /
 #           DT_UINT16                512     / unsigned short (16 bits)     /
 #           DT_UINT32                768     / unsigned int (32 bits)       /
@@ -59,8 +59,8 @@
 #           NIFTI_TYPE_FLOAT128     1536 /! 128 bit float = long double. /
 #           NIFTI_TYPE_COMPLEX128   1792 /! 128 bit complex = 2 64 bit floats. /
 #           NIFTI_TYPE_COMPLEX256   2048 /! 256 bit complex = 2 128 bit floats /
-# 
-#     
+#
+#
 task_name: ConformImage
 nipype_name: ConformImage
 nipype_module: mriqc.interfaces.common.conform_image
@@ -70,11 +70,11 @@ inputs:
   rename:
   # dict[str, str] - fields to rename in the Pydra interface
   types:
-  # dict[str, type] - override inferred types (use "mime-like" string for file-format types,
-  # e.g. 'medimage/nifti-gz'). For most fields the type will be correctly inferred
-  # from the nipype interface, but you may want to be more specific, particularly
-  # for file types, where specifying the format also specifies the file that will be
-  # passed to the field in the automatically generated unittests.
+    # dict[str, type] - override inferred types (use "mime-like" string for file-format types,
+    # e.g. 'medimage/nifti-gz'). For most fields the type will be correctly inferred
+    # from the nipype interface, but you may want to be more specific, particularly
+    # for file types, where specifying the format also specifies the file that will be
+    # passed to the field in the automatically generated unittests.
     in_file: generic/file
     # type=file|default=<undefined>: input image
   callable_defaults:
@@ -88,11 +88,11 @@ outputs:
   rename:
   # dict[str, str] - fields to rename in the Pydra interface
   types:
-  # dict[str, type] - override inferred types (use "mime-like" string for file-format types,
-  # e.g. 'medimage/nifti-gz'). For most fields the type will be correctly inferred
-  # from the nipype interface, but you may want to be more specific, particularly
-  # for file types, where specifying the format also specifies the file that will be
-  # passed to the field in the automatically generated unittests.
+    # dict[str, type] - override inferred types (use "mime-like" string for file-format types,
+    # e.g. 'medimage/nifti-gz'). For most fields the type will be correctly inferred
+    # from the nipype interface, but you may want to be more specific, particularly
+    # for file types, where specifying the format also specifies the file that will be
+    # passed to the field in the automatically generated unittests.
     out_file: generic/file
     # type=file: output conformed file
   callables:
@@ -103,28 +103,34 @@ outputs:
   requirements:
   # dict[str, list[str]] - input fields that are required to be provided for the output field to be present
 tests:
-- inputs:
-  # dict[str, str] - values to provide to inputs fields in the task initialisation
-  # (if not specified, will try to choose a sensible value)
-    in_file:
-    # type=file|default=<undefined>: input image
-    check_ras:
-    # type=bool|default=True: check that orientation is RAS
-    check_dtype:
-    # type=bool|default=True: check data type
-  imports:
-  # list[nipype2pydra.task.base.explicitimport] - list import statements required by the test, with each list item
-  # consisting of 'module', 'name', and optionally 'alias' keys
-  expected_outputs:
-  # dict[str, str] - expected values for selected outputs, noting that tests will typically
-  # be terminated before they complete for time-saving reasons, and therefore
-  # these values will be ignored, when running in CI
-  timeout: 10
-  # int - the value to set for the timeout in the generated test, 
-  # after which the test will be considered to have been initialised 
-  # successfully. Set to 0 to disable the timeout (warning, this could
-  # lead to the unittests taking a very long time to complete)
-  xfail: true
-  # bool - whether the unittest is expected to fail or not. Set to false
-  # when you are satisfied with the edits you have made to this file
+  - inputs:
+      # dict[str, str] - values to provide to inputs fields in the task initialisation
+      # (if not specified, will try to choose a sensible value)
+      in_file:
+      # type=file|default=<undefined>: input image
+      check_ras:
+      # type=bool|default=True: check that orientation is RAS
+      check_dtype:
+      # type=bool|default=True: check data type
+    imports:
+    # list[nipype2pydra.task.base.explicitimport] - list import statements required by the test, with each list item
+    # consisting of 'module', 'name', and optionally 'alias' keys
+    expected_outputs:
+    # dict[str, str] - expected values for selected outputs, noting that tests will typically
+    # be terminated before they complete for time-saving reasons, and therefore
+    # these values will be ignored, when running in CI
+    timeout: 10
+    # int - the value to set for the timeout in the generated test,
+    # after which the test will be considered to have been initialised
+    # successfully. Set to 0 to disable the timeout (warning, this could
+    # lead to the unittests taking a very long time to complete)
+    xfail: true
+    # bool - whether the unittest is expected to fail or not. Set to false
+    # when you are satisfied with the edits you have made to this file
 doctests: []
+find_replace:
+  - [config\.loggers\.interface, "logger"]
+  - [
+      messages\.SUSPICIOUS_DATA_TYPE,
+      '"Input image {in_file} has a suspicious data type: ''{dtype}''"',
+    ]

example-specs/workflow/mriqc/interfaces/datalad_identity_interface_callables.py

@@ -0,0 +1,91 @@
+# This file is used to manually specify the semi-automatic conversion of
+# 'mriqc.interfaces.DerivativesDataSink' from Nipype to Pydra.
+#
+# Please fill-in/edit the fields below where appropriate
+#
+# Docs
+# ----
+# 
+task_name: DerivativesDataSink
+nipype_name: DerivativesDataSink
+nipype_module: mriqc.interfaces
+inputs:
+  omit:
+  # list[str] - fields to omit from the Pydra interface
+  rename:
+  # dict[str, str] - fields to rename in the Pydra interface
+  types:
+  # dict[str, type] - override inferred types (use "mime-like" string for file-format types,
+  # e.g. 'medimage/nifti-gz'). For most fields the type will be correctly inferred
+  # from the nipype interface, but you may want to be more specific, particularly
+  # for file types, where specifying the format also specifies the file that will be
+  # passed to the field in the automatically generated unittests.
+    base_directory: generic/directory
+    # type=directory|default='': Path to the base directory for storing data.
+    in_file: generic/file+list-of
+    # type=inputmultiobject|default=[]: the object to be saved
+    source_file: generic/file+list-of
+    # type=inputmultiobject|default=[]: the source file(s) to extract entities from
+  callable_defaults:
+  # dict[str, str] - names of methods/callable classes defined in the adjacent `*_callables.py`
+  # to set as the `default` method of input fields
+  metadata:
+  # dict[str, dict[str, any]] - additional metadata to set on any of the input fields (e.g. out_file: position: 1)
+outputs:
+  omit:
+  # list[str] - fields to omit from the Pydra interface
+  rename:
+  # dict[str, str] - fields to rename in the Pydra interface
+  types:
+  # dict[str, type] - override inferred types (use "mime-like" string for file-format types,
+  # e.g. 'medimage/nifti-gz'). For most fields the type will be correctly inferred
+  # from the nipype interface, but you may want to be more specific, particularly
+  # for file types, where specifying the format also specifies the file that will be
+  # passed to the field in the automatically generated unittests.
+    out_file: generic/file+list-of
+    # type=outputmultiobject: 
+    out_meta: generic/file+list-of
+    # type=outputmultiobject: 
+  callables:
+  # dict[str, str] - names of methods/callable classes defined in the adjacent `*_callables.py`
+  # to set to the `callable` attribute of output fields
+  templates:
+  # dict[str, str] - `output_file_template` values to be provided to output fields
+  requirements:
+  # dict[str, list[str]] - input fields that are required to be provided for the output field to be present
+tests:
+- inputs:
+  # dict[str, str] - values to provide to inputs fields in the task initialisation
+  # (if not specified, will try to choose a sensible value)
+    base_directory:
+    # type=directory|default='': Path to the base directory for storing data.
+    check_hdr:
+    # type=bool|default=True: fix headers of NIfTI outputs
+    compress:
+    # type=inputmultiobject|default=[]: whether ``in_file`` should be compressed (True), uncompressed (False) or left unmodified (None, default).
+    data_dtype:
+    # type=str|default='': NumPy datatype to coerce NIfTI data to, or `source` tomatch the input file dtype
+    dismiss_entities:
+    # type=inputmultiobject|default=[]: a list entities that will not be propagated from the source file
+    in_file:
+    # type=inputmultiobject|default=[]: the object to be saved
+    meta_dict:
+    # type=dict|default={}: an input dictionary containing metadata
+    source_file:
+    # type=inputmultiobject|default=[]: the source file(s) to extract entities from
+  imports:
+  # list[nipype2pydra.task.base.explicitimport] - list import statements required by the test, with each list item
+  # consisting of 'module', 'name', and optionally 'alias' keys
+  expected_outputs:
+  # dict[str, str] - expected values for selected outputs, noting that tests will typically
+  # be terminated before they complete for time-saving reasons, and therefore
+  # these values will be ignored, when running in CI
+  timeout: 10
+  # int - the value to set for the timeout in the generated test, 
+  # after which the test will be considered to have been initialised 
+  # successfully. Set to 0 to disable the timeout (warning, this could
+  # lead to the unittests taking a very long time to complete)
+  xfail: true
+  # bool - whether the unittest is expected to fail or not. Set to false
+  # when you are satisfied with the edits you have made to this file
+doctests: []

example-specs/workflow/mriqc/interfaces/derivatives_data_sink.yaml

@@ -0,0 +1,34 @@
+"""Module to put any functions that are referred to in the "callables" section of DerivativesDataSink.yaml"""
+
+
+def compression_callable(output_dir, inputs, stdout, stderr):
+    outputs = _list_outputs(
+        output_dir=output_dir, inputs=inputs, stdout=stdout, stderr=stderr
+    )
+    return outputs["compression"]
+
+
+def fixed_hdr_callable(output_dir, inputs, stdout, stderr):
+    outputs = _list_outputs(
+        output_dir=output_dir, inputs=inputs, stdout=stdout, stderr=stderr
+    )
+    return outputs["fixed_hdr"]
+
+
+def out_file_callable(output_dir, inputs, stdout, stderr):
+    outputs = _list_outputs(
+        output_dir=output_dir, inputs=inputs, stdout=stdout, stderr=stderr
+    )
+    return outputs["out_file"]
+
+
+def out_meta_callable(output_dir, inputs, stdout, stderr):
+    outputs = _list_outputs(
+        output_dir=output_dir, inputs=inputs, stdout=stdout, stderr=stderr
+    )
+    return outputs["out_meta"]
+
+
+# Original source at L568 of <nipype-install>/interfaces/base/core.py
+def _list_outputs(inputs=None, stdout=None, stderr=None, output_dir=None):
+    return _results