Skip to content

Commit 830abb5

Browse files
tclosetclose
committed
finished changes to docs for now. Not all tutorials are working though
1 parent ac80065 commit 830abb5

File tree

9 files changed

+195
-60
lines changed

9 files changed

+195
-60
lines changed

‎.python-version‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎new-docs/source/examples/.python-version‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
wf12

‎new-docs/source/tutorial/2-advanced-execution.ipynb‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -395,9 +395,44 @@
395395
"source": [
396396
"## Environments\n",
397397
"\n",
398-
"Work in progress...\n",
398+
"For shell tasks, it is possible to specify that the command runs within a specific\n",
399+
"software environment, such as those provided by software containers (e.g. Docker or Apptainer).\n",
400+
"This is down by providing the environment to the submitter/execution call,"
401+
]
402+
},
403+
{
404+
"cell_type": "code",
405+
"execution_count": null,
406+
"metadata": {},
407+
"outputs": [],
408+
"source": [
409+
"import tempfile\n",
410+
"from pydra.tasks.mrtrix3.v3_0 import MrGrid\n",
411+
"from pydra.engine.environments import Docker\n",
412+
"\n",
413+
"test_dir = tempfile.mkdtemp()\n",
414+
"\n",
415+
"nifti_file = Nifti1.sample(test_dir, seed=0)\n",
399416
"\n",
400-
"See [Containers and Environments](../explanation/environments.rst) for more details."
417+
"# Instantiate the task definition, \"splitting\" over all NIfTI files in the test directory\n",
418+
"# by splitting the \"input\" input field over all files in the directory\n",
419+
"mrgrid = MrGrid(in_file=nifti_file, operation=\"regrid\", voxel=(0.5,0.5,0.5))\n",
420+
"\n",
421+
"# Run the task to resample all NIfTI files\n",
422+
"outputs = mrgrid(environment=Docker(image=\"mrtrix3/mrtrix3\", tag=\"latest\"))\n",
423+
"\n",
424+
"# Print the locations of the output files\n",
425+
"print(\"\\n\".join(str(p) for p in outputs.out_file))"
426+
]
427+
},
428+
{
429+
"cell_type": "markdown",
430+
"metadata": {},
431+
"source": [
432+
"Of course for this to work Docker needs to work and be configured for\n",
433+
"[sudo-less execution](https://docs.docker.com/engine/install/linux-postinstall/).\n",
434+
"See [Containers and Environments](../explanation/environments.rst) for more details on\n",
435+
"how to utilise containers and add support for other software environments."
401436
]
402437
},
403438
{

‎new-docs/source/tutorial/3-troubleshooting.ipynb‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -128,9 +128,8 @@
128128
"# This workflow will fail because we are trying to divide by 0\n",
129129
"wf = UnsafeDivisionWorkflow(a=10, b=5).split(denominator=[3, 2 ,0])\n",
130130
"\n",
131-
"if __name__ == \"__main__\":\n",
132-
" with Submitter(worker=\"cf\") as sub:\n",
133-
" result = sub(wf)\n",
131+
"with Submitter(worker=\"cf\") as sub:\n",
132+
" result = sub(wf)\n",
134133
" \n",
135134
"if result.errored:\n",
136135
" print(\"Workflow failed with errors:\\n\" + str(result.errors))\n",

‎new-docs/source/tutorial/6-workflow.ipynb‎

Lines changed: 124 additions & 0 deletions
Large diffs are not rendered by default.

‎new-docs/source/tutorial/7-canonical-form.ipynb‎

Lines changed: 11 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,13 @@
3131
"Python tasks in dataclass form are decorated by `pydra.design.python.define`\n",
3232
"with inputs listed as type annotations. Outputs are similarly defined in a nested class\n",
3333
"called `Outputs`. The function to be executed should be a staticmethod called `function`.\n",
34-
"Default values can also be set directly, as with Attrs classes.\n"
34+
"Default values can also be set directly, as with Attrs classes.\n",
35+
"\n",
36+
"In order to allow static type-checkers to check the type of outputs of tasks added\n",
37+
"to workflows, it is also necessary to explicitly extend from the `pydra.engine.specs.PythonDef`\n",
38+
"and `pydra.engine.specs.PythonOutputs` classes (they are otherwise set as bases by the\n",
39+
"`define` method implicitly). Thus the \"canonical form\" of Python task definition is as\n",
40+
"follows"
3541
]
3642
},
3743
{
@@ -47,7 +53,7 @@
4753
"\n",
4854
"\n",
4955
"@python.define\n",
50-
"class CanonicalPythonDef:\n",
56+
"class CanonicalPythonDef(PythonDef[\"CanonicalPythonDef.Outputs\"]):\n",
5157
" \"\"\"Canonical Python task definition class for testing\n",
5258
"\n",
5359
" Args:\n",
@@ -59,7 +65,7 @@
5965
" a: int\n",
6066
" b: float = 2.0 # set default value\n",
6167
"\n",
62-
" class Outputs:\n",
68+
" class Outputs(PythonOutputs):\n",
6369
" \"\"\"\n",
6470
" Args:\n",
6571
" c: Sum of a and b\n",
@@ -95,7 +101,7 @@
95101
"\n",
96102
"\n",
97103
"@python.define\n",
98-
"class CanonicalPythonDef:\n",
104+
"class CanonicalPythonDef(PythonDef[\"CanonicalPythonDef.Outputs\"]):\n",
99105
" \"\"\"Canonical Python task definition class for testing\n",
100106
"\n",
101107
" Args:\n",
@@ -107,7 +113,7 @@
107113
" a: int = python.arg(allowed_values=[1, 2, 3, 4, 5])\n",
108114
" b: float = python.arg(default=2.0, validator=attrs.validators.not_(0))\n",
109115
"\n",
110-
" class Outputs:\n",
116+
" class Outputs(PythonOutputs):\n",
111117
" \"\"\"\n",
112118
" Args:\n",
113119
" c: Sum of a and b\n",
@@ -125,51 +131,6 @@
125131
"pprint(fields_dict(CanonicalPythonDef.Outputs))"
126132
]
127133
},
128-
{
129-
"cell_type": "markdown",
130-
"metadata": {},
131-
"source": [
132-
"In order to allow static type-checkers to check the type of outputs of tasks added\n",
133-
"to workflows, it is also necessary to explicitly extend from the `pydra.engine.specs.PythonDef`\n",
134-
"and `pydra.engine.specs.PythonOutputs` classes (they are otherwise set as bases by the\n",
135-
"`define` method implicitly). Thus the \"canonical\" is as follows"
136-
]
137-
},
138-
{
139-
"cell_type": "code",
140-
"execution_count": null,
141-
"metadata": {},
142-
"outputs": [],
143-
"source": [
144-
"\n",
145-
"@python.define\n",
146-
"class CanonicalPythonDef(PythonDef[\"CanonicalPythonDef.Outputs\"]):\n",
147-
" \"\"\"Canonical Python task definition class for testing\n",
148-
"\n",
149-
" Args:\n",
150-
" a: First input\n",
151-
" to be inputted\n",
152-
" b: Second input\n",
153-
" \"\"\"\n",
154-
"\n",
155-
" a: int\n",
156-
" b: float = 2.0 # set default value\n",
157-
"\n",
158-
" class Outputs(PythonOutputs):\n",
159-
" \"\"\"\n",
160-
" Args:\n",
161-
" c: Sum of a and b\n",
162-
" d: Product of a and b\n",
163-
" \"\"\"\n",
164-
"\n",
165-
" c: float\n",
166-
" d: float\n",
167-
"\n",
168-
" @staticmethod\n",
169-
" def function(a, b):\n",
170-
" return a + b, a / b"
171-
]
172-
},
173134
{
174135
"cell_type": "markdown",
175136
"metadata": {},

‎pydra/engine/core.py‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -721,7 +721,12 @@ def clear_cache(cls):
721721
"""Clear the cache of constructed workflows"""
722722
cls._constructed.clear()
723723

724-
def add(self, task_spec: TaskDef[OutputsType], name=None) -> OutputsType:
724+
def add(
725+
self,
726+
task_def: TaskDef[OutputsType],
727+
name: str | None = None,
728+
environment: Environment | None = None,
729+
) -> OutputsType:
725730
"""Add a node to the workflow
726731
727732
Parameters
@@ -738,10 +743,17 @@ def add(self, task_spec: TaskDef[OutputsType], name=None) -> OutputsType:
738743
The outputs definition of the node
739744
"""
740745
if name is None:
741-
name = type(task_spec).__name__
746+
name = type(task_def).__name__
742747
if name in self._nodes:
743748
raise ValueError(f"Node with name {name!r} already exists in the workflow")
744-
node = Node[OutputsType](name=name, definition=task_spec, workflow=self)
749+
if environment and task_def._task_type != "shell":
750+
raise ValueError(
751+
"Environments can only be used with 'shell' tasks not "
752+
f"{task_def._task_type!r} tasks ({task_def!r})"
753+
)
754+
node = Node[OutputsType](
755+
name=name, definition=task_def, workflow=self, environment=environment
756+
)
745757
self._nodes[name] = node
746758
return node.lzout
747759

‎pydra/engine/node.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
if ty.TYPE_CHECKING:
1818
from .core import Workflow
19+
from .environments import Environment
1920
from pydra.engine.specs import TaskDef, TaskOutputs
2021

2122

@@ -41,6 +42,7 @@ class Node(ty.Generic[OutputType]):
4142

4243
name: str
4344
_definition: "TaskDef[OutputType]"
45+
_environment: "Environment | None" = None
4446
_workflow: "Workflow" = attrs.field(default=None, eq=False, hash=False)
4547
_lzout: OutputType | None = attrs.field(
4648
init=False, default=None, eq=False, hash=False

‎pydra/engine/submitter.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -541,13 +541,14 @@ def all_failed(self) -> bool:
541541
)
542542

543543
def _generate_tasks(self) -> ty.Iterable["Task[DefType]"]:
544-
if self.node.state is None:
544+
if not self.node.state:
545545
yield Task(
546546
definition=self.node._definition._resolve_lazy_inputs(
547547
workflow_inputs=self.workflow_inputs,
548548
graph=self.graph,
549549
),
550550
submitter=self.submitter,
551+
environment=self.node._environment,
551552
name=self.node.name,
552553
)
553554
else:
@@ -559,6 +560,7 @@ def _generate_tasks(self) -> ty.Iterable["Task[DefType]"]:
559560
state_index=index,
560561
),
561562
submitter=self.submitter,
563+
environment=self.node._environment,
562564
name=self.node.name,
563565
state_index=index,
564566
)

0 commit comments

Comments
 (0)