Skip to content

Commit 5cbb4df

Browse files
tclosetclose
committed
added section on workers
1 parent 9f7aa2b commit 5cbb4df

File tree

1 file changed

+45
-3
lines changed

1 file changed

+45
-3
lines changed

‎new-docs/source/tutorial/2-advanced-execution.ipynb‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,52 @@
2424
"\n",
2525
"- `ConcurrentFutures`\n",
2626
"- `SLURM`\n",
27+
"- `SGE`\n",
2728
"- `Dask` (experimental)\n",
2829
"- `Serial` (for debugging)\n",
2930
"\n",
30-
"Work in progress..."
31+
"By default, the *ConcurrentFutures* worker (abbreviated to \"cf\") will be used, which\n",
32+
"divides tasks across multiple processes. If you are using a high-performance cluster (HPC)\n",
33+
"then the [SLURM](https://slurm.schedmd.com/documentation.html) and\n",
34+
"[SGE](https://www.metagenomics.wiki/tools/hpc-sge) workers can be used to submit each\n",
35+
"workflow node as separate jobs to the HPC scheduler. When using a graphical debugger to\n",
36+
"debug workflow or Python tasks, the *Serial* worker is recommended. There is also an\n",
37+
"experimental [Dask](https://www.dask.org/) worker.\n",
38+
"\n",
39+
"To specify a worker, the abbreviation can be passed either as a string or using the\n",
40+
"class itself"
41+
]
42+
},
43+
{
44+
"cell_type": "code",
45+
"execution_count": 1,
46+
"metadata": {},
47+
"outputs": [
48+
{
49+
"ename": "TypeError",
50+
"evalue": "'typing.TypeVar' object is not subscriptable",
51+
"output_type": "error",
52+
"traceback": [
53+
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
54+
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
55+
"Cell \u001b[0;32mIn[1], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpydra\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdesign\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m python\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;129;43m@python\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdefine\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;43;01mdef\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;21;43mPower10\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mn\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m>\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m:\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mreturn\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\n\u001b[1;32m 7\u001b[0m pow10 \u001b[38;5;241m=\u001b[39m Power10()\u001b[38;5;241m.\u001b[39msplit(n\u001b[38;5;241m=\u001b[39m[\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m3\u001b[39m, \u001b[38;5;241m4\u001b[39m, \u001b[38;5;241m5\u001b[39m])\n",
56+
"File \u001b[0;32m~/git/workflows/pydra/pydra/design/python.py:124\u001b[0m, in \u001b[0;36mdefine\u001b[0;34m(wrapped, inputs, outputs, bases, outputs_bases, auto_attribs)\u001b[0m\n\u001b[1;32m 92\u001b[0m \u001b[38;5;129m@dataclass_transform\u001b[39m(\n\u001b[1;32m 93\u001b[0m kw_only_default\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 94\u001b[0m field_specifiers\u001b[38;5;241m=\u001b[39m(arg,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 103\u001b[0m auto_attribs: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 104\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPythonDef\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 105\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 106\u001b[0m \u001b[38;5;124;03m Create an interface for a function or a class.\u001b[39;00m\n\u001b[1;32m 107\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 122\u001b[0m \u001b[38;5;124;03m The task definition class for the Python function\u001b[39;00m\n\u001b[1;32m 123\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 124\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpydra\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mengine\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mspecs\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m PythonDef, PythonOutputs\n\u001b[1;32m 126\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmake\u001b[39m(wrapped: ty\u001b[38;5;241m.\u001b[39mCallable \u001b[38;5;241m|\u001b[39m \u001b[38;5;28mtype\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m PythonDef:\n\u001b[1;32m 127\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m inspect\u001b[38;5;241m.\u001b[39misclass(wrapped):\n",
57+
"File \u001b[0;32m~/git/workflows/pydra/pydra/engine/specs.py:1020\u001b[0m\n\u001b[1;32m 1010\u001b[0m argstr_formatted \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 1011\u001b[0m argstr_formatted\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m[ \u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m[\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 1012\u001b[0m \u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m ]\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m]\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1015\u001b[0m \u001b[38;5;241m.\u001b[39mstrip()\n\u001b[1;32m 1016\u001b[0m )\n\u001b[1;32m 1017\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m argstr_formatted\n\u001b[0;32m-> 1020\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpydra\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mengine\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m core \u001b[38;5;66;03m# noqa: E402\u001b[39;00m\n",
58+
"File \u001b[0;32m~/git/workflows/pydra/pydra/engine/core.py:60\u001b[0m\n\u001b[1;32m 56\u001b[0m DefType \u001b[38;5;241m=\u001b[39m ty\u001b[38;5;241m.\u001b[39mTypeVar(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDefType\u001b[39m\u001b[38;5;124m\"\u001b[39m, bound\u001b[38;5;241m=\u001b[39mTaskDef)\n\u001b[1;32m 57\u001b[0m OutputsType \u001b[38;5;241m=\u001b[39m ty\u001b[38;5;241m.\u001b[39mTypeVar(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOutputType\u001b[39m\u001b[38;5;124m\"\u001b[39m, bound\u001b[38;5;241m=\u001b[39mTaskOutputs)\n\u001b[0;32m---> 60\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mTask\u001b[39;00m(ty\u001b[38;5;241m.\u001b[39mGeneric[\u001b[43mDefType\u001b[49m\u001b[43m[\u001b[49m\u001b[43mOutputsType\u001b[49m\u001b[43m]\u001b[49m]):\n\u001b[1;32m 61\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;124;03m A base structure for the nodes in the processing graph.\u001b[39;00m\n\u001b[1;32m 63\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 66\u001b[0m \n\u001b[1;32m 67\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[1;32m 69\u001b[0m _api_version: \u001b[38;5;28mstr\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m0.0.1\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;66;03m# Should generally not be touched by subclasses\u001b[39;00m\n",
59+
"\u001b[0;31mTypeError\u001b[0m: 'typing.TypeVar' object is not subscriptable"
60+
]
61+
}
62+
],
63+
"source": [
64+
"from pydra.design import python\n",
65+
"\n",
66+
"@python.define\n",
67+
"def Power10(n: int) -> int:\n",
68+
" return 10 ** n\n",
69+
"\n",
70+
"pow10 = Power10().split(n=[1, 2, 3, 4, 5])\n",
71+
"\n",
72+
"p1, p2, p3, p4, p5 = pow10(worker=\"serial\")"
3173
]
3274
},
3375
{
@@ -68,7 +110,7 @@
68110
"source": [
69111
"from pathlib import Path\n",
70112
"import tempfile\n",
71-
"from fileformats.medimage import Nifti\n",
113+
"from fileformats.medimage import Nifti1\n",
72114
"from pydra.engine.submitter import Submitter\n",
73115
"from pydra.tasks.mrtrix3.v3_0 import MrGrid\n",
74116
"import nest_asyncio\n",
@@ -81,7 +123,7 @@
81123
"nifti_dir = test_dir / \"nifti\"\n",
82124
"nifti_dir.mkdir()\n",
83125
"for i in range(10):\n",
84-
" Nifti.sample(nifti_dir, seed=i)\n",
126+
" Nifti1.sample(nifti_dir, seed=i)\n",
85127
"\n",
86128
"VOXEL_SIZES = [0.5, 0.5, 0.5, 0.75, 0.75, 0.75, 1.0, 1.0, 1.0, 1.25]\n",
87129
"\n",

0 commit comments

Comments
 (0)