Skip to content

Commit 92fe97c

Browse files
tclosetclose
committed
debugged shell and python task tutorias
1 parent 75d429b commit 92fe97c

File tree

10 files changed

+433
-317
lines changed

10 files changed

+433
-317
lines changed

‎docs/source/tutorial/shell.ipynb‎

Lines changed: 195 additions & 70 deletions
Large diffs are not rendered by default.

‎docs/source/tutorial/task.ipynb‎

Lines changed: 31 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
},
1010
{
1111
"cell_type": "code",
12-
"execution_count": 5,
12+
"execution_count": 8,
1313
"metadata": {},
1414
"outputs": [
1515
{
@@ -44,7 +44,7 @@
4444
},
4545
{
4646
"cell_type": "code",
47-
"execution_count": 6,
47+
"execution_count": 9,
4848
"metadata": {},
4949
"outputs": [],
5050
"source": [
@@ -65,7 +65,7 @@
6565
},
6666
{
6767
"cell_type": "code",
68-
"execution_count": 8,
68+
"execution_count": 10,
6969
"metadata": {},
7070
"outputs": [],
7171
"source": [
@@ -113,24 +113,24 @@
113113
},
114114
{
115115
"cell_type": "code",
116-
"execution_count": 18,
116+
"execution_count": 12,
117117
"metadata": {},
118118
"outputs": [
119119
{
120120
"name": "stdout",
121121
"output_type": "stream",
122122
"text": [
123-
"[arg(name='a', type=<class 'int'>, default=EMPTY, help_string='First input to be inputted', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False),\n",
124-
" arg(name='b', type=<class 'float'>, default=EMPTY, help_string='Second input', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False),\n",
125-
" arg(name='function', type=typing.Callable, default=<function SampleSpec at 0x11ad1c900>, help_string='', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False)]\n",
126-
"[out(name='c', type=<class 'float'>, default=EMPTY, help_string='Sum of a and b', requires=[], converter=None, validator=None),\n",
127-
" out(name='d', type=<class 'float'>, default=EMPTY, help_string='Product of a and b', requires=[], converter=None, validator=None)]\n"
123+
"{'a': arg(name='a', type=<class 'int'>, default=EMPTY, help_string='First input to be inputted', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False),\n",
124+
" 'b': arg(name='b', type=<class 'float'>, default=EMPTY, help_string='Second input', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False),\n",
125+
" 'function': arg(name='function', type=typing.Callable, default=<function SampleSpec at 0x10d0253a0>, help_string='', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False)}\n",
126+
"{'c': out(name='c', type=<class 'float'>, default=EMPTY, help_string='Sum of a and b', requires=[], converter=None, validator=None),\n",
127+
" 'd': out(name='d', type=<class 'float'>, default=EMPTY, help_string='Product of a and b', requires=[], converter=None, validator=None)}\n"
128128
]
129129
}
130130
],
131131
"source": [
132132
"from pprint import pprint\n",
133-
"from pydra.engine.helpers import list_fields\n",
133+
"from pydra.engine.helpers import fields_dict\n",
134134
"\n",
135135
"@python.define(outputs=[\"c\", \"d\"])\n",
136136
"def SampleSpec(a: int, b: float) -> tuple[float, float]:\n",
@@ -147,8 +147,8 @@
147147
" \"\"\"\n",
148148
" return a + b, a * b\n",
149149
"\n",
150-
"pprint(list_fields(SampleSpec))\n",
151-
"pprint(list_fields(SampleSpec.Outputs))"
150+
"pprint(fields_dict(SampleSpec))\n",
151+
"pprint(fields_dict(SampleSpec.Outputs))"
152152
]
153153
},
154154
{
@@ -160,18 +160,18 @@
160160
},
161161
{
162162
"cell_type": "code",
163-
"execution_count": 19,
163+
"execution_count": 13,
164164
"metadata": {},
165165
"outputs": [
166166
{
167167
"name": "stdout",
168168
"output_type": "stream",
169169
"text": [
170-
"[arg(name='b', type=<class 'float'>, default=2.0, help_string='Second input', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False),\n",
171-
" arg(name='a', type=<class 'int'>, default=EMPTY, help_string='First input to be inputted', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False),\n",
172-
" arg(name='function', type=typing.Callable, default=<function SampleSpec.function at 0x11ad0b600>, help_string='', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False)]\n",
173-
"[out(name='c', type=<class 'float'>, default=EMPTY, help_string='Sum of a and b', requires=[], converter=None, validator=None),\n",
174-
" out(name='d', type=<class 'float'>, default=EMPTY, help_string='Product of a and b', requires=[], converter=None, validator=None)]\n"
170+
"{'a': arg(name='a', type=<class 'int'>, default=EMPTY, help_string='First input to be inputted', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False),\n",
171+
" 'b': arg(name='b', type=<class 'float'>, default=2.0, help_string='Second input', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False),\n",
172+
" 'function': arg(name='function', type=typing.Callable, default=<function SampleSpec.function at 0x10d024040>, help_string='', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False)}\n",
173+
"{'c': out(name='c', type=<class 'float'>, default=EMPTY, help_string='Sum of a and b', requires=[], converter=None, validator=None),\n",
174+
" 'd': out(name='d', type=<class 'float'>, default=EMPTY, help_string='Product of a and b', requires=[], converter=None, validator=None)}\n"
175175
]
176176
}
177177
],
@@ -204,8 +204,8 @@
204204
" def function(a, b):\n",
205205
" return a + b, a * b\n",
206206
"\n",
207-
"pprint(list_fields(SampleSpec))\n",
208-
"pprint(list_fields(SampleSpec.Outputs))"
207+
"pprint(fields_dict(SampleSpec))\n",
208+
"pprint(fields_dict(SampleSpec.Outputs))"
209209
]
210210
},
211211
{
@@ -217,23 +217,23 @@
217217
},
218218
{
219219
"cell_type": "code",
220-
"execution_count": 20,
220+
"execution_count": 14,
221221
"metadata": {},
222222
"outputs": [
223223
{
224224
"name": "stdout",
225225
"output_type": "stream",
226226
"text": [
227-
"[arg(name='a', type=<class 'int'>, default=EMPTY, help_string='First input to be inputted', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False),\n",
228-
" arg(name='b', type=<class 'float'>, default=EMPTY, help_string='Second input', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False),\n",
229-
" arg(name='function', type=typing.Callable, default=<function SampleSpec.function at 0x11ad1d300>, help_string='', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False)]\n",
230-
"[out(name='c', type=<class 'float'>, default=EMPTY, help_string='Sum of a and b', requires=[], converter=None, validator=None),\n",
231-
" out(name='d', type=<class 'float'>, default=EMPTY, help_string='Product of a and b', requires=[], converter=None, validator=None)]\n"
227+
"{'a': arg(name='a', type=<class 'int'>, default=EMPTY, help_string='First input to be inputted', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False),\n",
228+
" 'b': arg(name='b', type=<class 'float'>, default=EMPTY, help_string='Second input', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False),\n",
229+
" 'function': arg(name='function', type=typing.Callable, default=<function SampleSpec.function at 0x10d024180>, help_string='', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=<CopyMode.any: 15>, copy_collation=<CopyCollation.any: 0>, copy_ext_decomp=<ExtensionDecomposition.single: 1>, readonly=False)}\n",
230+
"{'c': out(name='c', type=<class 'float'>, default=EMPTY, help_string='Sum of a and b', requires=[], converter=None, validator=None),\n",
231+
" 'd': out(name='d', type=<class 'float'>, default=EMPTY, help_string='Product of a and b', requires=[], converter=None, validator=None)}\n"
232232
]
233233
}
234234
],
235235
"source": [
236-
"from pydra.engine.specs import PythonSpec\n",
236+
"from pydra.engine.specs import PythonSpec, PythonOutputs\n",
237237
"\n",
238238
"@python.define\n",
239239
"class SampleSpec(PythonSpec[\"SampleSpec.Outputs\"]):\n",
@@ -248,7 +248,8 @@
248248
" a: int\n",
249249
" b: float\n",
250250
"\n",
251-
" class Outputs:\n",
251+
" @python.outputs\n",
252+
" class Outputs(PythonOutputs):\n",
252253
" \"\"\"\n",
253254
" Args:\n",
254255
" c: Sum of a and b\n",
@@ -262,8 +263,8 @@
262263
" def function(a, b):\n",
263264
" return a + b, a * b\n",
264265
"\n",
265-
"pprint(list_fields(SampleSpec))\n",
266-
"pprint(list_fields(SampleSpec.Outputs))"
266+
"pprint(fields_dict(SampleSpec))\n",
267+
"pprint(fields_dict(SampleSpec.Outputs))"
267268
]
268269
},
269270
{

‎pydra/design/base.py‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -618,12 +618,32 @@ def ensure_field_objects(
618618
out.name = output_name
619619
if not out.help_string:
620620
out.help_string = output_helps.get(output_name, "")
621-
else:
621+
elif inspect.isclass(out):
622622
outputs[output_name] = out_type(
623623
type=out,
624624
name=output_name,
625625
help_string=output_helps.get(output_name, ""),
626626
)
627+
elif isinstance(out, dict):
628+
out_kwds = copy(out)
629+
if "help_string" not in out_kwds:
630+
out_kwds["help_string"] = output_helps.get(output_name, "")
631+
outputs[output_name] = out_type(
632+
name=output_name,
633+
**out_kwds,
634+
)
635+
elif isinstance(out, ty.Callable) and hasattr(out_type, "callable"):
636+
outputs[output_name] = out_type(
637+
name=output_name,
638+
type=ty.get_type_hints(out).get("return", ty.Any),
639+
callable=out,
640+
help_string=re.split(r"\n\s*\n", out.__doc__)[0] if out.__doc__ else "",
641+
)
642+
else:
643+
raise ValueError(
644+
f"Unrecognised value provided to outputs ({arg}), can be either {out_type} "
645+
"type" + (" or callable" if hasattr(out_type, "callable") else "")
646+
)
627647

628648
return inputs, outputs
629649

‎pydra/design/shell.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,11 @@ class out(Out):
128128

129129
callable: ty.Callable | None = None
130130

131+
def __attrs_post_init__(self):
132+
# Set type from return annotation of callable if not set
133+
if self.type is ty.Any and self.callable:
134+
self.type = ty.get_type_hints(self.callable).get("return", ty.Any)
135+
131136

132137
@attrs.define(kw_only=True)
133138
class outarg(Out, arg):
@@ -474,7 +479,7 @@ def parse_command_line_template(
474479
outputs = {}
475480
parts = template.split()
476481
executable = []
477-
for i, part in enumerate(parts):
482+
for i, part in enumerate(parts, start=1):
478483
if part.startswith("<") or part.startswith("-"):
479484
break
480485
executable.append(part)
@@ -484,7 +489,7 @@ def parse_command_line_template(
484489
executable = executable[0]
485490
if i == len(parts):
486491
return executable, inputs, outputs
487-
args_str = " ".join(parts[i:])
492+
args_str = " ".join(parts[i - 1 :])
488493
tokens = re.split(r"\s+", args_str.strip())
489494
arg_pattern = r"<([:a-zA-Z0-9_,\|\-\.\/\+]+(?:\?|=[^>]+)?)>"
490495
opt_pattern = r"--?[a-zA-Z0-9_]+"

‎pydra/engine/core.py‎

Lines changed: 20 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
ensure_list,
3535
record_error,
3636
PydraFileLock,
37-
parse_copyfile,
37+
list_fields,
3838
is_lazy,
3939
)
4040
from pydra.utils.hash import hash_function
@@ -80,6 +80,8 @@ class Task:
8080
name: str
8181
spec: TaskSpec
8282

83+
_inputs: dict[str, ty.Any] | None = None
84+
8385
def __init__(
8486
self,
8587
spec,
@@ -264,27 +266,7 @@ def output_names(self):
264266

265267
@property
266268
def generated_output_names(self):
267-
"""Get the names of the outputs generated by the task.
268-
If the spec doesn't have generated_output_names method,
269-
it uses output_names.
270-
The results depends on the input provided to the task
271-
"""
272-
output_klass = self.spec.Outputs
273-
if hasattr(output_klass, "_generated_output_names"):
274-
output = output_klass(
275-
**{f.name: attr.NOTHING for f in attr.fields(output_klass)}
276-
)
277-
# using updated input (after filing the templates)
278-
_inputs = deepcopy(self.spec)
279-
modified_inputs = template_update(_inputs, self.output_dir)
280-
if modified_inputs:
281-
_inputs = attr.evolve(_inputs, **modified_inputs)
282-
283-
return output._generated_output_names(
284-
inputs=_inputs, output_dir=self.output_dir
285-
)
286-
else:
287-
return self.output_names
269+
return self.output_names
288270

289271
@property
290272
def can_resume(self):
@@ -372,8 +354,10 @@ def __call__(
372354
res = self._run(rerun=rerun, environment=environment, **kwargs)
373355
return res
374356

375-
def _modify_inputs(self):
376-
"""This method modifies the inputs of the task ahead of its execution:
357+
@property
358+
def inputs(self) -> dict[str, ty.Any]:
359+
"""Resolve any template inputs of the task ahead of its execution:
360+
377361
- links/copies upstream files and directories into the destination tasks
378362
working directory as required select state array values corresponding to
379363
state index (it will try to leave them where they are unless specified or
@@ -383,46 +367,34 @@ def _modify_inputs(self):
383367
execution (they will be replaced after the task's execution with the
384368
original inputs to ensure the tasks checksums are consistent)
385369
"""
370+
if self._inputs is not None:
371+
return self._inputs
372+
386373
from pydra.utils.typing import TypeParser
387374

388-
orig_inputs = {
375+
self._inputs = {
389376
k: v for k, v in attrs_values(self.spec).items() if not k.startswith("_")
390377
}
391378
map_copyfiles = {}
392-
input_fields = attr.fields(type(self.spec))
393-
for name, value in orig_inputs.items():
394-
fld = getattr(input_fields, name)
395-
copy_mode, copy_collation = parse_copyfile(
396-
fld, default_collation=self.DEFAULT_COPY_COLLATION
397-
)
379+
for fld in list_fields(self.spec):
380+
name = fld.name
381+
value = self._inputs[name]
398382
if value is not attr.NOTHING and TypeParser.contains_type(
399383
FileSet, fld.type
400384
):
401385
copied_value = copy_nested_files(
402386
value=value,
403387
dest_dir=self.output_dir,
404-
mode=copy_mode,
405-
collation=copy_collation,
388+
mode=fld.copy_mode,
389+
collation=fld.copy_collation,
406390
supported_modes=self.SUPPORTED_COPY_MODES,
407391
)
408392
if value is not copied_value:
409393
map_copyfiles[name] = copied_value
410-
modified_inputs = template_update(
411-
self.spec, self.output_dir, map_copyfiles=map_copyfiles
412-
)
413-
assert all(m in orig_inputs for m in modified_inputs), (
414-
"Modified inputs contain fields not present in original inputs. "
415-
"This is likely a bug."
394+
self._inputs.update(
395+
template_update(self.spec, self.output_dir, map_copyfiles=map_copyfiles)
416396
)
417-
for name, orig_value in orig_inputs.items():
418-
try:
419-
value = modified_inputs[name]
420-
except KeyError:
421-
# Ensure we pass a copy not the original just in case inner
422-
# attributes are modified during execution
423-
value = deepcopy(orig_value)
424-
setattr(self.spec, name, value)
425-
return orig_inputs
397+
return self._inputs
426398

427399
def _populate_filesystem(self, checksum, output_dir):
428400
"""

‎pydra/engine/helpers.py‎

Lines changed: 5 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@ def list_fields(spec: "type[TaskSpec] | TaskSpec") -> list["Field"]:
4848
]
4949

5050

51+
def fields_dict(spec: "type[TaskSpec] | TaskSpec") -> dict[str, "Field"]:
52+
"""Returns the fields of a spec in a dictionary"""
53+
return {f.name: f for f in list_fields(spec)}
54+
55+
5156
# from .specs import MultiInputFile, MultiInputObj, MultiOutputObj, MultiOutputFile
5257

5358

@@ -529,41 +534,6 @@ async def __aexit__(self, exc_type, exc_value, traceback):
529534
return None
530535

531536

532-
def parse_copyfile(fld: attrs.Attribute, default_collation=FileSet.CopyCollation.any):
533-
"""Gets the copy mode from the 'copyfile' value from a field attribute"""
534-
copyfile = fld.metadata.get("copyfile", FileSet.CopyMode.any)
535-
if isinstance(copyfile, tuple):
536-
mode, collation = copyfile
537-
elif isinstance(copyfile, str):
538-
try:
539-
mode, collation = copyfile.split(",")
540-
except ValueError:
541-
mode = copyfile
542-
collation = default_collation
543-
else:
544-
collation = FileSet.CopyCollation[collation]
545-
mode = FileSet.CopyMode[mode]
546-
else:
547-
if copyfile is True:
548-
mode = FileSet.CopyMode.copy
549-
elif copyfile is False:
550-
mode = FileSet.CopyMode.link
551-
elif copyfile is None:
552-
mode = FileSet.CopyMode.any
553-
else:
554-
mode = copyfile
555-
collation = default_collation
556-
if not isinstance(mode, FileSet.CopyMode):
557-
raise TypeError(
558-
f"Unrecognised type for mode copyfile metadata of {fld}, {mode}"
559-
)
560-
if not isinstance(collation, FileSet.CopyCollation):
561-
raise TypeError(
562-
f"Unrecognised type for collation copyfile metadata of {fld}, {collation}"
563-
)
564-
return mode, collation
565-
566-
567537
def parse_format_string(fmtstr):
568538
"""Parse a argstr format string and return all keywords used in it."""
569539
identifier = r"[a-zA-Z_]\w*"

0 commit comments

Comments
 (0)