Skip to content

Commit bb1078d

Browse files
adi611adi611
committed
update psijworker
2 parents 85a05f6 + 99ab49a commit bb1078d

15 files changed

+292
-85
lines changed

‎.github/workflows/publish.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ jobs:
1212
runs-on: ubuntu-latest
1313

1414
steps:
15-
- uses: actions/checkout@v3
15+
- uses: actions/checkout@v4
1616
- name: Set up Python
1717
uses: actions/setup-python@v4
1818
with:

‎.github/workflows/release.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
runs-on: ubuntu-latest
1616
if: "!contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.head_commit.message, 'skip ci')"
1717
steps:
18-
- uses: actions/checkout@v3
18+
- uses: actions/checkout@v4
1919

2020
- name: Prepare repository
2121
# Fetch full git history and tags

‎.github/workflows/testdask.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424

2525
steps:
2626
- name: Checkout repository
27-
uses: actions/checkout@v3
27+
uses: actions/checkout@v4
2828
with:
2929
repository: ${{ github.repository }}
3030

‎.github/workflows/testpydra.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
build:
2323
runs-on: ubuntu-latest
2424
steps:
25-
- uses: actions/checkout@v3
25+
- uses: actions/checkout@v4
2626
with:
2727
fetch-depth: 0
2828
- uses: actions/setup-python@v4
@@ -80,7 +80,7 @@ jobs:
8080
name: archive
8181
path: archive/
8282
- name: Fetch repository
83-
uses: actions/checkout@v3
83+
uses: actions/checkout@v4
8484
if: matrix.install == 'repo'
8585

8686
- name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}

‎.github/workflows/testsingularity.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
echo "RELEASE_VERSION=v3.7.1" >> $GITHUB_ENV
2222
echo "NO_ET=TRUE" >> $GITHUB_ENV
2323
- name: Setup Singularity
24-
uses: actions/checkout@v3
24+
uses: actions/checkout@v4
2525
with:
2626
repository: hpcng/singularity
2727
ref: 'v3.7.1'
@@ -57,7 +57,7 @@ jobs:
5757

5858

5959
- name: Checkout Pydra repo
60-
uses: actions/checkout@v3
60+
uses: actions/checkout@v4
6161
with:
6262
repository: ${{ github.repository }}
6363
- name: Install pydra (test)

‎.github/workflows/testslurm.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
steps:
1616
- name: Disable etelemetry
1717
run: echo "NO_ET=TRUE" >> $GITHUB_ENV
18-
- uses: actions/checkout@v3
18+
- uses: actions/checkout@v4
1919
- name: Pull docker image
2020
run: |
2121
docker pull $DOCKER_IMAGE

‎pydra/engine/helpers.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,8 @@ def make_klass(spec):
261261
type=tp,
262262
**kwargs,
263263
)
264-
type_checker = TypeParser[newfield.type](newfield.type)
264+
checker_label = f"'{name}' field of {spec.name}"
265+
type_checker = TypeParser[newfield.type](newfield.type, label=checker_label)
265266
if newfield.type in (MultiInputObj, MultiInputFile):
266267
converter = attr.converters.pipe(ensure_list, type_checker)
267268
elif newfield.type in (MultiOutputObj, MultiOutputFile):
@@ -652,7 +653,11 @@ def argstr_formatting(argstr, inputs, value_updates=None):
652653
for fld in inp_fields:
653654
fld_name = fld[1:-1] # extracting the name form {field_name}
654655
fld_value = inputs_dict[fld_name]
655-
if fld_value is attr.NOTHING:
656+
fld_attr = getattr(attrs.fields(type(inputs)), fld_name)
657+
if fld_value is attr.NOTHING or (
658+
fld_value is False
659+
and TypeParser.matches_type(fld_attr.type, ty.Union[Path, bool])
660+
):
656661
# if value is NOTHING, nothing should be added to the command
657662
val_dict[fld_name] = ""
658663
else:

‎pydra/engine/helpers_file.py

Lines changed: 34 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ def template_update(inputs, output_dir, state_ind=None, map_copyfiles=None):
120120
field
121121
for field in attr_fields(inputs)
122122
if field.metadata.get("output_file_template")
123+
and getattr(inputs, field.name) is not False
123124
and all(
124125
getattr(inputs, required_field) is not attr.NOTHING
125126
for required_field in field.metadata.get("requires", ())
@@ -150,25 +151,19 @@ def template_update_single(
150151
# if input_dict_st with state specific value is not available,
151152
# the dictionary will be created from inputs object
152153
from ..utils.typing import TypeParser # noqa
153-
from pydra.engine.specs import LazyField
154-
155-
VALID_TYPES = (str, ty.Union[str, bool], Path, ty.Union[Path, bool], LazyField)
154+
from pydra.engine.specs import LazyField, OUTPUT_TEMPLATE_TYPES
156155

157156
if inputs_dict_st is None:
158157
inputs_dict_st = attr.asdict(inputs, recurse=False)
159158

160159
if spec_type == "input":
161160
inp_val_set = inputs_dict_st[field.name]
162-
if inp_val_set is not attr.NOTHING and not TypeParser.is_instance(
163-
inp_val_set, VALID_TYPES
164-
):
165-
raise TypeError(
166-
f"'{field.name}' field has to be a Path instance or a bool, but {inp_val_set} set"
167-
)
168161
if isinstance(inp_val_set, bool) and field.type in (Path, str):
169162
raise TypeError(
170163
f"type of '{field.name}' is Path, consider using Union[Path, bool]"
171164
)
165+
if inp_val_set is not attr.NOTHING and not isinstance(inp_val_set, LazyField):
166+
inp_val_set = TypeParser(ty.Union[OUTPUT_TEMPLATE_TYPES])(inp_val_set)
172167
elif spec_type == "output":
173168
if not TypeParser.contains_type(FileSet, field.type):
174169
raise TypeError(
@@ -178,22 +173,23 @@ def template_update_single(
178173
else:
179174
raise TypeError(f"spec_type can be input or output, but {spec_type} provided")
180175
# for inputs that the value is set (so the template is ignored)
181-
if spec_type == "input" and isinstance(inputs_dict_st[field.name], (str, Path)):
182-
return inputs_dict_st[field.name]
183-
elif spec_type == "input" and inputs_dict_st[field.name] is False:
184-
# if input fld is set to False, the fld shouldn't be used (setting NOTHING)
185-
return attr.NOTHING
186-
else: # inputs_dict[field.name] is True or spec_type is output
187-
value = _template_formatting(field, inputs, inputs_dict_st)
188-
# changing path so it is in the output_dir
189-
if output_dir and value is not attr.NOTHING:
190-
# should be converted to str, it is also used for input fields that should be str
191-
if type(value) is list:
192-
return [str(output_dir / Path(val).name) for val in value]
193-
else:
194-
return str(output_dir / Path(value).name)
195-
else:
176+
if spec_type == "input":
177+
if isinstance(inp_val_set, (Path, list)):
178+
return inp_val_set
179+
if inp_val_set is False:
180+
# if input fld is set to False, the fld shouldn't be used (setting NOTHING)
196181
return attr.NOTHING
182+
# inputs_dict[field.name] is True or spec_type is output
183+
value = _template_formatting(field, inputs, inputs_dict_st)
184+
# changing path so it is in the output_dir
185+
if output_dir and value is not attr.NOTHING:
186+
# should be converted to str, it is also used for input fields that should be str
187+
if type(value) is list:
188+
return [str(output_dir / Path(val).name) for val in value]
189+
else:
190+
return str(output_dir / Path(value).name)
191+
else:
192+
return attr.NOTHING
197193

198194

199195
def _template_formatting(field, inputs, inputs_dict_st):
@@ -204,16 +200,27 @@ def _template_formatting(field, inputs, inputs_dict_st):
204200
Allowing for multiple input values used in the template as longs as
205201
there is no more than one file (i.e. File, PathLike or string with extensions)
206202
"""
207-
from .specs import MultiInputObj, MultiOutputFile
208-
209203
# if a template is a function it has to be run first with the inputs as the only arg
210204
template = field.metadata["output_file_template"]
211205
if callable(template):
212206
template = template(inputs)
213207

214208
# as default, we assume that keep_extension is True
215-
keep_extension = field.metadata.get("keep_extension", True)
209+
if isinstance(template, (tuple, list)):
210+
formatted = [
211+
_string_template_formatting(field, t, inputs, inputs_dict_st)
212+
for t in template
213+
]
214+
else:
215+
assert isinstance(template, str)
216+
formatted = _string_template_formatting(field, template, inputs, inputs_dict_st)
217+
return formatted
218+
216219

220+
def _string_template_formatting(field, template, inputs, inputs_dict_st):
221+
from .specs import MultiInputObj, MultiOutputFile
222+
223+
keep_extension = field.metadata.get("keep_extension", True)
217224
inp_fields = re.findall(r"{\w+}", template)
218225
inp_fields_fl = re.findall(r"{\w+:[0-9.]+f}", template)
219226
inp_fields += [re.sub(":[0-9.]+f", "", el) for el in inp_fields_fl]

‎pydra/engine/run_pickled_function.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import pickle
2+
import pydra
3+
import sys
4+
5+
def run_pickled():
6+
with open('/pydra/pydra/engine/my_function.pkl', 'rb') as file:
7+
loaded_function = pickle.load(file)
8+
9+
result = loaded_function(rerun=False)
10+
11+
print(f'Result: {result}')
12+
13+
if __name__ == '__main__':
14+
run_pickled()

‎pydra/engine/run_pickled_function_2.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import pickle
2+
import pydra
3+
import sys
4+
5+
def run_pickled():
6+
with open('/pydra/pydra/engine/my_function.pkl', 'rb') as file:
7+
loaded_function = pickle.load(file)
8+
with open('/pydra/pydra/engine/taskmain.pkl', 'rb') as file:
9+
taskmain = pickle.load(file)
10+
with open('/pydra/pydra/engine/ind.pkl', 'rb') as file:
11+
ind = pickle.load(file)
12+
13+
result = loaded_function(taskmain, ind, rerun=False)
14+
15+
print(f'Result: {result}')
16+
17+
if __name__ == '__main__':
18+
run_pickled()

0 commit comments

Comments
 (0)