Skip to content

Commit a244aa3

Browse files
mr-cmr-c
committed
fix secondaryFiles
1 parent 7af5a74 commit a244aa3

File tree

5 files changed

+192
-104
lines changed

5 files changed

+192
-104
lines changed

‎cwl_utils/cwl_v1_0_expression_refactor.py‎

Lines changed: 50 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1222,37 +1222,21 @@ def process_level_reqs(
12221222
expression = get_expression(entry.entry, inputs, None)
12231223
if expression:
12241224
modified = True
1225-
if entry.entryname is None:
1226-
raise SourceLine(
1227-
req.listing,
1228-
listing_index,
1229-
raise_type=WorkflowException,
1230-
).makeError(
1231-
"`entryname` is required: {}".format(entry)
1225+
if entry.entryname is not None:
1226+
entryname_expr = get_expression(
1227+
entry.entryname, inputs, None
12321228
)
1233-
entryname_expr = get_expression(
1234-
entry.entryname, inputs, None
1235-
)
1236-
entryname = (
1237-
entry.entryname
1238-
if entryname_expr
1239-
else '"{}"'.format(entry.entryname)
1240-
)
1241-
d_target_type = ["File", "Directory"]
1242-
target = cwl.InputParameter(
1243-
id=None,
1244-
type=d_target_type,
1245-
)
1246-
etool_id = "_expression_{}_InitialWorkDirRequirement_{}".format(
1247-
step_name, listing_index
1248-
)
1249-
1250-
new_expression = (
1251-
"${var result; var entryname = "
1252-
+ entryname
1253-
+ "; var entry = "
1254-
+ entry.entry[2:-1]
1255-
+ """;
1229+
entryname = (
1230+
entry.entryname
1231+
if entryname_expr
1232+
else '"{}"'.format(entry.entryname)
1233+
)
1234+
new_expression = (
1235+
"${var result; var entryname = "
1236+
+ entryname
1237+
+ "; var entry = "
1238+
+ entry.entry[2:-1]
1239+
+ """;
12561240
if (typeof entry === 'string' || entry instanceof String) {
12571241
result = {"class": "File", "basename": entryname, "contents": entry} ;
12581242
if (typeof entryname === 'string' || entryname instanceof String) {
@@ -1262,7 +1246,18 @@ def process_level_reqs(
12621246
result = entry ;
12631247
}
12641248
return result; }"""
1249+
)
1250+
else:
1251+
new_expression = expression
1252+
d_target_type = ["File", "Directory"]
1253+
target = cwl.InputParameter(
1254+
id=None,
1255+
type=d_target_type,
1256+
)
1257+
etool_id = "_expression_{}_InitialWorkDirRequirement_{}".format(
1258+
step_name, listing_index
12651259
)
1260+
12661261
replace_clt_hintreq_expr_with_etool(
12671262
new_expression,
12681263
etool_id,
@@ -1371,8 +1366,8 @@ def traverse_CommandLineTool(
13711366
replace_step_clt_expr_with_etool(
13721367
expression, etool_id, parent, target, step, replace_etool
13731368
)
1374-
target_clt.arguments[index].valueFrom = "$(inputs.{})".format(
1375-
inp_id
1369+
target_clt.arguments[index] = cwl.CommandLineBinding(
1370+
valueFrom="$(inputs.{})".format(inp_id)
13761371
)
13771372
target_clt.inputs.append(
13781373
cwl.CommandInputParameter(
@@ -1555,13 +1550,29 @@ def traverse_CommandLineTool(
15551550
remove_JSReq(new_clt_step.run, skip_command_line1)
15561551
for new_outp in new_clt_step.run.outputs:
15571552
if new_outp.id.split("#")[-1] == outp_id:
1558-
if new_outp.outputBinding:
1559-
new_outp.outputBinding.outputEval = None
1560-
new_outp.outputBinding.loadContents = None
1561-
new_outp.type = cwl.CommandOutputArraySchema(
1562-
items="File",
1563-
type="array",
1564-
)
1553+
if isinstance(
1554+
new_outp,
1555+
(
1556+
cwl.WorkflowOutputParameter,
1557+
cwl.ExpressionToolOutputParameter,
1558+
),
1559+
):
1560+
new_outp.type = cwl.OutputArraySchema(
1561+
items="File", type="array"
1562+
)
1563+
elif isinstance(new_outp, cwl.CommandOutputParameter):
1564+
if new_outp.outputBinding:
1565+
new_outp.outputBinding.outputEval = None
1566+
new_outp.outputBinding.loadContents = None
1567+
new_outp.type = cwl.CommandOutputArraySchema(
1568+
items="File",
1569+
type="array",
1570+
)
1571+
else:
1572+
raise Exception(
1573+
"Unimplemented OutputParamter type: %s",
1574+
type(new_outp),
1575+
)
15651576
new_clt_step.in_ = copy.deepcopy(step.in_)
15661577
for inp in new_clt_step.in_:
15671578
inp.id = inp.id.split("/")[-1]

‎cwl_utils/cwl_v1_1_expression_refactor.py‎

Lines changed: 66 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -579,12 +579,18 @@ def empty_inputs(
579579
result[param.id.split("#")[-1]] = example_input(param.type)
580580
else:
581581
for param in process_or_step.in_:
582-
try:
583-
result[param.id.split("/")[-1]] = example_input(
584-
type_for_source(process_or_step.run, param.source, parent)
585-
)
586-
except WorkflowException:
587-
pass
582+
param_id = param.id.split("/")[-1]
583+
if param.source is None and param.valueFrom:
584+
result[param_id] = example_input("string")
585+
elif param.source is None and param.default:
586+
result[param_id] = param.default
587+
else:
588+
try:
589+
result[param_id] = example_input(
590+
type_for_source(process_or_step.run, param.source, parent)
591+
)
592+
except WorkflowException:
593+
pass
588594
return result
589595

590596

@@ -726,15 +732,17 @@ def process_workflow_inputs_and_outputs(
726732
raise_type=WorkflowException,
727733
).makeError(TOPLEVEL_FORMAT_EXPR_ERROR.format(param.id.split("#")[-1]))
728734
if param.secondaryFiles:
729-
if get_expression(param.secondaryFiles, inputs, EMPTY_FILE):
735+
if hasattr(param.secondaryFiles, "pattern") and get_expression(
736+
param.secondaryFiles.pattern, inputs, EMPTY_FILE
737+
):
730738
raise SourceLine(
731739
param.loadingOptions.original_doc,
732740
"secondaryFiles",
733741
raise_type=WorkflowException,
734742
).makeError(TOPLEVEL_SF_EXPR_ERROR.format(param.id.split("#")[-1]))
735743
elif isinstance(param.secondaryFiles, MutableSequence):
736744
for index2, entry in enumerate(param.secondaryFiles):
737-
if get_expression(entry, inputs, EMPTY_FILE):
745+
if get_expression(entry.pattern, inputs, EMPTY_FILE):
738746
raise SourceLine(
739747
param.loadingOptions.original_doc,
740748
index2,
@@ -1222,37 +1230,21 @@ def process_level_reqs(
12221230
expression = get_expression(entry.entry, inputs, None)
12231231
if expression:
12241232
modified = True
1225-
if entry.entryname is None:
1226-
raise SourceLine(
1227-
req.listing,
1228-
listing_index,
1229-
raise_type=WorkflowException,
1230-
).makeError(
1231-
"`entryname` is required: {}".format(entry)
1233+
if entry.entryname is not None:
1234+
entryname_expr = get_expression(
1235+
entry.entryname, inputs, None
12321236
)
1233-
entryname_expr = get_expression(
1234-
entry.entryname, inputs, None
1235-
)
1236-
entryname = (
1237-
entry.entryname
1238-
if entryname_expr
1239-
else '"{}"'.format(entry.entryname)
1240-
)
1241-
d_target_type = ["File", "Directory"]
1242-
target = cwl.WorkflowInputParameter(
1243-
id=None,
1244-
type=d_target_type,
1245-
)
1246-
etool_id = "_expression_{}_InitialWorkDirRequirement_{}".format(
1247-
step_name, listing_index
1248-
)
1249-
1250-
new_expression = (
1251-
"${var result; var entryname = "
1252-
+ entryname
1253-
+ "; var entry = "
1254-
+ entry.entry[2:-1]
1255-
+ """;
1237+
entryname = (
1238+
entry.entryname
1239+
if entryname_expr
1240+
else '"{}"'.format(entry.entryname)
1241+
)
1242+
new_expression = (
1243+
"${var result; var entryname = "
1244+
+ entryname
1245+
+ "; var entry = "
1246+
+ entry.entry[2:-1]
1247+
+ """;
12561248
if (typeof entry === 'string' || entry instanceof String) {
12571249
result = {"class": "File", "basename": entryname, "contents": entry} ;
12581250
if (typeof entryname === 'string' || entryname instanceof String) {
@@ -1262,7 +1254,18 @@ def process_level_reqs(
12621254
result = entry ;
12631255
}
12641256
return result; }"""
1257+
)
1258+
else:
1259+
new_expression = expression
1260+
d_target_type = ["File", "Directory"]
1261+
target = cwl.WorkflowInputParameter(
1262+
id=None,
1263+
type=d_target_type,
12651264
)
1265+
etool_id = "_expression_{}_InitialWorkDirRequirement_{}".format(
1266+
step_name, listing_index
1267+
)
1268+
12661269
replace_clt_hintreq_expr_with_etool(
12671270
new_expression,
12681271
etool_id,
@@ -1371,8 +1374,8 @@ def traverse_CommandLineTool(
13711374
replace_step_clt_expr_with_etool(
13721375
expression, etool_id, parent, target, step, replace_etool
13731376
)
1374-
target_clt.arguments[index].valueFrom = "$(inputs.{})".format(
1375-
inp_id
1377+
target_clt.arguments[index] = cwl.CommandLineBinding(
1378+
valueFrom="$(inputs.{})".format(inp_id)
13761379
)
13771380
target_clt.inputs.append(
13781381
cwl.CommandInputParameter(
@@ -1555,13 +1558,29 @@ def traverse_CommandLineTool(
15551558
remove_JSReq(new_clt_step.run, skip_command_line1)
15561559
for new_outp in new_clt_step.run.outputs:
15571560
if new_outp.id.split("#")[-1] == outp_id:
1558-
if new_outp.outputBinding:
1559-
new_outp.outputBinding.outputEval = None
1560-
new_outp.outputBinding.loadContents = None
1561-
new_outp.type = cwl.CommandOutputArraySchema(
1562-
items="File",
1563-
type="array",
1564-
)
1561+
if isinstance(
1562+
new_outp,
1563+
(
1564+
cwl.WorkflowOutputParameter,
1565+
cwl.ExpressionToolOutputParameter,
1566+
),
1567+
):
1568+
new_outp.type = cwl.OutputArraySchema(
1569+
items="File", type="array"
1570+
)
1571+
elif isinstance(new_outp, cwl.CommandOutputParameter):
1572+
if new_outp.outputBinding:
1573+
new_outp.outputBinding.outputEval = None
1574+
new_outp.outputBinding.loadContents = None
1575+
new_outp.type = cwl.CommandOutputArraySchema(
1576+
items="File",
1577+
type="array",
1578+
)
1579+
else:
1580+
raise Exception(
1581+
"Unimplemented OutputParamter type: %s",
1582+
type(new_outp),
1583+
)
15651584
new_clt_step.in_ = copy.deepcopy(step.in_)
15661585
for inp in new_clt_step.in_:
15671586
inp.id = inp.id.split("/")[-1]

‎cwl_utils/cwl_v1_2_expression_refactor.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -777,15 +777,17 @@ def process_workflow_inputs_and_outputs(
777777
raise_type=WorkflowException,
778778
).makeError(TOPLEVEL_FORMAT_EXPR_ERROR.format(param.id.split("#")[-1]))
779779
if param.secondaryFiles:
780-
if get_expression(param.secondaryFiles, inputs, EMPTY_FILE):
780+
if hasattr(param.secondaryFiles, "pattern") and get_expression(
781+
param.secondaryFiles.pattern, inputs, EMPTY_FILE
782+
):
781783
raise SourceLine(
782784
param.loadingOptions.original_doc,
783785
"secondaryFiles",
784786
raise_type=WorkflowException,
785787
).makeError(TOPLEVEL_SF_EXPR_ERROR.format(param.id.split("#")[-1]))
786788
elif isinstance(param.secondaryFiles, MutableSequence):
787789
for index2, entry in enumerate(param.secondaryFiles):
788-
if get_expression(entry, inputs, EMPTY_FILE):
790+
if get_expression(entry.pattern, inputs, EMPTY_FILE):
789791
raise SourceLine(
790792
param.loadingOptions.original_doc,
791793
index2,

‎cwl_utils/parser_v1_1.py‎

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -301,25 +301,53 @@ def load(self, doc, baseuri, loadingOptions, docRoot=None):
301301

302302

303303
class _SecondaryDSLLoader(_Loader):
304-
def __init__(self, items):
304+
def __init__(self, inner):
305305
# type: (_Loader) -> None
306-
self.items = items
306+
self.inner = inner
307307

308308
def load(self, doc, baseuri, loadingOptions, docRoot=None):
309309
# type: (Any, str, LoadingOptions, Optional[str]) -> Any
310+
r: List[Dict[str, Any]] = []
310311
if isinstance(doc, MutableSequence):
311-
r = [] # type: List[Any]
312312
for d in doc:
313313
if isinstance(d, str):
314-
r.append(d)
314+
if d.endswith("?"):
315+
r.append({"pattern": d[:-1], "required": False})
316+
else:
317+
r.append({"pattern": d})
318+
elif isinstance(d, dict):
319+
new_dict: Dict[str, Any] = {}
320+
if "pattern" in d:
321+
new_dict["pattern"] = d.pop("pattern")
322+
else:
323+
raise ValidationException(
324+
"Missing pattern in secondaryFiles specification entry: {}".format(
325+
d
326+
)
327+
)
328+
new_dict["required"] = (
329+
d.pop("required") if "required" in d else None
330+
)
331+
332+
if len(d):
333+
raise ValidationException(
334+
"Unallowed values in secondaryFiles specification entry: {}".format(
335+
d
336+
)
337+
)
338+
315339
else:
316-
raise ValidationException("Expected str or sequence of str")
317-
doc = r
340+
raise ValidationException(
341+
"Expected a string or sequence of (strings or mappings)."
342+
)
318343
elif isinstance(doc, str):
319-
pass
344+
if doc.endswith("?"):
345+
r.append({"pattern": doc[:-1], "required": False})
346+
else:
347+
r.append({"pattern": doc})
320348
else:
321349
raise ValidationException("Expected str or sequence of str")
322-
return doc
350+
return self.inner.load(r, baseuri, loadingOptions, docRoot)
323351

324352

325353
class _RecordLoader(_Loader):

0 commit comments

Comments
 (0)