Skip to content

Commit bc0cae7

Browse files
GlassOfWhiskeymr-c
GlassOfWhiskey
authored and
mr-c
committed
Reworking type system
1 parent f99751a commit bc0cae7

26 files changed

+399
-336
lines changed

‎cwltool/builder.py‎

Lines changed: 37 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
from cwl_utils import expression
1111
from cwl_utils.file_formats import check_format
12-
from cwl_utils.types import CWLObjectType, CWLOutputType
12+
from cwl_utils.types import CWLObjectType, CWLOutputType, CWLDirectoryType, CWLFileType
1313
from mypy_extensions import mypyc_attr
1414
from rdflib import Graph
1515
from ruamel.yaml.comments import CommentedMap
@@ -94,7 +94,7 @@ class Builder(HasReqsHints):
9494
def __init__(
9595
self,
9696
job: CWLObjectType,
97-
files: list[CWLObjectType],
97+
files: MutableSequence[CWLFileType | CWLDirectoryType],
9898
bindings: list[CWLObjectType],
9999
schemaDefs: MutableMapping[str, CWLObjectType],
100100
names: Names,
@@ -165,7 +165,7 @@ def build_job_script(self, commands: list[str]) -> str | None:
165165
return self.job_script_provider.build_job_script(self, commands)
166166
return None
167167

168-
def _capture_files(self, f: CWLObjectType) -> CWLObjectType:
168+
def _capture_files(self, f: CWLFileType | CWLDirectoryType) -> CWLFileType | CWLDirectoryType:
169169
self.files.append(f)
170170
return f
171171

@@ -355,13 +355,13 @@ def bind_input(
355355
)
356356
binding = {}
357357

358-
def _capture_files(f: CWLObjectType) -> CWLObjectType:
358+
def _capture_files(f: CWLFileType | CWLDirectoryType) -> CWLFileType | CWLDirectoryType:
359359
self.files.append(f)
360360
return f
361361

362362
if schema["type"] == "org.w3id.cwl.cwl.File":
363-
datum = cast(CWLObjectType, datum)
364-
self.files.append(datum)
363+
file_datum = cast(CWLFileType, datum)
364+
self.files.append(file_datum)
365365

366366
loadContents_sourceline: (
367367
None | MutableMapping[str, str | list[int]] | CWLObjectType
@@ -379,14 +379,16 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType:
379379
debug,
380380
):
381381
try:
382-
with self.fs_access.open(cast(str, datum["location"]), "rb") as f2:
383-
datum["contents"] = content_limit_respected_read(f2)
382+
with self.fs_access.open(file_datum["location"], "rb") as f2:
383+
file_datum["contents"] = content_limit_respected_read(f2)
384384
except Exception as e:
385-
raise Exception("Reading {}\n{}".format(datum["location"], e)) from e
385+
raise Exception(
386+
"Reading {}\n{}".format(file_datum["location"], e)
387+
) from e
386388

387389
if "secondaryFiles" in schema:
388-
if "secondaryFiles" not in datum:
389-
datum["secondaryFiles"] = []
390+
if "secondaryFiles" not in file_datum:
391+
file_datum["secondaryFiles"] = []
390392
sf_schema = aslist(schema["secondaryFiles"])
391393
elif not discover_secondaryFiles:
392394
sf_schema = [] # trust the inputs
@@ -395,7 +397,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType:
395397

396398
for num, sf_entry in enumerate(sf_schema):
397399
if "required" in sf_entry and sf_entry["required"] is not None:
398-
required_result = self.do_eval(sf_entry["required"], context=datum)
400+
required_result = self.do_eval(sf_entry["required"], context=file_datum)
399401
if not (isinstance(required_result, bool) or required_result is None):
400402
if sf_schema == schema["secondaryFiles"]:
401403
sf_item: Any = sf_schema[num]
@@ -417,15 +419,15 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType:
417419
if "$(" in sf_entry["pattern"] or "${" in sf_entry["pattern"]:
418420
sfpath = self.do_eval(sf_entry["pattern"], context=datum)
419421
else:
420-
sfpath = substitute(cast(str, datum["basename"]), sf_entry["pattern"])
422+
sfpath = substitute(file_datum["basename"], sf_entry["pattern"])
421423

422424
for sfname in aslist(sfpath):
423425
if not sfname:
424426
continue
425427
found = False
426428

427429
if isinstance(sfname, str):
428-
d_location = cast(str, datum["location"])
430+
d_location = file_datum["location"]
429431
if "/" in d_location:
430432
sf_location = (
431433
d_location[0 : d_location.rindex("/") + 1] + sfname
@@ -434,6 +436,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType:
434436
sf_location = d_location + sfname
435437
sfbasename = sfname
436438
elif isinstance(sfname, MutableMapping):
439+
sfname = cast(CWLFileType | CWLDirectoryType, sfname)
437440
sf_location = sfname["location"]
438441
sfbasename = sfname["basename"]
439442
else:
@@ -446,10 +449,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType:
446449
f"{type(sfname)!r} from {sf_entry['pattern']!r}."
447450
)
448451

449-
for d in cast(
450-
MutableSequence[MutableMapping[str, str]],
451-
datum["secondaryFiles"],
452-
):
452+
for d in file_datum["secondaryFiles"]:
453453
if not d.get("basename"):
454454
d["basename"] = d["location"][d["location"].rindex("/") + 1 :]
455455
if d["basename"] == sfbasename:
@@ -458,8 +458,8 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType:
458458
if not found:
459459

460460
def addsf(
461-
files: MutableSequence[CWLObjectType],
462-
newsf: CWLObjectType,
461+
files: MutableSequence[CWLFileType | CWLDirectoryType],
462+
newsf: CWLFileType | CWLDirectoryType,
463463
) -> None:
464464
for f in files:
465465
if f["location"] == newsf["location"]:
@@ -469,23 +469,19 @@ def addsf(
469469

470470
if isinstance(sfname, MutableMapping):
471471
addsf(
472-
cast(
473-
MutableSequence[CWLObjectType],
474-
datum["secondaryFiles"],
475-
),
476-
sfname,
472+
file_datum["secondaryFiles"],
473+
cast(CWLFileType | CWLDirectoryType, sfname),
477474
)
478475
elif discover_secondaryFiles and self.fs_access.exists(sf_location):
479476
addsf(
480-
cast(
481-
MutableSequence[CWLObjectType],
482-
datum["secondaryFiles"],
477+
file_datum["secondaryFiles"],
478+
CWLFileType(
479+
**{
480+
"location": sf_location,
481+
"basename": sfname,
482+
"class": "File",
483+
}
483484
),
484-
{
485-
"location": sf_location,
486-
"basename": sfname,
487-
"class": "File",
488-
},
489485
)
490486
elif sf_required:
491487
raise SourceLine(
@@ -495,12 +491,10 @@ def addsf(
495491
debug,
496492
).makeError(
497493
"Missing required secondary file '%s' from file object: %s"
498-
% (sfname, json_dumps(datum, indent=4))
494+
% (sfname, json_dumps(file_datum, indent=4))
499495
)
500496

501-
normalizeFilesDirs(
502-
cast(MutableSequence[CWLObjectType], datum["secondaryFiles"])
503-
)
497+
normalizeFilesDirs(file_datum["secondaryFiles"])
504498

505499
if "format" in schema:
506500
eval_format: Any = self.do_eval(schema["format"])
@@ -545,7 +539,7 @@ def addsf(
545539
)
546540
try:
547541
check_format(
548-
datum,
542+
file_datum,
549543
evaluated_format,
550544
self.formatgraph,
551545
)
@@ -556,21 +550,21 @@ def addsf(
556550
) from ve
557551

558552
visit_class(
559-
datum.get("secondaryFiles", []),
553+
file_datum.get("secondaryFiles", []),
560554
("File", "Directory"),
561555
self._capture_files,
562556
)
563557

564558
if schema["type"] == "org.w3id.cwl.cwl.Directory":
565-
datum = cast(CWLObjectType, datum)
559+
dir_datum = cast(CWLDirectoryType, datum)
566560
ll = schema.get("loadListing") or self.loadListing
567561
if ll and ll != "no_listing":
568562
get_listing(
569563
self.fs_access,
570-
datum,
564+
dir_datum,
571565
(ll == "deep_listing"),
572566
)
573-
self.files.append(datum)
567+
self.files.append(dir_datum)
574568

575569
if schema["type"] == "Any":
576570
visit_class(datum, ("File", "Directory"), self._capture_files)
@@ -595,9 +589,7 @@ def tostr(self, value: MutableMapping[str, str] | Any) -> str:
595589
match value:
596590
case {"class": "File" | "Directory" as class_name, **rest}:
597591
if "path" not in rest:
598-
raise WorkflowException(
599-
'{} object missing "path": {}'.format(class_name, value)
600-
)
592+
raise WorkflowException(f'{class_name} object missing "path": {value}')
601593
return str(rest["path"])
602594
case ScalarFloat():
603595
rep = RoundTripRepresenter()

‎cwltool/checker.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ def _get_type(tp: Any) -> Any:
2222

2323

2424
def check_types(
25-
srctype: SinkType,
26-
sinktype: SinkType,
25+
srctype: SinkType | None,
26+
sinktype: SinkType | None,
2727
linkMerge: str | None,
2828
valueFrom: str | None,
2929
) -> Literal["pass"] | Literal["warning"] | Literal["exception"]:
@@ -56,7 +56,7 @@ def check_types(
5656
raise WorkflowException(f"Unrecognized linkMerge enum {linkMerge!r}")
5757

5858

59-
def merge_flatten_type(src: SinkType) -> CWLOutputType:
59+
def merge_flatten_type(src: SinkType | None) -> CWLOutputType | None:
6060
"""Return the merge flattened type of the source type."""
6161
match src:
6262
case MutableSequence():
@@ -67,7 +67,9 @@ def merge_flatten_type(src: SinkType) -> CWLOutputType:
6767
return {"items": src, "type": "array"}
6868

6969

70-
def can_assign_src_to_sink(src: SinkType, sink: SinkType | None, strict: bool = False) -> bool:
70+
def can_assign_src_to_sink(
71+
src: SinkType | None, sink: SinkType | None, strict: bool = False
72+
) -> bool:
7173
"""
7274
Check for identical type specifications, ignoring extra keys like inputBinding.
7375
@@ -85,8 +87,8 @@ def can_assign_src_to_sink(src: SinkType, sink: SinkType | None, strict: bool =
8587
return False
8688
if src["type"] == "array" and sink["type"] == "array":
8789
return can_assign_src_to_sink(
88-
cast(MutableSequence[CWLOutputType], src["items"]),
89-
cast(MutableSequence[CWLOutputType], sink["items"]),
90+
cast(MutableSequence[CWLOutputType | None], src["items"]),
91+
cast(MutableSequence[CWLOutputType | None], sink["items"]),
9092
strict,
9193
)
9294
if src["type"] == "record" and sink["type"] == "record":

0 commit comments

Comments
 (0)