Merge remote-tracking branch 'refs/remotes/upstream/main' into on-error-abort

# Conflicts:
#	cwltool/context.py

Author: AlexTate

.github/workflows/ci-tests.yml

@@ -32,7 +32,7 @@ jobs:
     strategy:
       matrix:
         py-ver-major: [3]
-        py-ver-minor: [8, 9, 10, 11, 12]
+        py-ver-minor: [9, 10, 11, 12, 13]
         step: [lint, unit, bandit, mypy]
 
     env:
@@ -165,6 +165,7 @@ jobs:
     runs-on: ubuntu-22.04
 
     strategy:
+      fail-fast: false
       matrix:
         cwl-version: [v1.0, v1.1, v1.2]
         container: [docker, singularity, podman]

.gitignore

@@ -1,6 +1,8 @@
 # Generated during tests
 pytestdebug.log
 tmp/
+*.sif
+involucro
 
 # Python temps
 __pycache__/
@@ -59,4 +61,3 @@ cwltool/_version.py
 cwltool_deps
 docs/_build/
 docs/autoapi/
-

Makefile

@@ -24,7 +24,7 @@ MODULE=cwltool
 
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
-PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py) setup.py
+PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py tests/cwl-conformance/*.py) setup.py
 DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox<4' tox-pyenv auto-walrus \
 	isort wheel autoflake pyupgrade bandit -rlint-requirements.txt\
 	-rtest-requirements.txt -rmypy-requirements.txt -rdocs/requirements.txt
@@ -190,7 +190,7 @@ shellcheck: FORCE
 		cwltool-in-docker.sh
 
 pyupgrade: $(PYSOURCES)
-	pyupgrade --exit-zero-even-if-changed --py38-plus $^
+	pyupgrade --exit-zero-even-if-changed --py39-plus $^
 	auto-walrus $^
 
 release-test: FORCE

cwltool/argparser.py

@@ -3,19 +3,8 @@
 import argparse
 import os
 import urllib
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    List,
-    MutableMapping,
-    MutableSequence,
-    Optional,
-    Sequence,
-    Type,
-    Union,
-    cast,
-)
+from collections.abc import MutableMapping, MutableSequence, Sequence
+from typing import Any, Callable, Optional, Union, cast
 
 from .loghandler import _logger
 from .process import Process, shortname
@@ -719,7 +708,7 @@ def arg_parser() -> argparse.ArgumentParser:
     return parser
 
 
-def get_default_args() -> Dict[str, Any]:
+def get_default_args() -> dict[str, Any]:
     """Get default values of cwltool's command line options."""
     ap = arg_parser()
     args = ap.parse_args([])
@@ -733,7 +722,7 @@ class FSAction(argparse.Action):
 
     def __init__(
         self,
-        option_strings: List[str],
+        option_strings: list[str],
         dest: str,
         nargs: Any = None,
         urljoin: Callable[[str, str], str] = urllib.parse.urljoin,
@@ -771,7 +760,7 @@ class FSAppendAction(argparse.Action):
 
     def __init__(
         self,
-        option_strings: List[str],
+        option_strings: list[str],
         dest: str,
         nargs: Any = None,
         urljoin: Callable[[str, str], str] = urllib.parse.urljoin,
@@ -828,7 +817,7 @@ class AppendAction(argparse.Action):
 
     def __init__(
         self,
-        option_strings: List[str],
+        option_strings: list[str],
         dest: str,
         nargs: Any = None,
         **kwargs: Any,
@@ -860,7 +849,7 @@ def add_argument(
     toolparser: argparse.ArgumentParser,
     name: str,
     inptype: Any,
-    records: List[str],
+    records: list[str],
     description: str = "",
     default: Any = None,
     input_required: bool = True,
@@ -889,9 +878,9 @@ def add_argument(
             return None
 
     ahelp = description.replace("%", "%%")
-    action: Optional[Union[Type[argparse.Action], str]] = None
+    action: Optional[Union[type[argparse.Action], str]] = None
     atype: Optional[Any] = None
-    typekw: Dict[str, Any] = {}
+    typekw: dict[str, Any] = {}
 
     if inptype == "File":
         action = FileAction
@@ -904,6 +893,11 @@ def add_argument(
             action = DirectoryAppendAction
         else:
             action = AppendAction
+            items = inptype["items"]
+            if items == "int" or items == "long":
+                atype = int
+            elif items == "double" or items == "float":
+                atype = float
     elif isinstance(inptype, MutableMapping) and inptype["type"] == "enum":
         atype = str
     elif isinstance(inptype, MutableMapping) and inptype["type"] == "record":
@@ -958,8 +952,8 @@ def add_argument(
 def generate_parser(
     toolparser: argparse.ArgumentParser,
     tool: Process,
-    namemap: Dict[str, str],
-    records: List[str],
+    namemap: dict[str, str],
+    records: list[str],
     input_required: bool = True,
     urljoin: Callable[[str, str], str] = urllib.parse.urljoin,
     base_uri: str = "",

cwltool/builder.py

@@ -3,21 +3,9 @@
 import copy
 import logging
 import math
+from collections.abc import MutableMapping, MutableSequence
 from decimal import Decimal
-from typing import (
-    IO,
-    TYPE_CHECKING,
-    Any,
-    Callable,
-    Dict,
-    List,
-    MutableMapping,
-    MutableSequence,
-    Optional,
-    Type,
-    Union,
-    cast,
-)
+from typing import IO, TYPE_CHECKING, Any, Callable, Optional, Union, cast
 
 from cwl_utils import expression
 from cwl_utils.file_formats import check_format
@@ -55,7 +43,7 @@
     )
     from .pathmapper import PathMapper
 
-INPUT_OBJ_VOCAB: Dict[str, str] = {
+INPUT_OBJ_VOCAB: dict[str, str] = {
     "Any": "https://w3id.org/cwl/salad#Any",
     "File": "https://w3id.org/cwl/cwl#File",
     "Directory": "https://w3id.org/cwl/cwl#Directory",
@@ -107,16 +95,16 @@ class Builder(HasReqsHints):
     def __init__(
         self,
         job: CWLObjectType,
-        files: List[CWLObjectType],
-        bindings: List[CWLObjectType],
+        files: list[CWLObjectType],
+        bindings: list[CWLObjectType],
         schemaDefs: MutableMapping[str, CWLObjectType],
         names: Names,
-        requirements: List[CWLObjectType],
-        hints: List[CWLObjectType],
-        resources: Dict[str, Union[int, float]],
+        requirements: list[CWLObjectType],
+        hints: list[CWLObjectType],
+        resources: dict[str, Union[int, float]],
         mutation_manager: Optional[MutationManager],
         formatgraph: Optional[Graph],
-        make_fs_access: Type[StdFsAccess],
+        make_fs_access: type[StdFsAccess],
         fs_access: StdFsAccess,
         job_script_provider: Optional[DependenciesConfiguration],
         timeout: float,
@@ -172,19 +160,20 @@ def __init__(
         self.find_default_container: Optional[Callable[[], str]] = None
         self.container_engine = container_engine
 
-    def build_job_script(self, commands: List[str]) -> Optional[str]:
+    def build_job_script(self, commands: list[str]) -> Optional[str]:
+        """Use the job_script_provider to turn the commands into a job script."""
         if self.job_script_provider is not None:
             return self.job_script_provider.build_job_script(self, commands)
         return None
 
     def bind_input(
         self,
         schema: CWLObjectType,
-        datum: Union[CWLObjectType, List[CWLObjectType]],
+        datum: Union[CWLObjectType, list[CWLObjectType]],
         discover_secondaryFiles: bool,
-        lead_pos: Optional[Union[int, List[int]]] = None,
-        tail_pos: Optional[Union[str, List[int]]] = None,
-    ) -> List[MutableMapping[str, Union[str, List[int]]]]:
+        lead_pos: Optional[Union[int, list[int]]] = None,
+        tail_pos: Optional[Union[str, list[int]]] = None,
+    ) -> list[MutableMapping[str, Union[str, list[int]]]]:
         """
         Bind an input object to the command line.
 
@@ -200,8 +189,8 @@ def bind_input(
         if lead_pos is None:
             lead_pos = []
 
-        bindings: List[MutableMapping[str, Union[str, List[int]]]] = []
-        binding: Union[MutableMapping[str, Union[str, List[int]]], CommentedMap] = {}
+        bindings: list[MutableMapping[str, Union[str, list[int]]]] = []
+        binding: Union[MutableMapping[str, Union[str, list[int]]], CommentedMap] = {}
         value_from_expression = False
         if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping):
             binding = CommentedMap(schema["inputBinding"].items())
@@ -282,7 +271,7 @@ def bind_input(
                 and "itemSeparator" not in binding
             ):
                 st["inputBinding"] = {}
-            for k in ("secondaryFiles", "format", "streamable"):
+            for k in ("secondaryFiles", "format", "streamable", "loadContents"):
                 if k in schema:
                     st[k] = schema[k]
             if value_from_expression:
@@ -324,7 +313,7 @@ def bind_input(
 
             if schema["type"] == "record":
                 datum = cast(CWLObjectType, datum)
-                for f in cast(List[CWLObjectType], schema["fields"]):
+                for f in cast(list[CWLObjectType], schema["fields"]):
                     name = cast(str, f["name"])
                     if name in datum and datum[name] is not None:
                         bindings.extend(
@@ -349,7 +338,7 @@ def bind_input(
                         "type": schema["items"],
                         "inputBinding": b2,
                     }
-                    for k in ("secondaryFiles", "format", "streamable"):
+                    for k in ("secondaryFiles", "format", "streamable", "loadContents"):
                         if k in schema:
                             itemschema[k] = schema[k]
                     bindings.extend(
@@ -372,7 +361,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType:
                 self.files.append(datum)
 
                 loadContents_sourceline: Union[
-                    None, MutableMapping[str, Union[str, List[int]]], CWLObjectType
+                    None, MutableMapping[str, Union[str, list[int]]], CWLObjectType
                 ] = None
                 if binding and binding.get("loadContents"):
                     loadContents_sourceline = binding
@@ -513,7 +502,7 @@ def addsf(
                 if "format" in schema:
                     eval_format: Any = self.do_eval(schema["format"])
                     if isinstance(eval_format, str):
-                        evaluated_format: Union[str, List[str]] = eval_format
+                        evaluated_format: Union[str, list[str]] = eval_format
                     elif isinstance(eval_format, MutableSequence):
                         for index, entry in enumerate(eval_format):
                             message = None
@@ -541,7 +530,7 @@ def addsf(
                                 raise SourceLine(
                                     schema["format"], index, WorkflowException, debug
                                 ).makeError(message)
-                        evaluated_format = cast(List[str], eval_format)
+                        evaluated_format = cast(list[str], eval_format)
                     else:
                         raise SourceLine(schema, "format", WorkflowException, debug).makeError(
                             "An expression in the 'format' field must "
@@ -586,8 +575,8 @@ def addsf(
         # Position to front of the sort key
         if binding:
             for bi in bindings:
-                bi["position"] = cast(List[int], binding["position"]) + cast(
-                    List[int], bi["position"]
+                bi["position"] = cast(list[int], binding["position"]) + cast(
+                    list[int], bi["position"]
                 )
             bindings.append(binding)
 
@@ -618,7 +607,8 @@ def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str:
         else:
             return str(value)
 
-    def generate_arg(self, binding: CWLObjectType) -> List[str]:
+    def generate_arg(self, binding: CWLObjectType) -> list[str]:
+        """Convert an input binding to a list of command line arguments."""
         value = binding.get("datum")
         debug = _logger.isEnabledFor(logging.DEBUG)
         if "valueFrom" in binding:
@@ -648,7 +638,7 @@ def generate_arg(self, binding: CWLObjectType) -> List[str]:
                 argl = [itemSeparator.join([self.tostr(v) for v in value])]
             elif binding.get("valueFrom"):
                 value = [self.tostr(v) for v in value]
-                return cast(List[str], ([prefix] if prefix else [])) + cast(List[str], value)
+                return cast(list[str], ([prefix] if prefix else [])) + cast(list[str], value)
             elif prefix and value:
                 return [prefix]
             else: