ProcessGenerator: allow for pickling

cwltool/main.py

@@ -119,7 +119,6 @@ def _terminate_processes() -> None:
     continuing to execute while it kills the processes that they've
     spawned. This may occasionally lead to unexpected behaviour.
     """
-    global docker_exe
     # It's possible that another thread will spawn a new task while
     # we're executing, so it's not safe to use a for loop here.
     while processes_to_kill:

cwltool/process.py

@@ -1109,7 +1109,6 @@ def __str__(self) -> str:
 
 def uniquename(stem: str, names: Optional[set[str]] = None) -> str:
     """Construct a thread-unique name using the given stem as a prefix."""
-    global _names
     if names is None:
         names = _names
     c = 1

pyproject.toml

@@ -7,7 +7,7 @@ requires = [
     "types-psutil",
     "importlib_resources>=1.4;python_version<'3.9'",
     "ruamel.yaml>=0.16.0,<0.19",
-    "schema-salad>=8.7,<9",
+    "schema-salad>=8.9,<9",
     "cwl-utils>=0.32",
     "toml",
     "argcomplete>=1.12.0",

requirements.txt

@@ -1,7 +1,7 @@
 requests>=2.6.1
 ruamel.yaml>=0.16.0,<0.19
 rdflib>=4.2.2,<7.2
-schema-salad>=8.7,<9
+schema-salad>=8.9,<9
 prov==1.5.1
 mypy-extensions
 psutil>=5.6.6

setup.py

@@ -74,6 +74,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li
             "cwlprov/__init__.py",
             "cuda.py",  # for monkeypatch
             "run_job.py",
+            "procgenerator.py",  # for ProcessGenerator
             "cwlprov/writablebagfile.py",  # WritableBag is having issues
             "stdfsaccess.py",  # StdFsAccess needs to be subclassable
         )
@@ -150,7 +151,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li
         # https://github.com/ionrock/cachecontrol/issues/137
         "ruamel.yaml >= 0.16, < 0.19",
         "rdflib >= 4.2.2, < 7.2.0",
-        "schema-salad >= 8.7, < 9",
+        "schema-salad >= 8.9, < 9",
         "prov == 1.5.1",
         "mypy-extensions",
         "psutil >= 5.6.6",

tests/test_subclass_mypyc.py

@@ -13,11 +13,13 @@
 from cwltool.builder import Builder
 from cwltool.command_line_tool import CommandLineTool, ExpressionTool
 from cwltool.context import LoadingContext, RuntimeContext
+from cwltool.procgenerator import ProcessGenerator
 from cwltool.stdfsaccess import StdFsAccess
 from cwltool.update import INTERNAL_VERSION
 from cwltool.workflow import Workflow
 
 from .test_anon_types import snippet
+from .util import get_data
 
 
 @pytest.mark.parametrize("snippet", snippet)
@@ -52,6 +54,35 @@ def test_pickle_unpickle_workflow(snippet: CommentedMap) -> None:
     assert pickle.loads(stream)
 
 
+def test_pickle_processgenerator() -> None:
+    """We can pickle ProcessGenerator."""
+
+    pytoolgen = get_data("tests/wf/generator/pytoolgen.cwl")
+    a = ProcessGenerator(
+        CommentedMap(
+            {
+                "cwlVersion": "v1.0",
+                "id": f"file://{pytoolgen}",
+                "$namespaces": {"cwltool": "http://commonwl.org/cwltool#"},
+                "class": "cwltool:ProcessGenerator",
+                "inputs": {},
+                "outputs": {},
+                "run": {
+                    "id": f"file://{pytoolgen}#f9f617af-3088-4ade-bbf3-acd1d6f51355",
+                    "cwlVersion": "v1.0",
+                    "class": "CommandLineTool",
+                    "inputs": {},
+                    "outputs": {},
+                    "baseCommand": "echo",
+                },
+            },
+        ),
+        LoadingContext(),
+    )
+
+    assert pickle.dumps(a)
+
+
 def test_serialize_builder() -> None:
     """We can pickle Builder."""
     runtime_context = RuntimeContext()