drop kwargs from Factory()

Author: mr-c

cwltool/factory.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import
 
 import os
-from typing import Callable as tCallable # pylint: disable=unused-import
-from typing import (Any, # pylint: disable=unused-import
+from typing import Callable as tCallable   # pylint: disable=unused-import
+from typing import (Any,  # pylint: disable=unused-import
                     Dict, Optional, Text, Tuple, Union)
 
 from . import load_tool
@@ -29,44 +29,33 @@ def __init__(self, t, factory):  # type: (Process, Factory) -> None
 
     def __call__(self, **kwargs):
         # type: (**Any) -> Union[Text, Dict[Text, Text]]
-        runtimeContext = self.factory.runtimeContext.copy()
-        runtimeContext.basedir = os.getcwd()
-        out, status = self.factory.executor(self.t, kwargs, runtimeContext)
+        runtime_context = self.factory.runtime_context.copy()
+        runtime_context.basedir = os.getcwd()
+        out, status = self.factory.executor(self.t, kwargs, runtime_context)
         if status != "success":
             raise WorkflowStatus(out, status)
         else:
             return out
 
 class Factory(object):
     def __init__(self,
-                 executor=None,            # type: tCallable[...,Tuple[Dict[Text,Any], Text]]
-                 loadingContext=None,      # type: LoadingContext
-                 runtimeContext=None,      # type: RuntimeContext
-                 **kwargs
+                 executor=None,        # type: tCallable[...,Tuple[Dict[Text,Any], Text]]
+                 loading_context=None,  # type: LoadingContext
+                 runtime_context=None   # type: RuntimeContext
                 ):  # type: (...) -> None
         if executor is None:
             executor = SingleJobExecutor()
         self.executor = executor
-
-        new_exec_kwargs = get_default_args()
-        new_exec_kwargs.update(kwargs)
-        new_exec_kwargs.pop("job_order")
-        new_exec_kwargs.pop("workflow")
-        new_exec_kwargs.pop("outdir")
-
-        if loadingContext is None:
-            self.loadingContext = LoadingContext(new_exec_kwargs)
-        else:
-            self.loadingContext = loadingContext
-
-        if runtimeContext is None:
-            self.runtimeContext = RuntimeContext(new_exec_kwargs)
-        else:
-            self.runtimeContext = runtimeContext
+        self.loading_context = loading_context
+        if loading_context is None:
+            self.loading_context = LoadingContext()
+        self.runtime_context = runtime_context
+        if runtime_context is None:
+            self.runtime_context = RuntimeContext()
 
     def make(self, cwl):
         """Instantiate a CWL object from a CWl document."""
-        load = load_tool.load_tool(cwl, self.loadingContext)
+        load = load_tool.load_tool(cwl, self.loading_context)
         if isinstance(load, int):
             raise Exception("Error loading tool")
         return Callable(load, self)

tests/test_examples.py

@@ -19,6 +19,7 @@
 from cwltool.errors import WorkflowException
 from cwltool.main import main
 from cwltool.utils import onWindows, subprocess
+from cwltool.context import RuntimeContext
 
 from .util import (get_data, get_windows_safe_factory, needs_docker,
                    needs_singularity, windows_needs_docker)
@@ -152,16 +153,21 @@ def test_factory_bad_outputs(self):
 
     def test_default_args(self):
         f = cwltool.factory.Factory()
-        assert f.runtimeContext.use_container is True
-        assert f.runtimeContext.on_error == "stop"
+        assert f.runtime_context.use_container is True
+        assert f.runtime_context.on_error == "stop"
 
     def test_redefined_args(self):
-        f = cwltool.factory.Factory(use_container=False, on_error="continue")
-        assert f.runtimeContext.use_container is False
-        assert f.runtimeContext.on_error == "continue"
+        runtime_context = RuntimeContext()
+        runtime_context.use_container = False
+        runtime_context.on_error = "continue"
+        f = cwltool.factory.Factory(runtime_context=runtime_context)
+        assert f.runtime_context.use_container is False
+        assert f.runtime_context.on_error == "continue"
 
     def test_partial_scatter(self):
-        f = cwltool.factory.Factory(on_error="continue")
+        runtime_context = RuntimeContext()
+        runtime_context.on_error = "continue"
+        f = cwltool.factory.Factory(runtime_context=runtime_context)
         fail = f.make(get_data("tests/wf/scatterfail.cwl"))
         try:
             fail()
@@ -173,7 +179,9 @@ def test_partial_scatter(self):
             self.fail("Should have raised WorkflowStatus")
 
     def test_partial_output(self):
-        f = cwltool.factory.Factory(on_error="continue")
+        runtime_context = RuntimeContext()
+        runtime_context.on_error = "continue"
+        f = cwltool.factory.Factory(runtime_context=runtime_context)
         fail = f.make(get_data("tests/wf/wffail.cwl"))
         try:
             fail()
@@ -248,36 +256,32 @@ def loadref(base, p):
 
         sc.sort(key=lambda k: k["basename"])
 
-        self.assertEquals([{
-            "basename": "bar.cwl",
-            "nameroot": "bar",
-            "class": "File",
-            "nameext": ".cwl",
-            "location": "file:///example/bar.cwl"
-        },
-            {
-                "basename": "data.txt",
-                "nameroot": "data",
-                "class": "File",
-                "nameext": ".txt",
-                "location": "file:///example/data.txt"
-            },
-            {
-                "basename": "data2",
-                "class": "Directory",
-                "location": "file:///example/data2",
-                "listing": [{
-                    "basename": "data3.txt",
-                    "nameroot": "data3",
-                    "class": "File",
-                    "nameext": ".txt",
-                    "location": "file:///example/data3.txt",
-                    "secondaryFiles": [{
-                        "class": "File",
-                        "basename": "data5.txt",
-                        "location": "file:///example/data5.txt",
-                        "nameext": ".txt",
-                        "nameroot": "data5"
+        self.assertEquals([
+            {"basename": "bar.cwl",
+             "nameroot": "bar",
+             "class": "File",
+             "nameext": ".cwl",
+             "location": "file:///example/bar.cwl"},
+            {"basename": "data.txt",
+             "nameroot": "data",
+             "class": "File",
+             "nameext": ".txt",
+             "location": "file:///example/data.txt"},
+            {"basename": "data2",
+             "class": "Directory",
+             "location": "file:///example/data2",
+             "listing": [
+                 {"basename": "data3.txt",
+                  "nameroot": "data3",
+                  "class": "File",
+                  "nameext": ".txt",
+                  "location": "file:///example/data3.txt",
+                  "secondaryFiles": [
+                      {"class": "File",
+                       "basename": "data5.txt",
+                       "location": "file:///example/data5.txt",
+                       "nameext": ".txt",
+                       "nameroot": "data5"
                     }]
                 }]
             }, {
@@ -313,39 +317,27 @@ def test_trick_scandeps(self):
 
 class TestDedup(unittest.TestCase):
     def test_dedup(self):
-        ex = [{
-            "class": "File",
-            "location": "file:///example/a"
-        },
-            {
-                "class": "File",
-                "location": "file:///example/a"
-            },
-            {
-                "class": "File",
-                "location": "file:///example/d"
-            },
-            {
-                "class": "Directory",
-                "location": "file:///example/c",
-                "listing": [{
-                    "class": "File",
-                    "location": "file:///example/d"
-                }]
-            }]
-
-        self.assertEquals([{
-            "class": "File",
-            "location": "file:///example/a"
-        },
-            {
-                "class": "Directory",
-                "location": "file:///example/c",
-                "listing": [{
-                    "class": "File",
-                    "location": "file:///example/d"
-                }]
-            }], cwltool.pathmapper.dedup(ex))
+        ex = [{"class": "File",
+               "location": "file:///example/a"},
+              {"class": "File",
+               "location": "file:///example/a"},
+              {"class": "File",
+               "location": "file:///example/d"},
+              {"class": "Directory",
+               "location": "file:///example/c",
+               "listing": [
+                   {"class": "File",
+                    "location": "file:///example/d"}]}]
+
+        self.assertEquals([
+            {"class": "File",
+             "location": "file:///example/a"},
+            {"class": "Directory",
+             "location": "file:///example/c",
+             "listing": [
+                 {"class": "File",
+                  "location": "file:///example/d"}]}],
+            cwltool.pathmapper.dedup(ex))
 
 
 class TestTypeCompare(unittest.TestCase):
@@ -623,11 +615,9 @@ def test_print_dot(self):
 
 class TestCmdLine(unittest.TestCase):
     def get_main_output(self, new_args):
-        process = subprocess.Popen([
-                                       sys.executable,
-                                       "-m",
-                                       "cwltool"
-                                   ] + new_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        process = subprocess.Popen(
+            [sys.executable, "-m", "cwltool"] + new_args,
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
         stdout, stderr = process.communicate()
         return process.returncode, stdout.decode(), stderr.decode()
@@ -684,15 +674,15 @@ def test_issue_740_fixed(self):
 class TestChecksum(TestCmdLine):
 
     def test_compute_checksum(self):
-        f = cwltool.factory.Factory(compute_checksum=True,
-                use_container=onWindows())
+        runtime_context = RuntimeContext()
+        runtime_context.compute_checksum = True
+        runtime_context.use_container = onWindows()
+        f = cwltool.factory.Factory(runtime_context=runtime_context)
         echo = f.make(get_data("tests/wf/cat-tool.cwl"))
-        output = echo(file1={
-                "class": "File",
-                "location": get_data("tests/wf/whale.txt")
-            },
-            reverse=False
-        )
+        output = echo(
+            file1={"class": "File",
+                   "location": get_data("tests/wf/whale.txt")},
+            reverse=False)
         self.assertEquals(output['output']["checksum"], "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376")
 
     def test_no_compute_checksum(self):
@@ -721,5 +711,6 @@ def test_singularity_workflow(self):
         self.assertIn("completed success", stderr)
         self.assertEquals(error_code, 0)
 
+
 if __name__ == '__main__':
     unittest.main()

tests/test_parallel.py

@@ -1,12 +1,8 @@
 import json
 import unittest
 
-import pytest
-
-import cwltool
-import cwltool.factory
 from cwltool.executors import MultithreadedJobExecutor
-from cwltool.utils import onWindows
+from cwltool.context import RuntimeContext
 
 from .util import get_data, get_windows_safe_factory, windows_needs_docker
 
@@ -15,12 +11,15 @@ class TestParallel(unittest.TestCase):
     @windows_needs_docker
     def test_sequential_workflow(self):
         test_file = "tests/wf/count-lines1-wf.cwl"
-        f = get_windows_safe_factory(executor=MultithreadedJobExecutor())
-        echo = f.make(get_data(test_file))
-        self.assertEqual(echo(file1= {
-                "class": "File",
-                "location": get_data("tests/wf/whale.txt")
-            }),
+        executor = MultithreadedJobExecutor()
+        runtime_context = RuntimeContext()
+        runtime_context.select_resources = executor.select_resources
+        factory = get_windows_safe_factory(
+            executor=executor, runtime_context=runtime_context)
+        echo = factory.make(get_data(test_file))
+        self.assertEqual(
+            echo(file1={"class": "File",
+                        "location": get_data("tests/wf/whale.txt")}),
             {"count_output": 16})
 
     @windows_needs_docker

tests/util.py

@@ -10,20 +10,23 @@
 
 from cwltool.factory import Factory
 from cwltool.utils import onWindows, windows_default_container_id
+from cwltool.context import RuntimeContext, LoadingContext
 
-
-def get_windows_safe_factory(**execkwargs):
+def get_windows_safe_factory(runtime_context=None,  # type: RuntimeContext
+                             loading_context=None,  # type: LoadingContext
+                             executor=None          # type: Any
+                            ):  # type: (...) -> Factory
     if onWindows():
-        makekwargs = {'find_default_container': functools.partial(
-            force_default_container, windows_default_container_id),
-                      'use_container': True}
-        execkwargs['default_container'] = windows_default_container_id
-    else:
-        makekwargs = {}
-    return Factory(makekwargs=makekwargs, **execkwargs)
+        if not runtime_context:
+            runtime_context = RuntimeContext()
+        runtime_context.find_default_container = functools.partial(
+            force_default_container, windows_default_container_id)
+        runtime_context.use_container = True
+        runtime_context.default_container = windows_default_container_id
+    return Factory(executor, loading_context, runtime_context)
 
 def force_default_container(default_container_id, builder):
-   return default_container_id
+    return default_container_id
 
 def get_data(filename):
     filename = os.path.normpath(
@@ -51,6 +54,6 @@ def get_data(filename):
                                        "system path.")
 
 windows_needs_docker = pytest.mark.skipif(
-        onWindows() and not bool(distutils.spawn.find_executable('docker')),
-        reason="Running this test on MS Windows requires the docker executable "
-        "on the system path.")
+    onWindows() and not bool(distutils.spawn.find_executable('docker')),
+    reason="Running this test on MS Windows requires the docker executable "
+    "on the system path.")

tests/wf/count-lines1-wf.cwl

@@ -1,5 +1,8 @@
 class: Workflow
 cwlVersion: v1.0
+requirements:
+  ResourceRequirement:
+    ramMin: 100
 
 inputs:
   file1:

tests/wf/scatter-wf4.cwl

@@ -3,6 +3,9 @@ cwlVersion: v1.0
 $graph:
 - id: echo
   class: CommandLineTool
+  requirements:
+    ResourceRequirement:
+      ramMin: 10
   inputs:
     echo_in1:
       type: string