Merge branch 'master' into py3

Author: gijzelaerr

cwltool/draft2tool.py

@@ -61,14 +61,18 @@ def run(self, **kwargs):  # type: (**Any) -> None
                              e, exc_info=kwargs.get('debug'))
                 self.output_callback({}, "permanentFail")
 
-    def job(self, joborder, output_callback, **kwargs):
-        # type: (Dict[Text, Text], Callable[[Any, Any], Any], **Any) -> Generator[ExpressionTool.ExpressionJob, None, None]
-        builder = self._init_job(joborder, **kwargs)
+    def job(self,
+            job_order,  # type: Dict[Text, Text]
+            output_callbacks,  # type: Callable[[Any, Any], Any]
+            **kwargs  # type: Any
+            ):
+        # type: (...) -> Generator[ExpressionTool.ExpressionJob, None, None]
+        builder = self._init_job(job_order, **kwargs)
 
         j = ExpressionTool.ExpressionJob()
         j.builder = builder
         j.script = self.tool["expression"]
-        j.output_callback = output_callback
+        j.output_callback = output_callbacks
         j.requirements = self.requirements
         j.hints = self.hints
         j.outdir = None
@@ -166,8 +170,12 @@ def makePathMapper(self, reffiles, stagedir, **kwargs):
         dockerReq, _ = self.get_requirement("DockerRequirement")
         return PathMapper(reffiles, kwargs["basedir"], stagedir)
 
-    def job(self, joborder, output_callback, **kwargs):
-        # type: (Dict[Text, Text], Callable[..., Any], **Any) -> Generator[Union[CommandLineJob, CallbackJob], None, None]
+    def job(self,
+            job_order,  # type: Dict[Text, Text]
+            output_callbacks,  # type: Callable[[Any, Any], Any]
+            **kwargs  # type: Any
+            ):
+        # type: (...) -> Generator[Union[CommandLineJob, CallbackJob], None, None]
 
         jobname = uniquename(kwargs.get("name", shortname(self.tool.get("id", "job"))))
 
@@ -176,7 +184,7 @@ def job(self, joborder, output_callback, **kwargs):
             cacheargs["outdir"] = "/out"
             cacheargs["tmpdir"] = "/tmp"
             cacheargs["stagedir"] = "/stage"
-            cachebuilder = self._init_job(joborder, **cacheargs)
+            cachebuilder = self._init_job(job_order, **cacheargs)
             cachebuilder.pathmapper = PathMapper(cachebuilder.files,
                                                  kwargs["basedir"],
                                                  cachebuilder.stagedir,
@@ -223,7 +231,7 @@ def job(self, joborder, output_callback, **kwargs):
                     cachebuilder.outdir = jobcache
 
                 _logger.info("[job %s] Using cached output in %s", jobname, jobcache)
-                yield CallbackJob(self, output_callback, cachebuilder, jobcache)
+                yield CallbackJob(self, output_callbacks, cachebuilder, jobcache)
                 return
             else:
                 _logger.info("[job %s] Output of job will be cached in %s", jobname, jobcache)
@@ -232,19 +240,19 @@ def job(self, joborder, output_callback, **kwargs):
                 kwargs["outdir"] = jobcache
                 open(jobcachepending, "w").close()
 
-                def rm_pending_output_callback(output_callback, jobcachepending,
+                def rm_pending_output_callback(output_callbacks, jobcachepending,
                                                outputs, processStatus):
                     if processStatus == "success":
                         os.remove(jobcachepending)
-                    output_callback(outputs, processStatus)
+                    output_callbacks(outputs, processStatus)
 
-                output_callback = cast(
+                output_callbacks = cast(
                     Callable[..., Any],  # known bug in mypy
                     # https://github.com/python/mypy/issues/797
-                    partial(rm_pending_output_callback, output_callback,
+                    partial(rm_pending_output_callback, output_callbacks,
                             jobcachepending))
 
-        builder = self._init_job(joborder, **kwargs)
+        builder = self._init_job(job_order, **kwargs)
 
         reffiles = copy.deepcopy(builder.files)
 
@@ -266,7 +274,7 @@ def rm_pending_output_callback(output_callback, jobcachepending,
                           j.name,
                           self.tool.get("id", ""),
                           u" as part of %s" % kwargs["part_of"] if "part_of" in kwargs else "")
-            _logger.debug(u"[job %s] %s", j.name, json.dumps(joborder, indent=4))
+            _logger.debug(u"[job %s] %s", j.name, json.dumps(job_order, indent=4))
 
         builder.pathmapper = None
         make_path_mapper_kwargs = kwargs
@@ -378,7 +386,7 @@ def rm_pending_output_callback(output_callback, jobcachepending,
         j.collect_outputs = partial(
             self.collect_output_ports, self.tool["outputs"], builder,
             compute_checksum=kwargs.get("compute_checksum", True))
-        j.output_callback = output_callback
+        j.output_callback = output_callbacks
 
         yield j
 

cwltool/factory.py

@@ -34,10 +34,13 @@ def __call__(self, **kwargs):
 
 
 class Factory(object):
-    def __init__(self, makeTool=workflow.defaultMakeTool,
-                 executor=main.single_job_executor,
-                 **execkwargs):
-        # type: (tCallable[[Dict[Text, Any], Any], Process],tCallable[...,Tuple[Dict[Text,Any], Text]], **Any) -> None
+    def __init__(self,
+                 makeTool=workflow.defaultMakeTool,  # type: tCallable[[Any], Process]
+                 # should be tCallable[[Dict[Text, Any], Any], Process] ?
+                 executor=main.single_job_executor,  # type: tCallable[...,Tuple[Dict[Text,Any], Text]]
+                 **execkwargs  # type: Any
+                 ):
+        # type: (...) -> None
         self.makeTool = makeTool
         self.executor = executor
         self.execkwargs = execkwargs

cwltool/job.py

@@ -206,6 +206,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
 
         scr, _ = get_feature(self, "ShellCommandRequirement")
 
+        shouldquote = None  # type: Callable[[Any], Any]
         if scr:
             shouldquote = lambda x: False
         else:

cwltool/main.py

@@ -183,8 +183,11 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     return parser
 
 
-def single_job_executor(t, job_order_object, **kwargs):
-    # type: (Process, Dict[Text, Any], **Any) -> Tuple[Dict[Text, Any], Text]
+def single_job_executor(t,  # type: Process
+                        job_order_object,  # type: Dict[Text, Any]
+                        **kwargs  # type: Any
+                        ):
+    # type: (...) -> Tuple[Dict[Text, Any], Text]
     final_output = []
     final_status = []
 

cwltool/process.py

@@ -599,8 +599,12 @@ def visit(self, op):  # type: (Callable[[Dict[Text, Any]], None]) -> None
         op(self.tool)
 
     @abc.abstractmethod
-    def job(self, job_order, output_callbacks, **kwargs):
-        # type: (Dict[Text, Text], Callable[[Any, Any], Any], **Any) -> Generator[Any, None, None]
+    def job(self,
+            job_order,  # type: Dict[Text, Text]
+            output_callbacks,  # type: Callable[[Any, Any], Any]
+            **kwargs  # type: Any
+            ):
+        # type: (...) -> Generator[Any, None, None]
         return None
 
 

cwltool/workflow.py

@@ -22,8 +22,10 @@
 WorkflowStateItem = namedtuple('WorkflowStateItem', ['parameter', 'value'])
 
 
-def defaultMakeTool(toolpath_object, **kwargs):
-    # type: (Dict[Text, Any], **Any) -> Process
+def defaultMakeTool(toolpath_object,  # type: Dict[Text, Any]
+                    **kwargs  # type: Any
+                   ):
+    # type: (...) -> Process
     if not isinstance(toolpath_object, dict):
         raise WorkflowException(u"Not a dict: `%s`" % toolpath_object)
     if "class" in toolpath_object:
@@ -436,15 +438,19 @@ def __init__(self, toolpath_object, **kwargs):
 
         # TODO: statically validate data links instead of doing it at runtime.
 
-    def job(self, joborder, output_callback, **kwargs):
-        # type: (Dict[Text, Text], Callable[[Any, Any], Any], **Any) -> Generator
-        builder = self._init_job(joborder, **kwargs)
+    def job(self,
+            job_order,  # type: Dict[Text, Text]
+            output_callbacks,  # type: Callable[[Any, Any], Any]
+            **kwargs  # type: Any
+            ):
+        # type: (...) -> Generator[Any, None, None]
+        builder = self._init_job(job_order, **kwargs)
         wj = WorkflowJob(self, **kwargs)
         yield wj
 
         kwargs["part_of"] = u"workflow %s" % wj.name
 
-        for w in wj.job(builder.job, output_callback, **kwargs):
+        for w in wj.job(builder.job, output_callbacks, **kwargs):
             yield w
 
     def visit(self, op):
@@ -561,17 +567,23 @@ def receive_output(self, output_callback, jobout, processStatus):
                 processStatus = "permanentFail"
         output_callback(output, processStatus)
 
-    def job(self, joborder, output_callback, **kwargs):
-        # type: (Dict[Text, Any], Callable[...,Any], **Any) -> Generator
+    def job(self,
+            job_order,  # type: Dict[Text, Text]
+            output_callbacks,  # type: Callable[[Any, Any], Any]
+            **kwargs  # type: Any
+            ):
+        # type: (...) -> Generator[Any, None, None]
         for i in self.tool["inputs"]:
             p = i["id"]
             field = shortname(p)
-            joborder[field] = joborder[i["id"]]
-            del joborder[i["id"]]
+            job_order[field] = job_order[i["id"]]
+            del job_order[i["id"]]
 
         try:
-            for t in self.embedded_tool.job(joborder,
-                                            functools.partial(self.receive_output, output_callback),
+            for t in self.embedded_tool.job(job_order,
+                                            functools.partial(
+                                                self.receive_output,
+                                                output_callbacks),
                                             **kwargs):
                 yield t
         except WorkflowException:

requirements.txt

@@ -3,5 +3,5 @@ ruamel.yaml==0.13.7
 rdflib==4.2.1
 rdflib-jsonld==0.4.0
 shellescape==3.4.1
-schema-salad>=2.1.20161227155225,<3
+schema-salad>=2.1.20170208112505,<3
 typing==3.5.2.2 ; python_version>="2.7"

setup.py

@@ -47,8 +47,8 @@
           'ruamel.yaml >= 0.12.4',
           'rdflib >= 4.2.0, < 4.3.0',
           'shellescape >= 3.4.1, < 3.5',
-          'schema-salad >= 2.2.20170111180227, < 3',
-          'typing >= 3.5.2, < 3.6',
+          'schema-salad >= 2.2.20170208112505, < 3',
+          'typing >= 3.5.2, < 3.6' ,
           'six >= 1.10.0',
       ],
       test_suite='tests',

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 #envlist = py35-lint,py34-lint,py33-lint,py27-lint,py35-unit,py34-unit,py33-unit,py27-unit
-envlist = py27-lint, py27-unit, pip27-pipconflictchecker, py35-mypy, py36-lint, py35-lint, py34-lint, py33-lint
+envlist = py27-lint, py27-unit, py27-pipconflictchecker, py35-mypy, py36-lint, py35-lint, py34-lint, py33-lint
 skipsdist = True
 
 [tox:travis]
@@ -14,7 +14,7 @@ deps = -rrequirements.txt
 commands = make mypy
 whitelist_externals = make
 deps =
-  mypy-lang>=0.4.4
+  mypy>=0.470
   typed-ast
   -rrequirements.txt