sync types

Author: mr-c

cwltool/builder.py

@@ -34,9 +34,9 @@ class Builder(object):
     def __init__(self):  # type: () -> None
         self.names = None  # type: avro.schema.Names
         self.schemaDefs = None  # type: Dict[str,Dict[unicode, Any]]
-        self.files = None  # type: List[Dict[str, str]]
+        self.files = None  # type: List[Dict[unicode, unicode]]
         self.fs_access = None  # type: StdFsAccess
-        self.job = None  # type: Dict[str, Any]
+        self.job = None  # type: Dict[unicode, Union[Dict[unicode, Any], List, unicode]]
         self.requirements = None  # type: List[Dict[str,Any]]
         self.outdir = None  # type: str
         self.tmpdir = None  # type: str

cwltool/cwlrdf.py

@@ -6,7 +6,7 @@
 from typing import Any, Union, Dict, IO
 
 def makerdf(workflow, wf, ctx):
-    # type: (Union[str, unicode], Dict[str,Any], Loader.ContextType) -> Graph
+    # type: (Union[str, unicode], Dict[unicode, Any], Loader.ContextType) -> Graph
     prefixes = {}
     for k,v in ctx.iteritems():
         if isinstance(v, dict):
@@ -31,7 +31,7 @@ def makerdf(workflow, wf, ctx):
     return g
 
 def printrdf(workflow, wf, ctx, sr, stdout):
-    # type: (Union[str, unicode], Dict[str, Any], Loader.ContextType, str, IO[Any]) -> None
+    # type: (Union[str, unicode], Dict[unicode, Any], Loader.ContextType, str, IO[Any]) -> None
     stdout.write(makerdf(workflow, wf, ctx).serialize(format=sr))
 
 def lastpart(uri):  # type: (Any) -> str
@@ -172,7 +172,7 @@ def dot_without_parameters(g, stdout):  # type: (Graph, IO[Any]) -> None
 
 
 def printdot(workflow, wf, ctx, stdout, include_parameters=False):
-    # type: (Union[str, unicode], Dict[str, Any], Loader.ContextType, Any, bool) -> None
+    # type: (Union[str, unicode], Dict[unicode, Any], Loader.ContextType, Any, bool) -> None
     g = makerdf(workflow, wf, ctx)
 
     stdout.write("digraph {")

cwltool/draft2tool.py

@@ -30,7 +30,7 @@
 
 class ExpressionTool(Process):
     def __init__(self, toolpath_object, **kwargs):
-        # type: (Dict[str,List[None]], **Any) -> None
+        # type: (Dict[unicode, Any], **Any) -> None
         super(ExpressionTool, self).__init__(toolpath_object, **kwargs)
 
     class ExpressionJob(object):
@@ -53,7 +53,7 @@ def run(self, **kwargs):  # type: (**Any) -> None
                 self.output_callback({}, "permanentFail")
 
     def job(self, joborder, output_callback, **kwargs):
-        # type: (Dict[str,str], str, Callable[[Any, Any], Any], **Any) -> Generator[ExpressionTool.ExpressionJob, None, None]
+        # type: (Dict[unicode, unicode], Callable[[Any, Any], Any], **Any) -> Generator[ExpressionTool.ExpressionJob, None, None]
         builder = self._init_job(joborder, **kwargs)
 
         j = ExpressionTool.ExpressionJob()
@@ -113,14 +113,14 @@ def run(self, **kwargs):
 
 class CommandLineTool(Process):
     def __init__(self, toolpath_object, **kwargs):
-        # type: (Dict[str,Any], **Any) -> None
+        # type: (Dict[unicode, Any], **Any) -> None
         super(CommandLineTool, self).__init__(toolpath_object, **kwargs)
 
     def makeJobRunner(self):  # type: () -> CommandLineJob
         return CommandLineJob()
 
     def makePathMapper(self, reffiles, **kwargs):
-        # type: (Set[str], str, **Any) -> PathMapper
+        # type: (Set[unicode], **Any) -> PathMapper
         dockerReq, _ = self.get_requirement("DockerRequirement")
         try:
             if dockerReq and kwargs.get("use_container"):
@@ -132,7 +132,7 @@ def makePathMapper(self, reffiles, **kwargs):
                 raise WorkflowException(u"Missing input file %s" % e)
 
     def job(self, joborder, output_callback, **kwargs):
-        # type: (Dict[str,str], str, Callable[..., Any], **Any) -> Generator[Union[CommandLineJob, CallbackJob], None, None]
+        # type: (Dict[unicode, unicode], Callable[..., Any], **Any) -> Generator[Union[CommandLineJob, CallbackJob], None, None]
 
         jobname = uniquename(kwargs.get("name", shortname(self.tool.get("id", "job"))))
 
@@ -149,7 +149,7 @@ def job(self, joborder, output_callback, **kwargs):
             if docker_req and kwargs.get("use_container") is not False:
                 dockerimg = docker_req.get("dockerImageId") or docker_req.get("dockerPull")
                 cmdline = ["docker", "run", dockerimg] + cmdline
-            keydict = {"cmdline": cmdline}
+            keydict = {u"cmdline": cmdline}
 
             for _,f in cachebuilder.pathmapper.items():
                 st = os.stat(f[0])
@@ -200,7 +200,7 @@ def rm_pending_output_callback(output_callback, jobcachepending,
 
         builder = self._init_job(joborder, **kwargs)
 
-        reffiles = set((f["path"] for f in builder.files))
+        reffiles = set((f[u"path"] for f in builder.files))
 
         j = self.makeJobRunner()
         j.builder = builder
@@ -292,9 +292,9 @@ def _check_adjust(f):  # type: (Dict[str,Any]) -> Dict[str,Any]
         yield j
 
     def collect_output_ports(self, ports, builder, outdir):
-        # type: (Set[Dict[str,Any]], Builder, str) -> Dict[str,Union[str,List[Any],Dict[str,Any]]]
+        # type: (Set[Dict[str,Any]], Builder, str) -> Dict[unicode, Union[unicode, List[Any], Dict[unicode, Any]]]
         try:
-            ret = {}  # type: Dict[str,Union[str,List[Any],Dict[str,Any]]]
+            ret = {}  # type: Dict[unicode, Union[unicode, List[Any], Dict[unicode, Any]]]
             custom_output = os.path.join(outdir, "cwl.output.json")
             if builder.fs_access.exists(custom_output):
                 with builder.fs_access.open(custom_output, "r") as f:
@@ -323,7 +323,7 @@ def collect_output_ports(self, ports, builder, outdir):
             raise WorkflowException("Error validating output record, " + str(e) + "\n in " + json.dumps(ret, indent=4))
 
     def collect_output(self, schema, builder, outdir):
-        # type: (Dict[str,Any], Builder, str) -> Union[Dict[str, Any], List[Union[Dict[str, Any], str]]]
+        # type: (Dict[str,Any], Builder, str) -> Union[Dict[unicode, Any], List[Union[Dict[unicode, Any], unicode]]]
         r = []  # type: List[Any]
         if "outputBinding" in schema:
             binding = schema["outputBinding"]

cwltool/expression.py

@@ -19,7 +19,7 @@ def jshead(engineConfig, rootvars):
     return u"\n".join(engineConfig + [u"var %s = %s;" % (k, json.dumps(v, indent=4)) for k, v in rootvars.items()])
 
 def exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image):
-    # type: (Dict[str,Any], Dict[str,str], List[Dict[str, Any]], str, str, Any, bool) -> sandboxjs.JSON
+    # type: (Dict[str, Any], Dict[unicode, Union[Dict, List, unicode]], List[Dict[str, Any]], str, str, Any, bool) -> sandboxjs.JSON
 
     if ex["engine"] == "https://w3id.org/cwl/cwl#JavascriptEngine":
         engineConfig = []  # type: List[unicode]
@@ -126,7 +126,7 @@ def param_interpolate(ex, obj, strip=True):
 
 def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
             context=None, pull_image=True, timeout=None):
-    # type: (Any, Dict[str,str], List[Dict[str,Any]], str, str, Dict[str, Union[int, str]], Any, bool, int) -> Any
+    # type: (Union[dict, unicode], Dict[unicode, Union[Dict, List, unicode]], List[Dict[str, Any]], str, str, Dict[str, Union[int, str]], Any, bool, int) -> Any
 
     runtime = resources.copy()
     runtime["tmpdir"] = tmpdir
@@ -140,7 +140,7 @@ def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
 
     if isinstance(ex, dict) and "engine" in ex and "script" in ex:
         return exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image)
-    if isinstance(ex, basestring):
+    if isinstance(ex, (str, unicode)):
         for r in requirements:
             if r["class"] == "InlineJavascriptRequirement":
                 return sandboxjs.interpolate(str(ex), jshead(r.get("expressionLib", []), rootvars),

cwltool/job.py

@@ -42,7 +42,7 @@ class CommandLineJob(object):
 
     def __init__(self):  # type: () -> None
         self.builder = None  # type: Builder
-        self.joborder = None  # type: Dict[str,str]
+        self.joborder = None  # type: Dict[unicode, Union[Dict[unicode, Any], List, unicode]]
         self.stdin = None  # type: str
         self.stdout = None  # type: str
         self.successCodes = None  # type: Iterable[int]

cwltool/load_tool.py

@@ -44,7 +44,7 @@ def fetch_document(argsworkflow):
 
 def validate_document(document_loader, workflowobj, uri,
                       enable_dev=False, strict=True, preprocess_only=False):
-    # type: (Loader, Dict[unicode, Any], unicode, bool, bool, bool) -> Tuple[Loader, Names, Any, Dict[str, str], unicode]
+    # type: (Loader, Dict[unicode, Any], unicode, bool, bool, bool) -> Tuple[Loader, Names, Dict[unicode, Any], Dict[unicode, Any], unicode]
     """Validate a CWL document."""
     jobobj = None
     if "cwl:tool" in workflowobj:
@@ -83,6 +83,8 @@ def validate_document(document_loader, workflowobj, uri,
 
     workflowobj["id"] = fileuri
     processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)
+    if not isinstance(processobj, dict):
+        raise validate.ValidationException("Workflow must be a dict.")
 
     if not metadata:
         metadata = {"$namespaces": processobj.get("$namespaces", {}),
@@ -99,14 +101,14 @@ def validate_document(document_loader, workflowobj, uri,
             processobj, document_loader, fileuri, enable_dev, metadata)
 
     if jobobj:
-        metadata["cwl:defaults"] = jobobj
+        metadata[u"cwl:defaults"] = jobobj
 
     return document_loader, avsc_names, processobj, metadata, uri
 
 
 def make_tool(document_loader, avsc_names, processobj, metadata, uri, makeTool,
               kwargs):
-    # type: (Loader, Names, Dict[str, Any], Dict[str, Any], unicode, Callable[..., Process], Dict[str, Any]) -> Process
+    # type: (Loader, Names, Dict[unicode, Any], Dict[unicode, Any], unicode, Callable[..., Process], Dict[str, Any]) -> Process
     """Make a Python CWL object."""
     resolveduri = document_loader.resolve_ref(uri)[0]
 
@@ -120,7 +122,7 @@ def make_tool(document_loader, avsc_names, processobj, metadata, uri, makeTool,
                     urlparse.urldefrag(i["id"])[1] for i in resolveduri
                     if "id" in i))
     else:
-        processobj = cast(Dict[str, Any], resolveduri)
+        processobj = cast(Dict[unicode, Any], resolveduri)
 
     kwargs = kwargs.copy()
     kwargs.update({

cwltool/main.py

@@ -152,7 +152,7 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
 
 
 def single_job_executor(t, job_order_object, **kwargs):
-    # type: (Process, Dict[str,Any], **Any) -> Union[str,Dict[str,str]]
+    # type: (Process, Dict[unicode, Any], **Any) -> Union[str,Dict[str,str]]
     final_output = []
     final_status = []
 
@@ -238,7 +238,7 @@ def __call__(self, parser, namespace, values, option_string=None):
 
 
 def generate_parser(toolparser, tool, namemap):
-    # type: (argparse.ArgumentParser, Process,Dict[str,str]) -> argparse.ArgumentParser
+    # type: (argparse.ArgumentParser, Process, Dict[unicode, unicode]) -> argparse.ArgumentParser
     toolparser.add_argument("job_order", nargs="?", help="Job input json file")
     namemap["job_order"] = "job_order"
 
@@ -298,8 +298,9 @@ def generate_parser(toolparser, tool, namemap):
         else:
             typekw = {}
 
-        toolparser.add_argument(flag + name, required=required,
-                help=ahelp, action=action, default=default, **typekw)
+        toolparser.add_argument(  # type: ignore
+                args=flag + name, required=required, help=ahelp, action=action,
+                default=default, **typekw)
 
     return toolparser
 
@@ -313,9 +314,9 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
         loader = Loader({})
     else:
         jobloaderctx = {
-                "path": {"@type": "@id"},
-                "format": {"@type": "@id"},
-                "id": "@id"}
+                u"path": {u"@type": u"@id"},
+                u"format": {u"@type": u"@id"},
+                u"id": u"@id"}
         jobloaderctx.update(t.metadata.get("$namespaces", {}))
         loader = Loader(jobloaderctx)
 
@@ -339,7 +340,7 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
         toolparser = None
     else:
         input_basedir = args.basedir if args.basedir else os.getcwd()
-        namemap = {}  # type: Dict[str,str]
+        namemap = {}  # type: Dict[unicode, unicode]
         toolparser = generate_parser(argparse.ArgumentParser(prog=args.workflow), t, namemap)
         if toolparser:
             if args.tool_help:
@@ -419,7 +420,7 @@ def makeRelative(u):
 
     stdout.write(json.dumps(deps, indent=4))
 
-def flatten_deps(d, files):
+def flatten_deps(d, files):  # type: (Any, Set[unicode]) -> None
     if isinstance(d, list):
         for s in d:
             flatten_deps(s, files)
@@ -428,41 +429,44 @@ def flatten_deps(d, files):
         if "secondaryFiles" in d:
             flatten_deps(d["secondaryFiles"], files)
 
-def find_run(d, runs):
+def find_run(d, runs):  # type: (Any, Set[unicode]) -> None
     if isinstance(d, list):
         for s in d:
             find_run(s, runs)
     elif isinstance(d, dict):
-        if "run" in d and isinstance(d["run"], basestring):
+        if "run" in d and isinstance(d["run"], (str, unicode)):
             runs.add(d["run"])
         for s in d.values():
             find_run(s, runs)
 
 def replace_refs(d, rewrite, stem, newstem):
+    # type: (Any, Dict[unicode, unicode], unicode, unicode) -> None
     if isinstance(d, list):
         for s,v in enumerate(d):
-            if isinstance(v, basestring) and v.startswith(stem):
+            if isinstance(v, (str, unicode)) and v.startswith(stem):
                 d[s] = newstem + v[len(stem):]
             else:
                 replace_refs(v, rewrite, stem, newstem)
     elif isinstance(d, dict):
-        if "run" in d and isinstance(d["run"], basestring):
+        if "run" in d and isinstance(d["run"], (str, unicode)):
             d["run"] = rewrite[d["run"]]
         for s,v in d.items():
-            if isinstance(v, basestring) and v.startswith(stem):
+            if isinstance(v, (str, unicode)) and v.startswith(stem):
                 d[s] = newstem + v[len(stem):]
             replace_refs(v, rewrite, stem, newstem)
 
 def print_pack(document_loader, processobj, uri, metadata):
+    # type: (Loader, Any, unicode, Dict[unicode, unicode]) -> str
     def loadref(b, u):
+        # type: (unicode, unicode) -> Union[Dict, List, unicode]
         return document_loader.resolve_ref(u, base_url=b)[0]
     deps = process.scandeps(uri, processobj,
                             set(("run",)), set(), loadref)
 
     fdeps = set((uri,))
     flatten_deps(deps, fdeps)
 
-    runs = set()
+    runs = set()  # type: Set[unicode]
     for f in fdeps:
         find_run(document_loader.idx[f], runs)
 
@@ -476,9 +480,10 @@ def loadref(b, u):
     for r in runs:
         rewrite[r] = "#" + shortname(r)
 
-    packed = {"$graph": [], "cwlVersion": metadata["cwlVersion"]}
+    packed = {"$graph": [], "cwlVersion": metadata["cwlVersion"]
+            }  # type: Dict[unicode, Any]
     for r,v in rewrite.items():
-        dc = copy.deepcopy(document_loader.idx[r])
+        dc = cast(Dict[unicode, Any], copy.deepcopy(document_loader.idx[r]))
         dc["id"] = v
         dc["name"] = v
         replace_refs(dc, rewrite, r+"/" if "#" in r else r+"#", v+"/")
@@ -508,7 +513,7 @@ def main(argsl=None,
          stderr=sys.stderr,
          versionfunc=versionstring,
          job_order_object=None):
-    # type: (List[str], argparse.namespace, Callable[..., Union[str, Dict[str, str]]], Callable[..., Process], Callable[[Dict[str, int]], Dict[str, int]], IO[Any], IO[Any], IO[Any], Callable[[], unicode], Union[int, Tuple[Dict[str, Any], str]]) -> int
+    # type: (List[str], argparse.Namespace, Callable[..., Union[str, Dict[str, str]]], Callable[..., Process], Callable[[Dict[str, int]], Dict[str, int]], IO[Any], IO[Any], IO[Any], Callable[[], unicode], Union[int, Tuple[Dict[str, Any], str]]) -> int
 
     _logger.removeHandler(defaultStreamHandler)
     stderr_handler = logging.StreamHandler(stderr)
@@ -611,14 +616,15 @@ def main(argsl=None,
 
         if args.tmp_outdir_prefix != 'tmp':
             # Use user defined temp directory (if it exists)
-            args.tmp_outdir_prefix = os.path.abspath(args.tmp_outdir_prefix)
+            setattr(args, 'tmp_outdir_prefix',
+                    os.path.abspath(args.tmp_outdir_prefix))
             if not os.path.exists(args.tmp_outdir_prefix):
                 _logger.error("Intermediate output directory prefix doesn't exist.")
                 return 1
 
         if args.tmpdir_prefix != 'tmp':
             # Use user defined prefix (if the folder exists)
-            args.tmpdir_prefix = os.path.abspath(args.tmpdir_prefix)
+            setattr(args, 'tmpdir_prefix', os.path.abspath(args.tmpdir_prefix))
             if not os.path.exists(args.tmpdir_prefix):
                 _logger.error("Temporary directory prefix doesn't exist.")
                 return 1
@@ -633,12 +639,13 @@ def main(argsl=None,
             return job_order_object
 
         if args.cachedir:
-            args.cachedir = os.path.abspath(args.cachedir)
-            args.move_outputs = False
+            setattr(args, 'cachedir', os.path.abspath(args.cachedir))
+            setattr(args, 'move_outputs', False)
 
         try:
-            args.tmp_outdir_prefix = args.cachedir if args.cachedir else args.tmp_outdir_prefix
-            args.basedir = job_order_object[1]
+            setattr(args, 'tmp_outdir_prefix',
+                    args.cachedir if args.cachedir else args.tmp_outdir_prefix)
+            setattr(args, 'basedir', job_order_object[1])
             del args.workflow
             del args.job_order
             out = executor(tool, job_order_object[0],