Merge branch 'master' into testing-windows

Author: mr-c

.travis.yml

@@ -2,17 +2,17 @@ sudo: false
 language: python
 python:
   - 2.7
-    #  - 3.4
+  - 3.3
+  - 3.4
   - 3.5
-    #  - 3.6
+  - 3.6
 os:
   - linux
-
 install:
   - pip install tox-travis
 script: tox
-notifications:
-  email: false
 branches:
   only:
     - master
+notifications:
+  email: false

Makefile

@@ -161,7 +161,7 @@ mypy2: ${PYSOURCES}
 	ln -s $(shell python -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2and3/schema_salad
 	MYPYPATH=$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
-		 --warn-redundant-casts --warn-unused-ignores \
+		 --warn-redundant-casts \
 		 cwltool
 
 mypy3: ${PYSOURCES}
@@ -172,7 +172,7 @@ mypy3: ${PYSOURCES}
 	ln -s $(shell python3 -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2and3/schema_salad
 	MYPYPATH=$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
-		 --warn-redundant-casts --warn-unused-ignores \
+		 --warn-redundant-casts \
 		 cwltool
 
 FORCE:

cwltool/builder.py

@@ -61,8 +61,8 @@ def __init__(self):  # type: () -> None
         self.job_script_provider = None  # type: Any
 
     def build_job_script(self, commands):
-        # type: (List[str]) -> Text
-        build_job_script_method = getattr(self.job_script_provider, "build_job_script", None)  # type: Callable[[Builder, List[str]], Text]
+        # type: (List[bytes]) -> Text
+        build_job_script_method = getattr(self.job_script_provider, "build_job_script", None)  # type: Callable[[Builder, List[bytes]], Text]
         if build_job_script_method:
             return build_job_script_method(self, commands)
         else:

cwltool/docker.py

@@ -5,6 +5,7 @@
 import subprocess
 import sys
 import tempfile
+from io import open
 from typing import Dict, List, Text
 
 import requests
@@ -56,7 +57,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
                 found = True
         elif "dockerFile" in dockerRequirement:
             dockerfile_dir = str(tempfile.mkdtemp())
-            with open(os.path.join(dockerfile_dir, "Dockerfile"), str("wb")) as df:
+            with open(os.path.join(dockerfile_dir, "Dockerfile"), "wb") as df:
                 df.write(dockerRequirement["dockerFile"].encode('utf-8'))
             cmd = ["docker", "build", "--tag=%s" %
                    str(dockerRequirement["dockerImageId"]), dockerfile_dir]
@@ -70,7 +71,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
             if not dry_run:
                 if os.path.exists(dockerRequirement["dockerLoad"]):
                     _logger.info(u"Loading docker image from %s", dockerRequirement["dockerLoad"])
-                    with open(dockerRequirement["dockerLoad"], str("rb")) as f:
+                    with open(dockerRequirement["dockerLoad"], "rb") as f:
                         loadproc = subprocess.Popen(cmd, stdin=f, stdout=sys.stderr)
                 else:
                     loadproc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=sys.stderr)

cwltool/expression.py

@@ -23,7 +23,7 @@ def jshead(engineConfig, rootvars):
     # TODO: need to make sure the `rootvars dict`
     # contains no bytes type in the first place.
     if six.PY3:
-        rootvars = bytes2str_in_dicts(rootvars)  # type -> ignore
+        rootvars = bytes2str_in_dicts(rootvars)  # type: ignore
 
     return u"\n".join(engineConfig + [u"var %s = %s;" % (k, json.dumps(v, indent=4)) for k, v in rootvars.items()])
 

cwltool/job.py

@@ -10,6 +10,7 @@
 import subprocess
 import sys
 import tempfile
+from io import open
 from typing import (IO, Any, Callable, Dict, Iterable, List, MutableMapping, Text,
                     Tuple, Union, cast)
 
@@ -23,6 +24,7 @@
 from .pathmapper import PathMapper
 from .process import (UnsupportedRequirement, empty_subtree, get_feature,
                       stageFiles)
+from .utils import bytes2str_in_dicts
 
 _logger = logging.getLogger("cwltool")
 
@@ -229,6 +231,7 @@ def _execute(self, runtime, env, rm_tmpdir=True, move_outputs="move"):
                 relink_initialworkdir(self.generatemapper, inplace_update=self.inplace_update)
 
             outputs = self.collect_outputs(self.outdir)
+            outputs = bytes2str_in_dicts(outputs)  # type: ignore
 
         except OSError as e:
             if e.errno == 2:
@@ -414,7 +417,7 @@ def run(self, pull_image=True, rm_container=True,
 
 
 def _job_popen(
-        commands,  # type: List[str]
+        commands,  # type: List[bytes]
         stdin_path,  # type: Text
         stdout_path,  # type: Text
         stderr_path,  # type: Text
@@ -477,9 +480,8 @@ def _job_popen(
             job_script_contents = SHELL_COMMAND_TEMPLATE
 
         env_copy = {}
-        key = None  # type: Union[Text, bytes]
+        key = None  # type: Any
         for key in env:
-            key = key.encode("utf-8")
             env_copy[key] = env[key]
 
         job_description = dict(

cwltool/load_tool.py

@@ -41,8 +41,7 @@ def fetch_document(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
     # type: (...) -> Tuple[Loader, CommentedMap, Text]
     """Retrieve a CWL document."""
 
-    document_loader = Loader(jobloaderctx,
-                             fetcher_constructor=fetcher_constructor)
+    document_loader = Loader(jobloaderctx, fetcher_constructor=fetcher_constructor)  # type: ignore
 
     uri = None  # type: Text
     workflowobj = None  # type: CommentedMap

cwltool/main.py

@@ -506,7 +506,7 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
 
     _jobloaderctx = jobloaderctx.copy()
     _jobloaderctx.update(t.metadata.get("$namespaces", {}))
-    loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor)
+    loader = Loader(_jobloaderctx, fetcher_constructor=fetcher_constructor)  # type: ignore
 
     if len(args.job_order) == 1 and args.job_order[0][0] != "-":
         job_order_file = args.job_order[0]

cwltool/process.py

@@ -13,6 +13,7 @@
 import tempfile
 import uuid
 from collections import Iterable
+from io import open
 from functools import cmp_to_key
 from typing import (Any, Callable, Dict, Generator, List, Set, Text,
                     Tuple, Union, cast)

cwltool/stdfsaccess.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 import glob
 import os
-from typing import BinaryIO, List, Text, IO
+from typing import BinaryIO, List, Union, Text, IO, overload
 
 from .utils import onWindows
 
@@ -29,7 +29,17 @@ def _abs(self, p):  # type: (Text) -> Text
     def glob(self, pattern):  # type: (Text) -> List[Text]
         return [file_uri(str(self._abs(l))) for l in glob.glob(self._abs(pattern))]
 
-    def open(self, fn, mode):  # type: (Text, str) -> IO[bytes]
+    # overload is related to mypy type checking and in no way
+    # modifies the behaviour of the function.
+    @overload
+    def open(self, fn, mode='rb'):  # type: (Text, str) -> IO[bytes]
+        pass
+
+    @overload
+    def open(self, fn, mode='r'):  # type: (Text, str) -> IO[str]
+        pass
+
+    def open(self, fn, mode):
         return open(self._abs(fn), mode)
 
     def exists(self, fn):  # type: (Text) -> bool