Merge pull request #442 from manu-chroma/unicode_literal_usage

Working towards python3 compatible codebase

Author: mr-c

.gitignore

@@ -15,3 +15,18 @@ eggs/
 *~
 \#*\#
 .desktop
+
+# virtualenv
+venv/
+venv3/
+
+# pycharm
+.idea/
+
+# typshed repo
+typeshed/2and3/schema_salad
+typeshed/2and3/ruamel/yaml
+
+
+#mypy
+.mypy_cache/

Makefile

@@ -152,26 +152,26 @@ list-author-emails:
 	@git log --format='%aN,%aE' | sort -u | grep -v 'root'
 
 
-mypy: ${PYSOURCES}
-	rm -Rf typeshed/2.7/ruamel/yaml
+mypy2: ${PYSOURCES}
+	rm -Rf typeshed/2and3/ruamel/yaml
 	ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
-		typeshed/2.7/ruamel/yaml
-	rm -Rf typeshed/2.7/schema_salad
+		typeshed/2and3/ruamel/yaml
+	rm -Rf typeshed/2and3/schema_salad
 	ln -s $(shell python -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
-		typeshed/2.7/schema_salad
-	MYPYPATH=typeshed/2.7 mypy --py2 --disallow-untyped-calls \
-		 --warn-redundant-casts --warn-unused-ignores --fast-parser \
+		typeshed/2and3/schema_salad
+	MYPYPATH=$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
+		 --warn-redundant-casts --warn-unused-ignores \
 		 cwltool
 
 mypy3: ${PYSOURCES}
-	rm -Rf typeshed/3/ruamel/yaml
+	rm -Rf typeshed/2and3/ruamel/yaml
 	ln -s $(shell python3 -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
-		typeshed/3/ruamel/yaml
-	rm -Rf typeshed/3/schema_salad
+		typeshed/2and3/ruamel/yaml
+	rm -Rf typeshed/2and3/schema_salad
 	ln -s $(shell python3 -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
-		typeshed/3/schema_salad
-	MYPYPATH=typeshed/3 mypy --disallow-untyped-calls \
-		 --warn-redundant-casts --warn-unused-ignores --fast-parser \
+		typeshed/2and3/schema_salad
+	MYPYPATH=$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
+		 --warn-redundant-casts --warn-unused-ignores \
 		 cwltool
 
 FORCE:

cwlref-runner/setup.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-
+from __future__ import absolute_import
 import os
 
 from setuptools import setup, find_packages

cwltool.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import absolute_import
 """Convienance entry point for cwltool.
 
 This can be used instead of the recommended method of `./setup.py install`

cwltool/__init__.py

@@ -1 +1,2 @@
+from __future__ import absolute_import
 __author__ = '[email protected]'

cwltool/__main__.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import sys
 
 from . import main

cwltool/builder.py

@@ -1,6 +1,8 @@
+from __future__ import absolute_import
 import copy
-from typing import Any, Callable, Text, Type, Union
+from typing import Any, Callable, Dict, List, Text, Type, Union
 
+import six
 from six import iteritems, string_types
 
 import avro
@@ -15,6 +17,11 @@
 from .stdfsaccess import StdFsAccess
 from .utils import aslist
 
+# if six.PY3:
+# AvroSchemaFromJSONData = avro.schema.SchemaFromJSONData
+# else:
+AvroSchemaFromJSONData = avro.schema.make_avsc_object
+
 CONTENT_LIMIT = 64 * 1024
 
 
@@ -86,7 +93,7 @@ def bind_input(self, schema, datum, lead_pos=None, tail_pos=None):
                 elif isinstance(t, dict) and "name" in t and self.names.has_name(t["name"], ""):
                     avsc = self.names.get_name(t["name"], "")
                 else:
-                    avsc = avro.schema.make_avsc_object(t, self.names)
+                    avsc = AvroSchemaFromJSONData(t, self.names)
                 if validate.validate(avsc, datum):
                     schema = copy.deepcopy(schema)
                     schema["type"] = t

cwltool/cwlrdf.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from typing import IO, Any, Dict, Text
 
 from rdflib import Graph

cwltool/docker.py

@@ -1,10 +1,11 @@
+from __future__ import absolute_import
 import logging
 import os
 import re
 import subprocess
 import sys
 import tempfile
-from typing import Text
+from typing import Dict, List, Text
 
 import requests
 
@@ -21,7 +22,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
         dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
 
     for ln in subprocess.check_output(
-            ["docker", "images", "--no-trunc", "--all"]).splitlines():
+            ["docker", "images", "--no-trunc", "--all"]).decode('utf-8').splitlines():
         try:
             m = re.match(r"^([^ ]+)\s+([^ ]+)\s+([^ ]+)", ln)
             sp = dockerRequirement["dockerImageId"].split(":")
@@ -46,7 +47,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
             pass
 
     if not found and pull_image:
-        cmd = []  # type: List[str]
+        cmd = []  # type: List[Text]
         if "dockerPull" in dockerRequirement:
             cmd = ["docker", "pull", str(dockerRequirement["dockerPull"])]
             _logger.info(Text(cmd))
@@ -55,8 +56,8 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
                 found = True
         elif "dockerFile" in dockerRequirement:
             dockerfile_dir = str(tempfile.mkdtemp())
-            with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as df:
-                df.write(dockerRequirement["dockerFile"])
+            with open(os.path.join(dockerfile_dir, "Dockerfile"), str("w")) as df:
+                df.write(dockerRequirement["dockerFile"].encode('utf-8'))
             cmd = ["docker", "build", "--tag=%s" %
                    str(dockerRequirement["dockerImageId"]), dockerfile_dir]
             _logger.info(Text(cmd))
@@ -69,7 +70,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
             if not dry_run:
                 if os.path.exists(dockerRequirement["dockerLoad"]):
                     _logger.info(u"Loading docker image from %s", dockerRequirement["dockerLoad"])
-                    with open(dockerRequirement["dockerLoad"], "rb") as f:
+                    with open(dockerRequirement["dockerLoad"], str("rb")) as f:
                         loadproc = subprocess.Popen(cmd, stdin=f, stdout=sys.stderr)
                 else:
                     loadproc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=sys.stderr)

cwltool/docker_uid.py

@@ -1,7 +1,8 @@
 from __future__ import print_function
+from __future__ import absolute_import
 
 import subprocess
-from typing import Text
+from typing import List, Text
 
 
 def docker_vm_uid():  # type: () -> int
@@ -91,6 +92,7 @@ def cmd_output_to_int(cmd):  # type: (List[Text]) -> int
         except ValueError:
             # ValueError is raised if int conversion fails
             return None
+    return None
 
 
 def boot2docker_uid():  # type: () -> int