Merge pull request #166 from common-workflow-language/flake-mypy-auto

add auto flake8 & mypy tests for pull requests

Author: mr-c

.travis.yml

@@ -0,0 +1,13 @@
+sudo: false
+language: python
+python:
+  - 2.7
+  - 3.5
+os:
+  - linux
+
+install:
+  - pip install tox-travis
+script: tox
+notifications:
+  email: false

Makefile

@@ -26,8 +26,8 @@ MODULE=cwltool
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
-DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257
-DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pep257 sloccount
+DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8
+DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pep257 sloccount python-flake8
 VERSION=1.0.$(shell date +%Y%m%d%H%M%S --date=`git log --first-parent \
 	--max-count=1 --format=format:%cI`)
 mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -149,10 +149,10 @@ list-author-emails:
 
 mypy: ${PYSOURCES}
 	rm -Rf typeshed/2.7/ruamel/yaml
-	ln -s $(shell python -c 'import ruamel.yaml; import os.path; print os.path.dirname(ruamel.yaml.__file__)') \
+	ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
 		typeshed/2.7/ruamel/yaml
 	rm -Rf typeshed/2.7/schema_salad
-	ln -s $(shell python -c 'import schema_salad; import os.path; print os.path.dirname(schema_salad.__file__)') \
+	ln -s $(shell python -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2.7/schema_salad
 	MYPYPATH=typeshed/2.7 mypy --py2 --disallow-untyped-calls \
 		 --warn-redundant-casts --warn-unused-ignores cwltool

cwltool/builder.py

@@ -86,10 +86,9 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
                         b2 = copy.deepcopy(binding)
                         b2["datum"] = item
                     bindings.extend(
-                            self.bind_input(
-                                {"type": schema["items"],
-                                 "inputBinding": b2},
-                                item, lead_pos=n, tail_pos=tail_pos))
+                        self.bind_input(
+                            {"type": schema["items"], "inputBinding": b2},
+                            item, lead_pos=n, tail_pos=tail_pos))
                 binding = None
 
             if schema["type"] == "File":

cwltool/cwltest.py

@@ -43,7 +43,7 @@ def compare(a, b):  # type: (Any, Any) -> bool
                     raise CompareFail(u"%s does not end with %s" %(b[comp], a[comp]))
                 # ignore empty collections
                 b = {k: v for k, v in b.iteritems()
-                        if not isinstance(v, (list, dict)) or len(v) > 0}
+                    if not isinstance(v, (list, dict)) or len(v) > 0}
             elif a.get("class") == "Directory":
                 if len(a["listing"]) != len(b["listing"]):
                     return False
@@ -215,8 +215,8 @@ def main():  # type: () -> int
         _logger.error("Tests interrupted")
 
     if failures == 0 and unsupported == 0:
-         _logger.info("All tests passed")
-         return 0
+        _logger.info("All tests passed")
+        return 0
     elif failures == 0 and unsupported > 0:
         _logger.warn("%i tests passed, %i unsupported features", total - unsupported, unsupported)
         return 0

cwltool/draft2tool.py

@@ -109,11 +109,11 @@ def __init__(self, job, output_callback, cachebuilder, jobcache):
 
     def run(self, **kwargs):
         # type: (**Any) -> None
-        self.output_callback(self.job.collect_output_ports(self.job.tool["outputs"],
-                                                           self.cachebuilder,
-                                                           self.outdir,
-                                                           kwargs.get("compute_checksum", True)),
-                                            "success")
+        self.output_callback(self.job.collect_output_ports(
+            self.job.tool["outputs"],
+            self.cachebuilder,
+            self.outdir,
+            kwargs.get("compute_checksum", True)), "success")
 
 # map files to assigned path inside a container. We need to also explicitly
 # walk over input as implicit reassignment doesn't reach everything in builder.bindings
@@ -221,16 +221,17 @@ def job(self, joborder, output_callback, **kwargs):
                 os.makedirs(jobcache)
                 kwargs["outdir"] = jobcache
                 open(jobcachepending, "w").close()
+
                 def rm_pending_output_callback(output_callback, jobcachepending,
                                                outputs, processStatus):
                     if processStatus == "success":
                         os.remove(jobcachepending)
                     output_callback(outputs, processStatus)
                 output_callback = cast(
-                        Callable[..., Any],  # known bug in mypy
-                        # https://github.com/python/mypy/issues/797
-                        partial(rm_pending_output_callback, output_callback,
-                            jobcachepending))
+                    Callable[..., Any],  # known bug in mypy
+                    # https://github.com/python/mypy/issues/797
+                    partial(rm_pending_output_callback, output_callback,
+                        jobcachepending))
 
         builder = self._init_job(joborder, **kwargs)
 
@@ -358,7 +359,8 @@ def rm_pending_output_callback(output_callback, jobcachepending,
 
         j.pathmapper = builder.pathmapper
         j.collect_outputs = partial(
-                self.collect_output_ports, self.tool["outputs"], builder, compute_checksum=kwargs.get("compute_checksum", True))
+            self.collect_output_ports, self.tool["outputs"], builder,
+            compute_checksum=kwargs.get("compute_checksum", True))
         j.output_callback = output_callback
 
         yield j
@@ -380,12 +382,11 @@ def collect_output_ports(self, ports, builder, outdir, compute_checksum=True):
                         ret[fragment] = self.collect_output(port, builder, outdir, fs_access, compute_checksum=compute_checksum)
                     except Exception as e:
                         _logger.debug(
-                                u"Error collecting output for parameter '%s'"
-                                % shortname(port["id"]), exc_info=True)
+                            u"Error collecting output for parameter '%s'"
+                            % shortname(port["id"]), exc_info=True)
                         raise WorkflowException(
-                                u"Error collecting output for " \
-                                        "parameter '%s': %s" %
-                                        (shortname(port["id"]), e))
+                            u"Error collecting output for parameter '%s': %s"
+                            % (shortname(port["id"]), e))
 
             if ret:
                 adjustFileObjs(ret,
@@ -507,6 +508,7 @@ def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=Tr
             out = {}
             for f in schema["type"]["fields"]:
                 out[shortname(f["name"])] = self.collect_output(  # type: ignore
-                        f, builder, outdir, fs_access, compute_checksum=compute_checksum)
+                    f, builder, outdir, fs_access,
+                    compute_checksum=compute_checksum)
             return out
         return r

cwltool/expression.py

@@ -31,8 +31,7 @@ def exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image):
             "self": context,
             "runtime": {
                 "tmpdir": tmpdir,
-                "outdir": outdir
-                }
+                "outdir": outdir }
         }
         return sandboxjs.execjs(ex["script"], jshead(engineConfig, rootvars))
 
@@ -66,7 +65,7 @@ def __init__(self):  # type: ()->None
 
             _logger.debug(u"Invoking expression engine %s with %s",
                           runtime + aslist(r["engineCommand"]),
-                                           json.dumps(inp, indent=4))
+                          json.dumps(inp, indent=4))
 
             sp = subprocess.Popen(runtime + aslist(r["engineCommand"]),
                              shell=False,
@@ -85,7 +84,7 @@ def __init__(self):  # type: ()->None
 seg_symbol = r"""\w+"""
 seg_single = r"""\['([^']|\\')+'\]"""
 seg_double = r"""\["([^"]|\\")+"\]"""
-seg_index  = r"""\[[0-9]+\]"""
+seg_index = r"""\[[0-9]+\]"""
 segments = r"(\.%s|%s|%s|%s)" % (seg_symbol, seg_single, seg_double, seg_index)
 segment_re = re.compile(segments, flags=re.UNICODE)
 param_re = re.compile(r"\$\((%s)%s*\)" % (seg_symbol, segments), flags=re.UNICODE)
@@ -132,10 +131,9 @@ def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
     runtime["outdir"] = outdir
 
     rootvars = {
-            "inputs": jobinput,
-            "self": context,
-            "runtime": runtime
-        }
+        "inputs": jobinput,
+        "self": context,
+        "runtime": runtime }
 
     if isinstance(ex, dict) and "engine" in ex and "script" in ex:
         return exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image)

cwltool/job.py

@@ -79,8 +79,8 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
             p = self.pathmapper.mapper(knownfile)
             if p.type == "File" and not os.path.isfile(p[0]):
                 raise WorkflowException(
-                u"Input file %s (at %s) not found or is not a regular file."
-                % (knownfile, self.pathmapper.mapper(knownfile)[0]))
+                    u"Input file %s (at %s) not found or is not a regular "
+                    "file." % (knownfile, self.pathmapper.mapper(knownfile)[0]))
 
         img_id = None
         env = None  # type: MutableMapping[str, str]
@@ -176,6 +176,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
                                             self.outdir, separateDirs=False)
                 _logger.debug(u"[job %s] initial work dir %s", self.name,
                               json.dumps({p: generatemapper.mapper(p) for p in generatemapper.files()}, indent=4))
+
                 def linkoutdir(src, tgt):
                     # Need to make the link to the staged file (may be inside
                     # the container)

cwltool/load_tool.py

@@ -54,8 +54,8 @@ def _convert_stdstreams_to_files(workflowobj):
                     if out['type'] == streamtype:
                         if 'outputBinding' in out:
                             raise ValidationException(
-                                    "Not allowed to specify outputBinding when"
-                                    " using %s shortcut." % streamtype)
+                                "Not allowed to specify outputBinding when"
+                                " using %s shortcut." % streamtype)
                         if streamtype in workflowobj:
                             filename = workflowobj[streamtype]
                         else:
@@ -104,7 +104,7 @@ def validate_document(document_loader, workflowobj, uri,
             del workflowobj["@graph"]
 
     (document_loader, avsc_names) = \
-            process.get_schema(workflowobj["cwlVersion"])[:2]
+        process.get_schema(workflowobj["cwlVersion"])[:2]
 
     if isinstance(avsc_names, Exception):
         raise avsc_names

cwltool/main.py

@@ -140,8 +140,9 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
 
     parser.add_argument("--tool-help", action="store_true", help="Print command line help for tool")
 
-    parser.add_argument("--relative-deps", choices=['primary', 'cwd'], default="primary",
-                         help="When using --print-deps, print paths relative to primary file or current working directory.")
+    parser.add_argument("--relative-deps", choices=['primary', 'cwd'],
+        default="primary", help="When using --print-deps, print paths "
+        "relative to primary file or current working directory.")
 
     parser.add_argument("--enable-dev", action="store_true",
                         help="Allow loading and running development versions "
@@ -306,8 +307,8 @@ def generate_parser(toolparser, tool, namemap):
 
         ahelp = inp.get("description", "").replace("%", "%%")
         action = None  # type: Union[argparse.Action,str]
-        atype = None # type: Any
-        default = None # type: Any
+        atype = None  # type: Any
+        default = None  # type: Any
 
         if inptype == "File":
             action = cast(argparse.Action, FileAction)
@@ -345,8 +346,8 @@ def generate_parser(toolparser, tool, namemap):
             typekw = {}
 
         toolparser.add_argument(  # type: ignore
-                flag + name, required=required, help=ahelp, action=action,
-                default=default, **typekw)
+            flag + name, required=required, help=ahelp, action=action,
+            default=default, **typekw)
 
     return toolparser
 
@@ -361,10 +362,10 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
         loader = Loader({})
     else:
         jobloaderctx = {
-                u"path": {u"@type": u"@id"},
-                u"location": {u"@type": u"@id"},
-                u"format": {u"@type": u"@id"},
-                u"id": u"@id"}
+            u"path": {u"@type": u"@id"},
+            u"location": {u"@type": u"@id"},
+            u"format": {u"@type": u"@id"},
+            u"id": u"@id"}
         jobloaderctx.update(t.metadata.get("$namespaces", {}))
         loader = Loader(jobloaderctx)
 
@@ -458,9 +459,9 @@ def printdeps(obj, document_loader, stdout, relative_deps, uri, basedir=None):
     def loadref(b, u):
         return document_loader.fetch(urlparse.urljoin(b, u))
 
-    sf = scandeps(basedir if basedir else uri, obj,
-                          set(("$import", "run")),
-                          set(("$include", "$schemas", "path", "location")), loadref)
+    sf = scandeps(
+        basedir if basedir else uri, obj, set(("$import", "run")),
+        set(("$include", "$schemas", "path", "location")), loadref)
     if sf:
         deps["secondaryFiles"] = sf
 
@@ -471,6 +472,7 @@ def loadref(b, u):
             base = "file://" + os.getcwd()
         else:
             raise Exception(u"Unknown relative_deps %s" % relative_deps)
+
         def makeRelative(ob):
             u = ob.get("location", ob.get("path"))
             if ":" in u.split("/")[0] and not u.startswith("file://"):
@@ -521,8 +523,7 @@ def print_pack(document_loader, processobj, uri, metadata):
     def loadref(b, u):
         # type: (unicode, unicode) -> Union[Dict, List, unicode]
         return document_loader.resolve_ref(u, base_url=b)[0]
-    deps = scandeps(uri, processobj,
-                            set(("run",)), set(), loadref)
+    deps = scandeps(uri, processobj, set(("run",)), set(), loadref)
 
     fdeps = set((uri,))
     flatten_deps(deps, fdeps)

cwltool/pathmapper.py

@@ -161,14 +161,14 @@ def setup(self, referenced_files, basedir):
 
         # Dereference symbolic links
         for path, (ab, tgt, type) in self._pathmap.items():
-            if type != "File": # or not os.path.exists(ab):
+            if type != "File":  # or not os.path.exists(ab):
                 continue
             deref = ab
             st = os.lstat(deref)
             while stat.S_ISLNK(st.st_mode):
                 rl = os.readlink(deref)
                 deref = rl if os.path.isabs(rl) else os.path.join(
-                        os.path.dirname(deref), rl)
+                    os.path.dirname(deref), rl)
                 st = os.lstat(deref)
 
             self._pathmap[path] = MapperEnt(deref, tgt, "File")