Merge branch 'master' into wilke/fix-pull-image

Author: mr-c

MANIFEST.in

@@ -9,6 +9,7 @@ include tests/wf/*
 include tests/override/*
 include tests/checker_wf/*
 include tests/subgraph/*
+include tests/trs/*
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/v1.0/*.md

Makefile

@@ -27,7 +27,7 @@ MODULE=cwltool
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
 DEVPKGS=pycodestyle diff_cover autopep8 pylint coverage pydocstyle flake8 \
-	pytest pytest-xdist isort wheel
+	pytest-xdist==1.27.0 isort wheel -rtest-requirements.txt
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
 	   python-flake8 python-mock shellcheck
 VERSION=1.0.$(shell TZ=UTC git log --first-parent --max-count=1 \
@@ -51,7 +51,7 @@ help: Makefile
 
 ## install-dep : install most of the development dependencies via pip
 install-dep:
-	pip install --upgrade $(DEVPKGS) -rtest-requirements.txt
+	pip install --upgrade $(DEVPKGS)
 
 ## install-deb-dep: install most of the dev dependencies via apt-get
 install-deb-dep:

README.rst

@@ -498,7 +498,7 @@ Running tests locally
 
 -  Running basic tests ``(/tests)``:
 
-To run the basis tests after installing `cwltool` execute the following:
+To run the basic tests after installing `cwltool` execute the following:
 
 .. code:: bash
 

appveyor.yml

@@ -32,12 +32,12 @@ install:
   - ps: 'Install-Product node 0.12 x64'
   - "set PATH=%PYTHON%\\Scripts;%PATH%"
   - "%PYTHON%\\python.exe -m pip install -U pip setuptools^>=20.3 wheel"
-  - "%PYTHON%\\python.exe -m pip install -U codecov pytest-xdist pytest-cov galaxy-lib -rtest-requirements.txt"
+  - "%PYTHON%\\python.exe -m pip install -U codecov -rtest-requirements.txt pytest-xdist==1.27 "
     # Note the use of a `^` to escape the `>`
 
 build_script:
   - "%PYTHON%\\python.exe -m pip install -rrequirements.txt"
-  - "%PYTHON%\\python.exe -m pip install -e ."
+  - "%PYTHON%\\python.exe -m pip install -e .[deps]"
 
 test_script:
   - |

cwl-docker.sh

@@ -1,2 +1,3 @@
 #!/bin/sh
+# Changing the line below? Update https://cloud.docker.com/u/commonworkflowlanguage/repository/docker/commonworkflowlanguage/cwltool
 exec docker run -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp -v "$PWD":"$PWD" -w="$PWD" commonworkflowlanguage/cwltool "$@"

cwltool/builder.py

@@ -207,11 +207,16 @@ def bind_input(self,
         value_from_expression = False
         if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping):
             binding = CommentedMap(schema["inputBinding"].items())
-            assert binding is not None
 
             bp = list(aslist(lead_pos))
             if "position" in binding:
-                bp.extend(aslist(binding["position"]))
+                position = binding["position"]
+                if isinstance(position, str):   # no need to test the CWL Version
+                                                # the schema for v1.0 only allow ints
+                    binding['position'] = self.do_eval(position, context=datum)
+                    bp.append(binding['position'])
+                else:
+                    bp.extend(aslist(binding['position']))
             else:
                 bp.append(0)
             bp.extend(aslist(tail_pos))
@@ -232,7 +237,6 @@ def bind_input(self,
                     avsc = self.names.get_name(t["name"], "")
                 if not avsc:
                     avsc = make_avsc_object(convert_to_dict(t), self.names)
-                assert avsc is not None
                 if validate.validate(avsc, datum):
                     schema = copy.deepcopy(schema)
                     schema["type"] = t
@@ -373,7 +377,6 @@ def tostr(self, value):  # type: (Any) -> Text
                 # docker_req is none only when there is no dockerRequirement
                 # mentioned in hints and Requirement
                 path = docker_windows_path_adjust(value["path"])
-                assert path is not None
                 return path
             return value["path"]
         else:
@@ -418,8 +421,7 @@ def generate_arg(self, binding):  # type: (Dict[Text, Any]) -> List[Text]
             if sep:
                 args.extend([prefix, self.tostr(j)])
             else:
-                assert prefix is not None
-                args.append(prefix + self.tostr(j))
+                args.append("" if not prefix else prefix + self.tostr(j))
 
         return [a for a in args if a is not None]
 

cwltool/command_line_tool.py

@@ -97,6 +97,7 @@ def __init__(self,
 
         def run(self, runtimeContext):  # type: (RuntimeContext) -> None
             try:
+                normalizeFilesDirs(self.builder.job)
                 ev = self.builder.do_eval(self.script)
                 normalizeFilesDirs(ev)
                 self.output_callback(ev, "success")
@@ -156,7 +157,8 @@ def revmap_file(builder, outdir, f):
         if "basename" not in f:
             f["basename"] = os.path.basename(path)
 
-        assert builder.pathmapper is not None
+        if not builder.pathmapper:
+            raise ValueError("Do not call revmap_file using a builder that doesn't have a pathmapper.")
         revmap_f = builder.pathmapper.reversemap(path)
 
         if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"):
@@ -203,7 +205,8 @@ def check_adjust(builder, file_o):
     doesn't reach everything in builder.bindings
     """
 
-    assert builder.pathmapper is not None
+    if not builder.pathmapper:
+            raise ValueError("Do not call check_adjust using a builder that doesn't have a pathmapper.")
     file_o["path"] = docker_windows_path_adjust(
         builder.pathmapper.mapper(file_o["location"])[1])
     dn, bn = os.path.split(file_o["path"])
@@ -299,7 +302,7 @@ def job(self,
         if runtimeContext.cachedir and enableReuse:
             cachecontext = runtimeContext.copy()
             cachecontext.outdir = "/out"
-            cachecontext.tmpdir = "/tmp"
+            cachecontext.tmpdir = "/tmp"  # nosec
             cachecontext.stagedir = "/stage"
             cachebuilder = self._init_job(job_order, cachecontext)
             cachebuilder.pathmapper = PathMapper(cachebuilder.files,
@@ -353,7 +356,8 @@ def job(self,
 
             keydictstr = json_dumps(keydict, separators=(',', ':'),
                                     sort_keys=True)
-            cachekey = hashlib.md5(keydictstr.encode('utf-8')).hexdigest()
+            cachekey = hashlib.md5(  # nosec
+                keydictstr.encode('utf-8')).hexdigest()
 
             _logger.debug("[job %s] keydictstr is %s -> %s", jobname,
                           keydictstr, cachekey)
@@ -476,24 +480,24 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
         if self.tool.get("stdin"):
             with SourceLine(self.tool, "stdin", validate.ValidationException, debug):
                 j.stdin = builder.do_eval(self.tool["stdin"])
-                assert j.stdin is not None
-                reffiles.append({"class": "File", "path": j.stdin})
+                if j.stdin:
+                    reffiles.append({"class": "File", "path": j.stdin})
 
         if self.tool.get("stderr"):
             with SourceLine(self.tool, "stderr", validate.ValidationException, debug):
                 j.stderr = builder.do_eval(self.tool["stderr"])
-                assert j.stderr is not None
-                if os.path.isabs(j.stderr) or ".." in j.stderr:
-                    raise validate.ValidationException(
-                        "stderr must be a relative path, got '%s'" % j.stderr)
+                if j.stderr:
+                    if os.path.isabs(j.stderr) or ".." in j.stderr:
+                        raise validate.ValidationException(
+                            "stderr must be a relative path, got '%s'" % j.stderr)
 
         if self.tool.get("stdout"):
             with SourceLine(self.tool, "stdout", validate.ValidationException, debug):
                 j.stdout = builder.do_eval(self.tool["stdout"])
-                assert j.stdout is not None
-                if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout:
-                    raise validate.ValidationException(
-                        "stdout must be a relative path, got '%s'" % j.stdout)
+                if j.stdout:
+                    if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout:
+                        raise validate.ValidationException(
+                            "stdout must be a relative path, got '%s'" % j.stdout)
 
         if debug:
             _logger.debug(u"[job %s] command line bindings is %s", j.name,
@@ -546,7 +550,7 @@ def register_reader(f):
             adjustDirObjs(builder.files, register_reader)
             adjustDirObjs(builder.bindings, register_reader)
 
-        timelimit, _ = self.get_requirement("TimeLimit")
+        timelimit, _ = self.get_requirement("ToolTimeLimit")
         if timelimit is not None:
             with SourceLine(timelimit, "timelimit", validate.ValidationException, debug):
                 j.timelimit = builder.do_eval(timelimit["timelimit"])
@@ -592,12 +596,17 @@ def collect_output_ports(self,
                              ports,                  # type: Set[Dict[Text, Any]]
                              builder,                # type: Builder
                              outdir,                 # type: Text
+                             rcode,                  # type: int
                              compute_checksum=True,  # type: bool
                              jobname="",             # type: Text
                              readers=None            # type: Dict[Text, Any]
                             ):  # type: (...) -> OutputPorts
         ret = {}  # type: OutputPorts
         debug = _logger.isEnabledFor(logging.DEBUG)
+        cwl_version = self.metadata.get(
+            "http://commonwl.org/cwltool#original_cwlVersion", None)
+        if cwl_version != "v1.0":
+            builder.resources["exitCode"] = rcode
         try:
             fs_access = builder.make_fs_access(outdir)
             custom_output = fs_access.join(outdir, "cwl.output.json")
@@ -706,14 +715,13 @@ def collect_output(self,
                         if ll and ll != "no_listing":
                             get_listing(fs_access, files, (ll == "deep_listing"))
                     else:
-                        with fs_access.open(rfile["location"], "rb") as f:
-                            contents = b""
-                            if binding.get("loadContents") or compute_checksum:
-                                contents = content_limit_respected_read_bytes(f)
-                            if binding.get("loadContents"):
-                                files["contents"] = contents.decode("utf-8")
-                            if compute_checksum:
-                                checksum = hashlib.sha1()
+                        if binding.get("loadContents"):
+                            with fs_access.open(rfile["location"], "rb") as f:
+                                files["contents"] = content_limit_respected_read_bytes(f).decode("utf-8")
+                        if compute_checksum:
+                            with fs_access.open(rfile["location"], "rb") as f:
+                                checksum = hashlib.sha1()  # nosec
+                                contents = f.read(1024 * 1024)
                                 while contents != b"":
                                     checksum.update(contents)
                                     contents = f.read(1024 * 1024)

cwltool/cwlrdf.py

@@ -27,7 +27,6 @@ def printrdf(wflow, ctx, style):  # type: (Process, ContextType, Text) -> Text
     rdf = gather(wflow, ctx).serialize(format=style, encoding='utf-8')
     if not rdf:
         return u""
-    assert rdf is not None
     return rdf.decode('utf-8')
 
 

cwltool/docker.py

@@ -155,7 +155,7 @@ def get_image(docker_requirement,     # type: Dict[Text, Text]
                 else:
                     loadproc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
                                                 stdout=sys.stderr)
-                    assert loadproc.stdin is not None
+                    assert loadproc.stdin is not None  # nosec
                     _logger.info(u"Sending GET request to %s", docker_requirement["dockerLoad"])
                     req = requests.get(docker_requirement["dockerLoad"], stream=True)
                     size = 0
@@ -259,7 +259,7 @@ def add_writable_directory_volume(self,
                 self.append_volume(runtime, new_dir, volume.target,
                                    writable=True)
             elif not os.path.exists(host_outdir_tgt):
-                os.makedirs(host_outdir_tgt, 0o0755)
+                os.makedirs(host_outdir_tgt)
         else:
             if self.inplace_update:
                 self.append_volume(runtime, volume.resolved, volume.target,
@@ -296,7 +296,8 @@ def create_runtime(self,
             runtime = [u"docker", u"run", u"-i"]
         self.append_volume(runtime, os.path.realpath(self.outdir),
                            self.builder.outdir, writable=True)
-        self.append_volume(runtime, os.path.realpath(self.tmpdir), "/tmp",
+        tmpdir = "/tmp"  # nosec
+        self.append_volume(runtime, os.path.realpath(self.tmpdir), tmpdir,
                            writable=True)
         self.add_volumes(self.pathmapper, runtime, any_path_okay=True,
                          secret_store=runtimeContext.secret_store,

cwltool/executors.py

@@ -167,17 +167,17 @@ def run_jobs(self,
                         self.output_dirs.add(job.outdir)
                     if runtime_context.research_obj is not None:
                         if not isinstance(process, Workflow):
-                            runtime_context.prov_obj = process.provenance_object
+                            prov_obj = process.provenance_object
                         else:
-                            runtime_context.prov_obj = job.prov_obj
-                        assert runtime_context.prov_obj
-                        runtime_context.prov_obj.evaluate(
-                            process, job, job_order_object,
-                            runtime_context.research_obj)
-                        process_run_id =\
-                            runtime_context.prov_obj.record_process_start(
-                                process, job)
-                        runtime_context = runtime_context.copy()
+                            prov_obj = job.prov_obj
+                        if prov_obj:
+                            runtime_context.prov_obj = prov_obj
+                            prov_obj.evaluate(
+                                process, job, job_order_object,
+                                runtime_context.research_obj)
+                            process_run_id =\
+                                prov_obj.record_process_start(process, job)
+                            runtime_context = runtime_context.copy()
                         runtime_context.process_run_id = process_run_id
                     job.run(runtime_context)
                 else: