Merge branch 'master' into parallel-exec

Author: Anton Khodak

appveyor.yml

@@ -31,6 +31,7 @@ install:
   - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
 
 build_script:
+  - "%CMD_IN_ENV% pip install -U setuptools pip"
   - "%CMD_IN_ENV% pip install ."
 
 

cwltool/argparser.py

@@ -4,6 +4,7 @@
 from typing import Any, Dict, Text, Tuple, Union
 
 from . import load_tool, workflow
+from .argparser import get_default_args
 from .executors import SingleJobExecutor
 from .process import Process
 
@@ -44,7 +45,13 @@ def __init__(self,
         if executor is None:
             executor = SingleJobExecutor()
         self.executor = executor
-        self.execkwargs = execkwargs
+
+        kwargs = get_default_args()
+        kwargs.pop("job_order")
+        kwargs.pop("workflow")
+        kwargs.pop("outdir")
+        kwargs.update(execkwargs)
+        self.execkwargs = kwargs
 
     def make(self, cwl):
         """Instantiate a CWL object from a CWl document."""

cwltool/factory.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
+
 import codecs
+import datetime
 import functools
 import io
 import json
@@ -11,22 +13,22 @@
 import subprocess
 import sys
 import tempfile
-from threading import Lock
 from io import open
+from threading import Lock
 from typing import (IO, Any, Callable, Dict, Iterable, List, MutableMapping, Text,
-                    Tuple, Union, cast)
+                    Union, cast)
 
 import shellescape
 
-from .utils import copytree_with_merge, docker_windows_path_adjust, onWindows
 from . import docker
 from .builder import Builder
 from .docker_id import docker_vm_id
 from .errors import WorkflowException
 from .pathmapper import PathMapper, ensure_writable
-from .process import (UnsupportedRequirement, empty_subtree, get_feature,
+from .process import (UnsupportedRequirement, get_feature,
                       stageFiles)
 from .utils import bytes2str_in_dicts
+from .utils import copytree_with_merge, docker_windows_path_adjust, onWindows
 
 _logger = logging.getLogger("cwltool")
 
@@ -367,8 +369,10 @@ def add_volumes(self, pathmapper, runtime):
                         docker_windows_path_adjust(vol.target)))
 
     def run(self, pull_image=True, rm_container=True,
+            record_container_id=False, cidfile_dir="",
+            cidfile_prefix="",
             rm_tmpdir=True, move_outputs="move", **kwargs):
-        # type: (bool, bool, bool, Text, **Any) -> None
+        # type: (bool, bool, bool, Text, Text, bool, Text, **Any) -> None
 
         (docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
 
@@ -467,6 +471,26 @@ def run(self, pull_image=True, rm_container=True,
         # directory." but spec might change to designated temp directory.
         # runtime.append("--env=HOME=/tmp")
         runtime.append(u"--env=HOME=%s" % self.builder.outdir)
+        
+        # add parameters to docker to write a container ID file
+        if record_container_id:
+            if cidfile_dir != "":
+                if not os.path.isdir(cidfile_dir):
+                    _logger.error("--cidfile-dir %s error:\n%s", cidfile_dir,
+                                  cidfile_dir + " is not a directory or "
+                                  "directory doesn't exist, please check it first")
+                    exit(2)
+                if not os.path.exists(cidfile_dir):
+                    _logger.error("--cidfile-dir %s error:\n%s", cidfile_dir,
+                                  "directory doesn't exist, please create it first")
+                    exit(2)
+            else:
+                cidfile_dir = os.getcwd()
+            cidfile_name = datetime.datetime.now().strftime("%Y%m%d%H%M%S-%f")+".cid"
+            if cidfile_prefix != "":
+                cidfile_name = str(cidfile_prefix + "-" + cidfile_name)
+            cidfile_path = os.path.join(cidfile_dir, cidfile_name)
+            runtime.append(u"--cidfile=%s" % cidfile_path)
 
         for t, v in self.environment.items():
             runtime.append(u"--env=%s=%s" % (t, v))

cwltool/job.py

@@ -34,7 +34,7 @@ do
 	# use pip2.7 and pip3 in separate loop runs
 	pip${PYTHON_VERSION} install -U setuptools wheel pip
 	pip${PYTHON_VERSION} install .
-	pip${PYTHON_VERSION} install "cwltest>=1.0.20180130081614"
+	pip${PYTHON_VERSION} install -U "cwltest>=1.0.20180130081614"
 	pushd common-workflow-language
 	# shellcheck disable=SC2154
 	if [[ "$version" = *dev* ]]

cwltool/main.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env cwl-runner
 class: Workflow
 cwlVersion: v1.0
 requirements:

jenkins.bash

@@ -1,3 +1,4 @@
+#!/usr/bin/env cwl-runner
 class: Workflow
 cwlVersion: v1.0
 requirements:

tests/checker_wf/broken-wf.cwl

@@ -1,3 +1,4 @@
+#!/usr/bin/env cwl-runner
 cwlVersion: v1.0
 class: CommandLineTool
 baseCommand: cat

tests/checker_wf/broken-wf2.cwl

@@ -1,3 +1,4 @@
+#!/usr/bin/env cwl-runner
 cwlVersion: v1.0
 class: CommandLineTool
 baseCommand: echo