Merge branch 'master' into fix/concat-vf

Author: Anton Khodak

appveyor.yml

@@ -31,6 +31,7 @@ install:
   - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
 
 build_script:
+  - "%CMD_IN_ENV% pip install -U setuptools pip"
   - "%CMD_IN_ENV% pip install ."
 
 

cwltool/argparser.py

@@ -49,6 +49,27 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                          default=True, help="Do not delete Docker container used by jobs after they exit",
                          dest="rm_container")
 
+    cidgroup = parser.add_argument_group("Options for recording the Docker "
+        "container identifier into a file")
+    cidgroup.add_argument("--record-container-id", action="store_true",
+                          default=False,
+                          help="If enabled, store the Docker container ID into a file. "
+                          "See --cidfile-dir to specify the directory.",
+                          dest="record_container_id")
+
+    cidgroup.add_argument("--cidfile-dir", type=Text,
+                        help="Directory for storing the Docker container ID file. "
+                             "The default is the current directory",
+                        default="",
+                        dest="cidfile_dir")
+
+    cidgroup.add_argument("--cidfile-prefix", type=Text,
+                        help="Specify a prefix to the container ID filename. "
+                             "Final file name will be followed by a timestamp. "
+                             "The default is no prefix.",
+                        default="",
+                        dest="cidfile_prefix")
+
     parser.add_argument("--tmpdir-prefix", type=Text,
                         help="Path prefix for temporary directories",
                         default="tmp")
@@ -127,6 +148,9 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
     exgroup.add_argument("--debug", action="store_true", help="Print even more logging")
 
+    parser.add_argument("--timestamps", action="store_true", help="Add "
+                        "timestamps to the errors, warnings, and "
+                        "notifications.")
     parser.add_argument("--js-console", action="store_true", help="Enable javascript console output")
     parser.add_argument("--user-space-docker-cmd",
                         help="(Linux/OS X only) Specify a user space docker "
@@ -370,4 +394,4 @@ def generate_parser(toolparser, tool, namemap, records):
         default = inp.get("default", None)
         add_argument(toolparser, name, inptype, records, description, default)
 
-    return toolparser
+    return toolparser

cwltool/job.py

@@ -11,6 +11,7 @@
 import subprocess
 import sys
 import tempfile
+import datetime
 from io import open
 from typing import (IO, Any, Callable, Dict, Iterable, List, MutableMapping, Text,
                     Tuple, Union, cast)
@@ -363,8 +364,10 @@ def add_volumes(self, pathmapper, runtime):
                         docker_windows_path_adjust(vol.target)))
 
     def run(self, pull_image=True, rm_container=True,
+            record_container_id=False, cidfile_dir="",
+            cidfile_prefix="",
             rm_tmpdir=True, move_outputs="move", **kwargs):
-        # type: (bool, bool, bool, Text, **Any) -> None
+        # type: (bool, bool, bool, Text, Text, bool, Text, **Any) -> None
 
         (docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
 
@@ -463,6 +466,26 @@ def run(self, pull_image=True, rm_container=True,
         # directory." but spec might change to designated temp directory.
         # runtime.append("--env=HOME=/tmp")
         runtime.append(u"--env=HOME=%s" % self.builder.outdir)
+        
+        # add parameters to docker to write a container ID file
+        if record_container_id:
+            if cidfile_dir != "":
+                if not os.path.isdir(cidfile_dir):
+                    _logger.error("--cidfile-dir %s error:\n%s", cidfile_dir,
+                                  cidfile_dir + " is not a directory or "
+                                  "directory doesn't exist, please check it first")
+                    exit(2)
+                if not os.path.exists(cidfile_dir):
+                    _logger.error("--cidfile-dir %s error:\n%s", cidfile_dir,
+                                  "directory doesn't exist, please create it first")
+                    exit(2)
+            else:
+                cidfile_dir = os.getcwd()
+            cidfile_name = datetime.datetime.now().strftime("%Y%m%d%H%M%S-%f")+".cid"
+            if cidfile_prefix != "":
+                cidfile_name = str(cidfile_prefix + "-" + cidfile_name)
+            cidfile_path = os.path.join(cidfile_dir, cidfile_name)
+            runtime.append(u"--cidfile=%s" % cidfile_path)
 
         for t, v in self.environment.items():
             runtime.append(u"--env=%s=%s" % (t, v))

cwltool/main.py

@@ -424,6 +424,7 @@ def main(argsl=None,  # type: List[str]
                      'cachedir': None,
                      'quiet': False,
                      'debug': False,
+                     'timestamps': False,
                      'js_console': False,
                      'version': False,
                      'enable_dev': False,
@@ -452,6 +453,10 @@ def main(argsl=None,  # type: List[str]
             _logger.setLevel(logging.WARN)
         if args.debug:
             _logger.setLevel(logging.DEBUG)
+        if args.timestamps:
+            formatter = logging.Formatter("[%(asctime)s] %(message)s",
+                                          "%Y-%m-%d %H:%M:%S")
+            stderr_handler.setFormatter(formatter)
 
         if args.version:
             print(versionfunc())

tests/checker_wf/broken-wf.cwl

@@ -1,3 +1,4 @@
+#!/usr/bin/env cwl-runner
 class: Workflow
 cwlVersion: v1.0
 requirements:

tests/checker_wf/broken-wf2.cwl

@@ -1,3 +1,4 @@
+#!/usr/bin/env cwl-runner
 class: Workflow
 cwlVersion: v1.0
 requirements:

tests/checker_wf/cat.cwl

@@ -1,3 +1,4 @@
+#!/usr/bin/env cwl-runner
 cwlVersion: v1.0
 class: CommandLineTool
 baseCommand: cat

tests/checker_wf/echo.cwl

@@ -1,3 +1,4 @@
+#!/usr/bin/env cwl-runner
 cwlVersion: v1.0
 class: CommandLineTool
 baseCommand: echo

tests/checker_wf/functional-wf.cwl

@@ -1,3 +1,4 @@
+#!/usr/bin/env cwl-runner
 class: Workflow
 cwlVersion: v1.0
 requirements:

tests/echo.cwl

@@ -1,3 +1,4 @@
+#!/usr/bin/env cwl-runner
 cwlVersion: v1.0
 class: CommandLineTool
 inputs: