implement logging timestamps and record container ID in a file

Author: MichaelTong

cwltool/job.py

@@ -11,6 +11,7 @@
 import subprocess
 import sys
 import tempfile
+import datetime
 from io import open
 from typing import (IO, Any, Callable, Dict, Iterable, List, MutableMapping, Text,
                     Tuple, Union, cast)
@@ -363,6 +364,8 @@ def add_volumes(self, pathmapper, runtime):
                         docker_windows_path_adjust(vol.target)))
 
     def run(self, pull_image=True, rm_container=True,
+            record_container_id=False, cidfile_dir="/tmp/",
+            cidfile_prefix="",
             rm_tmpdir=True, move_outputs="move", **kwargs):
         # type: (bool, bool, bool, Text, **Any) -> None
 
@@ -463,6 +466,25 @@ def run(self, pull_image=True, rm_container=True,
         # directory." but spec might change to designated temp directory.
         # runtime.append("--env=HOME=/tmp")
         runtime.append(u"--env=HOME=%s" % self.builder.outdir)
+        
+        # add parameters to docker to write a container ID file
+        if record_container_id:
+            if cidfile_dir:
+                if not os.path.isdir(cidfile_dir):
+                    _logger.error("--cidfile-dir %s error:\n%s", cidfile_dir,
+                                  cidfile_dir + " is not a directory or directory doesn't exist, please check it first")
+                    exit(-1)
+                if not os.path.exists(cidfile_dir):
+                    _logger.error("--cidfile-dir %s error:\n%s", cidfile_dir,
+                                  "directory doesn't exist, please create it first")
+                    exit(-1)
+            else:
+                cidfile_dir = "/tmp/"
+            cidfile_name = datetime.datetime.now().strftime("%Y%m%d%H%M%S-%f")+".cid"
+            if cidfile_prefix != "":
+                cidfile_name = cidfile_prefix + "-" + cidfile_name
+            cidfile_path = cidfile_dir + cidfile_name
+            runtime.append(u"--cidfile=%s" % cidfile_path)
 
         for t, v in self.environment.items():
             runtime.append(u"--env=%s=%s" % (t, v))

cwltool/main.py

@@ -84,6 +84,20 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                          default=True, help="Do not delete Docker container used by jobs after they exit",
                          dest="rm_container")
 
+    parser.add_argument("--record-container-id", action="store_true", default=False,
+                        help="If enabled, a file with suffix \".cid\" will be created storing the container ID under CIDFILE_DIR",
+                        dest="record_container_id")
+
+    parser.add_argument("--cidfile-dir", type=Text,
+                        help="Directory for storing cidfiles. Default at /tmp/",
+                        default="/tmp/",
+                        dest="cidfile_dir")
+
+    parser.add_argument("--cidfile-prefix", type=Text,
+                        help="Give a prefix to cidfile. Final file name will be followed by a timestamp. Default empty.",
+                        default="",
+                        dest="cidfile_prefix")
+
     parser.add_argument("--tmpdir-prefix", type=Text,
                         help="Path prefix for temporary directories",
                         default="tmp")
@@ -161,6 +175,7 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     exgroup.add_argument("--verbose", action="store_true", help="Default logging")
     exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
     exgroup.add_argument("--debug", action="store_true", help="Print even more logging")
+    parser.add_argument("--logtstp", action="store_true", help="Print timestamps with logging")
 
     parser.add_argument("--js-console", action="store_true", help="Enable javascript console output")
     parser.add_argument("--user-space-docker-cmd",
@@ -774,6 +789,7 @@ def main(argsl=None,  # type: List[str]
                      'cachedir': None,
                      'quiet': False,
                      'debug': False,
+                     'logtstp': False,
                      'js_console': False,
                      'version': False,
                      'enable_dev': False,
@@ -802,6 +818,10 @@ def main(argsl=None,  # type: List[str]
             _logger.setLevel(logging.WARN)
         if args.debug:
             _logger.setLevel(logging.DEBUG)
+        if args.logtstp:
+            formatter = logging.Formatter("[%(asctime)s] %(message)s",
+                                          "%Y-%m-%d %H:%M:%S")
+            stderr_handler.setFormatter(formatter)
 
         if args.version:
             print(versionfunc())