first attempt at saving log for provenance

Author: mr-c

cwltool/executors.py

@@ -98,9 +98,10 @@ def execute(self,
 
         if runtime_context.rm_tmpdir:
             if runtime_context.cachedir is None:
-                output_dirs = self.output_dirs # type: Iterable[Any]
+                output_dirs = self.output_dirs  # type: Iterable[Any]
             else:
-                output_dirs = filter(lambda x: not x.startswith(runtime_context.cachedir), self.output_dirs)
+                output_dirs = filter(lambda x: not x.startswith(
+                    runtime_context.cachedir), self.output_dirs)
             cleanIntermediate(output_dirs)
 
         if self.final_output and self.final_status:

cwltool/job.py

@@ -344,7 +344,7 @@ def _execute(self,
 
         if self.generatemapper and runtimeContext.secret_store:
             # Delete any runtime-generated files containing secrets.
-            for f, p in self.generatemapper.items():
+            for _, p in self.generatemapper.items():
                 if p.type == "CreateFile":
                     if runtimeContext.secret_store.has_secret(p.resolved):
                         host_outdir = self.outdir
@@ -509,7 +509,7 @@ def _job_popen(
         job_script_contents=None,  # type: Text
         timelimit=None,            # type: int
         name=None                  # type: Text
-       ):  # type: (...) -> int
+        ):  # type: (...) -> int
 
     if not job_script_contents and not FORCE_SHELLED_POPEN:
 

cwltool/main.py

@@ -465,6 +465,7 @@ def main(argsl=None,                   # type: List[str]
     _logger.addHandler(stderr_handler)
     # pre-declared for finally block
     workflowobj = None
+    prov_handler = None  # type: Optional[logger.FileHandler]
     try:
         if args is None:
             if argsl is None:
@@ -493,6 +494,14 @@ def main(argsl=None,                   # type: List[str]
         rdflib_logger = logging.getLogger("rdflib.term")
         rdflib_logger.addHandler(stderr_handler)
         rdflib_logger.setLevel(logging.ERROR)
+        if args.provenance:
+            try:
+                os.makedirs(args.provenance)
+            except os.error:
+                pass
+            prov_handler = logging.FileHandler(
+                os.path.join(args.provenance, "log"))
+            _logger.addHandler(prov_handler)
         if args.quiet:
             _logger.setLevel(logging.WARN)
         if runtimeContext.debug:
@@ -502,6 +511,8 @@ def main(argsl=None,                   # type: List[str]
             formatter = logging.Formatter("[%(asctime)s] %(message)s",
                                           "%Y-%m-%d %H:%M:%S")
             stderr_handler.setFormatter(formatter)
+            if prov_handler:
+                prov_handler.setFormatter(formatter)
 
         if args.version:
             print(versionfunc())
@@ -774,6 +785,9 @@ def loc_to_path(obj):
             runtimeContext.research_obj.close(args.provenance)
 
         _logger.removeHandler(stderr_handler)
+        if prov_handler:
+            prov_handler.close()
+            _logger.removeHandler(prov_handler)
         _logger.addHandler(defaultStreamHandler)