add stderr support

Author: mr-c

cwltool/cwltest.py

@@ -114,7 +114,7 @@ def run_test(args, i, t):  # type: (argparse.Namespace, Any, Dict[str,str]) -> i
     if "output" in t:
         checkkeys = ["output"]
     else:
-        checkkeys = ["args", "stdin", "stdout", "createfiles"]
+        checkkeys = ["args", "stdin", "stderr", "stdout", "createfiles"]
 
     for key in checkkeys:
         try:

cwltool/draft2tool.py

@@ -206,6 +206,7 @@ def rm_pending_output_callback(output_callback, jobcachepending,
         j.builder = builder
         j.joborder = builder.job
         j.stdin = None
+        j.stderr = None
         j.stdout = None
         j.successCodes = self.tool.get("successCodes")
         j.temporaryFailCodes = self.tool.get("temporaryFailCodes")
@@ -227,6 +228,11 @@ def rm_pending_output_callback(output_callback, jobcachepending,
             j.stdin = builder.do_eval(self.tool["stdin"])
             reffiles.add(j.stdin)
 
+        if self.tool.get("stderr"):
+            j.stderr = builder.do_eval(self.tool["stderr"])
+            if os.path.isabs(j.stderr) or ".." in j.stderr:
+                raise validate.ValidationException("stderr must be a relative path")
+
         if self.tool.get("stdout"):
             j.stdout = builder.do_eval(self.tool["stdout"])
             if os.path.isabs(j.stdout) or ".." in j.stdout:

cwltool/job.py

@@ -44,6 +44,7 @@ def __init__(self):  # type: () -> None
         self.builder = None  # type: Builder
         self.joborder = None  # type: Dict[str,str]
         self.stdin = None  # type: str
+        self.stderr = None  # type: str
         self.stdout = None  # type: str
         self.successCodes = None  # type: Iterable[int]
         self.temporaryFailCodes = None  # type: Iterable[int]
@@ -137,6 +138,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
                         env[key] = value
 
         stdin = None  # type: Union[IO[Any],int]
+        stderr = None  # type: IO[Any]
         stdout = None  # type: IO[Any]
 
         scr, _ = get_feature(self, "ShellCommandRequirement")
@@ -146,12 +148,13 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
         else:
             shouldquote = needs_shell_quoting_re.search
 
-        _logger.info(u"[job %s] %s$ %s%s%s",
+        _logger.info(u"[job %s] %s$ %s%s%s%s",
                      self.name,
                      self.outdir,
                      " \\\n    ".join([shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line)]),
                      u' < %s' % (self.stdin) if self.stdin else '',
-                     u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '')
+                     u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '',
+                     u' \2> %s' % os.path.join(self.outdir, self.stderr) if self.stderr else '')
 
         if dry_run:
             return (self.outdir, {})
@@ -178,6 +181,15 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
             else:
                 stdin = subprocess.PIPE
 
+            if self.stderr:
+                abserr = os.path.join(self.outdir, self.stderr)
+                dnerr = os.path.dirname(abserr)
+                if dnerr and not os.path.exists(dnerr):
+                    os.makedirs(dnerr)
+                stderr = open(absout, "wb")
+            else:
+                stderr = sys.stderr
+
             if self.stdout:
                 absout = os.path.join(self.outdir, self.stdout)
                 dn = os.path.dirname(absout)
@@ -191,6 +203,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
                                   shell=False,
                                   close_fds=True,
                                   stdin=stdin,
+                                  stderr=stderr,
                                   stdout=stdout,
                                   env=env,
                                   cwd=self.outdir)
@@ -203,6 +216,9 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
             if isinstance(stdin, file):
                 stdin.close()
 
+            if stderr is not sys.stderr:
+                stderr.close()
+
             if stdout is not sys.stderr:
                 stdout.close()
 

cwltool/main.py

@@ -183,6 +183,8 @@ def output_callback(out, processStatus):
         a = {"args": job.command_line}
         if job.stdin:
             a["stdin"] = job.pathmapper.mapper(job.stdin)[1]
+        if job.stderr:
+            a["stderr"] = job.stderr
         if job.stdout:
             a["stdout"] = job.stdout
         if job.generatefiles: