enable use inside Jupyter notebooks (#1272)

* make default stdin, stdout, stderr replaceable by runtimeContext
* add test for replacing default stdout & stderr

Author: giannisdoukas

.gitignore

@@ -42,6 +42,7 @@ lib/
 .cache/
 cache/
 .coverage
+.coverage.*
 coverage.xml
 htmlcov
 output

cwltool/context.py

@@ -1,7 +1,7 @@
 """Shared context objects that replace use of kwargs."""
 import copy
 import threading
-from typing import Any, Callable, Dict, Iterable, List, Optional, Union
+from typing import Any, Callable, Dict, Iterable, List, Optional, Union, IO, TextIO
 
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 from ruamel.yaml.comments import CommentedMap
@@ -143,7 +143,8 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
         self.process_run_id = None  # type: Optional[str]
         self.prov_obj = None  # type: Optional[ProvenanceProfile]
         self.mpi_config = MpiConfig()  # type: MpiConfig
-
+        self.default_stdout = None  # type: Optional[Union[IO[bytes], TextIO]]
+        self.default_stderr = None  # type: Optional[Union[IO[bytes], TextIO]]
         super(RuntimeContext, self).__init__(kwargs)
 
     def copy(self):

cwltool/job.py

@@ -386,6 +386,8 @@ def _execute(
                 timelimit=self.timelimit,
                 name=self.name,
                 monitor_function=monitor_function,
+                default_stdout=runtimeContext.default_stdout,
+                default_stderr=runtimeContext.default_stderr,
             )
 
             if rcode in self.successCodes:
@@ -915,6 +917,8 @@ def _job_popen(
     timelimit: Optional[int] = None,
     name: Optional[str] = None,
     monitor_function=None,  # type: Optional[Callable[[subprocess.Popen[str]], None]]
+    default_stdout=None,  # type: Optional[Union[IO[bytes], TextIO]]
+    default_stderr=None,  # type: Optional[Union[IO[bytes], TextIO]]
 ) -> int:
 
     if job_script_contents is None and not FORCE_SHELLED_POPEN:
@@ -923,11 +927,11 @@ def _job_popen(
         if stdin_path is not None:
             stdin = open(stdin_path, "rb")
 
-        stdout = sys.stderr  # type: Union[IO[bytes], TextIO]
+        stdout = default_stdout if default_stdout is not None else sys.stderr  # type: Union[IO[bytes], TextIO]
         if stdout_path is not None:
             stdout = open(stdout_path, "wb")
 
-        stderr = sys.stderr  # type: Union[IO[bytes], TextIO]
+        stderr = default_stderr if default_stderr is not None else sys.stderr  # type: Union[IO[bytes], TextIO]
         if stderr_path is not None:
             stderr = open(stderr_path, "wb")
 
@@ -972,13 +976,13 @@ def terminate():  # type: () -> None
         if tm is not None:
             tm.cancel()
 
-        if isinstance(stdin, IOBase):
+        if isinstance(stdin, IOBase) and hasattr(stdin, 'close'):
             stdin.close()
 
-        if stdout is not sys.stderr:
+        if stdout is not sys.stderr and hasattr(stdout, 'close'):
             stdout.close()
 
-        if stderr is not sys.stderr:
+        if stderr is not sys.stderr and hasattr(stderr, 'close'):
             stderr.close()
 
         return rcode

tests/test_context.py

@@ -0,0 +1,25 @@
+import subprocess
+
+from cwltool.context import RuntimeContext
+from .util import get_windows_safe_factory, get_data, windows_needs_docker
+
+
+@windows_needs_docker
+def test_replace_default_stdout_stderr():
+    import sys
+    # break stdout & stderr
+    original_stdout = sys.stdout
+    original_stderr = sys.stderr
+
+    sys.stdout = ''
+    sys.stderr = ''
+
+    runtime_context = RuntimeContext()
+    runtime_context.default_stdout = subprocess.DEVNULL
+    runtime_context.default_stderr = subprocess.DEVNULL
+    factory = get_windows_safe_factory(runtime_context=runtime_context)
+    echo = factory.make(get_data("tests/echo.cwl"))
+
+    assert echo(inp="foo") == {"out": "foo\n"}
+    sys.stdout = original_stdout
+    sys.stderr = original_stderr