--write-summary to redirect final JSON output (#1773)

Co-authored-by: Jake Fennick <[email protected]>
Co-authored-by: Michael R. Crusoe <[email protected]>

Author: jfennick

cwltool/argparser.py

@@ -390,6 +390,15 @@ def arg_parser() -> argparse.ArgumentParser:
         "--debug", action="store_true", help="Print even more logging"
     )
 
+    parser.add_argument(
+        "--write-summary",
+        "-w",
+        type=str,
+        help="Path to write the final output JSON object to. Default is stdout.",
+        default="",
+        dest="write_summary",
+    )
+
     parser.add_argument(
         "--strict-memory-limit",
         action="store_true",

cwltool/cwlrdf.py

@@ -1,6 +1,6 @@
 import urllib
 from codecs import StreamWriter
-from typing import Any, Dict, Iterator, Optional, TextIO, Union, cast
+from typing import IO, Any, Dict, Iterator, Optional, TextIO, Union, cast
 
 from rdflib import Graph
 from rdflib.query import ResultRow
@@ -217,7 +217,7 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non
 def printdot(
     wf: Process,
     ctx: ContextType,
-    stdout: Union[TextIO, StreamWriter],
+    stdout: IO[str],
 ) -> None:
     cwl_viewer = CWLViewer(printrdf(wf, ctx, "n3"))  # type: CWLViewer
     stdout.write(cwl_viewer.dot().replace(f"{wf.metadata['id']}#", ""))

cwltool/main.py

@@ -39,7 +39,13 @@
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import Loader, file_uri, uri_file_path
 from schema_salad.sourceline import cmap, strip_dup_lineno
-from schema_salad.utils import ContextType, FetcherCallableType, json_dumps, yaml_no_ts
+from schema_salad.utils import (
+    ContextType,
+    FetcherCallableType,
+    json_dump,
+    json_dumps,
+    yaml_no_ts,
+)
 
 import ruamel.yaml
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
@@ -415,7 +421,7 @@ def init_job_order(
     args: argparse.Namespace,
     process: Process,
     loader: Loader,
-    stdout: Union[TextIO, StreamWriter],
+    stdout: IO[str],
     print_input_deps: bool = False,
     relative_deps: str = "primary",
     make_fs_access: Callable[[str], StdFsAccess] = StdFsAccess,
@@ -438,7 +444,7 @@ def init_job_order(
             file_uri(os.getcwd()) + "/",
         )
         if args.tool_help:
-            toolparser.print_help(cast(IO[str], stdout))
+            toolparser.print_help(stdout)
             exit(0)
         cmd_line = vars(toolparser.parse_args(args.job_order))
         for record_name in records:
@@ -564,7 +570,7 @@ def make_relative(base: str, obj: CWLObjectType) -> None:
 def printdeps(
     obj: CWLObjectType,
     document_loader: Loader,
-    stdout: Union[TextIO, StreamWriter],
+    stdout: IO[str],
     relative_deps: str,
     uri: str,
     basedir: Optional[str] = None,
@@ -577,7 +583,7 @@ def printdeps(
     elif relative_deps == "cwd":
         base = os.getcwd()
     visit_class(deps, ("File", "Directory"), functools.partial(make_relative, base))
-    print(json_dumps(deps, indent=4, default=str), file=stdout)
+    json_dump(deps, stdout, indent=4, default=str)
 
 
 def prov_deps(
@@ -641,10 +647,10 @@ def print_pack(
     """Return a CWL serialization of the CWL document in JSON."""
     packed = pack(loadingContext, uri)
     if len(cast(Sized, packed["$graph"])) > 1:
-        return json_dumps(packed, indent=4, default=str)
-    return json_dumps(
-        cast(MutableSequence[CWLObjectType], packed["$graph"])[0], indent=4, default=str
-    )
+        target = packed
+    else:
+        target = cast(MutableSequence[CWLObjectType], packed["$graph"])[0]
+    return json_dumps(target, indent=4, default=str)
 
 
 def supported_cwl_versions(enable_dev: bool) -> List[str]:
@@ -763,9 +769,7 @@ def setup_loadingContext(
     return loadingContext
 
 
-def make_template(
-    tool: Process,
-) -> None:
+def make_template(tool: Process, target: IO[str]) -> None:
     """Make a template CWL input object for the give Process."""
 
     def my_represent_none(
@@ -783,7 +787,7 @@ def my_represent_none(
     yaml.block_seq_indent = 2
     yaml.dump(
         generate_input_template(tool),
-        sys.stdout,
+        target,
     )
 
 
@@ -945,7 +949,7 @@ def check_working_directories(
 
 def print_targets(
     tool: Process,
-    stdout: Union[TextIO, StreamWriter],
+    stdout: IO[str],
     loading_context: LoadingContext,
     prefix: str = "",
 ) -> None:
@@ -982,7 +986,7 @@ def main(
     args: Optional[argparse.Namespace] = None,
     job_order_object: Optional[CWLObjectType] = None,
     stdin: IO[Any] = sys.stdin,
-    stdout: Optional[Union[TextIO, StreamWriter]] = None,
+    stdout: Optional[IO[str]] = None,
     stderr: IO[Any] = sys.stderr,
     versionfunc: Callable[[], str] = versionstring,
     logger_handler: Optional[logging.Handler] = None,
@@ -992,17 +996,18 @@ def main(
     runtimeContext: Optional[RuntimeContext] = None,
     input_required: bool = True,
 ) -> int:
-    if not stdout:  # force UTF-8 even if the console is configured differently
+    if stdout is None:  # force UTF-8 even if the console is configured differently
         if hasattr(sys.stdout, "encoding") and sys.stdout.encoding.upper() not in (
             "UTF-8",
             "UTF8",
         ):
             if hasattr(sys.stdout, "detach"):
                 stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
             else:
-                stdout = getwriter("utf-8")(sys.stdout)  # type: ignore
+                stdout = getwriter("utf-8")(sys.stdout)  # type: ignore[assignment,arg-type]
         else:
             stdout = sys.stdout
+        stdout = cast(IO[str], stdout)
 
     _logger.removeHandler(defaultStreamHandler)
     stderr_handler = logger_handler
@@ -1151,15 +1156,13 @@ def main(
                 )
 
             if args.print_pre:
-                print(
-                    json_dumps(
-                        processobj,
-                        indent=4,
-                        sort_keys=True,
-                        separators=(",", ": "),
-                        default=str,
-                    ),
-                    file=stdout,
+                json_dump(
+                    processobj,
+                    stdout,
+                    indent=4,
+                    sort_keys=True,
+                    separators=(",", ": "),
+                    default=str,
                 )
                 return 0
 
@@ -1179,7 +1182,7 @@ def main(
                     raise main_missing_exc
 
             if args.make_template:
-                make_template(tool)
+                make_template(tool, stdout)
                 return 0
 
             if args.validate:
@@ -1225,15 +1228,13 @@ def main(
             if args.print_subgraph:
                 if "name" in tool.tool:
                     del tool.tool["name"]
-                print(
-                    json_dumps(
-                        tool.tool,
-                        indent=4,
-                        sort_keys=True,
-                        separators=(",", ": "),
-                        default=str,
-                    ),
-                    file=stdout,
+                json_dump(
+                    tool.tool,
+                    stdout,
+                    indent=4,
+                    sort_keys=True,
+                    separators=(",", ": "),
+                    default=str,
                 )
                 return 0
 
@@ -1398,12 +1399,15 @@ def loc_to_path(obj: CWLObjectType) -> None:
                 # Unsetting the Generation from final output object
                 visit_class(out, ("File",), MutationManager().unset_generation)
 
-                print(
-                    json_dumps(out, indent=4, ensure_ascii=False, default=str),
-                    file=stdout,
-                )
-                if hasattr(stdout, "flush"):
-                    stdout.flush()
+                if args.write_summary:
+                    with open(args.write_summary, "w") as output_file:
+                        json_dump(
+                            out, output_file, indent=4, ensure_ascii=False, default=str
+                        )
+                else:
+                    json_dump(out, stdout, indent=4, ensure_ascii=False, default=str)
+                    if hasattr(stdout, "flush"):
+                        stdout.flush()
 
             if status != "success":
                 _logger.warning("Final process status is %s", status)

tests/test_examples.py

@@ -1348,6 +1348,31 @@ def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
     assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists()
 
 
+def test_write_summary(tmp_path: Path) -> None:
+    """Test --write-summary."""
+    commands = [
+        get_data("tests/wf/no-parameters-echo.cwl"),
+    ]
+    error_code, stdout, stderr = get_main_output(commands)
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert error_code == 0, stderr
+
+    final_output_path = str(tmp_path / "final-output.json")
+    commands_no = [
+        "--write-summary",
+        final_output_path,
+        get_data("tests/wf/no-parameters-echo.cwl"),
+    ]
+    error_code, stdout_no, stderr = get_main_output(commands_no)
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert error_code == 0, stderr
+
+    with open(final_output_path) as f:
+        final_output_str = f.read()
+
+    assert len(stdout_no) + len(final_output_str) == len(stdout)
+
+
 @needs_docker
 def test_compute_checksum() -> None:
     runtime_context = RuntimeContext()