Allow software requirements to modify env.

First do basic env setup, run the modification commands in a script
and record the environment. Have the run_job script then use this
environment (ignoring some variables).

Author: rupertnash

Makefile

@@ -116,7 +116,7 @@ pylint_report.txt: $(PYSOURCES)
 		$^ -j0> $@ || true
 
 diff_pylint_report: pylint_report.txt
-	diff-quality --violations=pylint pylint_report.txt
+	diff-quality --compare-branch=main --violations=pylint pylint_report.txt
 
 .coverage: testcov
 
@@ -136,10 +136,10 @@ coverage-report: .coverage
 	coverage report
 
 diff-cover: coverage.xml
-	diff-cover $^
+	diff-cover --compare-branch=main $^
 
 diff-cover.html: coverage.xml
-	diff-cover $^ --html-report $@
+	diff-cover --compare-branch=main $^ --html-report $@
 
 ## test        : run the ${MODULE} test suite
 test: $(PYSOURCES)

README.rst

@@ -316,6 +316,13 @@ This option allows one to specify a dependency resolver's configuration file.
 This file may be specified as either XML or YAML and very simply describes various
 plugins to enable to "resolve" ``SoftwareRequirement`` dependencies.
 
+Using these hints will allow cwltool to modify the environment in
+which your tool runs, for example by loading one or more environment
+modules. The enviroment is constructed as normal (i.e. standard CWL
+runtime enviroment, ``EnvVarRequirement``, ``--preserve-environment``
+are applied), then this is modified. This currently means that you
+cannot override variables set in this way.
+
 To discuss some of these plugins and how to configure them, first consider the
 following ``hint`` definition for an example CWL tool.
 
@@ -427,7 +434,7 @@ So consider the resolvers configuration file
     base_path: ./tests/test_deps_env
     mapping_files: ./tests/test_deps_mapping.yml
 
-And the corresponding mapping configuraiton file (`tests/test_deps_mapping.yml`):
+And the corresponding mapping configuration file (`tests/test_deps_mapping.yml`):
 
 .. code:: yaml
 

cwltool/job.py

@@ -36,6 +36,7 @@
 from schema_salad.utils import json_dump, json_dumps
 from typing_extensions import TYPE_CHECKING
 
+from . import run_job
 from .builder import Builder, HasReqsHints
 from .context import RuntimeContext
 from .errors import UnsupportedRequirement, WorkflowException
@@ -62,57 +63,7 @@
 FORCE_SHELLED_POPEN = os.getenv("CWLTOOL_FORCE_SHELL_POPEN", "0") == "1"
 
 SHELL_COMMAND_TEMPLATE = """#!/bin/bash
-python "run_job.py" "job.json"
-"""
-
-PYTHON_RUN_SCRIPT = """
-import json
-import os
-import sys
-try:
-    import subprocess32 as subprocess  # type: ignore
-except Exception:
-    import subprocess
-
-with open(sys.argv[1], "r") as f:
-    popen_description = json.load(f)
-    commands = popen_description["commands"]
-    cwd = popen_description["cwd"]
-    env = popen_description["env"]
-    env["PATH"] = os.environ.get("PATH")
-    stdin_path = popen_description["stdin_path"]
-    stdout_path = popen_description["stdout_path"]
-    stderr_path = popen_description["stderr_path"]
-    if stdin_path is not None:
-        stdin = open(stdin_path, "rb")
-    else:
-        stdin = subprocess.PIPE
-    if stdout_path is not None:
-        stdout = open(stdout_path, "wb")
-    else:
-        stdout = sys.stderr
-    if stderr_path is not None:
-        stderr = open(stderr_path, "wb")
-    else:
-        stderr = sys.stderr
-    sp = subprocess.Popen(commands,
-                          shell=False,
-                          close_fds=True,
-                          stdin=stdin,
-                          stdout=stdout,
-                          stderr=stderr,
-                          env=env,
-                          cwd=cwd)
-    if sp.stdin:
-        sp.stdin.close()
-    rcode = sp.wait()
-    if stdin is not subprocess.PIPE:
-        stdin.close()
-    if stdout is not sys.stderr:
-        stdout.close()
-    if stderr is not sys.stderr:
-        stderr.close()
-    sys.exit(rcode)
+python3 "run_job.py" "job.json"
 """
 
 
@@ -988,8 +939,7 @@ def terminate():  # type: () -> None
             with open(job_script, "wb") as _:
                 _.write(job_script_contents.encode("utf-8"))
             job_run = os.path.join(job_dir, "run_job.py")
-            with open(job_run, "wb") as _:
-                _.write(PYTHON_RUN_SCRIPT.encode("utf-8"))
+            shutil.copyfile(run_job.__file__, job_run)
             sproc = subprocess.Popen(  # nosec
                 ["bash", job_script.encode("utf-8")],
                 shell=False,  # nosec

cwltool/run_job.py

@@ -0,0 +1,96 @@
+"""Only used when there is a job script or CWLTOOL_FORCE_SHELL_POPEN=1."""
+import json
+import os
+import subprocess  # nosec
+import sys
+from typing import BinaryIO, Dict, List, Optional, TextIO, Union
+
+
+def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[str, str]:
+    """Update the provided environment variables dictiony by running the script."""
+    res = subprocess.run(["bash", script], shell=False, env=cwl_env)  # nosec
+    if res.returncode != 0:
+        sys.stderr.write(
+            "Error while using SoftwareRequirements to modify environment\n"
+        )
+        return cwl_env
+    env = cwl_env.copy()
+    with open("output_environment.bash") as env_file:
+        for line in env_file:
+            key, val = line.split("=", 1)
+            if key in ("_", "PWD", "SHLVL"):
+                # Skip some variables that are meaningful to the shell
+                continue
+            env[key] = val[:-1]  # remove trailing newline
+    return env
+
+
+def main(argv: List[str]) -> int:
+    """
+    Read in the configuration JSON and execute the commands.
+
+    The first argument is the path to the JSON dictionary file containing keys:
+      "commands": an array of strings that represents the command line to run
+      "cwd": A string specifying which directory to run in
+      "env": a dictionary of strings containing the environment variables to set
+      "stdin_path": a string (or a null) giving the path that should be piped to STDIN
+      "stdout_path": a string (or a null) giving the path that should receive the STDOUT
+      "stderr_path": a string (or a null) giving the path that should receive the STDERR
+
+    The second argument is optional, it specifes a shell script to execute prior,
+      and the environment variables it sets will be combined with the environment
+      variables from the "env" key in the JSON dictionary from the first argument.
+    """
+    with open(argv[1]) as f:
+        popen_description = json.load(f)
+        commands = popen_description["commands"]
+        cwd = popen_description["cwd"]
+        env = popen_description["env"]
+        env["PATH"] = os.environ.get("PATH")
+        stdin_path = popen_description["stdin_path"]
+        stdout_path = popen_description["stdout_path"]
+        stderr_path = popen_description["stderr_path"]
+        if stdin_path is not None:
+            stdin: Union[BinaryIO, int] = open(stdin_path, "rb")
+        else:
+            stdin = subprocess.PIPE
+        if stdout_path is not None:
+            stdout: Union[BinaryIO, TextIO] = open(stdout_path, "wb")
+        else:
+            stdout = sys.stderr
+        if stderr_path is not None:
+            stderr: Union[BinaryIO, TextIO] = open(stderr_path, "wb")
+        else:
+            stderr = sys.stderr
+
+        try:
+            env_script: Optional[str] = argv[2]
+        except IndexError:
+            env_script = None
+        if env_script is not None:
+            env = handle_software_environment(env, env_script)
+
+        sp = subprocess.Popen(  # nosec
+            commands,
+            shell=False,
+            close_fds=True,
+            stdin=stdin,
+            stdout=stdout,
+            stderr=stderr,
+            env=env,
+            cwd=cwd,
+        )
+        if sp.stdin:
+            sp.stdin.close()
+        rcode = sp.wait()
+        if not isinstance(stdin, int):
+            stdin.close()
+        if stdout is not sys.stderr:
+            stdout.close()
+        if stderr is not sys.stderr:
+            stderr.close()
+    return rcode
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

cwltool/software_requirements.py

@@ -30,8 +30,11 @@
 
 COMMAND_WITH_DEPENDENCIES_TEMPLATE = string.Template(
     """#!/bin/bash
+cat > modify_environment.bash <<'EOF'
 $handle_dependencies
-python3 "run_job.py" "job.json"
+env > output_environment.bash
+EOF
+python3 "run_job.py" "job.json" "modify_environment.bash"
 """
 )