Merge pull request #719 from common-workflow-language/singularity_userns

Singularity userns support

Author: mr-c

cwltool/docker.py

@@ -4,22 +4,25 @@
 import os
 import re
 import shutil
-import subprocess
 import sys
 import tempfile
 from io import open
-
 import datetime
+import threading
+
 import requests
 from typing import (Dict, List, Text, Any, MutableMapping, Set)
-import threading
 
 from .docker_id import docker_vm_id
 from .errors import WorkflowException
 from .job import ContainerCommandLineJob
 from .pathmapper import PathMapper, ensure_writable
 from .secrets import SecretStore
 from .utils import docker_windows_path_adjust, onWindows
+if os.name == 'posix' and sys.version_info[0] < 3:
+    import subprocess32 as subprocess  # type: ignore
+else:
+    import subprocess  # type: ignore
 
 _logger = logging.getLogger("cwltool")
 

cwltool/docker_id.py

@@ -1,8 +1,12 @@
 from __future__ import print_function
 from __future__ import absolute_import
-
-import subprocess
+import os
+import sys
 from typing import List, Text, Tuple
+if os.name == 'posix' and sys.version_info[0] < 3:
+    import subprocess32 as subprocess  # type: ignore
+else:
+    import subprocess  # type: ignore
 
 
 def docker_vm_id():  # type: () -> Tuple[int, int]

cwltool/job.py

@@ -9,7 +9,6 @@
 import re
 import shutil
 import stat
-import subprocess
 import sys
 import tempfile
 from abc import ABCMeta, abstractmethod
@@ -28,6 +27,11 @@
 from .secrets import SecretStore
 from .utils import bytes2str_in_dicts
 from .utils import copytree_with_merge, onWindows
+if os.name == 'posix' and sys.version_info[0] < 3:
+    import subprocess32 as subprocess  # type: ignore
+else:
+    import subprocess  # type: ignore
+
 
 _logger = logging.getLogger("cwltool")
 

cwltool/sandboxjs.py

@@ -5,20 +5,23 @@
 import os
 import re
 import select
-import subprocess
 import threading
 import sys
 from io import BytesIO
 from typing import Any, Dict, List, Mapping, Text, Tuple, Union
-from .utils import onWindows
-from pkg_resources import resource_stream
-
 import six
+from pkg_resources import resource_stream
+from .utils import onWindows
 
 try:
     import queue  # type: ignore
 except ImportError:
     import Queue as queue  # type: ignore
+if os.name == 'posix' and sys.version_info[0] < 3:
+    import subprocess32 as subprocess  # type: ignore
+else:
+    import subprocess  # type: ignore
+
 
 class JavascriptException(Exception):
     pass
@@ -194,7 +197,7 @@ def terminate():
 
     PROCESS_FINISHED_STR = "r1cepzbhUTxtykz5XTC4\n"
 
-    def process_finished(): # type: () -> bool
+    def process_finished():  # type: () -> bool
         return stdout_buf.getvalue().decode('utf-8').endswith(PROCESS_FINISHED_STR) and \
             stderr_buf.getvalue().decode('utf-8').endswith(PROCESS_FINISHED_STR)
 
@@ -363,4 +366,4 @@ def fn_linenum():  # type: () -> Text
         return json.loads(stdout)
     except ValueError as e:
         raise JavascriptException(u"%s\nscript was:\n%s\nstdout was: '%s'\nstderr was: '%s'\n" %
-                                    (e, fn_linenum(), stdout, stderr))
+                                  (e, fn_linenum(), stdout, stderr))

cwltool/singularity.py

@@ -1,22 +1,41 @@
+"""Support for executing Docker containers using Singularity."""
 from __future__ import absolute_import
-
 import logging
 import os
 import re
 import shutil
-import subprocess
 import sys
-from io import open
-
-from typing import (Dict, List, Text, Optional, MutableMapping, Any)
-
+from io import open  # pylint: disable=redefined-builtin
+from typing import (Dict, List, Text, Optional, MutableMapping)
 from .errors import WorkflowException
 from .job import ContainerCommandLineJob
 from .pathmapper import PathMapper, ensure_writable
 from .process import (UnsupportedRequirement)
 from .utils import docker_windows_path_adjust
+if os.name == 'posix' and sys.version_info[0] < 3:
+    from subprocess32 import (check_call, check_output,  # pylint: disable=import-error
+                              CalledProcessError, DEVNULL, PIPE, Popen,
+                              TimeoutExpired)
+elif os.name == 'posix':
+    from subprocess import (check_call, check_output,  # type: ignore
+                            CalledProcessError, DEVNULL, PIPE, Popen,
+                            TimeoutExpired)
+else:  # we're not on Unix, so none of this matters
+    pass
 
 _logger = logging.getLogger("cwltool")
+_USERNS = None
+
+def _singularity_supports_userns():  # type: ()->bool
+    global _USERNS  # pylint: disable=global-statement
+    if _USERNS is None:
+        try:
+            _USERNS = "No valid /bin/sh" in Popen(
+                [u"singularity", u"exec", u"--userns", u"/etc", u"true"],
+                stderr=PIPE, stdout=DEVNULL).communicate(timeout=60)[1]
+        except TimeoutExpired:
+            _USERNS = False
+    return _USERNS
 
 
 class SingularityCommandLineJob(ContainerCommandLineJob):
@@ -82,7 +101,7 @@ def get_image(dockerRequirement,  # type: Dict[Text, Text]
                        str(dockerRequirement["dockerPull"])]
                 _logger.info(Text(cmd))
                 if not dry_run:
-                    subprocess.check_call(cmd, stdout=sys.stderr)
+                    check_call(cmd, stdout=sys.stderr)
                     found = True
 
         return found
@@ -107,8 +126,8 @@ def get_from_requirements(self,
         if r:
             errmsg = None
             try:
-                subprocess.check_output(["singularity", "--version"])
-            except subprocess.CalledProcessError as err:
+                check_output(["singularity", "--version"])
+            except CalledProcessError as err:
                 errmsg = "Cannot execute 'singularity --version' {}".format(err)
             except OSError as err:
                 errmsg = "'singularity' executable not found: {}".format(err)
@@ -154,7 +173,7 @@ def add_volumes(self, pathmapper, runtime, stage_output):
             elif vol.type == "WritableFile":
                 if self.inplace_update:
                     runtime.append(u"--bind")
-                    runtime.append("{}:{}:rw".format(
+                    runtime.append(u"{}:{}:rw".format(
                         docker_windows_path_adjust(vol.resolved),
                         docker_windows_path_adjust(containertgt)))
                 else:
@@ -166,7 +185,7 @@ def add_volumes(self, pathmapper, runtime, stage_output):
                 else:
                     if self.inplace_update:
                         runtime.append(u"--bind")
-                        runtime.append("{}:{}:rw".format(
+                        runtime.append(u"{}:{}:rw".format(
                             docker_windows_path_adjust(vol.resolved),
                             docker_windows_path_adjust(containertgt)))
                     else:
@@ -176,7 +195,7 @@ def add_volumes(self, pathmapper, runtime, stage_output):
                 with open(createtmp, "wb") as tmp:
                     tmp.write(vol.resolved.encode("utf-8"))
                 runtime.append(u"--bind")
-                runtime.append("{}:{}:ro".format(
+                runtime.append(u"{}:{}:ro".format(
                     docker_windows_path_adjust(createtmp),
                     docker_windows_path_adjust(vol.target)))
 
@@ -192,7 +211,9 @@ def create_runtime(self,
         """ Returns the Singularity runtime list of commands and options."""
 
         runtime = [u"singularity", u"--quiet", u"exec", u"--contain", u"--pid",
-                   u"--ipc"]  # , u"--userns"]
+                   u"--ipc"]
+        if _singularity_supports_userns():
+            runtime.append(u"--userns")
         runtime.append(u"--bind")
         runtime.append(u"{}:{}:rw".format(
             docker_windows_path_adjust(os.path.realpath(self.outdir)),

setup.py

@@ -58,7 +58,9 @@
           'six >= 1.8.0',
       ],
       extras_require={
-          ':python_version<"3"': [ 'pathlib2' ],
+          ':python_version<"3" and platform_system=="Linux"':
+          ['subprocess32 == 3.5.0rc1'],
+          ':python_version<"3"': ['pathlib2'],
           'deps': ["galaxy-lib >= 17.09.3"]
       },
       python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4',