Add bones of idea without actually running the script.

Author: DailyDreaming

cwltool/docker.py

@@ -7,6 +7,7 @@
 import shutil
 import subprocess  # nosec
 import sys
+import tempfile
 import threading
 from distutils import spawn
 from io import StringIO, open  # pylint: disable=redefined-builtin
@@ -102,6 +103,9 @@ def __init__(
         super(DockerCommandLineJob, self).__init__(
             builder, joborder, make_path_mapper, requirements, hints, name
         )
+        # TODO: Unused; Implement for docker as well.
+        self.universal_file_bindmount_dir = tempfile.mkdtemp(suffix='-cwl-docker-mnt')
+        self.bindings_map = []
 
     @staticmethod
     def get_image(

cwltool/job.py

@@ -4,10 +4,12 @@
 import logging
 import os
 import re
+import resource
 import shutil
 import subprocess  # nosec
 import sys
 import tempfile
+import textwrap
 import threading
 import time
 import uuid
@@ -529,6 +531,15 @@ def run(
         tmpdir_lock: Optional[threading.Lock] = None,
     ) -> None:
 
+        # attempt to set an "unlimited" (-1) heap size for this process
+        # (& thus commandline size) on any system that supports it
+        # TODO: Do containers inherit the processes's limits?
+        #  Can they be configured from outside of the container?
+        try:
+            resource.setrlimit(resource.RLIMIT_DATA, (-1, -1))
+        except Exception:
+            pass
+
         if tmpdir_lock:
             with tmpdir_lock:
                 if not os.path.exists(self.tmpdir):
@@ -589,6 +600,20 @@ def run(
 
 class ContainerCommandLineJob(JobBase, metaclass=ABCMeta):
     """Commandline job using containers."""
+    def __init__(
+        self,
+        builder: Builder,
+        joborder: CWLObjectType,
+        make_path_mapper: Callable[..., PathMapper],
+        requirements: List[CWLObjectType],
+        hints: List[CWLObjectType],
+        name: str,
+    ) -> None:
+        super(JobBase, self).__init__(
+            builder, joborder, make_path_mapper, requirements, hints, name
+        )
+        self.universal_file_bindmount_dir = None
+        self.bindings_map = None
 
     @abstractmethod
     def get_from_requirements(
@@ -710,6 +735,41 @@ def add_volumes(
                 )
                 pathmapper.update(key, new_path, vol.target, vol.type, vol.staged)
 
+        # Dir of individual file inputs for the job (all named as uuid4).
+        # This creates the same dir inside of the container as exists outside of it,
+        # Overlayfs must be supported/enabled (which should always be true for CWL).
+        src = dst = self.universal_file_bindmount_dir
+        runtime.append(f"--bind={src}:{dst}:rw")
+
+        # Make a TSV of the file mappings.
+        mapping_tsv = os.path.join(self.universal_file_bindmount_dir, 'mapping.tsv')
+        with open(mapping_tsv, 'w') as f:
+            # 1. Sort by the destination path, which should sort alphabetically
+            #    and by shortest path first.
+            # 2. Then, when we go to hardlink the files, we
+            #    should then just be able to hardlink them in order.
+            for (src, dst, writable) in sorted(self.bindings_map, key=lambda x: len(x[1])):
+                f.write('\t'.join((src, dst, writable)) + '\n')
+
+        # Make the script that uses the TSV file mappings to hardlink everything
+        # inside of the container to where the job expects to find them.
+        # This script needs to be the first thing run inside of the container.
+        linking_script = os.path.join(self.universal_file_bindmount_dir, 'hard_linking_script.py')
+        # TODO: Write in bash instead.  All images might not have python.
+        with open(linking_script, 'w') as f:
+            f.write(textwrap.dedent(f"""
+                import os
+
+                with open('{mapping_tsv}', 'r') as f:
+                    for line in f:
+                        src, dst, writable = line.split('\\t')
+                        os.makedirs(os.path.dirname(dst), exist_ok=True)
+                        os.link(src, dst)
+                        # TODO: set the permissions on the file here after linking
+
+                """[1:]))
+        os.chmod(linking_script, 0o777)
+
     def run(
         self,
         runtimeContext: RuntimeContext,

cwltool/singularity.py

@@ -5,6 +5,8 @@
 import re
 import shutil
 import sys
+import tempfile
+from uuid import uuid4
 from distutils import spawn
 from subprocess import (  # nosec
     DEVNULL,
@@ -102,6 +104,8 @@ def __init__(
         super(SingularityCommandLineJob, self).__init__(
             builder, joborder, make_path_mapper, requirements, hints, name
         )
+        self.universal_file_bindmount_dir = tempfile.mkdtemp(suffix='-cwl-singularity-mnt')
+        self.bindings_map = []
 
     @staticmethod
     def get_image(
@@ -278,18 +282,25 @@ def get_from_requirements(
 
         return os.path.abspath(cast(str, r["dockerImageId"]))
 
-    @staticmethod
     def append_volume(
-        runtime: List[str], source: str, target: str, writable: bool = False
+        self, runtime: List[str], source: str, target: str, writable: bool = False
     ) -> None:
-        runtime.append("--bind")
-        runtime.append(
-            "{}:{}:{}".format(
-                docker_windows_path_adjust(source),
-                docker_windows_path_adjust(target),
-                "rw" if writable else "ro",
-            )
-        )
+        src = docker_windows_path_adjust(source)
+        dst = docker_windows_path_adjust(target)
+        writable = "rw" if writable else "ro"
+
+        # use only "os.path.isfile(source)" for Windows? check on this...
+        if os.path.isfile(source) or os.path.isfile(src):
+            bindmount_path = os.path.join(self.universal_file_bindmount_dir, str(uuid4()))
+            os.link(src, bindmount_path)
+            self.bindings_map.append((bindmount_path, dst, writable))
+            # don't add a bind arg for the shared self.universal_file_bindmount_dir
+            # here but at the very end
+        else:
+            # TODO: We can still bind enough dirs to exceed the max command line length.
+            #  Not sure how to handle this, since outputs deposited in mounted dirs
+            #  need to be there after the run.
+            runtime.append(f"--bind={src}:{dst}:{writable}")
 
     def add_file_or_directory_volume(
         self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str]