Exclude children of subdirectories from symlink (#1690)

Author: jfennick

cwltool/job.py

@@ -79,7 +79,7 @@ def relink_initialworkdir(
     container_outdir: str,
     inplace_update: bool = False,
 ) -> None:
-    for _, vol in pathmapper.items():
+    for _, vol in pathmapper.items_exclude_children():
         if not vol.staged:
             continue
 

cwltool/pathmapper.py

@@ -4,6 +4,7 @@
 import stat
 import urllib
 import uuid
+from pathlib import Path
 from typing import Dict, Iterator, List, Optional, Tuple, cast
 
 from schema_salad.exceptions import ValidationException
@@ -191,6 +192,16 @@ def files(self) -> List[str]:
     def items(self) -> List[Tuple[str, MapperEnt]]:
         return list(self._pathmap.items())
 
+    def items_exclude_children(self) -> List[Tuple[str, MapperEnt]]:
+        newitems = {}
+        keys = [key for key, entry in self.items()]
+        for key, entry in self.items():
+            parents = Path(key).parents
+            if any([Path(key_) in parents for key_ in keys]):
+                continue
+            newitems[key] = entry
+        return list(newitems.items())
+
     def reversemap(
         self,
         target: str,

cwltool/process.py

@@ -263,8 +263,9 @@ def stage_files(
     fix_conflicts: bool = False,
 ) -> None:
     """Link or copy files to their targets. Create them as needed."""
+    items = pathmapper.items() if not symlink else pathmapper.items_exclude_children()
     targets = {}  # type: Dict[str, MapperEnt]
-    for key, entry in pathmapper.items():
+    for key, entry in items:
         if "File" not in entry.type:
             continue
         if entry.target not in targets:
@@ -287,7 +288,7 @@ def stage_files(
                     % (targets[entry.target].resolved, entry.resolved, entry.target)
                 )
 
-    for key, entry in pathmapper.items():
+    for key, entry in items:
         if not entry.staged:
             continue
         if not os.path.exists(os.path.dirname(entry.target)):

tests/test_iwdr.py

@@ -25,6 +25,25 @@ def test_empty_file_creation() -> None:
     assert err_code == 0
 
 
+def test_passthrough_successive(tmp_path: Path) -> None:
+    """An empty file can be successively passed through a subdir of InitialWorkingDirectory."""
+    err_code, _, _ = get_main_output(
+        [
+            "--outdir",
+            str(tmp_path),
+            get_data("tests/wf/iwdr-passthrough-successive.cwl"),
+        ]
+    )
+    assert err_code == 0
+    children = sorted(
+        tmp_path.glob("*")
+    )  # This input directory should be left pristine.
+    assert len(children) == 1
+    subdir = tmp_path / children[0]
+    assert len(sorted(subdir.glob("*"))) == 1
+    assert (subdir / "file").exists()
+
+
 @needs_docker
 def test_directory_literal_with_real_inputs_inside(tmp_path: Path) -> None:
     """Cope with unmoveable files in the output directory created by Docker+IWDR."""

tests/wf/iwdr-passthrough-successive.cwl

@@ -0,0 +1,71 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: Workflow
+
+inputs: []
+
+steps:
+  # Create a test directory structure; could be done outside CWL and passed in as input.
+  # This input directory should be left pristine.
+  mkdirs:
+    run:
+      class: CommandLineTool
+      baseCommand: [bash, '-c', 'mkdir dir dir/subdir && touch dir/subdir/file', '-']
+      inputs: []
+      outputs:
+        mkdirs_out:
+          type: Directory
+          outputBinding:
+            glob: dir
+    in: []
+    out: [mkdirs_out]
+
+  # Given an input directory, emit a subdirectory as output.
+  passthrough1:
+    run:
+      class: CommandLineTool
+      requirements:
+      - class: InitialWorkDirRequirement
+        listing:
+        - entry: $(inputs.passthrough1_in)
+          writable: false
+      baseCommand: ["true"]
+      inputs:
+        passthrough1_in:
+          type: Directory
+      outputs:
+        passthrough1_subdir:
+          type: Directory
+          outputBinding:
+            glob: $(inputs.passthrough1_in.basename)/subdir
+    in:
+      passthrough1_in: mkdirs/mkdirs_out
+    out: [passthrough1_subdir]
+
+  # Given a (sub-)directory, emit it unchanged.
+  passthrough2:
+    run:
+      class: CommandLineTool
+      requirements:
+      - class: InitialWorkDirRequirement
+        listing:
+        - entry: $(inputs.passthrough2_in)
+          writable: false
+      baseCommand: ["true"]
+      inputs:
+        passthrough2_in:
+          type: Directory
+      outputs:
+        passthrough2_subdir:
+          type: Directory
+          outputBinding:
+            glob: $(inputs.passthrough2_in.basename)
+    in:
+      passthrough2_in: passthrough1/passthrough1_subdir
+    out: [passthrough2_subdir]
+
+outputs:
+  out:
+    type: Directory
+    outputSource: passthrough2/passthrough2_subdir