Fix algorithm for generating output names when output names clash (#1342)

Author: pvanheus

cwltool/process.py

@@ -276,12 +276,13 @@ def stage_files(
             targets[entry.target] = entry
         elif targets[entry.target].resolved != entry.resolved:
             if fix_conflicts:
-                tgt = entry.target
+                # find first key that does not clash with an existing entry in targets
+                # start with entry.target + '_' + 2 and then keep incrementing the number till there is no clash
                 i = 2
-                tgt = "%s_%s" % (tgt, i)
+                tgt = "%s_%s" % (entry.target, i)
                 while tgt in targets:
                     i += 1
-                    tgt = "%s_%s" % (tgt, i)
+                    tgt = "%s_%s" % (entry.target, i)
                 targets[tgt] = pathmapper.update(
                     key, entry.resolved, tgt, entry.type, entry.staged
                 )

tests/scatter_numbers.cwl

@@ -0,0 +1,55 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.1
+class: Workflow
+
+inputs:
+  range:
+    type: int
+outputs:
+  output:
+    type: File[]
+    outputSource:
+      generate_files/output
+
+steps:
+  generate_list:
+    requirements:
+      - class: InlineJavascriptRequirement
+    run:
+      class: ExpressionTool
+      inputs:
+        max: 
+          type: int
+          default: 100
+      outputs:
+        numbers:
+          type: int[]
+      expression: |
+        ${
+          var numberList = Array.apply(null, Array(inputs.max)).map(function(_, i) { return i});
+          return { "numbers": numberList } 
+         }
+    in:
+      max: range
+    out:
+      - numbers
+  generate_files:
+    requirements:
+      - class: ScatterFeatureRequirement
+    scatter: number
+    run:
+      class: CommandLineTool
+      inputs:
+        number:
+          type: int
+          inputBinding:
+            position: 10
+      baseCommand: [ echo ]
+      stdout: output.txt
+      outputs:
+        output:
+          type: stdout
+    in:
+      number: generate_list/numbers
+    out:
+      - output

tests/test_examples.py

@@ -1261,3 +1261,23 @@ def test_v1_0_arg_empty_prefix_separate_false() -> None:
     )
     assert "completed success" in stderr
     assert error_code == 0
+
+
+def test_scatter_output_filenames(tmpdir: py.path.local) -> None:
+    """If a scatter step produces identically named output then confirm that the final output is renamed correctly."""
+    cwd = tmpdir.chdir()
+    rtc = RuntimeContext()
+    rtc.outdir = str(cwd)
+    factory = cwltool.factory.Factory(runtime_context=rtc)
+    output_names = ['output.txt', 'output.txt_2', 'output.txt_3']
+    scatter_workflow = factory.make(get_data("tests/scatter_numbers.cwl"))
+    result = scatter_workflow(range=3)
+    assert 'output' in result
+    
+    locations = sorted([element['location'] for element in result['output']])
+
+    assert(
+        locations[0].endswith('output.txt') and
+        locations[1].endswith('output.txt_2') and
+        locations[2].endswith('output.txt_3')
+    ), "Locations {} do not end with {}".format(locations, output_names)