Merge pull request #4 from bogdang989/feature/optional-secondary

Feature/optional secondary

Author: tetron

CommandLineTool.yml

@@ -77,6 +77,7 @@ $graph:
           input object documents.
         * Clarify behavior of `glob` for absolute paths and symlinks.
         * Clarify behavior of `glob` to include directories.
+        * `secondaryFiles` can now be explicitly marked as `required` or not.
 
       See also the [CWL Workflow Description, v1.1.0-dev1 changelog](Workflow.html#Changelog).
 
@@ -207,7 +208,7 @@ $graph:
         `InputParameter.default` field) must be applied before evaluating the
         expression.
 
-        If the value of the associated input parameter is `null`, `valueFrom` is 
+        If the value of the associated input parameter is `null`, `valueFrom` is
         not evaluated and nothing is added to the command line.
 
         When a binding is part of the `CommandLineTool.arguments` field,
@@ -249,7 +250,7 @@ $graph:
           items: string
       doc: |
         Find files or directories relative to the output directory, using POSIX
-        glob(3) pathname matching.  If an array is provided, find files or 
+        glob(3) pathname matching.  If an array is provided, find files or
         directories that match any pattern in the array.  If an expression is
         provided, the expression must return a string or an array of strings,
         which will then be evaluated as one or more glob patterns.  Must only
@@ -932,14 +933,14 @@ $graph:
     - name: listing
       type:
         - type: array
-          items: 
+          items:
             - "null"
             - File
             - type: array
               items:
                 - File
                 - Directory
-            - Directory 
+            - Directory
             - Dirent
             - string
             - Expression

Process.yml

@@ -237,7 +237,9 @@ $graph:
         - "null"
         - type: array
           items: [File, Directory]
-      jsonldPredicate: "cwl:secondaryFiles"
+      jsonldPredicate:
+        _id: "cwl:secondaryFiles"
+        secondaryFilesDSL: true
       doc: |
         A list of additional files or directories that are associated with the
         primary file and must be transferred alongside the primary file.
@@ -458,20 +460,25 @@ $graph:
     secondaryFiles:
       type:
         - "null"
-        - string
-        - Expression
+        - SecondaryFileSchema
         - type: array
-          items: [string, Expression]
-      jsonldPredicate: "cwl:secondaryFiles"
+          items: SecondaryFileSchema
+      jsonldPredicate:
+        _id: "cwl:secondaryFiles"
+        secondaryFilesDSL: true
       doc: |
         Only valid when `type: File` or is an array of `items: File`.
 
-        Provides a pattern or expression specifying files or directories that
-        must be included alongside the primary file. An implementation must append 
-        these files/directories to the `secondaryFiles` property of the primary file.
-        These files/directories must be transferred alongside the primary file.
-        An implementation may fail workflow execution if an expected secondary file
-        does not exist.
+        Provides a pattern or expression specifying files or
+        directories that should be included alongside the primary
+        file.  Secondary files may be required or optional.  When not
+        explicitly specified, secondary files specified for `inputs`
+        are required and `outputs` are optional.  An implementation
+        must include matching Files and Directories in the
+        `secondaryFiles` property of the primary file.  These Files
+        and Directories must be transferred and staged alongside the
+        primary file.  An implementation may fail workflow execution
+        if a required secondary file does not exist.
 
         If the value is an expression, the value of `self` in the expression
         must be the primary input or output File object to which this binding
@@ -482,6 +489,8 @@ $graph:
         `path` or `location` and `basename` fields set, or an array consisting
         of strings or File or Directory objects.  It is legal to reference an
         unchanged File or Directory object taken from input as a secondaryFile.
+        The expression may return "null" in which case there is no secondaryFile
+        from that expression.
 
         To work on non-filename-preserving storage systems, portable tool
         descriptions should avoid constructing new values from `location`, but
@@ -492,11 +501,13 @@ $graph:
         it specifies that the following pattern should be applied to the path
         of the primary file to yield a filename relative to the primary File:
 
-          1. If string begins with one or more caret `^` characters, for each
+          1. If string ends with `?` character, remove the last `?` and mark
+            the resulting secondary file as optional.
+          2. If string begins with one or more caret `^` characters, for each
             caret, remove the last file extension from the path (the last
             period `.` and all following characters).  If there are no file
             extensions, the path is unchanged.
-          2. Append the remainder of the string to the end of the file path.
+          3. Append the remainder of the string to the end of the file path.
 
     streamable:
       type: boolean?
@@ -856,3 +867,50 @@ $graph:
         type: array
         items: CommandInputSchema
       doc: The list of type definitions.
+
+- name: SecondaryFileSchema
+  type: record
+  fields:
+    - name: pattern
+      type:
+        - string
+        - Expression
+      doc: |
+        Provides a pattern or expression specifying files or directories that
+        should be included alongside the primary file.
+
+        If the value is an expression, the value of `self` in the expression
+        must be the primary input or output File object to which this binding
+        applies.  The `basename`, `nameroot` and `nameext` fields must be
+        present in `self`.  For `CommandLineTool` outputs the `path` field must
+        also be present.  The expression must return a filename string relative
+        to the path to the primary File, a File or Directory object with either
+        `path` or `location` and `basename` fields set, or an array consisting
+        of strings or File or Directory objects.  It is legal to reference an
+        unchanged File or Directory object taken from input as a secondaryFile.
+        The expression may return "null" in which case there is no secondaryFile
+        from that expression.
+
+        To work on non-filename-preserving storage systems, portable tool
+        descriptions should avoid constructing new values from `location`, but
+        should construct relative references using `basename` or `nameroot`
+        instead.
+
+        If a value in `secondaryFiles` is a string that is not an expression,
+        it specifies that the following pattern should be applied to the path
+        of the primary file to yield a filename relative to the primary File:
+
+          1. If string begins with one or more caret `^` characters, for each
+            caret, remove the last file extension from the path (the last
+            period `.` and all following characters).  If there are no file
+            extensions, the path is unchanged.
+          2. If string ends with `?` character, remove the last `?` and mark
+            the resulting secondary file as optional.
+          3. Append the remainder of the string to the end of the file path.
+    - name: required
+      type: ["null", boolean, string, Expression]
+      doc: |
+        An implementation must not fail workflow execution if `required` is
+        set to `false` and the expected secondary file does not exist.
+        Default value for `required` field is `true` for secondary files on
+        input and `false` for secondary files on output.

Workflow.yml

@@ -71,6 +71,7 @@ $graph:
         * `WorkflowStepInput` now has a `label` field
         * [Addition](#Requirements_and_hints) of `cwl:requirments` field to
           input object documents
+        * `secondaryFiles` can now be explicitly marked as `required` or not.
 
       See also the [CWL Command Line Tool Description, v1.1.0-dev1 changelog](CommandLineTool.html#Changelog).
 
@@ -331,7 +332,7 @@ $graph:
         1. `null` if there is no `source` field
         2. the value of the parameter(s) specified in the `source` field when this
         workflow input parameter **is not** specified in this workflow step's `scatter` field.
-        3. an element of the parameter specified in the `source` field when this workflow input 
+        3. an element of the parameter specified in the `source` field when this workflow input
         parameter **is** specified in this workflow step's `scatter` field.
 
         The value of `inputs` in the parameter reference or expression must be

conformance_tests.yaml

@@ -1366,7 +1366,15 @@
   tool: tests/docker-array-secondaryfiles.cwl
   label: filesarray_secondaryfiles
   id: 100
-  doc: Test secondaryFiles on array of files.
+  doc: Test required, optional and null secondaryFiles on array of files.
+  tags: [ docker, inline_javascript, shell_command, command_line_tool ]
+
+- job: tests/docker-array-secondaryfiles-job2.json
+  should_fail: true
+  tool: tests/docker-array-secondaryfiles.cwl
+  label: filesarray_secondaryfiles2
+  id: 100
+  doc: Test required, optional and null secondaryFiles on array of files.
   tags: [ docker, inline_javascript, shell_command, command_line_tool ]
 
 - job: tests/dir7.yml

tests/docker-array-secondaryfiles-job2.json

@@ -0,0 +1,13 @@
+{
+    "fasta_path": [
+        {
+            "class": "File",
+            "location": "ref.fasta"
+        },
+        {
+            "class": "File",
+            "location": "ref2.fasta"
+        }
+    ],
+    "require_dat": true
+}

tests/docker-array-secondaryfiles.cwl

@@ -16,13 +16,25 @@ inputs:
       type: array
       items: File
     secondaryFiles:
-      - .fai
+      - pattern: .fai
+        required: true
+      - pattern: .crai
+        required: false
+      - .bai?
+      - "${ if (inputs.require_dat) {return '.dat'} else {return null} }"
+      - "${ return null; }"
+      - pattern: .dat2
+        required: $(inputs.require_dat)
+  require_dat: boolean?
 
 outputs:
   bai_list:
     type: File
     outputBinding:
       glob: "fai.list"
+    secondaryFiles:
+      - .bai?
+      - pattern: "${ return null }"
 
 arguments:
   - valueFrom: ${