Merge commit 'cb7f18c050334af7b964e84e03f25b975be0880f' into stagefiles

Author: Peter Amstutz

cwltool/schemas/draft-4/CommandLineTool.yml

@@ -79,30 +79,6 @@ $graph:
     - {$include: concepts.md}
     - {$include: invocation.md}
 
-- type: record
-  name: FileDef
-  doc: |
-    Define a file that must be placed in the designated output directory
-    prior to executing the command line tool.  May be the result of executing
-    an expression, such as building a configuration file from a template.
-  fields:
-    - name: "filename"
-      type: ["string", "#Expression"]
-      doc: "The name of the file to create in the output directory."
-    - name: "fileContent"
-      type: ["string", "#Expression"]
-      doc: |
-        If the value is a string literal or an expression which evaluates to a
-        string, a new file must be created with the string as the file contents.
-
-        If the value is an expression that evaluates to a File object, this
-        indicates the referenced file should be added to the designated output
-        directory prior to executing the tool.
-
-        Files added in this way may be read-only, and may be provided
-        by bind mounts or file system links to avoid
-        unnecessary copying of the input file.
-
 
 - type: record
   name: EnvironmentDef
@@ -628,20 +604,53 @@ $graph:
 
 - name: DirentExt
   type: record
+  doc: |
+    Define a file or subdirectory that must be placed in the designated output
+    directory prior to executing the command line tool.  May be the result of
+    executing an expression, such as building a configuration file from a
+    template.
   fields:
     - name: entryname
       type: [string, Expression]
       jsonldPredicate:
         _id: cwl:entryname
+      doc: |
+        The name of the file or subdirectory to create in the output directory.
     - name: entry
       type: [string, Expression]
       jsonldPredicate:
         _id: cwl:entry
+      doc: |
+        If the value is a string literal or an expression which evaluates to a
+        string, a new file must be created with the string as the file contents.
 
+        If the value is an expression that evaluates to a `File` object, this
+        indicates the referenced file should be added to the designated output
+        directory prior to executing the tool.
+
+        If the value is an expression that evaluates to a `Dirent` object, this
+        indicates that the File or Directory in `entry` should be added to the
+        designated output directory with the name in `entryname`.
+
+        If `writable` is false, the file may be made available using a bind
+        mount or file system link to avoid unnecessary copying of the input
+        file.
+    - name: writable
+      type: boolean?
+      doc: |
+        If true, the file or directory must be writable by the tool.  Changes
+        to the file or directory must be isolated and not visible by any other
+        CommandLineTool process.  This may be implemented by making a copy of
+        the original file or directory.  Default false (files and directories
+        read-only by default).
 
 - name: InitialWorkDirRequirement
   type: record
   extends: ProcessRequirement
+  doc:
+    Define a list of files and subdirectories that must be created by the
+    workflow platform in the designated output directory prior to executing the
+    command line tool.
   fields:
     - name: listing
       type:
@@ -653,24 +662,12 @@ $graph:
         _id: "cwl:listing"
         mapSubject: entryname
         mapPredicate: entry
+      doc: |
+        The list of files or subdirectories that must be placed in the
+        designated output directory prior to executing the command line tool.
 
-
-- name: CreateFileRequirement
-  type: record
-  extends: "#ProcessRequirement"
-  doc: |
-    *Deprecated*, superceded by InitialWorkDirRequirement.
-
-    Define a list of files that must be created by the workflow
-    platform in the designated output directory prior to executing the command
-    line tool.  See `FileDef` for details.
-  fields:
-    - name: fileDef
-      type:
-        type: "array"
-        items: "#FileDef"
-      doc: The list of files.
-
+        May be an expression.  If so, the expression return value must validate
+        as `{type: array, items: [File, Dirent]}`.
 
 - name: EnvVarRequirement
   type: record

cwltool/schemas/draft-4/Process.yml

@@ -74,16 +74,21 @@ $graph:
         implementation is unable to retrieve the file content stored at a
         remote resource (due to unsupported protocol, access denied, or other
         issue) it must signal an error.
+
+        If the `path` field is provided but the `location` field is not, an
+        implementation may assign the value of the `path` field to `location`,
+        then follow the rules above.
       jsonldPredicate:
         _id: "@id"
         _type: "@id"
     - name: path
       type: string?
       doc: |
         The local path where the File is made available prior to executing a
-        CommandLineTool.  This field must not be used in any other context.  The
-        command line tool being executed must be able to to access the file at
-        `path` using the POSIX `open(2)` syscall.
+        CommandLineTool.  This must be set by the implementation.  This field
+        must not be used in any other context.  The command line tool being
+        executed must be able to to access the file at `path` using the POSIX
+        `open(2)` syscall.
       jsonldPredicate:
         "_id": "cwl:path"
         "_type": "@id"
@@ -211,16 +216,26 @@ $graph:
         A URI that identifies the directory resource.  This may be a relative
         reference, in which case it must be resolved using the base URI of the
         document.  The location may refer to a local or remote resource.  If
-        the `listing` field is not set, the implementation must use the URI to
-        retrieve directory listing.  If an implementation is unable to retrieve
-        the file content stored at a remote resource (due to unsupported
-        protocol, access denied, or other issue) it must signal an error.
+        the `listing` field is not set, the implementation must use the
+        location URI to retrieve directory listing.  If an implementation is
+        unable to retrieve the directory listing stored at a remote resource (due to
+        unsupported protocol, access denied, or other issue) it must signal an
+        error.
+
+        If the `path` field is provided but the `location` field is not, an
+        implementation may assign the value of the `path` field to `location`,
+        then follow the rules above.
       jsonldPredicate:
         _id: "@id"
         _type: "@id"
     - name: path
       type: string?
-      doc: The path to the directory.
+      doc: |
+        The local path where the Directory is made available prior to executing a
+        CommandLineTool.  This must be set by the implementation.  This field
+        must not be used in any other context.  The command line tool being
+        executed must be able to to access the directory at `path` using the POSIX
+        `opendir(2)` syscall.
       jsonldPredicate:
         _id: "cwl:path"
         _type: "@id"

cwltool/schemas/draft-4/conformance_test_draft-4.yaml

@@ -77,28 +77,39 @@
   tool: draft-4/cat3-tool-mediumcut.cwl
   doc: Test command execution in Docker with stdout redirection
 
-# - args: [egrep]
-#   stderr: error.txt
-#   job:
-#   tool: draft-4/egrep-stderr.cwl
-#   doc: Test command line with stderr redirection
-
-# - args: [egrep]
-#   job:
-#   tool: draft-4/egrep-stderr-shortcut.cwl
-#   doc: Test command line with stderr redirection, brief syntax
-
-# - args: [egrep]
-#   stderr: std.err
-#   output:
-#     output_file:
-#       class: File
-#       size: 84
-#       checksum: sha1$cec7b8746a78c42060c96505887449bca0142976
-#       location: std.err
-#   job:
-#   tool: draft-4/egrep-stderr-mediumcut.cwl
-#   doc: Test command line with stderr redirection, named brief syntax
+- job:
+  output: {
+    "output_file": {
+        "checksum": "sha1$cec7b8746a78c42060c96505887449bca0142976",
+        "size": 84,
+        "location": "error.txt",
+        "class": "File"
+    }
+  }
+  tool: draft-4/egrep-stderr.cwl
+  doc: Test command line with stderr redirection
+
+- job:
+  output: {
+    "output_file": {
+        "checksum": "sha1$cec7b8746a78c42060c96505887449bca0142976",
+        "size": 84,
+        "location": "",
+        "class": "File"
+    }
+  }
+  tool: draft-4/egrep-stderr-shortcut.cwl
+  doc: Test command line with stderr redirection, brief syntax
+
+- output:
+    output_file:
+      class: File
+      size: 84
+      checksum: sha1$cec7b8746a78c42060c96505887449bca0142976
+      location: std.err
+  job:
+  tool: draft-4/egrep-stderr-mediumcut.cwl
+  doc: Test command line with stderr redirection, named brief syntax
 
 - job: draft-4/cat-job.json
   output:
@@ -738,23 +749,23 @@
   doc: Test directory input in Docker
 
 - job: draft-4/dir4-job.yml
-  output:
+  output: {
     "outlist": {
-        "size": 20,
+        "checksum": "sha1$2ab6f189e84753c05a23413fbf6b6fbf4c53489f",
+        "size": 90,
         "location": "output.txt",
-        "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592",
         "class": "File"
     }
+  }
   tool: draft-4/dir4.cwl
   doc: Test directories in secondaryFiles
 
 - job: draft-4/dir-job.yml
   output: {
     "outlist": {
         "checksum": "sha1$907a866a3e0b7f1fc5a2222531c5fb9063704438",
-        "path": "/home/peter/work/common-workflow-language/draft-4/draft-4/output.txt",
         "size": 33,
-        "location": "file:///home/peter/work/common-workflow-language/draft-4/draft-4/output.txt",
+        "location": "output.txt",
         "class": "File"
     }
   }

cwltool/schemas/draft-4/draft-4/dir4-job.yml

@@ -4,7 +4,7 @@ inf:
   secondaryFiles:
     - class: File
       location: index.py
-    - entryname: testdir
+    - entryname: xtestdir
       entry:
         class: Directory
         location: testdir