Merge commit '79ef5c323e7316edbd903e09ec06525ea48d5e4a'

Author: Peter Amstutz

cwltool/schemas/README.md

@@ -7,9 +7,9 @@ portability of data analysis workflows.  Our goal is to create specifications
 that enable data scientists to describe analysis tools and workflows that are
 powerful, easy to use, portable, and support reproducibility.
 
-CWL builds on technologies such as [JSON-LD](http://json-ld.org) and
-[Avro](https://avro.apache.org/) for data modeling and
-[Docker](http://docker.com) for portable runtime environments.
+CWL builds on technologies such as [JSON-LD](http://json-ld.org)
+for data modeling and [Docker](http://docker.com) for portable runtime\
+environments.
 
 CWL is designed to express workflows for data-intensive science, such as
 Bioinformatics, Medical Imaging, Chemistry, Physics, and Astronomy.
@@ -64,6 +64,15 @@ https://ci.commonwl.org
 
 [Github repository of example tools and workflows.](https://github.com/common-workflow-language/workflows)
 
+## Support
+
+The best place to ask a question about all things CWL is on
+[Biostars](https://www.biostars.org/t/cwl/).
+
+[![Biostars
+CWL](https://www.biostars.org/static/biostar2.logo.png)](https://www.biostars.org/t/cwl/)
+</a>
+
 ## Development and testing
 
 [The CWL project is on Github.](https://github.com/common-workflow-language/common-workflow-language)
@@ -91,7 +100,7 @@ and send a pull request!
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/common-workflow-language/common-workflow-language?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 Your CWL Community Engineer, Michael R. Crusoe, publishes a blog about his work
-with weekly updates at http://mrc.commonwl.org.
+with updates at http://mrc.commonwl.org.
 
 ### Code of Conduct
 
@@ -142,6 +151,7 @@ Conduct](https://github.com/common-workflow-language/common-workflow-language/bl
 * Maxim Mikheev <[email protected]>
 * Tim Pierce <[email protected]>
 * Josh Randall <[email protected]>
+* Janko Simonović <[email protected]>
 * Stian Soiland-Reyes <[email protected]>
 * Luka Stojanovic <[email protected]>
 * Nebojša Tijanić <[email protected]>

cwltool/schemas/draft-3/conformance_test_draft-3.yaml

@@ -276,23 +276,33 @@
         "secondaryFiles": [
             {
                 "path": "input.txt.idx1",
-                "class": "File"
+                "class": "File",
+                "checksum": "sha1$553f3a09003a9f69623f03bec13c0b078d706023",
+                "size": 1500
             },
             {
                 "path": "input.idx2",
-                "class": "File"
+                "class": "File",
+                "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                "size": 0
             },
             {
                 "path": "input.txt.idx3",
-                "class": "File"
+                "class": "File",
+                "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                "size": 0
             },
             {
                 "path": "input.txt.idx4",
-                "class": "File"
+                "class": "File",
+                "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                "size": 0
             },
             {
                 "path": "input.txt.idx5",
-                "class": "File"
+                "class": "File",
+                "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                "size": 0
             }
         ],
         "size": 1111
@@ -583,7 +593,9 @@
   output: {
     "foo": {
         "path": "foo",
-        "class": "File"
+        "class": "File",
+        "checksum": "sha1$f1d2d2f924e986ac86fdf7b36c94bcdf32beec15",
+        "size": 4
     }
   }
   tool: draft-3/test-cwl-out.cwl

cwltool/schemas/site/userguide-v1.0-input.json

@@ -45,6 +45,11 @@
                     "class": "File", 
                     "location": "../v1.0/examples/record-job2.yml"
                 }, 
+                {
+                    "basename": "record-job3.yml", 
+                    "class": "File", 
+                    "location": "../v1.0/examples/record-job3.yml"
+                }, 
                 {
                     "basename": "echo-job.yml", 
                     "class": "File", 
@@ -145,4 +150,4 @@
     ], 
     "class": "File", 
     "location": "../v1.0/UserGuide.yml"
-}
+}

cwltool/schemas/v1.0/UserGuide.yml

@@ -85,7 +85,7 @@
 
       Available primitive types are *string*, *int*, *long*, *float*, *double*,
       and *null*; complex types are *array* and *record*; in addition there are
-      special types *File* and *Any*.
+      special types *File*, *Directory* and *Any*.
 
       The following example demonstrates some input parameters with different
       types and appearing on the command line in different ways:
@@ -601,7 +601,7 @@
 
       *record-job3.yml*
       ```
-    - $include: examples/record-job2.yml
+    - $include: examples/record-job3.yml
     - |
       ```
 
@@ -681,7 +681,7 @@
       Sometimes you need to create a file on the fly from input parameters,
       such as tools which expect to read their input configuration from a file
       rather than the command line parameters.  To do this, use
-      `CreateFileRequirement`.
+      `InitialWorkDirRequirement`.
 
       *createfile.cwl*
       ```
@@ -712,7 +712,7 @@
       Normally, input files are located in a read-only directory separate from
       the output directory.  This causes problems if the underlying tool
       expects to write its output files alongside the input file in the same
-      directory.  You use `CreateFileRequirement` to stage input files into the
+      directory.  You use `InitialWorkDirRequirement` to stage input files into the
       output directory.  In this example, we use a Javascript expression to
       extract the base name of the input file from its leading directory path.
 

cwltool/schemas/v1.0/Workflow.yml

@@ -62,7 +62,7 @@ $graph:
             type: int
             doc: Second input parameter
           ```
-        * The common field 'description` has been renamed to `doc`.
+        * The common field `description` has been renamed to `doc`.
 
       ## Purpose
 

cwltool/schemas/v1.0/concepts.md

@@ -95,7 +95,7 @@ https://github.com/common-workflow-language/common-workflow-language/tree/master
 
 If an object contains an `id` field, that is used to uniquely identify the
 object in that document.  The value of the `id` field must be unique over the
-entire document.  Identifiers may be resolved relative to other the document
+entire document.  Identifiers may be resolved relative to either the document
 base and/or other identifiers following the rules are described in the
 [Schema Salad specification](SchemaSalad.html#Identifier_resolution).
 

cwltool/schemas/v1.0/conformance_test_v1.0.yaml

@@ -45,7 +45,7 @@
     }
   job: v1.0/cat-job.json
   tool: v1.0/template-tool.cwl
-  doc: Test CreateFileRequirement ExpressionEngineRequirement.engineConfig feature
+  doc: Test InitialWorkDirRequirement ExpressionEngineRequirement.engineConfig feature
 
 - job: v1.0/cat-job.json
   output:
@@ -327,29 +327,39 @@
             {
                 "location": "input.txt.idx1",
                 "class": "File",
+                "checksum": "sha1$553f3a09003a9f69623f03bec13c0b078d706023",
+                "size": 1500
             },
             {
                 "location": "input.idx2",
                 "class": "File",
+                "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                "size": 0
             },
             {
                 "location": "input.txt.idx3",
                 "class": "File",
+                "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                "size": 0
             },
             {
                 "location": "input.txt.idx4",
                 "class": "File",
+                "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                "size": 0
             },
             {
                 "location": "input.txt.idx5",
                 "class": "File",
+                "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                "size": 0
             }
         ],
         "size": 1111
     }
   tool: "v1.0/search.cwl#main"
   doc: |
-    Test CreateFileRequirement linking input files and capturing secondaryFiles
+    Test InitialWorkDirRequirement linking input files and capturing secondaryFiles
     on input and output.
 
 - job: v1.0/rename-job.json
@@ -361,7 +371,7 @@
       size: 1111
   tool: v1.0/rename.cwl
   doc: |
-    Test CreateFileRequirement with expression in filename.
+    Test InitialWorkDirRequirement with expression in filename.
 
 - job: v1.0/wc-job.json
   output:
@@ -633,7 +643,9 @@
   output: {
     "foo": {
         "location": "foo",
-        "class": "File"
+        "class": "File",
+        "checksum": "sha1$f1d2d2f924e986ac86fdf7b36c94bcdf32beec15",
+        "size": 4
     }
   }
   tool: v1.0/test-cwl-out.cwl
@@ -713,7 +725,7 @@
         "class": "File"
     }
   tool: v1.0/dir.cwl
-  doc: Test directory input
+  doc: Test directory input with parameter reference
 
 - job: v1.0/dir-job.yml
   output:
@@ -733,11 +745,15 @@
         "listing": [
             {
               "class": "File",
-              "location": "hello.txt"
+              "location": "hello.txt",
+                "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b",
+                "size": 13
             },
             {
               "class": "File",
-              "location": "goodbye.txt"
+              "location": "goodbye.txt",
+                "checksum": "sha1$dd0a4c4c49ba43004d6611771972b6cf969c1c01",
+                "size": 24
             }
         ],
     }
@@ -809,3 +825,45 @@
         size: 21
   tool: v1.0/nameroot.cwl
   doc: Test nameroot/nameext expression in arguments, stdout
+
+- job: v1.0/dir-job.yml
+  output:
+    "outlist": {
+        "size": 20,
+        "location": "output.txt",
+        "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592",
+        "class": "File"
+    }
+  tool: v1.0/dir6.cwl
+  doc: Test directory input with inputBinding
+
+- job: v1.0/nested-array-job.yml
+  output:
+    echo:
+        checksum: sha1$3f786850e387550fdab836ed7e6dc881de23001b
+        location: echo.txt
+        class: File
+        size: 2
+  tool: v1.0/nested-array.cwl
+  doc: Test command line generation of array-of-arrays
+
+- job: v1.0/empty.json
+  output: {}
+  tool: v1.0/envvar.cwl
+  doc: Test $HOME and $TMPDIR are set correctly
+
+- job: v1.0/empty.json
+  output: {}
+  tool: v1.0/envvar2.cwl
+  doc: Test $HOME and $TMPDIR are set correctly in Docker
+
+- job: v1.0/empty.json
+  output:
+    "out": {
+        "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a",
+        "location": "whatever.txt",
+        "class": "File",
+        "size": 2
+    }
+  tool: "v1.0/js-expr-req-wf.cwl#wf"
+  doc: Test that expressionLib requirement of individual tool step overrides expressionLib of workflow.

cwltool/schemas/v1.0/v1.0/bwa-mem-tool.cwl

@@ -7,6 +7,8 @@ class: CommandLineTool
 hints:
   - class: ResourceRequirement
     coresMin: 4
+  - class: DockerRequirement
+    dockerPull: python
 
 inputs:
   - id: reference

cwltool/schemas/v1.0/v1.0/dir.cwl

@@ -9,7 +9,6 @@ outputs:
     type: File
     outputBinding:
       glob: output.txt
-baseCommand: []
 arguments: ["cd", "$(inputs.indir.path)",
   {shellQuote: false, valueFrom: "&&"},
   "find", ".",

cwltool/schemas/v1.0/v1.0/dir2.cwl

@@ -11,7 +11,6 @@ outputs:
     type: File
     outputBinding:
       glob: output.txt
-baseCommand: []
 arguments: ["cd", "$(inputs.indir.path)",
   {shellQuote: false, valueFrom: "&&"},
   "find", ".",