Bump to 1.2.0-dev2 (#1262)

* Bump to 1.2.0-dev2

* Round up fractional cores.

Remove package_data from setup.py which is confusingly redundant with MANIFEST.in

* Add v1.1+ extensions.yml file

Author: tetron

MANIFEST.in

@@ -24,16 +24,18 @@ include cwltool/schemas/v1.1.0-dev1/*.yml
 include cwltool/schemas/v1.1.0-dev1/*.md
 include cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.yml
 include cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.md
-include cwltool/schemas/v1.2.0-dev1/*.yml
-include cwltool/schemas/v1.2.0-dev1/*.md
-include cwltool/schemas/v1.2.0-dev1/salad/schema_salad/metaschema/*.yml
-include cwltool/schemas/v1.2.0-dev1/salad/schema_salad/metaschema/*.md
+include cwltool/schemas/v1.2.0-dev2/*.yml
+include cwltool/schemas/v1.2.0-dev2/*.md
+include cwltool/schemas/v1.2.0-dev2/salad/schema_salad/metaschema/*.yml
+include cwltool/schemas/v1.2.0-dev2/salad/schema_salad/metaschema/*.md
 include cwltool/cwlNodeEngine.js
 include cwltool/cwlNodeEngineJSConsole.js
 include cwltool/cwlNodeEngineWithContext.js
 include cwltool/extensions.yml
+include cwltool/extensions-v1.1.yml
 include cwltool/jshint/jshint_wrapper.js
 include cwltool/jshint/jshint.js
+include cwltool/hello.simg
 prune cwltool/schemas/v1.0/salad/typeshed
 prune cwltool/schemas/v1.0/salad/schema_salad/tests
 prune cwltool/schemas/v1.1.0-dev1/salad/typeshed

cwltool/builder.py

@@ -1,6 +1,7 @@
 import copy
 import logging
 import os
+import math
 from typing import (
     IO,
     Any,
@@ -544,13 +545,18 @@ def do_eval(self, ex, context=None, recursive=False, strip_whitespace=True):
             if isinstance(ex, MutableSequence):
                 return [self.do_eval(v, context, recursive) for v in ex]
 
+        resources = self.resources
+        if self.resources and "cores" in self.resources:
+            resources = copy.copy(resources)
+            resources["cores"] = int(math.ceil(resources["cores"]))
+
         return expression.do_eval(
             ex,
             self.job,
             self.requirements,
             self.outdir,
             self.tmpdir,
-            self.resources,
+            resources,
             context=context,
             timeout=self.timeout,
             debug=self.debug,

cwltool/extensions-v1.1.yml

@@ -0,0 +1,46 @@
+$base: http://commonwl.org/cwltool#
+$namespaces:
+  cwl: "https://w3id.org/cwl/cwl#"
+$graph:
+- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
+
+- name: Secrets
+  type: record
+  inVocab: false
+  extends: cwl:ProcessRequirement
+  fields:
+    class:
+      type: string
+      doc: "Always 'Secrets'"
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+    secrets:
+      type: string[]
+      doc: |
+        List one or more input parameters that are sensitive (such as passwords)
+        which will be deliberately obscured from logging.
+      jsonldPredicate:
+        "_type": "@id"
+        refScope: 0
+
+
+- name: ProcessGenerator
+  type: record
+  inVocab: true
+  extends: cwl:Process
+  documentRoot: true
+  fields:
+    - name: class
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+      type: string
+    - name: run
+      type: [string, cwl:Process]
+      jsonldPredicate:
+        _id: "cwl:run"
+        _type: "@id"
+        subscope: run
+      doc: |
+        Specifies the process to run.

cwltool/main.py

@@ -609,10 +609,17 @@ def setup_schema(
         custom_schema_callback()
     elif args.enable_ext:
         res = pkg_resources.resource_stream(__name__, "extensions.yml")
-        use_custom_schema("v1.0", "http://commonwl.org/cwltool", res.read())
+        ext10 = res.read()
+        res = pkg_resources.resource_stream(__name__, "extensions-v1.1.yml")
+        ext11 = res.read()
+        use_custom_schema("v1.0", "http://commonwl.org/cwltool", ext10)
+        use_custom_schema("v1.1", "http://commonwl.org/cwltool", ext11)
+        use_custom_schema("v1.2.0-dev2", "http://commonwl.org/cwltool", ext11)
         res.close()
     else:
         use_standard_schema("v1.0")
+        use_standard_schema("v1.1")
+        use_standard_schema("v1.2.0-dev2")
 
 
 def setup_provenance(

cwltool/schemas/v1.2.0-dev1/CommandLineTool-standalone.yml renamed to cwltool/schemas/v1.2.0-dev2/CommandLineTool-standalone.yml

@@ -10,10 +10,10 @@ $graph:
   type: documentation
   doc:
     - |
-      # Common Workflow Language (CWL) Command Line Tool Description, v1.2.0-dev1
+      # Common Workflow Language (CWL) Command Line Tool Description, v1.2.0-dev2
 
       This version:
-        * https://w3id.org/cwl/v1.2.0-dev1/
+        * https://w3id.org/cwl/v1.2.0-dev2/
 
       Current version:
         * https://w3id.org/cwl/
@@ -37,21 +37,26 @@ $graph:
     - {$include: intro.md}
 
     - |
-      ## Introduction to the CWL Command Line Tool standard v1.2.0-dev1
+      ## Introduction to the CWL Command Line Tool standard v1.2.0-dev2
 
       This specification represents the latest development release from the
-      CWL group.  Since the v1.1 release, v1.2.0-dev1 introduces the
+      CWL group.  Since the v1.1 release, v1.2.0-dev2 introduces the
       following updates to the CWL Command Line Tool standard.
-      Documents should use `cwlVersion: v1.2.0-dev1` to make use of new
-      syntax and features introduced in v1.2.0-dev1.  Existing v1.1 documents
+      Documents should use `cwlVersion: v1.2.0-dev2` to make use of new
+      syntax and features introduced in v1.2.0-dev2.  Existing v1.1 documents
       should be trivially updatable by changing `cwlVersion`, however
       CWL documents that relied on previously undefined or
       underspecified behavior may have slightly different behavior in
-      v1.2.0-dev1.
+      v1.2.0-dev2.
 
       ## Changelog
 
-      See also the [CWL Workflow Description, v1.2.0-dev1 changelog](Workflow.html#Changelog).
+      * `coresMin` and `coresMax` of
+          [ResourceRequirement](#ResourceRequirement) may now request fractional CPUs.
+      * (CommandLineTool)[#CommandLineTool] can now express `intent`
+        with an identifier for the type of computational operation.
+
+      See also the [CWL Workflow Description, v1.2.0-dev2 changelog](Workflow.html#Changelog).
 
       ## Purpose
 
@@ -874,14 +879,22 @@ $graph:
         If the value is a string literal or an expression which evaluates to a
         string, a new file must be created with the string as the file contents.
 
-        If the value is an expression that evaluates to a `File` object, this
-        indicates the referenced file should be added to the designated output
-        directory prior to executing the tool.
+        If the value is an expression that evaluates to a `File` or `Directory`
+        object, this indicates the referenced file or directory should be added
+        to the designated output directory prior to executing the tool.
+
+        If the value is an expression that evaluates to an array of `File` or
+        `Directory` objects, this indicates the referenced files or directories
+        should be added to the designated output directory prior to executing
+        the tool, with ignoring the value in `entryname`.
 
         If the value is an expression that evaluates to a `Dirent` object, this
         indicates that the File or Directory in `entry` should be added to the
         designated output directory with the name in `entryname`.
 
+        If the value is an expression that evaluates to `null`, nothing is added
+        to the designated output directory from this entry.
+
         If `writable` is false, the file may be made available using a bind
         mount or file system link to avoid unnecessary copying of the input
         file.
@@ -934,7 +947,7 @@ $graph:
         designated output directory prior to executing the command line tool.
 
         May be an expression. If so, the expression return value must validate as
-        `{type: array, items: ["null", File, File[], Directory, Directory[], Dirent]}`.
+        `{type: array, items: ["null", File, Directory, Dirent, {type: array, items: [File, Directory]}]}`.
 
         Files or Directories which are listed in the input parameters and
         appear in the `InitialWorkDirRequirement` listing must have their
@@ -1018,12 +1031,40 @@ $graph:
         "_id": "@type"
         "_type": "@vocab"
     - name: coresMin
-      type: ["null", long, Expression]
-      doc: Minimum reserved number of CPU cores (default is 1)
+      type: ["null", long, float, Expression]
+      doc: |
+        Minimum reserved number of CPU cores (default is 1).
+
+        May be a fractional value to indicate to a scheduling
+        algorithm that one core can be allocated to multiple
+        jobs. For example, a value of 0.25 indicates that up to 4
+        jobs may run in parallel on 1 core.  A value of 1.25 means
+        that up to 3 jobs can run on a 4 core system (4/1.25 ≈ 3).
+
+        Processes can only share a core allocation if the sum of each
+        of their `ramMax`, `tmpdirMax`, and `outdirMax` requests also
+        do not exceed the capacity of the node.
+
+        Processes sharing a core must have the same level of isolation
+        (typically a container or VM) that they would normally.
+
+        The reported number of CPU cores reserved for the process,
+        which is available to expressions on the CommandLineTool as
+        `runtime.cores`, must be a non-zero integer, and may be
+        calculated by rounding up the cores request to the next whole
+        number.
+
+        Scheduling systems may allocate fractional CPU resources by
+        setting quotas or scheduling weights.  Scheduling systems that
+        do not support fractional CPUs may round up the request to the
+        next whole number.
 
     - name: coresMax
-      type: ["null", int, Expression]
-      doc: Maximum reserved number of CPU cores
+      type: ["null", long, float, Expression]
+      doc:  |
+        Maximum reserved number of CPU cores.
+
+        See `coresMin` for discussion about fractional CPU requests.
 
     - name: ramMin
       type: ["null", long, Expression]

cwltool/schemas/v1.2.0-dev1/CommandLineTool.yml renamed to cwltool/schemas/v1.2.0-dev2/CommandLineTool.yml

@@ -8,7 +8,7 @@ $namespaces:
 
 $graph:
 
-- name: "Common Workflow Language, v1.2.0-dev1"
+- name: "Common Workflow Language, v1.2.0-dev2"
   type: documentation
   doc: {$include: concepts.md}
 
@@ -41,6 +41,7 @@ $graph:
     - cwl:v1.1.0-dev1   # a dash is required by the semver 2.0 rules
     - cwl:v1.1
     - cwl:v1.2.0-dev1
+    - cwl:v1.2.0-dev2
 
 - name: CWLType
   type: enum
@@ -843,6 +844,25 @@ $graph:
       jsonldPredicate:
         "_id": "cwl:cwlVersion"
         "_type": "@vocab"
+    - name: intent
+      type: string[]?
+      jsonldPredicate:
+        _type: "@id"
+        identity: true
+      doc: |
+        An identifier for the type of computational operation, of this Process.
+        Especially useful for "class: Operation", but can also be used for
+        CommandLineTool, Workflow, or ExpressionTool.
+
+        If provided, then this must be an IRI of a concept node that
+        represents the type of operation, preferrably defined within an ontology.
+
+        For example, in the domain of bioinformatics, one can use an IRI from
+        the EDAM Ontology's [Operation concept nodes](http://edamontology.org/operation_0004),
+        like [Alignment](http://edamontology.org/operation_2928),
+        or [Clustering](http://edamontology.org/operation_3432); or a more
+        specific Operation concept like
+        [Split read mapping](http://edamontology.org/operation_3199).
 
 - name: InlineJavascriptRequirement
   type: record
@@ -940,6 +960,20 @@ $graph:
         set to `false` and the expected secondary file does not exist.
         Default value for `required` field is `true` for secondary files on
         input and `false` for secondary files on output.
+  doc: |
+    Secondary files are specified using the following micro-DSL for secondary files:
+
+    * If the value is a string, it is transformed to an object with two fields
+      `pattern` and `required`
+    * By default, the value of `required` is `null`
+      (this indicates default behavior, which may be based on the context)
+    * If the value ends with a question mark `?` the question mark is
+      stripped off and the value of the field `required` is set to `False`
+    * The remaining value is assigned to the field `pattern`
+
+    For implementation details and examples, please see
+    [this section](SchemaSalad.html#Domain_Specific_Language_for_secondary_files)
+    in the Schema Salad specification.
 
 - name: LoadListingRequirement
   type: record