initial schema for a loop construct prototype

Based upon @GlassOfWhiskey 's work in common-workflow-language/common-workflow-language#495 (comment)
With comments from @tetron @mr-c

Author: mr-c

MANIFEST.in

@@ -57,6 +57,7 @@ include cwltool/cwlNodeEngineJSConsole.js
 include cwltool/cwlNodeEngineWithContext.js
 include cwltool/extensions.yml
 include cwltool/extensions-v1.1.yml
+include cwltool/extensions-v1.2.yml
 include cwltool/jshint/jshint_wrapper.js
 include cwltool/jshint/jshint.js
 include cwltool/hello.simg

cwltool/extensions-v1.2.yml

@@ -0,0 +1,249 @@
+$base: http://commonwl.org/cwltool#
+$namespaces:
+  cwl: "https://w3id.org/cwl/cwl#"
+$graph:
+- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
+
+- name: Secrets
+  type: record
+  inVocab: false
+  extends: cwl:ProcessRequirement
+  fields:
+    class:
+      type: string
+      doc: "Always 'Secrets'"
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+    secrets:
+      type: string[]
+      doc: |
+        List one or more input parameters that are sensitive (such as passwords)
+        which will be deliberately obscured from logging.
+      jsonldPredicate:
+        "_type": "@id"
+        refScope: 0
+
+
+- name: ProcessGenerator
+  type: record
+  inVocab: true
+  extends: cwl:Process
+  documentRoot: true
+  fields:
+    - name: class
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+      type: string
+    - name: run
+      type: [string, cwl:Process]
+      jsonldPredicate:
+        _id: "cwl:run"
+        _type: "@id"
+        subscope: run
+      doc: |
+        Specifies the process to run.
+
+- name: MPIRequirement
+  type: record
+  inVocab: false
+  extends: cwl:ProcessRequirement
+  doc: |
+    Indicates that a process requires an MPI runtime.
+  fields:
+    - name: class
+      type: string
+      doc: "Always 'MPIRequirement'"
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+    - name: processes
+      type: [int, cwl:Expression]
+      doc: |
+        The number of MPI processes to start. If you give a string,
+        this will be evaluated as a CWL Expression and it must
+        evaluate to an integer.
+
+- name: CUDARequirement
+  type: record
+  extends: cwl:ProcessRequirement
+  inVocab: false
+  doc: |
+    Require support for NVIDA CUDA (GPU hardware acceleration).
+  fields:
+    class:
+      type: string
+      doc: 'cwltool:CUDARequirement'
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
+    cudaVersionMin:
+      type: string
+      doc: |
+        Minimum CUDA version to run the software, in X.Y format.  This
+        corresponds to a CUDA SDK release.  When running directly on
+        the host (not in a container) the host must have a compatible
+        CUDA SDK (matching the exact version, or, starting with CUDA
+        11.3, matching major version).  When run in a container, the
+        container image should provide the CUDA runtime, and the host
+        driver is injected into the container.  In this case, because
+        CUDA drivers are backwards compatible, it is possible to
+        use an older SDK with a newer driver across major versions.
+
+        See https://docs.nvidia.com/deploy/cuda-compatibility/ for
+        details.
+    cudaComputeCapability:
+      type:
+        - 'string'
+        - 'string[]'
+      doc: |
+        CUDA hardware capability required to run the software, in X.Y
+        format.
+
+        * If this is a single value, it defines only the minimum
+          compute capability.  GPUs with higher capability are also
+          accepted.
+
+        * If it is an array value, then only select GPUs with compute
+          capabilities that explicitly appear in the array.
+    cudaDeviceCountMin:
+      type: ['null', int, cwl:Expression]
+      default: 1
+      doc: |
+        Minimum number of GPU devices to request.  If not specified,
+        same as `cudaDeviceCountMax`.  If neither are specified,
+        default 1.
+    cudaDeviceCountMax:
+      type: ['null', int, cwl:Expression]
+      doc: |
+        Maximum number of GPU devices to request.  If not specified,
+        same as `cudaDeviceCountMin`.
+
+- name: LoopInput
+  type: record
+  fields:
+    id:
+      type: string?
+      jsonldPredicate: "@id"
+      doc: "It must reference the `id` of one of the elements in the `in` field of the step."
+    source:
+      doc: |
+        Specifies one or more of the step output parameters that will
+        provide input to the loop iterations after the first one (inputs
+        of the first iteration are the step input parameters).
+      jsonldPredicate:
+        "_id": "cwl:source"
+        "_type": "@id"
+        refScope: 2
+      type:
+        - string?
+        - string[]?
+    linkMerge:
+      type: cwl:LinkMergeMethod?
+      jsonldPredicate: "cwl:linkMerge"
+      default: merge_nested
+      doc: |
+        The method to use to merge multiple inbound links into a single array.
+        If not specified, the default method is "merge_nested".
+    pickValue:
+      type: ["null", cwl:PickValueMethod]
+      jsonldPredicate: "cwl:pickValue"
+      doc: |
+        The method to use to choose non-null elements among multiple sources.
+    default:
+      type: ["null", Any]
+      doc: |
+        The default value for this parameter to use if either there is no
+        `source` field, or the value produced by the `source` is `null`.  The
+        default must be applied prior to scattering or evaluating `valueFrom`.
+      jsonldPredicate:
+        _id: "sld:default"
+        noLinkCheck: true
+    valueFrom:
+      type:
+        - "null"
+        - string
+        - cwl:Expression
+      jsonldPredicate: "cwl:valueFrom"
+      doc: |
+        To use valueFrom, [StepInputExpressionRequirement](#StepInputExpressionRequirement) must
+        be specified in the workflow or workflow step requirements.
+
+        If `valueFrom` is a constant string value, use this as the value for
+        this input parameter.
+
+        If `valueFrom` is a parameter reference or expression, it must be
+        evaluated to yield the actual value to be assiged to the input field.
+
+        The `self` value in the parameter reference or expression must be
+        `null` if there is no `source` field, or the value of the
+        parameter(s) specified in the `source` field.
+
+        The value of `inputs` in the parameter reference or expression must be
+        the input object to the last iteration of the workflow step after
+        assigning the `source` values and then applying `default`.
+
+        The value of `outputs` in the parameter reference or expression
+        must be the output object of the last step execution.
+
+- name: Loop
+  type: record
+  extends: cwl:ProcessRequirement
+  inVocab: false
+  doc: |
+    Prototype to enable workflow-level looping of a step.
+
+    Valid only under `requirements` of a https://www.commonwl.org/v1.2/Workflow.html#WorkflowStep.
+
+    `loop_when` is an expansion of the CWL v1.2 `when` construct which controls
+    conditional execution and thus when using `Loop` do not also specify a `when` as
+    the `loop_when` expanded definition is intended to replace `when` in a
+    future version of the CWL standards.
+
+    Loops are evaluated after any specified `scatter`ing; the implicit gathering
+    is done with the results of the scatter-jobs (after the `loop`s have finished).
+  fields:
+    class:
+      type: string
+      doc: 'cwltool:Loop'
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
+    loop_when:
+      type: cwl:Expression
+      doc: |
+        Only run the step while the expression evaluates to `true`.
+        If `false` and no iteration has been performed, the step is skipped.
+
+        A skipped step produces a `null` on each output.
+
+        The `inputs` value in the expression must be the step input object.
+
+        The `outputs` value in the expression must be the step output object
+        produced by the execution of the last iteration of this step, except
+        for the first execution when the `outputs` object is populated with
+        `null`s for each output declared in this step's `out` section.
+
+        It is an error if this expression returns a value other than `true` or `false`.
+    loop:
+      type: LoopInput[]?
+      jsonldPredicate:
+        _id: "cwltool:loop"
+        mapSubject: id
+        mapPredicate: source
+      doc: |
+        Defines the input parameters of the loop iterations after the first one
+        (inputs of the first iteration are the step input parameters). If no
+        `loop` rule is specified for a given step `in` field, the initial value
+        is kept constant among all iterations.
+    outputMethod:
+      type:
+        type: enum
+        name: LoopOutputModes
+        symbols: [ last, all ]
+      doc:
+        - Specify the desired method of dealing with loop outputs
+        - Default. Propagates only the last computed element to the subsequent steps when the loop terminates.
+        - Propagates a single array with all output values to the subsequent steps when the loop terminates.
+

cwltool/main.py

@@ -661,9 +661,11 @@ def setup_schema(
             ext10 = res.read().decode("utf-8")
         with pkg_resources.resource_stream(__name__, "extensions-v1.1.yml") as res:
             ext11 = res.read().decode("utf-8")
+        with pkg_resources.resource_stream(__name__, "extensions-v1.2.yml") as res:
+            ext12 = res.read().decode("utf-8")
         use_custom_schema("v1.0", "http://commonwl.org/cwltool", ext10)
         use_custom_schema("v1.1", "http://commonwl.org/cwltool", ext11)
-        use_custom_schema("v1.2", "http://commonwl.org/cwltool", ext11)
+        use_custom_schema("v1.2", "http://commonwl.org/cwltool", ext12)
         use_custom_schema("v1.2.0-dev1", "http://commonwl.org/cwltool", ext11)
         use_custom_schema("v1.2.0-dev2", "http://commonwl.org/cwltool", ext11)
         use_custom_schema("v1.2.0-dev3", "http://commonwl.org/cwltool", ext11)

tests/loop.cwl

@@ -0,0 +1,71 @@
+cwlVersion: v1.2
+class: Workflow
+
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+
+inputs:
+  i1: Any
+outputs:
+  o1:
+    type: Any
+    outputSource: subworkflow/o1
+steps:
+  A:
+    run: A.cwl
+    in:
+      in1: in1
+    out: [p1, p2]
+  B:
+    run: B.cwl
+    in:
+      p1: A/p1
+    out: [p3]
+  C:
+    run: C.cwl
+    in:
+      p2: A/p2
+    out: [p4]
+  subworkflow:
+    run:
+      class: Workflow
+      inputs:
+        p3: Any
+        p4: Any
+      outputs:
+        o1:
+          type: Any
+          outputSource: E/o1
+        p3:
+          type: Any
+          outputSource: F/p3
+      steps:
+        D:
+          run: D.cwl
+          in:
+            p3: p3
+          out: [p4]
+        E:
+          run: E.cwl
+          in:
+            p4:
+              source:
+              - p4
+              - D/p4
+              pickValue: the_only_non_null
+          out: [o1, p5]
+        F:
+          run: F.cwl
+          in:
+            p5: E/p5
+          out: [p3]
+    in:
+      p3: B/p3
+      p4: C/p4
+    out: [o1, p3]
+    requirements:
+      cwltool:Loop:
+        loop_when: $(outputs.o1 !== null)
+        loop:
+          p3: p3
+        outputMethod: last