abstract operation (#1255)

* Update to schema with Operation defined

* Accept and check for abstract operations

* Since we only support Python 3 now, can lift the pytest pin.

Author: tetron

cwltool/checker.py

@@ -397,8 +397,7 @@ def check_all_types(src_dict, sinks, sourceField, param_to_step):
     return validation
 
 
-def is_conditional_step(param_to_step: Dict[str, Dict[str, Any]],
-                        parm_id: str) -> bool:
+def is_conditional_step(param_to_step: Dict[str, Dict[str, Any]], parm_id: str) -> bool:
     source_step = param_to_step.get(parm_id)
     if source_step is not None:
         if source_step.get("when") is not None:

cwltool/command_line_tool.py

@@ -162,6 +162,17 @@ def job(
         yield job
 
 
+class AbstractOperation(Process):
+    def job(
+        self,
+        job_order,  # type: Mapping[str, str]
+        output_callbacks,  # type: Callable[[Any, Any], Any]
+        runtimeContext,  # type: RuntimeContext
+    ):
+        # type: (...) -> Generator[ExpressionTool.ExpressionJob, None, None]
+        raise WorkflowException("Abstract operation cannot be executed.")
+
+
 def remove_path(f):  # type: (Dict[str, Any]) -> None
     if "path" in f:
         del f["path"]

cwltool/docker.py

@@ -224,18 +224,16 @@ def append_volume(runtime, source, target, writable=False):
         # type: (List[str], str, str, bool) -> None
         """Add binding arguments to the runtime list."""
         options = [
-            'type=bind',
-            'source=' + source,
-            'target=' + target,
+            "type=bind",
+            "source=" + source,
+            "target=" + target,
         ]
         if not writable:
-            options.append('readonly')
+            options.append("readonly")
         output = StringIO()
         csv.writer(output).writerow(options)
         mount_arg = output.getvalue().strip()
-        runtime.append(
-            "--mount={}".format(mount_arg)
-        )
+        runtime.append("--mount={}".format(mount_arg))
         # Unlike "--volume", "--mount" will fail if the volume doesn't already exist.
         if not os.path.exists(source):
             os.mkdir(source)

cwltool/executors.py

@@ -7,11 +7,22 @@
 import threading
 from abc import ABCMeta, abstractmethod
 from threading import Lock
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+    MutableMapping,
+)
 
 import psutil
 
 from schema_salad.validate import ValidationException
+from schema_salad.sourceline import SourceLine
 
 from .command_line_tool import CallbackJob
 from .context import RuntimeContext, getdefault
@@ -68,6 +79,14 @@ def execute(
         if not runtime_context.basedir:
             raise WorkflowException("Must provide 'basedir' in runtimeContext")
 
+        def check_for_abstract_op(tool: MutableMapping[str, Any]) -> None:
+            if tool["class"] == "Operation":
+                raise SourceLine(tool, "class", WorkflowException).makeError(
+                    "Workflow has unrunnable abstract Operation"
+                )
+
+        process.visit(check_for_abstract_op)
+
         finaloutdir = None  # Type: Optional[str]
         original_outdir = runtime_context.outdir
         if isinstance(original_outdir, str):

cwltool/main.py

@@ -782,7 +782,10 @@ def main(
     input_required: bool = True,
 ) -> int:
     if not stdout:  # force UTF-8 even if the console is configured differently
-        if hasattr(sys.stdout, "encoding") and sys.stdout.encoding != "UTF-8":
+        if hasattr(sys.stdout, "encoding") and sys.stdout.encoding.upper() not in (
+            "UTF-8",
+            "UTF8",
+        ):
             if hasattr(sys.stdout, "detach"):
                 stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
             else:

cwltool/process.py

@@ -119,6 +119,7 @@ def filter(self, record):  # type: (logging.LogRecord) -> bool
     "CommandLineTool.yml",
     "CommonWorkflowLanguage.yml",
     "Process.yml",
+    "Operation.yml",
     "concepts.md",
     "contrib.md",
     "intro.md",

cwltool/schemas/v1.2.0-dev1/Operation.yml

@@ -0,0 +1,90 @@
+saladVersion: v1.1
+$base: "https://w3id.org/cwl/cwl#"
+
+$namespaces:
+  cwl: "https://w3id.org/cwl/cwl#"
+
+$graph:
+
+- name: OperationInputParameter
+  type: record
+  extends: InputParameter
+  docParent: "#Operation"
+  doc: |
+    Describe an input parameter of an operation.
+  fields:
+    - name: type
+      type:
+        - CWLType
+        - InputRecordSchema
+        - InputEnumSchema
+        - InputArraySchema
+        - string
+        - type: array
+          items:
+            - CWLType
+            - InputRecordSchema
+            - InputEnumSchema
+            - InputArraySchema
+            - string
+      jsonldPredicate:
+        "_id": "sld:type"
+        "_type": "@vocab"
+        refScope: 2
+        typeDSL: True
+      doc: |
+        Specify valid types of data that may be assigned to this parameter.
+
+- name: OperationOutputParameter
+  type: record
+  extends: OutputParameter
+  docParent: "#Operation"
+  doc: |
+    Describe an output parameter of an operation.
+  fields:
+    - name: type
+      type:
+        - CWLType
+        - OutputRecordSchema
+        - OutputEnumSchema
+        - OutputArraySchema
+        - string
+        - type: array
+          items:
+            - CWLType
+            - OutputRecordSchema
+            - OutputEnumSchema
+            - OutputArraySchema
+            - string
+      jsonldPredicate:
+        "_id": "sld:type"
+        "_type": "@vocab"
+        refScope: 2
+        typeDSL: True
+      doc: |
+        Specify valid types of data that may be assigned to this parameter.
+
+- type: record
+  name: Operation
+  extends: Process
+  documentRoot: true
+  specialize:
+    - specializeFrom: InputParameter
+      specializeTo: OperationInputParameter
+    - specializeFrom: OutputParameter
+      specializeTo: OperationOutputParameter
+  doc: |
+    This record describes an abstract operation.  It is a potential
+    step of a workflow that has not yet been bound to a concrete
+    implementation.  It specifies an input and output signature, but
+    does not provide enough information to be executed.  An
+    implementation (or other tooling) may provide a means of binding
+    an Operation to a concrete process (such as Workflow,
+    CommandLineTool, or ExpressionTool) with a compatible signature.
+
+  fields:
+    - name: class
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+      type: string

cwltool/schemas/v1.2.0-dev1/Workflow.yml

@@ -24,12 +24,12 @@ $graph:
     - |
       # Abstract
 
-      One way to define a workflow is: an analysis task represented by a
-      directed graph describing a sequence of operations that transform an
-      input data set to output. This specification defines the Common Workflow
-      Language (CWL) Workflow description, a vendor-neutral standard for
-      representing workflows intended to be portable across a variety of
-      computing platforms.
+      This specification defines the Common Workflow Language (CWL)
+      Workflow description, a vendor-neutral standard for representing
+      analysis tasks where a sequence of operations are described
+      using a directed graph of operations to transform input to
+      output.  CWL is portable across a variety of computing
+      platforms.
 
     - {$include: intro.md}
 
@@ -734,3 +734,5 @@ $graph:
       jsonldPredicate:
         "_id": "@type"
         "_type": "@vocab"
+
+- {$import: Operation.yml}

cwltool/schemas/v1.2.0-dev1/salad/schema_salad/metaschema/sfdsl_res.yml

@@ -0,0 +1,34 @@
+- |
+  ## Domain Specific Language for secondary files
+
+  Fields may be tagged `secondaryFilesDSL: true` in `jsonldPredicate`.  If so, the field is expanded using the
+  following micro-DSL for secondary files:
+
+  * If the value is a string, it is transformed to an object with two fields `pattern` and `required`
+  * By default, the value of `required` is `null` (this indicates default behavior, which may be based on the context)
+  * If the value ends with a question mark `?` the question mark is
+    stripped off and the value of the field `required` is set to `False`
+  * The remaining value is assigned to the field `pattern`
+
+  ### Type DSL example
+
+  Given the following schema:
+
+  ```
+- $include: sfdsl_res_schema.yml
+- |
+  ```
+
+  Process the following example:
+
+  ```
+- $include: sfdsl_res_src.yml
+- |
+  ```
+
+  This becomes:
+
+  ```
+- $include: sfdsl_res_proc.yml
+- |
+  ```

cwltool/schemas/v1.2.0-dev1/salad/schema_salad/metaschema/sfdsl_res_proc.yml

@@ -0,0 +1,21 @@
+[
+    {
+        "secondaryFiles": {
+          "pattern": ".bai",
+          "required": null
+    },
+    {
+        "secondaryFiles": {
+          "pattern": ".bai",
+          "required": false
+    },
+    {
+        "secondaryFiles": {
+          "pattern": ".bai?"
+    },
+    {
+        "secondaryFiles": {
+          "pattern": ".bai?",
+          "required": true
+    },
+]