feat: Extended schema for 'example 1' (step inputs & outputs)

Author: Alan Christie

tests/test_decoder_minimal.py renamed to tests/test_decoder.py

@@ -8,13 +8,21 @@
 
 from workflow import decoder
 
+# Pre-load some example workflows...
 _MINIMAL_WORKFLOW_FILE: str = os.path.join(
     os.path.dirname(__file__), "workflow-definitions", "minimal.yaml"
 )
 with open(_MINIMAL_WORKFLOW_FILE, "r", encoding="utf8") as workflow_file:
-    _MINIMAL_WORKFLOW: Dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
+    _MINIMAL_WORKFLOW: Dict[str, Any] = yaml.safe_load(workflow_file)
 assert _MINIMAL_WORKFLOW
 
+_SHORTCUT_EXAMPLE_1_WORKFLOW_FILE: str = os.path.join(
+    os.path.dirname(__file__), "workflow-definitions", "shortcut-example-1.yaml"
+)
+with open(_SHORTCUT_EXAMPLE_1_WORKFLOW_FILE, "r", encoding="utf8") as workflow_file:
+    _SHORTCUT_EXAMPLE_1_WORKFLOW: Dict[str, Any] = yaml.safe_load(workflow_file)
+assert _SHORTCUT_EXAMPLE_1_WORKFLOW
+
 
 def test_validate_minimal():
     # Arrange
@@ -60,3 +68,13 @@ def test_validate_name_with_spaces():
     assert (
         error == "'workflow with spaces' does not match '^[a-z][a-z0-9-]{,63}(?<!-)$'"
     )
+
+
+def test_validate_shortcut_example_1():
+    # Arrange
+
+    # Act
+    error = decoder.validate_schema(_SHORTCUT_EXAMPLE_1_WORKFLOW)
+
+    # Assert
+    assert error is None

tests/test_workflow_validator.py

@@ -6,7 +6,7 @@
 
 pytestmark = pytest.mark.unit
 
-from tests.test_decoder_minimal import _MINIMAL_WORKFLOW
+from tests.test_decoder import _MINIMAL_WORKFLOW
 from workflow.workflow_validator import ValidationLevel, WorkflowValidator
 
 

tests/workflow-definitions/shortcut-example-1.yaml

@@ -0,0 +1,22 @@
+---
+kind: DataManagerWorkflow
+kind-version: "2024.1"
+name: experimental-workflow
+description: An experimental workflow
+steps:
+- name: step-1
+  description: The first step
+  specification: '{"collection":"im-test","job":"process-a","version":"1.0.0", "variables":{"x": 7}}'
+  outputs:
+  - output: 'outputFile'
+    as: 'a.sdf'
+- name: step-2
+  specification: '{"collection":"im-test","job":"process-b","version":"1.0.0", "variables":{"y": 22}}'
+  inputs:
+  - input: 'inputFile'
+    from:
+      step: step-1
+      output: 'outputFile'
+  outputs:
+  - output: 'outputFile'
+    as: 'b.sdf'

workflow/workflow-schema.yaml

@@ -42,6 +42,54 @@ definitions:
   rfc1035-label-name:
     type: string
     pattern: ^[a-z][a-z0-9-]{,63}(?<!-)$
+    description: >-
+      A value compatible with Kubernetes variables
+      to allow it to be used ins Pod Label
+
+  variable-name:
+    type: string
+    pattern: ^[a-zA-Z_][a-zA-Z0-9_]{,63}$
+    description: >-
+      A Job/Step variable name, as used in the Data Manager Job Specification
+
+  # Declaration of a step from anotehr step
+  from-step-output:
+    type: object
+    additionalProperties: false
+    properties:
+      step:
+        $ref: '#/definitions/rfc1035-label-name'
+      output:
+        $ref: '#/definitions/variable-name'
+    required:
+    - step
+    - output
+
+  # A Step input (from an output of a prior step)
+  step-input-from-step:
+    type: object
+    additionalProperties: false
+    properties:
+      input:
+        $ref: '#/definitions/variable-name'
+      from:
+        $ref: '#/definitions/from-step-output'
+    required:
+    - input
+
+  # A Step output (with an 'as' - a declared value)
+  step-output-as:
+    type: object
+    additionalProperties: false
+    properties:
+      output:
+        $ref: '#/definitions/variable-name'
+      as:
+        type: string
+        description: The value to set the variable to
+    required:
+    - output
+    - as
 
   step:
     type: object
@@ -54,6 +102,16 @@ definitions:
       specification:
         type: string
         description: The Data Manager Job Specification, a JSON string
+      inputs:
+        type: array
+        items:
+          oneOf:
+          - $ref: "#/definitions/step-input-from-step"
+      outputs:
+        type: array
+        items:
+          oneOf:
+          - $ref: "#/definitions/step-output-as"
     required:
     - name
     - specification