refactor: Add support for simple python rdkit/butina example

Author: Alan Christie

tests/job-definitions/job-definitions.yaml

@@ -116,3 +116,11 @@ jobs:
             type: string
             default: clustered.sdf
             pattern: "^[A-Za-z0-9_/\\.\\-]+\\.(smi|sdf)$"
+
+  rdkit-molprops:
+    command: >-
+      rdkit-molprops.py --inputFile {{ inputFile }} --outputFile {{ outputFile }} --name {{ name }} --value {{ value }}
+
+  cluster-butina:
+    command: >-
+      rdkit-molprops.py --inputFile {{ inputFile }} --outputFile {{ outputFile }} --name {{ name }} --value {{ value }}

tests/jobs/cluster-butina.py

@@ -214,3 +214,34 @@ def test_workflow_engine_shortcut_example_1(basic_engine):
     # This test should generate a file in the simulated project directory
     assert project_file_exists(output_file_a)
     assert project_file_exists(output_file_b)
+
+
+@pytest.mark.skip(reason="The engine does not currently create the required variables")
+def test_workflow_engine_simple_python_molprops(basic_engine):
+    # Arrange
+    da, md = basic_engine
+    # Make sure files that should be generated by the test
+    # do not exist before we run the test.
+    output_file_a = "a.sdf"
+    assert not project_file_exists(output_file_a)
+    output_file_b = "b.sdf"
+    assert not project_file_exists(output_file_b)
+
+    # Act
+    r_wfid = start_workflow(
+        md, da, "simple-python-molprops", {"candidateMolecules": "C"}
+    )
+
+    # Assert
+    wait_for_workflow(da, r_wfid)
+    # Additional, detailed checks...
+    # Check we only have one RunningWorkflowStep, and it succeeded
+    response = da.get_running_workflow_steps(running_workflow_id=r_wfid)
+    assert response["count"] == 2
+    assert response["running_workflow_steps"][0]["done"]
+    assert response["running_workflow_steps"][0]["success"]
+    assert response["running_workflow_steps"][1]["done"]
+    assert response["running_workflow_steps"][1]["success"]
+    # This test should generate a file in the simulated project directory
+    assert project_file_exists(output_file_a)
+    assert project_file_exists(output_file_b)

tests/jobs/rdkik-molprops.py

@@ -102,3 +102,23 @@ def test_validate_shortcut_example_1():
     # Assert
     assert error.error_num == 0
     assert error.error_msg is None
+
+
+def test_validate_simple_python_molprops():
+    # Arrange
+    workflow_file: str = os.path.join(
+        os.path.dirname(__file__), "workflow-definitions", "simple-python-molprops.yaml"
+    )
+    with open(workflow_file, "r", encoding="utf8") as workflow_file:
+        workflow: dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
+    assert workflow
+
+    # Act
+    error = WorkflowValidator.validate(
+        level=ValidationLevel.CREATE,
+        workflow_definition=workflow,
+    )
+
+    # Assert
+    assert error.error_num == 0
+    assert error.error_msg is None

tests/test_workflow_engine_examples.py

@@ -0,0 +1,57 @@
+---
+kind: DataManagerWorkflow
+kind-version: "2024.1"
+name: python-workflow
+description: An simple python experimental workflow
+variables:
+  inputs:
+  - name: candidateMolecules
+  - type: squonk/x-smiles
+  outputs:
+  - name: clusteredMolecules
+    from:
+    - step: step2
+      output: outputFile
+    as: clustered-molecules.smi
+
+steps:
+- name: step1
+  description: Add column 1
+  specification: >-
+    {
+      "collection": "workflow-engine-unit-test-jobs",
+      "job": "rdkit-molprops",
+      "version": "1.0.0",
+      "variables": {
+        "name": "col1",
+        "value": 123
+      }
+    }
+  inputs:
+  - input: inputFile
+    from:
+      workflow-input: candidateMolecules
+  outputs:
+  - output: outputFile
+    as: __step1__out.smi
+
+- name: step2
+  Description: Add column 2
+  specification: >-
+    {
+      "collection": "workflow-engine-unit-test-jobs",
+      "job": "cluster-butina",
+      "version":"1.0.0",
+      "variables": {
+        "name":"col2",
+        "value":"999"
+      }
+    }
+  inputs:
+  - input: inputFile
+    from:
+      step: step1
+      output: outputFile
+  outputs:
+  - output: outputFile
+    as: __step2__out.smi

tests/test_workflow_validator_for_create_level.py

@@ -27,6 +27,13 @@ properties:
     type: array
     items:
       $ref: "#/definitions/step"
+  variables:
+    type: object
+    properties:
+      inputs:
+        type: array
+        items:
+          $ref: "#/definitions/workflow-input-parameter"
 required:
 - kind
 - kind-version
@@ -46,21 +53,35 @@ definitions:
       A value compatible with Kubernetes variables
       to allow it to be used ins Pod Label
 
-  variable-name:
+  parameter-name:
     type: string
-    pattern: ^[a-zA-Z_][a-zA-Z0-9_]{,63}$
-    description: >-
-      A Job/Step variable name, as used in the Data Manager Job Specification
+    pattern: ^[a-zA-Z_][a-zA-Z0-9_-]*$
 
-  # Declaration of a step from anotehr step
+  workflow-input-parameter:
+    type: object
+    properties:
+      name:
+        $ref: '#/definitions/parameter-name'
+
+  # Declaration of a value from a workflow input (variable)
+  from-workflow-input:
+    type: object
+    additionalProperties: false
+    properties:
+      workflow-input:
+        $ref: '#/definitions/parameter-name'
+    required:
+    - workflow-input
+
+  # Declaration of a value from another step
   from-step-output:
     type: object
     additionalProperties: false
     properties:
       step:
         $ref: '#/definitions/rfc1035-label-name'
       output:
-        $ref: '#/definitions/variable-name'
+        $ref: '#/definitions/parameter-name'
     required:
     - step
     - output
@@ -71,22 +92,33 @@ definitions:
     additionalProperties: false
     properties:
       input:
-        $ref: '#/definitions/variable-name'
+        $ref: '#/definitions/parameter-name'
       from:
         $ref: '#/definitions/from-step-output'
     required:
     - input
 
+  step-input-from-workflow:
+    type: object
+    additionalProperties: false
+    properties:
+      input:
+        $ref: '#/definitions/parameter-name'
+      from:
+        $ref: '#/definitions/from-workflow-input'
+    required:
+    - input
+
   # A Step output (with an 'as' - a declared value)
   step-output-as:
     type: object
     additionalProperties: false
     properties:
       output:
-        $ref: '#/definitions/variable-name'
+        $ref: '#/definitions/parameter-name'
       as:
         type: string
-        description: The value to set the variable to
+        description: The value to set the parameter to
     required:
     - output
     - as
@@ -105,12 +137,13 @@ definitions:
       inputs:
         type: array
         items:
-          oneOf:
+          anyOf:
           - $ref: "#/definitions/step-input-from-step"
+          - $ref: "#/definitions/step-input-from-workflow"
       outputs:
         type: array
         items:
-          oneOf:
+          anyOf:
           - $ref: "#/definitions/step-output-as"
     required:
     - name

tests/workflow-definitions/simple-python-molprops.yaml

@@ -338,16 +338,20 @@ def _validate_step_command(
         #
         # 1. The RunningWorkflow
         # 2. The Workflow
-        # 3. The Job Specification
+        # 3. The Job Step Specification Variables
         #
         # If variable 'x' is defined in all three then the RunningWorkflow's
         # value must be used.
 
+        # Get any variables from the step specification.
         all_variables = step_spec.pop("variables") if "variables" in step_spec else {}
+        # Merge workflow variables on top of these
         if workflow_variables:
             all_variables |= workflow_variables
+        # Merge running workflow variables on top of these
         if running_workflow_variables:
             all_variables |= running_workflow_variables
+
         message, success = decode(
             job["command"], all_variables, "command", TextEncoding.JINJA2_3_0
         )