Merge pull request #19 from InformaticsMatters/workflow-options

Basic workflow options

Author: alanbchristie

tests/test_decoder.py

@@ -56,6 +56,15 @@
     )
 assert _STEP_SPECIFICATION_VARIABLE_NAMES_WORKFLOW
 
+_WORKFLOW_OPTIONS_WORKFLOW_FILE: str = os.path.join(
+    os.path.dirname(__file__),
+    "workflow-definitions",
+    "workflow-options.yaml",
+)
+with open(_WORKFLOW_OPTIONS_WORKFLOW_FILE, "r", encoding="utf8") as workflow_file:
+    _WORKFLOW_OPTIONS: Dict[str, Any] = yaml.safe_load(workflow_file)
+assert _WORKFLOW_OPTIONS
+
 
 def test_validate_schema_for_minimal():
     # Arrange
@@ -133,6 +142,16 @@ def test_validate_schema_for_step_specification_variable_names():
     assert error is None
 
 
+def test_validate_schema_for_workflow_options():
+    # Arrange
+
+    # Act
+    error = decoder.validate_schema(_WORKFLOW_OPTIONS)
+
+    # Assert
+    assert error is None
+
+
 def test_get_workflow_variables_for_smiple_python_molprops():
     # Arrange
 

tests/test_workflow_validator_for_run_level.py

@@ -131,6 +131,88 @@ def test_validate_simple_python_molprops():
     assert error.error_msg is None
 
 
+def test_validate_simple_python_molprops_with_options_when_missing_required():
+    # Arrange
+    workflow_file: str = os.path.join(
+        os.path.dirname(__file__),
+        "workflow-definitions",
+        "simple-python-molprops-with-options.yaml",
+    )
+    with open(workflow_file, "r", encoding="utf8") as workflow_file:
+        workflow: dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
+    assert workflow
+    variables = {
+        "candidateMolecules": "input.sdf",
+        "clusteredMolecules": "output.sdf",
+    }
+
+    # Act
+    error = WorkflowValidator.validate(
+        level=ValidationLevel.RUN,
+        workflow_definition=workflow,
+        variables=variables,
+    )
+
+    # Assert
+    assert error.error_num == 7
+    assert error.error_msg == [
+        "Missing workflow variable values for: rdkitPropertyValue"
+    ]
+
+
+def test_validate_simple_python_molprops_with_options():
+    # Arrange
+    workflow_file: str = os.path.join(
+        os.path.dirname(__file__),
+        "workflow-definitions",
+        "simple-python-molprops-with-options.yaml",
+    )
+    with open(workflow_file, "r", encoding="utf8") as workflow_file:
+        workflow: dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
+    assert workflow
+    variables = {
+        "candidateMolecules": "input.sdf",
+        "clusteredMolecules": "output.sdf",
+        "rdkitPropertyName": "col1",
+        "rdkitPropertyValue": 123,
+    }
+
+    # Act
+    error = WorkflowValidator.validate(
+        level=ValidationLevel.RUN,
+        workflow_definition=workflow,
+        variables=variables,
+    )
+
+    # Assert
+    assert error.error_num == 0
+    assert error.error_msg is None
+
+
+def test_validate_simple_python_molprops_with_missing_input():
+    # Arrange
+    workflow_file: str = os.path.join(
+        os.path.dirname(__file__), "workflow-definitions", "simple-python-molprops.yaml"
+    )
+    with open(workflow_file, "r", encoding="utf8") as workflow_file:
+        workflow: dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
+    assert workflow
+    variables = {"clusteredMolecules": "output.sdf"}
+
+    # Act
+    error = WorkflowValidator.validate(
+        level=ValidationLevel.RUN,
+        workflow_definition=workflow,
+        variables=variables,
+    )
+
+    # Assert
+    assert error.error_num == 7
+    assert error.error_msg == [
+        "Missing workflow variable values for: candidateMolecules"
+    ]
+
+
 def test_validate_duplicate_workflow_variable_names():
     # Arrange
     workflow_file: str = os.path.join(

tests/workflow-definitions/duplicate-step-names.yaml

@@ -2,12 +2,15 @@
 kind: DataManagerWorkflow
 kind-version: "2024.1"
 name: duplicate-step-names
+
 steps:
+
 - name: step-1
   specification:
     collection: a
     job: b
     version: '1.0.0'
+
 - name: step-1
   specification:
     collection: a

tests/workflow-definitions/duplicate-workflow-variable-names.yaml

@@ -15,6 +15,7 @@ variables:
     as: clustered-molecules.smi
 
 steps:
+
 - name: step1
   description: Add column 1
   specification:

tests/workflow-definitions/example-nop-fail.yaml

@@ -4,7 +4,9 @@ kind-version: "2024.1"
 name: nop-fail
 description: >-
   A workflow with one step that fails
+
 steps:
+
 - name: step-1
   specification:
     collection: workflow-engine-unit-test-jobs

tests/workflow-definitions/example-smiles-to-file.yaml

@@ -5,7 +5,9 @@ name: smiles-to-file
 description: >-
   A workflow with one step that uses variables.
   The step takes an input string and the Job creates a file from it.
+
 steps:
+
 - name: step-1
   specification:
     collection: workflow-engine-unit-test-jobs

tests/workflow-definitions/example-two-step-nop.yaml

@@ -5,12 +5,15 @@ name: two-step-nop
 description: >-
   A workflow with two steps.
   The steps do nothing, take no arguments, and simply return success.
+
 steps:
+
 - name: step-1
   specification:
     collection: workflow-engine-unit-test-jobs
     job: nop
     version: "1.0.0"
+
 - name: step-2
   specification:
     collection: workflow-engine-unit-test-jobs

tests/workflow-definitions/minimal.yaml

@@ -2,7 +2,9 @@
 kind: DataManagerWorkflow
 kind-version: "2024.1"
 name: workflow-minimal
+
 steps:
+
 - name: step-1
   specification:
     collection: a

tests/workflow-definitions/shortcut-example-1.yaml

@@ -3,7 +3,9 @@ kind: DataManagerWorkflow
 kind-version: "2024.1"
 name: shortcut-example-1
 description: The shortcut example 1 workflow
+
 steps:
+
 - name: example-1-step-1
   description: The first step
   specification:
@@ -13,6 +15,7 @@ steps:
   outputs:
   - output: 'outputFile'
     as: 'a.sdf'
+
 - name: example-1-step-2
   description: The first step
   specification:

tests/workflow-definitions/simple-python-molprops-with-options.yaml

@@ -0,0 +1,59 @@
+---
+kind: DataManagerWorkflow
+kind-version: "2024.1"
+name: python-workflow
+description: A simple python experimental workflow
+variables:
+  inputs:
+  - name: candidateMolecules
+    type: squonk/x-smiles
+  outputs:
+  - name: clusteredMolecules
+    from:
+      step: step2
+      output: outputFile
+    as: clustered-molecules.smi
+  options:
+  - name: rdkitPropertyName
+    default: name
+    as:
+    - option: name
+      step: step1
+  - name: rdkitPropertyValue
+    as:
+    - option: value
+      step: step1
+
+steps:
+
+- name: step1
+  description: Add column 1
+  specification:
+    collection: workflow-engine-unit-test-jobs
+    job: rdkit-molprops
+    version: "1.0.0"
+  inputs:
+  - input: inputFile
+    from:
+      workflow-input: candidateMolecules
+  outputs:
+  - output: outputFile
+    as: step1.out.smi
+
+- name: step2
+  description: Add column 2
+  specification:
+    collection: workflow-engine-unit-test-jobs
+    job: cluster-butina
+    version: "1.0.0"
+    variables:
+      name: "col2"
+      value: "999"
+  inputs:
+  - input: inputFile
+    from:
+      step: step1
+      output: outputFile
+  outputs:
+  - output: outputFile
+    as: step2.out.smi