fix: Better testing of validation (job must be present)

Author: Alan Christie

tests/test_decoder.py

@@ -1,5 +1,5 @@
 import os
-from typing import Any, Dict
+from typing import Any
 
 import pytest
 import yaml
@@ -13,21 +13,21 @@
     os.path.dirname(__file__), "workflow-definitions", "minimal.yaml"
 )
 with open(_MINIMAL_WORKFLOW_FILE, "r", encoding="utf8") as workflow_file:
-    _MINIMAL_WORKFLOW: Dict[str, Any] = yaml.safe_load(workflow_file)
+    _MINIMAL_WORKFLOW: dict[str, Any] = yaml.safe_load(workflow_file)
 assert _MINIMAL_WORKFLOW
 
 _SHORTCUT_EXAMPLE_1_WORKFLOW_FILE: str = os.path.join(
     os.path.dirname(__file__), "workflow-definitions", "shortcut-example-1.yaml"
 )
 with open(_SHORTCUT_EXAMPLE_1_WORKFLOW_FILE, "r", encoding="utf8") as workflow_file:
-    _SHORTCUT_EXAMPLE_1_WORKFLOW: Dict[str, Any] = yaml.safe_load(workflow_file)
+    _SHORTCUT_EXAMPLE_1_WORKFLOW: dict[str, Any] = yaml.safe_load(workflow_file)
 assert _SHORTCUT_EXAMPLE_1_WORKFLOW
 
 _SIMPLE_PYTHON_MOLPROPS_WORKFLOW_FILE: str = os.path.join(
     os.path.dirname(__file__), "workflow-definitions", "simple-python-molprops.yaml"
 )
 with open(_SIMPLE_PYTHON_MOLPROPS_WORKFLOW_FILE, "r", encoding="utf8") as workflow_file:
-    _SIMPLE_PYTHON_MOLPROPS_WORKFLOW: Dict[str, Any] = yaml.safe_load(workflow_file)
+    _SIMPLE_PYTHON_MOLPROPS_WORKFLOW: dict[str, Any] = yaml.safe_load(workflow_file)
 assert _SIMPLE_PYTHON_MOLPROPS_WORKFLOW
 
 _SIMPLE_PYTHON_MOLPROPS_WITH_OPTIONS_WORKFLOW_FILE: str = os.path.join(
@@ -38,7 +38,7 @@
 with open(
     _SIMPLE_PYTHON_MOLPROPS_WITH_OPTIONS_WORKFLOW_FILE, "r", encoding="utf8"
 ) as workflow_file:
-    _SIMPLE_PYTHON_MOLPROPS_WITH_OPTIONS_WORKFLOW: Dict[str, Any] = yaml.safe_load(
+    _SIMPLE_PYTHON_MOLPROPS_WITH_OPTIONS_WORKFLOW: dict[str, Any] = yaml.safe_load(
         workflow_file
     )
 assert _SIMPLE_PYTHON_MOLPROPS_WITH_OPTIONS_WORKFLOW
@@ -49,7 +49,7 @@
     "simple-python-parallel.yaml",
 )
 with open(_SIMPLE_PYTHON_PARALLEL_FILE, "r", encoding="utf8") as workflow_file:
-    _SIMPLE_PYTHON_PARALLEL_WORKFLOW: Dict[str, Any] = yaml.safe_load(workflow_file)
+    _SIMPLE_PYTHON_PARALLEL_WORKFLOW: dict[str, Any] = yaml.safe_load(workflow_file)
 assert _SIMPLE_PYTHON_PARALLEL_WORKFLOW
 
 _STEP_SPECIFICATION_VARIABLE_NAMES_WORKFLOW_FILE: str = os.path.join(
@@ -60,7 +60,7 @@
 with open(
     _STEP_SPECIFICATION_VARIABLE_NAMES_WORKFLOW_FILE, "r", encoding="utf8"
 ) as workflow_file:
-    _STEP_SPECIFICATION_VARIABLE_NAMES_WORKFLOW: Dict[str, Any] = yaml.safe_load(
+    _STEP_SPECIFICATION_VARIABLE_NAMES_WORKFLOW: dict[str, Any] = yaml.safe_load(
         workflow_file
     )
 assert _STEP_SPECIFICATION_VARIABLE_NAMES_WORKFLOW

tests/test_workflow_validator_for_create_level.py

@@ -10,6 +10,13 @@
 from tests.wapi_adapter import UnitTestWorkflowAPIAdapter
 from workflow.workflow_validator import ValidationLevel, WorkflowValidator
 
+_NO_SUCH_JOB_WORKFLOW_FILE: str = os.path.join(
+    os.path.dirname(__file__), "workflow-definitions", "no-such-job.yaml"
+)
+with open(_NO_SUCH_JOB_WORKFLOW_FILE, "r", encoding="utf8") as workflow_file:
+    _NO_SUCH_JOB_WORKFLOW: dict[str, Any] = yaml.safe_load(workflow_file)
+assert _NO_SUCH_JOB_WORKFLOW
+
 
 @pytest.fixture
 def wapi():
@@ -33,6 +40,22 @@ def test_validate_minimal(wapi):
     assert error.error_msg is None
 
 
+def test_validate_no_such_job(wapi):
+    # Arrange
+    wapi_adapter = wapi
+
+    # Act
+    error = WorkflowValidator.validate(
+        level=ValidationLevel.CREATE,
+        workflow_definition=_NO_SUCH_JOB_WORKFLOW,
+        wapi_adapter=wapi_adapter,
+    )
+
+    # Assert
+    assert error.error_num == 0
+    assert error.error_msg is None
+
+
 def test_validate_example_nop_file(wapi):
     # Arrange
     wapi_adapter = wapi

tests/test_workflow_validator_for_run_level.py

@@ -9,13 +9,39 @@
 from tests.wapi_adapter import UnitTestWorkflowAPIAdapter
 from workflow.workflow_validator import ValidationLevel, WorkflowValidator
 
+_NO_SUCH_JOB_WORKFLOW_FILE: str = os.path.join(
+    os.path.dirname(__file__), "workflow-definitions", "no-such-job.yaml"
+)
+with open(_NO_SUCH_JOB_WORKFLOW_FILE, "r", encoding="utf8") as workflow_file:
+    _NO_SUCH_JOB_WORKFLOW: dict[str, Any] = yaml.safe_load(workflow_file)
+assert _NO_SUCH_JOB_WORKFLOW
+
 
 @pytest.fixture
 def wapi():
     wapi_adapter = UnitTestWorkflowAPIAdapter()
     yield wapi_adapter
 
 
+def test_validate_no_such_job(wapi):
+    # Arrange
+    wapi_adapter = wapi
+
+    # Act
+    error = WorkflowValidator.validate(
+        level=ValidationLevel.RUN,
+        workflow_definition=_NO_SUCH_JOB_WORKFLOW,
+        wapi_adapter=wapi_adapter,
+    )
+
+    # Assert
+    assert error.error_num != 0
+    assert error.error_msg != None
+    assert len(error.error_msg) == 1
+    assert error.error_msg[0].startswith("Step ")
+    assert error.error_msg[0].endswith(" is not present")
+
+
 def test_validate_example_nop_file(wapi):
     # Arrange
     wapi_adapter = wapi

tests/test_workflow_validator_for_tag_level.py

@@ -9,13 +9,36 @@
 from tests.wapi_adapter import UnitTestWorkflowAPIAdapter
 from workflow.workflow_validator import ValidationLevel, WorkflowValidator
 
+_NO_SUCH_JOB_WORKFLOW_FILE: str = os.path.join(
+    os.path.dirname(__file__), "workflow-definitions", "no-such-job.yaml"
+)
+with open(_NO_SUCH_JOB_WORKFLOW_FILE, "r", encoding="utf8") as workflow_file:
+    _NO_SUCH_JOB_WORKFLOW: dict[str, Any] = yaml.safe_load(workflow_file)
+assert _NO_SUCH_JOB_WORKFLOW
+
 
 @pytest.fixture
 def wapi():
     wapi_adapter = UnitTestWorkflowAPIAdapter()
     yield wapi_adapter
 
 
+def test_validate_no_such_job(wapi):
+    # Arrange
+    wapi_adapter = wapi
+
+    # Act
+    error = WorkflowValidator.validate(
+        level=ValidationLevel.TAG,
+        workflow_definition=_NO_SUCH_JOB_WORKFLOW,
+        wapi_adapter=wapi_adapter,
+    )
+
+    # Assert
+    assert error.error_num == 0
+    assert error.error_msg is None
+
+
 def test_validate_example_nop_file(wapi):
     # Arrange
     wapi_adapter = wapi

tests/wapi_adapter.py

@@ -280,8 +280,10 @@ def get_instance(self, *, instance_id: str) -> tuple[dict[str, Any], int]:
     def get_job(
         self, *, collection: str, job: str, version: str
     ) -> tuple[dict[str, Any], int]:
-        assert collection == _JOB_DEFINITIONS["collection"]
-        assert job in _JOB_DEFINITIONS["jobs"]
+        if collection != _JOB_DEFINITIONS["collection"]:
+            return {}, 0
+        if job not in _JOB_DEFINITIONS["jobs"]:
+            return {}, 0
         assert version
 
         jd = _JOB_DEFINITIONS["jobs"][job]

tests/workflow-definitions/no-such-job.yaml

@@ -0,0 +1,12 @@
+---
+kind: DataManagerWorkflow
+kind-version: "2025.2"
+name: workflow-minimal
+
+steps:
+
+- name: step-1
+  specification:
+    collection: a
+    job: no-such-job
+    version: "1.0.0"

workflow/workflow_validator.py

@@ -171,6 +171,7 @@ def _validate_run_level(
             )
 
         # All of the jobs must be known to the DM
+        errors: list[str] = []
         for step_name in get_step_names(workflow_definition):
             step_spec = get_step_specification(workflow_definition, step_name)
             j_collection: str = step_spec["collection"]
@@ -182,11 +183,10 @@ def _validate_run_level(
                 version=j_version,
             )
             if not job:
-                return ValidationResult(
-                    error_num=9,
-                    error_msg=[
-                        f"Step {step_name} Job ({j_collection}|{j_job}|{j_version}) is not present"
-                    ],
+                errors.append(
+                    f"Step {step_name} Job ({j_collection}|{j_job}|{j_version}) is not present"
                 )
+        if errors:
+            return ValidationResult(error_num=9, error_msg=errors)
 
         return _VALIDATION_SUCCESS