Skip to content

Commit 3423c0f

Browse files
author
Alan Christie
committed
feat: Add RUN level validation testing
1 parent e88b36b commit 3423c0f

File tree

4 files changed

+222
-72
lines changed

4 files changed

+222
-72
lines changed

‎tests/test_workflow_validator.py‎

Lines changed: 0 additions & 44 deletions
This file was deleted.

‎tests/test_workflow_validator_for_create_level.py‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
import os
2+
from typing import Any
3+
4+
import pytest
5+
import yaml
6+
7+
pytestmark = pytest.mark.unit
8+
9+
from tests.test_decoder import _MINIMAL_WORKFLOW
10+
from workflow.workflow_validator import ValidationLevel, WorkflowValidator
11+
12+
13+
def test_validate_minimal():
14+
# Arrange
15+
16+
# Act
17+
error = WorkflowValidator.validate(
18+
level=ValidationLevel.CREATE,
19+
workflow_definition=_MINIMAL_WORKFLOW,
20+
)
21+
22+
# Assert
23+
assert error.error_num == 0
24+
assert error.error_msg is None
25+
26+
27+
def test_validate_example_nop_file():
28+
# Arrange
29+
workflow_file: str = os.path.join(
30+
os.path.dirname(__file__), "workflow-definitions", "example-nop-fail.yaml"
31+
)
32+
with open(workflow_file, "r", encoding="utf8") as workflow_file:
33+
workflow: dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
34+
assert workflow
35+
36+
# Act
37+
error = WorkflowValidator.validate(
38+
level=ValidationLevel.CREATE,
39+
workflow_definition=workflow,
40+
)
41+
42+
# Assert
43+
assert error.error_num == 0
44+
assert error.error_msg is None
45+
46+
47+
def test_validate_example_smiles_to_file():
48+
# Arrange
49+
workflow_file: str = os.path.join(
50+
os.path.dirname(__file__), "workflow-definitions", "example-smiles-to-file.yaml"
51+
)
52+
with open(workflow_file, "r", encoding="utf8") as workflow_file:
53+
workflow: dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
54+
assert workflow
55+
56+
# Act
57+
error = WorkflowValidator.validate(
58+
level=ValidationLevel.CREATE,
59+
workflow_definition=workflow,
60+
)
61+
62+
# Assert
63+
assert error.error_num == 0
64+
assert error.error_msg is None
65+
66+
67+
def test_validate_example_tow_step_nop():
68+
# Arrange
69+
workflow_file: str = os.path.join(
70+
os.path.dirname(__file__), "workflow-definitions", "example-two-step-nop.yaml"
71+
)
72+
with open(workflow_file, "r", encoding="utf8") as workflow_file:
73+
workflow: dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
74+
assert workflow
75+
76+
# Act
77+
error = WorkflowValidator.validate(
78+
level=ValidationLevel.CREATE,
79+
workflow_definition=workflow,
80+
)
81+
82+
# Assert
83+
assert error.error_num == 0
84+
assert error.error_msg is None
85+
86+
87+
def test_validate_shortcut_example_1():
88+
# Arrange
89+
workflow_file: str = os.path.join(
90+
os.path.dirname(__file__), "workflow-definitions", "shortcut-example-1.yaml"
91+
)
92+
with open(workflow_file, "r", encoding="utf8") as workflow_file:
93+
workflow: dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
94+
assert workflow
95+
96+
# Act
97+
error = WorkflowValidator.validate(
98+
level=ValidationLevel.CREATE,
99+
workflow_definition=workflow,
100+
)
101+
102+
# Assert
103+
assert error.error_num == 0
104+
assert error.error_msg is None

‎tests/test_workflow_validator_for_run_level.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import os
2+
from typing import Any
3+
4+
import pytest
5+
import yaml
6+
7+
pytestmark = pytest.mark.unit
8+
9+
from workflow.workflow_validator import ValidationLevel, WorkflowValidator
10+
11+
12+
def test_validate_example_nop_file():
13+
# Arrange
14+
workflow_file: str = os.path.join(
15+
os.path.dirname(__file__), "workflow-definitions", "example-nop-fail.yaml"
16+
)
17+
with open(workflow_file, "r", encoding="utf8") as workflow_file:
18+
workflow: dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
19+
assert workflow
20+
21+
# Act
22+
error = WorkflowValidator.validate(
23+
level=ValidationLevel.RUN,
24+
workflow_definition=workflow,
25+
)
26+
27+
# Assert
28+
assert error.error_num == 0
29+
assert error.error_msg is None
30+
31+
32+
def test_validate_example_smiles_to_file():
33+
# Arrange
34+
workflow_file: str = os.path.join(
35+
os.path.dirname(__file__), "workflow-definitions", "example-smiles-to-file.yaml"
36+
)
37+
with open(workflow_file, "r", encoding="utf8") as workflow_file:
38+
workflow: dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
39+
assert workflow
40+
41+
# Act
42+
error = WorkflowValidator.validate(
43+
level=ValidationLevel.RUN,
44+
workflow_definition=workflow,
45+
)
46+
47+
# Assert
48+
assert error.error_num == 0
49+
assert error.error_msg is None
50+
51+
52+
def test_validate_example_tow_step_nop():
53+
# Arrange
54+
workflow_file: str = os.path.join(
55+
os.path.dirname(__file__), "workflow-definitions", "example-two-step-nop.yaml"
56+
)
57+
with open(workflow_file, "r", encoding="utf8") as workflow_file:
58+
workflow: dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
59+
assert workflow
60+
61+
# Act
62+
error = WorkflowValidator.validate(
63+
level=ValidationLevel.RUN,
64+
workflow_definition=workflow,
65+
)
66+
67+
# Assert
68+
assert error.error_num == 0
69+
assert error.error_msg is None
70+
71+
72+
def test_validate_shortcut_example_1():
73+
# Arrange
74+
workflow_file: str = os.path.join(
75+
os.path.dirname(__file__), "workflow-definitions", "shortcut-example-1.yaml"
76+
)
77+
with open(workflow_file, "r", encoding="utf8") as workflow_file:
78+
workflow: dict[str, Any] = yaml.load(workflow_file, Loader=yaml.FullLoader)
79+
assert workflow
80+
81+
# Act
82+
error = WorkflowValidator.validate(
83+
level=ValidationLevel.RUN,
84+
workflow_definition=workflow,
85+
)
86+
87+
# Assert
88+
assert error.error_num == 0
89+
assert error.error_msg is None

‎workflow/workflow_validator.py‎

Lines changed: 29 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -72,35 +72,36 @@ def _validate_run_level(
7272
assert workflow_definition
7373
del workflow_inputs
7474

75-
# RUN level requires that the specification is a valid JSON string.
75+
# RUN level requires that each step specification is a valid JSON string.
7676
# and contains properties for 'collection', 'job', and 'version'.
77-
try:
78-
specification = json.loads(workflow_definition["specification"])
79-
except json.decoder.JSONDecodeError as e:
80-
return ValidationResult(
81-
error_num=1,
82-
error_msg=[
83-
f"Error decoding specification, which is not valid JSON: {e}"
84-
],
85-
)
86-
except TypeError as e:
87-
return ValidationResult(
88-
error_num=2,
89-
error_msg=[
90-
f"Error decoding specification, which is not valid JSON: {e}"
91-
],
92-
)
93-
expected_keys: set[str] = {"collection", "job", "version"}
94-
missing_keys: list[str] = []
95-
missing_keys.extend(
96-
expected_key
97-
for expected_key in expected_keys
98-
if expected_key not in specification
99-
)
100-
if missing_keys:
101-
return ValidationResult(
102-
error_num=2,
103-
error_msg=[f"Specification is missing: {', '.join(missing_keys)}"],
77+
for step in workflow_definition["steps"]:
78+
try:
79+
specification = json.loads(step["specification"])
80+
except json.decoder.JSONDecodeError as e:
81+
return ValidationResult(
82+
error_num=2,
83+
error_msg=[
84+
f"Error decoding specification, which is not valid JSON: {e}"
85+
],
86+
)
87+
except TypeError as e:
88+
return ValidationResult(
89+
error_num=3,
90+
error_msg=[
91+
f"Error decoding specification, which is not valid JSON: {e}"
92+
],
93+
)
94+
expected_keys: set[str] = {"collection", "job", "version"}
95+
missing_keys: list[str] = []
96+
missing_keys.extend(
97+
expected_key
98+
for expected_key in expected_keys
99+
if expected_key not in specification
104100
)
101+
if missing_keys:
102+
return ValidationResult(
103+
error_num=2,
104+
error_msg=[f"Specification is missing: {', '.join(missing_keys)}"],
105+
)
105106

106107
return _VALIDATION_SUCCESS

0 commit comments

Comments
 (0)