refactor: Further API refactoring (DM-friendly responses)

Author: Alan Christie

tests/instance_launcher.py

@@ -71,7 +71,7 @@ def launch(self, launch_parameters: LaunchParameters) -> LaunchResult:
 
         # We're passed a RunningWorkflowStep ID but a record is expected to have been
         # created bt the caller, we simply create instance records.
-        response = self._api_adapter.get_running_workflow_step(
+        response, _ = self._api_adapter.get_running_workflow_step(
             running_workflow_step_id=launch_parameters.running_workflow_step_id
         )
         assert "running_workflow_step" in response
@@ -83,7 +83,7 @@ def launch(self, launch_parameters: LaunchParameters) -> LaunchResult:
         task_id = "task-00000000-0000-0000-0000-000000000001"
 
         # Apply variables to the step's Job command.
-        job = self._api_adapter.get_job(
+        job, _ = self._api_adapter.get_job(
             collection=launch_parameters.specification["collection"],
             job=launch_parameters.specification["job"],
             version="do-not-care",

tests/test_test_api_adapter.py

@@ -11,7 +11,7 @@ def test_get_nop_job():
     utaa = UnitTestWorkflowAPIAdapter()
 
     # Act
-    jd = utaa.get_job(
+    jd, _ = utaa.get_job(
         collection="workflow-engine-unit-test-jobs", job="nop", version="1.0.0"
     )
 
@@ -24,7 +24,9 @@ def test_get_unknown_workflow():
     utaa = UnitTestWorkflowAPIAdapter()
 
     # Act
-    wfd = utaa.get_workflow(workflow_id="workflow-00000000-0000-0000-0000-000000000001")
+    wfd, _ = utaa.get_workflow(
+        workflow_id="workflow-00000000-0000-0000-0000-000000000001"
+    )
 
     # Assert
     assert wfd == {}
@@ -48,7 +50,7 @@ def test_get_workflow():
     wfid = response["id"]
 
     # Act
-    wf = utaa.get_workflow(workflow_id=wfid)
+    wf, _ = utaa.get_workflow(workflow_id=wfid)
 
     # Assert
     assert wf["workflow"]["name"] == "blah"
@@ -85,7 +87,7 @@ def test_get_running_workflow():
     rwfid = response["id"]
 
     # Act
-    response = utaa.get_running_workflow(running_workflow_id=rwfid)
+    response, _ = utaa.get_running_workflow(running_workflow_id=rwfid)
 
     # Assert
     rwf = response["running_workflow"]
@@ -110,7 +112,7 @@ def test_set_running_workflow_done_when_success():
     utaa.set_running_workflow_done(running_workflow_id=rwfid, success=True)
 
     # Assert
-    response = utaa.get_running_workflow(running_workflow_id=rwfid)
+    response, _ = utaa.get_running_workflow(running_workflow_id=rwfid)
     assert response["running_workflow"]["done"]
     assert response["running_workflow"]["success"]
     assert response["running_workflow"]["error"] is None
@@ -135,7 +137,7 @@ def test_set_running_workflow_done_when_failed():
     )
 
     # Assert
-    response = utaa.get_running_workflow(running_workflow_id=rwfid)
+    response, _ = utaa.get_running_workflow(running_workflow_id=rwfid)
     assert response["running_workflow"]["done"]
     assert not response["running_workflow"]["success"]
     assert response["running_workflow"]["error"] == 1
@@ -154,7 +156,7 @@ def test_create_running_workflow_step():
     )
 
     # Act
-    response = utaa.create_running_workflow_step(
+    response, _ = utaa.create_running_workflow_step(
         running_workflow_id=response["id"], step="step-1"
     )
 
@@ -172,7 +174,7 @@ def test_set_running_workflow_step_done_when_success():
         project_id=TEST_PROJECT_ID,
         variables={},
     )
-    response = utaa.create_running_workflow_step(
+    response, _ = utaa.create_running_workflow_step(
         running_workflow_id=response["id"], step="step-1"
     )
     rwfsid = response["id"]
@@ -181,7 +183,7 @@ def test_set_running_workflow_step_done_when_success():
     utaa.set_running_workflow_step_done(running_workflow_step_id=rwfsid, success=True)
 
     # Assert
-    response = utaa.get_running_workflow_step(running_workflow_step_id=rwfsid)
+    response, _ = utaa.get_running_workflow_step(running_workflow_step_id=rwfsid)
     assert response["running_workflow_step"]["done"]
     assert response["running_workflow_step"]["success"]
     assert response["running_workflow_step"]["error"] is None
@@ -198,7 +200,7 @@ def test_set_running_workflow_step_done_when_failed():
         project_id=TEST_PROJECT_ID,
         variables={},
     )
-    response = utaa.create_running_workflow_step(
+    response, _ = utaa.create_running_workflow_step(
         running_workflow_id=response["id"], step="step-1"
     )
     rwfsid = response["id"]
@@ -209,7 +211,7 @@ def test_set_running_workflow_step_done_when_failed():
     )
 
     # Assert
-    response = utaa.get_running_workflow_step(running_workflow_step_id=rwfsid)
+    response, _ = utaa.get_running_workflow_step(running_workflow_step_id=rwfsid)
     assert response["running_workflow_step"]["done"]
     assert not response["running_workflow_step"]["success"]
     assert response["running_workflow_step"]["error"] == 1
@@ -228,13 +230,13 @@ def test_get_running_workflow_step():
         variables={},
     )
     rwfid = response["id"]
-    response = utaa.create_running_workflow_step(
+    response, _ = utaa.create_running_workflow_step(
         running_workflow_id=rwfid, step="step-1"
     )
     rwfsid = response["id"]
 
     # Act
-    response = utaa.get_running_workflow_step(running_workflow_step_id=rwfsid)
+    response, _ = utaa.get_running_workflow_step(running_workflow_step_id=rwfsid)
 
     # Assert
     rwfs = response["running_workflow_step"]
@@ -261,7 +263,7 @@ def test_create_and_get_instance():
     instance_id = response["id"]
 
     # Act
-    response = utaa.get_instance(instance_id=instance_id)
+    response, _ = utaa.get_instance(instance_id=instance_id)
 
     # Assert
     assert response["running_workflow_step"] == "r-workflow-step-000"

tests/test_test_instance_launcher.py

@@ -36,7 +36,7 @@ def test_launch_nop(basic_launcher):
         project_id=TEST_PROJECT_ID,
         variables={},
     )
-    response = utaa.create_running_workflow_step(
+    response, _ = utaa.create_running_workflow_step(
         running_workflow_id=response["id"], step="step-1"
     )
     rwfsid = response["id"]
@@ -73,7 +73,7 @@ def test_launch_nop_fail(basic_launcher):
         variables={},
     )
     rwfid = response["id"]
-    response = utaa.create_running_workflow_step(
+    response, _ = utaa.create_running_workflow_step(
         running_workflow_id=response["id"], step="step-1"
     )
     rwfsid = response["id"]
@@ -113,7 +113,7 @@ def test_launch_smiles_to_file(basic_launcher):
         variables={},
     )
     rwfid = response["id"]
-    response = utaa.create_running_workflow_step(
+    response, _ = utaa.create_running_workflow_step(
         running_workflow_id=response["id"], step="step-1"
     )
     rwfsid = response["id"]

tests/test_workflow_engine_examples.py

@@ -110,7 +110,7 @@ def wait_for_workflow(
     attempts = 0
     done = False
     while not done:
-        response = da.get_running_workflow(running_workflow_id=r_wfid)
+        response, _ = da.get_running_workflow(running_workflow_id=r_wfid)
         assert "running_workflow" in response
         r_wf = response["running_workflow"]
         if r_wf["done"]:

tests/wapi_adapter.py

@@ -83,7 +83,10 @@ def get_workflow(self, *, workflow_id: str) -> dict[str, Any]:
             workflow = Unpickler(pickle_file).load()
         UnitTestWorkflowAPIAdapter.lock.release()
 
-        return {"workflow": workflow[workflow_id]} if workflow_id in workflow else {}
+        response = (
+            {"workflow": workflow[workflow_id]} if workflow_id in workflow else {}
+        )
+        return response, 0
 
     def get_running_workflow(self, *, running_workflow_id: str) -> dict[str, Any]:
         UnitTestWorkflowAPIAdapter.lock.acquire()
@@ -93,7 +96,7 @@ def get_running_workflow(self, *, running_workflow_id: str) -> dict[str, Any]:
 
         if running_workflow_id not in running_workflow:
             return {}
-        return {"running_workflow": running_workflow[running_workflow_id]}
+        return {"running_workflow": running_workflow[running_workflow_id]}, 0
 
     def set_running_workflow_done(
         self,
@@ -140,7 +143,7 @@ def create_running_workflow_step(
             Pickler(pickle_file).dump(running_workflow_step)
         UnitTestWorkflowAPIAdapter.lock.release()
 
-        return {"id": running_workflow_step_id}
+        return {"id": running_workflow_step_id}, 0
 
     def get_running_workflow_step(
         self, *, running_workflow_step_id: str
@@ -151,10 +154,10 @@ def get_running_workflow_step(
         UnitTestWorkflowAPIAdapter.lock.release()
 
         if running_workflow_step_id not in running_workflow_step:
-            return {}
+            return {}, 0
         return {
             "running_workflow_step": running_workflow_step[running_workflow_step_id]
-        }
+        }, 0
 
     def set_running_workflow_step_done(
         self,
@@ -178,26 +181,14 @@ def set_running_workflow_step_done(
             Pickler(pickle_file).dump(running_workflow_step)
         UnitTestWorkflowAPIAdapter.lock.release()
 
-    def get_running_workflow_steps(self, *, running_workflow_id: str) -> dict[str, Any]:
-        UnitTestWorkflowAPIAdapter.lock.acquire()
-        with open(_RUNNING_WORKFLOW_STEP_PICKLE_FILE, "rb") as pickle_file:
-            running_workflow_step = Unpickler(pickle_file).load()
-        UnitTestWorkflowAPIAdapter.lock.release()
-
-        steps = []
-        for key, value in running_workflow_step.items():
-            if value["running_workflow"] == running_workflow_id:
-                item = {"running_workflow_step": value, "id": key}
-                steps.append(item)
-        return {"count": len(steps), "running_workflow_steps": steps}
-
     def get_instance(self, *, instance_id: str) -> dict[str, Any]:
         UnitTestWorkflowAPIAdapter.lock.acquire()
         with open(_INSTANCE_PICKLE_FILE, "rb") as pickle_file:
             instances = Unpickler(pickle_file).load()
         UnitTestWorkflowAPIAdapter.lock.release()
 
-        return {} if instance_id not in instances else instances[instance_id]
+        response = {} if instance_id not in instances else instances[instance_id]
+        return response, 0
 
     def get_job(self, *, collection: str, job: str, version: str) -> dict[str, Any]:
         assert collection == _JOB_DEFINITIONS["collection"]
@@ -208,7 +199,7 @@ def get_job(self, *, collection: str, job: str, version: str) -> dict[str, Any]:
         response = {"command": jd["command"]}
         if "variables" in jd:
             response["variables"] = jd["variables"]
-        return response
+        return response, 0
 
     # Methods required for the UnitTestInstanceLauncher and other (internal) logic
     # but not exposed to (or required by) the Workflow Engine...
@@ -279,3 +270,16 @@ def create_instance(self, *, running_workflow_step_id: str) -> dict[str, Any]:
         UnitTestWorkflowAPIAdapter.lock.release()
 
         return {"id": instance_id}
+
+    def get_running_workflow_steps(self, *, running_workflow_id: str) -> dict[str, Any]:
+        UnitTestWorkflowAPIAdapter.lock.acquire()
+        with open(_RUNNING_WORKFLOW_STEP_PICKLE_FILE, "rb") as pickle_file:
+            running_workflow_step = Unpickler(pickle_file).load()
+        UnitTestWorkflowAPIAdapter.lock.release()
+
+        steps = []
+        for key, value in running_workflow_step.items():
+            if value["running_workflow"] == running_workflow_id:
+                item = {"running_workflow_step": value, "id": key}
+                steps.append(item)
+        return {"count": len(steps), "running_workflow_steps": steps}

workflow/workflow_abc.py

@@ -79,16 +79,29 @@ def get_workflow(
         self,
         *,
         workflow_id: str,
-    ) -> dict[str, Any]:
+    ) -> tuple[dict[str, Any], int]:
         """Get a Workflow Record by ID."""
         # If present this should return:
         # {
-        #    "workflow": <workflow>,
+        #    "workflow": {
+        #       "steps": [
+        #          {
+        #            "name": "step-name"
+        #          }
+        #       ]
+        #    }
         # }
         # If not present an empty dictionary should be returned.
+        #
+        # The 'int' in the return tuple here (and elsewhere in this ABC)
+        # is an HTTP status code to simplify the DM implementation,
+        # and allow it to re-use any 'views.py' function that may be defined.
+        # This value is ignored by the Engine.
 
     @abstractmethod
-    def get_running_workflow(self, *, running_workflow_id: str) -> dict[str, Any]:
+    def get_running_workflow(
+        self, *, running_workflow_id: str
+    ) -> tuple[dict[str, Any], int]:
         """Get a RunningWorkflow Record"""
         # Should return:
         # {
@@ -124,7 +137,7 @@ def create_running_workflow_step(
         *,
         running_workflow_id: str,
         step: str,
-    ) -> dict[str, Any]:
+    ) -> tuple[dict[str, Any], int]:
         """Create a RunningWorkflowStep Record (from a RunningWorkflow)"""
         # Should return:
         # {
@@ -134,7 +147,7 @@ def create_running_workflow_step(
     @abstractmethod
     def get_running_workflow_step(
         self, *, running_workflow_step_id: str
-    ) -> dict[str, Any]:
+    ) -> tuple[dict[str, Any], int]:
         """Get a RunningWorkflowStep Record"""
         # Should return:
         # {
@@ -162,29 +175,7 @@ def set_running_workflow_step_done(
         If not successful an error code and message should be provided."""
 
     @abstractmethod
-    def get_running_workflow_steps(self, *, running_workflow_id: str) -> dict[str, Any]:
-        """Gets all the RunningWorkflowStep Records (for a RunningWorkflow)"""
-        # Should return:
-        # {
-        #    "count": 1,
-        #    "running_workflow_steps": [
-        #       {
-        #           "id": "r-workflow-step-00000000-0000-0000-0000-000000000001",
-        #           "running_workflow_step": {
-        #               "step:": "step-1234",
-        #               "done": False,
-        #               "success": false,
-        #               "error": None,
-        #               "error_msg": None,
-        #               "workflow": "workflow-00000000-0000-0000-0000-000000000001",
-        #           }
-        #       ...
-        #    ]
-        # }
-        # If there are not steps an empty dictionary should be returned and a count of 0
-
-    @abstractmethod
-    def get_instance(self, *, instance_id: str) -> dict[str, Any]:
+    def get_instance(self, *, instance_id: str) -> tuple[dict[str, Any], int]:
         """Get an Instance Record"""
         # Should return:
         # {
@@ -200,7 +191,7 @@ def get_job(
         collection: str,
         job: str,
         version: str,
-    ) -> dict[str, Any]:
+    ) -> tuple[dict[str, Any], int]:
         """Get a Job"""
         # If not present an empty dictionary should be returned.