attempting to fix remaining errors showing up in GHA

Author: tclose

.github/workflows/testpydra.yml

@@ -80,7 +80,7 @@ jobs:
 
       - name: Pytest
         run: |
-          pytest -vs -n auto --doctest-modules \
+          pytest -vs -n auto pydra --doctest-modules \
                  --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
 
       - name: Upload to codecov

.github/workflows/testslurm.yml

@@ -53,7 +53,7 @@ jobs:
         docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test] && python -c 'import pydra.engine; print(pydra.engine.__version__)'"
     - name: Run pytest
       run: |
-        docker exec slurm bash -c "pytest --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'"
+        docker exec slurm bash -c "pytest pydra/engine/tests/test_workflow.py pydra/engine/tests/test_submitter.py --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'"
     - name: Upload to codecov
       run: |
         docker exec slurm bash -c "pip install urllib3==1.26.6"

new-docs/source/tutorial/4-python.ipynb

@@ -22,7 +22,8 @@
    "source": [
     "from pydra.design import python\n",
     "\n",
-    "# Note that we use CamelCase as the return of the is a class\n",
+    "\n",
+    "# Note that we use PascalCase because the object returned by the decorator is actually a class\n",
     "@python.define\n",
     "def MyFirstTaskDef(a, b):\n",
     "    \"\"\"Sample function for testing\"\"\"\n",
@@ -72,6 +73,7 @@
     "    \"\"\"Sample function for testing\"\"\"\n",
     "    return a + b, a - b\n",
     "\n",
+    "\n",
     "named_output_task = NamedOutputTaskDef(a=2, b=1)\n",
     "\n",
     "outputs = named_output_task()\n",
@@ -123,13 +125,13 @@
    "source": [
     "from pydra.design import python\n",
     "\n",
-    "# Note that we use CamelCase as the function is translated to a class\n",
     "\n",
     "@python.define\n",
     "def MyTypedTask(a: int, b: float) -> float:\n",
     "    \"\"\"Sample function for testing\"\"\"\n",
     "    return a + b\n",
     "\n",
+    "\n",
     "try:\n",
     "    # 1.5 is not an integer so this should raise a TypeError\n",
     "    my_typed_task = MyTypedTask(a=1.5, b=2.0)\n",
@@ -164,7 +166,8 @@
     "from pprint import pprint\n",
     "from pydra.engine.helpers import fields_dict\n",
     "\n",
-    "@python.define\n",
+    "\n",
+    "@python.define(outputs=[\"c\", \"d\"])\n",
     "def DocStrDef(a: int, b: float) -> tuple[float, float]:\n",
     "    \"\"\"Sample function for testing\n",
     "\n",
@@ -179,6 +182,7 @@
     "    \"\"\"\n",
     "    return a + b, a * b\n",
     "\n",
+    "\n",
     "pprint(fields_dict(DocStrDef))\n",
     "pprint(fields_dict(DocStrDef.Outputs))"
    ]

new-docs/source/tutorial/tst.py

@@ -294,10 +294,16 @@ def __call__(
                         )
             raise
         if result.errored:
+            if result.errors:
+                time_of_crash = result.errors["time of crash"]
+                error_message = "\n".join(result.errors["error message"])
+            else:
+                time_of_crash = "UNKNOWN"
+                error_message = "NOT RETRIEVED"
             raise RuntimeError(
-                f"Task {self} failed @ {result.errors['time of crash']} with the "
+                f"Task {self} failed @ {time_of_crash} with the "
                 "following errors:\n"
-                + "\n".join(result.errors["error message"])
+                + "\n".join(error_message)
                 + (
                     "To inspect, please load the pickled task object from here: "
                     f"{result.output_dir}/_task.pklz"

pydra/engine/specs.py

@@ -818,8 +818,7 @@ def get_runnable_tasks(self, graph: DiGraph) -> list["Task[DefType]"]:
                         break
                 if is_runnable:
                     runnable.append(self.blocked.pop(index))
-        self.queued.update({t.state_index: t for t in runnable})
-        return list(self.queued.values())
+        return runnable
 
 
 async def prepare_runnable(runnable):

pydra/engine/submitter.py

@@ -1395,9 +1395,9 @@ def PassOdds(x):
     pass_odds = PassOdds().split("x", x=[1, 2, 3, 4, 5])
 
     with pytest.raises(Exception):
-        pass_odds(cache_dir=tmp_path, worker="cf")
+        pass_odds(cache_dir=tmp_path, worker="cf", n_procs=3)
     with pytest.raises(Exception):
-        pass_odds(cache_dir=tmp_path, worker="cf")
+        pass_odds(cache_dir=tmp_path, worker="cf", n_procs=3)
 
     out, err = capfd.readouterr()
     stdout_lines = out.splitlines()

pydra/engine/tests/test_task.py

@@ -4467,14 +4467,11 @@ def test_rerun_errored(plugin_parallel, tmp_path, capfd):
     """Test rerunning a workflow containing errors.
     Only the errored tasks and workflow should be rerun"""
 
-    class EvenException(Exception):
-        pass
-
     @python.define
     def PassOdds(x):
         if x % 2 == 0:
             print(f"x={x}, running x%2 = {x % 2} (even error)\n")
-            raise EvenException("even error")
+            raise ValueError("even error")
         else:
             print(f"x={x}, running x%2 = {x % 2}\n")
             return x
@@ -4498,9 +4495,6 @@ def WorkyPassOdds(x):
     out, err = capfd.readouterr()
     stdout_lines = out.splitlines()
 
-    with open("/Users/tclose/Desktop/pytest-stdout.txt", "w") as f:
-        f.write(out)
-
     tasks_run = 0
     errors_found = 0