added pydra module path to the sys path in run_pickled to avoid import issues when running tests from inside the namespace package

Author: tclose

docker-scripts-for-tests/start-docker-slurm.sh

@@ -7,7 +7,7 @@ NO_ET=TRUE
 docker pull $DOCKER_IMAGE
 
 # Start image
-docker run -itd -h slurmctl --cap-add sys_admin -d --name slurm -v $PKG_DIR:/pydra -e NO_ET=$NO_ET $DOCKER_IMAGE
+docker run --rm -itd -h slurmctl --cap-add sys_admin -d --name slurm -v $PKG_DIR:/pydra -e NO_ET=$NO_ET $DOCKER_IMAGE
 
 # Display previous jobs with sacct
 echo "Allowing ports/daemons time to start" && sleep 10

pydra/engine/run_pickled.py

@@ -1,14 +1,19 @@
-import pickle
+import cloudpickle as cp
 import sys
+from pathlib import Path
 from pydra.engine.helpers import load_and_run
 
+# To avoid issues when running pytest, where the namespace package "pydra" is dropped in
+# the pickling process due to it being run from inside the source tree
+sys.path.append(str(Path(__file__).parent.parent))
+
 
 def run_pickled(*file_paths, rerun=False):
     loaded_objects = []
 
     for file_path in file_paths:
         with open(file_path, "rb") as file:
-            loaded_objects.append(pickle.load(file))
+            loaded_objects.append(cp.load(file))
 
     if len(loaded_objects) == 1:
         result = loaded_objects[0](rerun=rerun)

pydra/engine/tests/test_submitter.py

@@ -10,20 +10,20 @@
 import pytest
 from pydra.design import workflow
 from fileformats.generic import Directory
+from pydra.engine.core import Task
+from pydra.engine.submitter import Submitter
+from pydra.engine.workers import DebugWorker
+from pydra.design import python
+from pathlib import Path
+from datetime import datetime
+from pydra.engine.specs import Result
 from .utils import (
     need_sge,
     need_slurm,
     BasicWorkflow,
     BasicWorkflowWithThreadCount,
     BasicWorkflowWithThreadCountConcurrent,
 )
-from ..core import Task
-from ..submitter import Submitter
-from ..workers import DebugWorker
-from pydra.design import python
-from pathlib import Path
-from datetime import datetime
-from pydra.engine.specs import Result
 
 
 @python.define

pydra/engine/workers.py

@@ -974,16 +974,13 @@ async def run(
         -------
         None
         """
-        import pickle
-        from pathlib import Path
-
         jex = self.psij.JobExecutor.get_instance(self.subtype)
         absolute_path = Path(__file__).parent
 
         cache_dir = task.cache_dir
         file_path = cache_dir / "runnable_function.pkl"
         with open(file_path, "wb") as file:
-            pickle.dump(task.run, file)
+            cp.dump(task.run, file)
         func_path = absolute_path / "run_pickled.py"
         spec = self.make_spec("python", [func_path, file_path])