RF: Use importlib[._]resources to access resources, not __file__

Author: effigies

pydra/engine/audit.py

@@ -1,12 +1,16 @@
 """Module to keep track of provenance information."""
 import os
-from pathlib import Path
 import json
 import attr
 from ..utils.messenger import send_message, make_message, gen_uuid, now, AuditFlag
 from .helpers import ensure_list, gather_runtime_info, hash_file
 from .specs import attr_fields, File, Directory
 
+try:
+    import importlib_resources
+except ImportError:
+    import importlib.resources as importlib_resources  # type: ignore
+
 
 class Audit:
     """Handle provenance tracking and resource utilization."""
@@ -133,9 +137,8 @@ def audit_message(self, message, flags=None):
 
         """
         if self.develop:
-            with open(
-                Path(os.path.dirname(__file__)) / ".." / "schema/context.jsonld"
-            ) as fp:
+            context_file = importlib_resources.files("pydra") / "schema/context.jsonld"
+            with context_file.open() as fp:
                 context = json.load(fp)
         else:
             context = {

pydra/engine/tests/conftest.py

@@ -0,0 +1,15 @@
+import pytest
+
+try:
+    import importlib_resources
+except ImportError:
+    import importlib.resources as importlib_resources
+
+
+@pytest.fixture(scope="package")
+def data_tests_dir():
+    test_nii = importlib_resources.files("pydra").joinpath(
+        "engine", "tests", "data_tests"
+    )
+    with importlib_resources.as_file(test_nii) as path:
+        yield path

pydra/engine/tests/test_boutiques.py

@@ -18,8 +18,6 @@
     reason="requires docker and bosh",
 )
 
-Infile = Path(__file__).resolve().parent / "data_tests" / "test.nii.gz"
-
 pytestmark = pytest.mark.skip()
 
 
@@ -30,10 +28,10 @@
     "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"]
 )
 @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter])
-def test_boutiques_1(maskfile, plugin, results_function, tmpdir):
+def test_boutiques_1(maskfile, plugin, results_function, tmpdir, data_tests_dir):
     """simple task to run fsl.bet using BoshTask"""
     btask = BoshTask(name="NA", zenodo_id="1482743")
-    btask.inputs.infile = Infile
+    btask.inputs.infile = data_tests_dir / "test.nii.gz"
     btask.inputs.maskfile = maskfile
     btask.cache_dir = tmpdir
     res = results_function(btask, plugin)
@@ -50,12 +48,12 @@ def test_boutiques_1(maskfile, plugin, results_function, tmpdir):
 @no_win
 @need_bosh_docker
 @pytest.mark.flaky(reruns=3)
-def test_boutiques_spec_1():
+def test_boutiques_spec_1(data_tests_dir):
     """testing spec: providing input/output fields names"""
     btask = BoshTask(
         name="NA",
         zenodo_id="1482743",
-        infile=Infile,
+        infile=data_tests_dir / "test.nii.gz",
         maskfile="test_brain.nii.gz",
         input_spec_names=["infile", "maskfile"],
         output_spec_names=["outfile", "out_outskin_off"],
@@ -75,12 +73,12 @@ def test_boutiques_spec_1():
 @no_win
 @need_bosh_docker
 @pytest.mark.flaky(reruns=3)
-def test_boutiques_spec_2():
+def test_boutiques_spec_2(data_tests_dir):
     """testing spec: providing partial input/output fields names"""
     btask = BoshTask(
         name="NA",
         zenodo_id="1482743",
-        infile=Infile,
+        infile=data_tests_dir / "test.nii.gz",
         maskfile="test_brain.nii.gz",
         input_spec_names=["infile"],
         output_spec_names=[],
@@ -101,11 +99,11 @@ def test_boutiques_spec_2():
 @pytest.mark.parametrize(
     "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"]
 )
-def test_boutiques_wf_1(maskfile, plugin, tmpdir):
+def test_boutiques_wf_1(maskfile, plugin, tmpdir, infile):
     """wf with one task that runs fsl.bet using BoshTask"""
     wf = Workflow(name="wf", input_spec=["maskfile", "infile"])
     wf.inputs.maskfile = maskfile
-    wf.inputs.infile = Infile
+    wf.inputs.infile = infile
     wf.cache_dir = tmpdir
 
     wf.add(
@@ -134,11 +132,11 @@ def test_boutiques_wf_1(maskfile, plugin, tmpdir):
 @pytest.mark.parametrize(
     "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"]
 )
-def test_boutiques_wf_2(maskfile, plugin, tmpdir):
+def test_boutiques_wf_2(maskfile, plugin, tmpdir, infile):
     """wf with two BoshTasks (fsl.bet and fsl.stats) and one ShellTask"""
     wf = Workflow(name="wf", input_spec=["maskfile", "infile"])
     wf.inputs.maskfile = maskfile
-    wf.inputs.infile = Infile
+    wf.inputs.infile = infile
     wf.cache_dir = tmpdir
 
     wf.add(

pydra/engine/tests/test_dockertask.py

@@ -509,19 +509,17 @@ def test_wf_docker_1_dockerflag(plugin, tmpdir):
 @no_win
 @need_docker
 @pytest.mark.skip(reason="we probably don't want to support bindings as an input")
-def test_wf_docker_2pre(plugin, tmpdir):
+def test_wf_docker_2pre(plugin, tmpdir, data_tests_dir):
     """a workflow with two connected task that run python scripts
     the first one creates a text file and the second one reads the file
     """
 
-    scripts_dir = os.path.join(os.path.dirname(__file__), "data_tests")
-
     cmd1 = ["python", "/scripts/saving.py", "-f", "/outputs/tmp.txt"]
     dt = DockerTask(
         name="save",
         image="python:3.7-alpine",
         executable=cmd1,
-        bindings=[(str(tmpdir), "/outputs"), (scripts_dir, "/scripts", "ro")],
+        bindings=[(str(tmpdir), "/outputs"), (str(data_tests_dir), "/scripts", "ro")],
         strip=True,
     )
     res = dt(plugin=plugin)
@@ -531,13 +529,11 @@ def test_wf_docker_2pre(plugin, tmpdir):
 @no_win
 @need_docker
 @pytest.mark.skip(reason="we probably don't want to support bindings as an input")
-def test_wf_docker_2(plugin, tmpdir):
+def test_wf_docker_2(plugin, tmpdir, data_tests_dir):
     """a workflow with two connected task that run python scripts
     the first one creates a text file and the second one reads the file
     """
 
-    scripts_dir = os.path.join(os.path.dirname(__file__), "data_tests")
-
     wf = Workflow(name="wf", input_spec=["cmd1", "cmd2"])
     wf.inputs.cmd1 = ["python", "/scripts/saving.py", "-f", "/outputs/tmp.txt"]
     wf.inputs.cmd2 = ["python", "/scripts/loading.py", "-f"]
@@ -546,7 +542,10 @@ def test_wf_docker_2(plugin, tmpdir):
             name="save",
             image="python:3.7-alpine",
             executable=wf.lzin.cmd1,
-            bindings=[(str(tmpdir), "/outputs"), (scripts_dir, "/scripts", "ro")],
+            bindings=[
+                (str(tmpdir), "/outputs"),
+                (str(data_tests_dir), "/scripts", "ro"),
+            ],
             strip=True,
         )
     )
@@ -556,7 +555,10 @@ def test_wf_docker_2(plugin, tmpdir):
             image="python:3.7-alpine",
             executable=wf.lzin.cmd2,
             args=wf.save.lzout.stdout,
-            bindings=[(str(tmpdir), "/outputs"), (scripts_dir, "/scripts", "ro")],
+            bindings=[
+                (str(tmpdir), "/outputs"),
+                (str(data_tests_dir), "/scripts", "ro"),
+            ],
             strip=True,
         )
     )

pydra/engine/tests/test_shelltask.py

@@ -4157,7 +4157,7 @@ def no_fsl():
 
 
 @pytest.mark.skipif(no_fsl(), reason="fsl is not installed")
-def test_fsl():
+def test_fsl(data_tests_dir):
     """mandatory field added to fields, value provided"""
 
     _xor_inputs = [
@@ -4385,7 +4385,7 @@ def change_name(file):
     )
 
     # TODO: not sure why this has to be string
-    in_file = Path(__file__).parent / "data_tests" / "test.nii.gz"
+    in_file = data_tests_dir / "test.nii.gz"
 
     # separate command into exec + args
     shelly = ShellCommandTask(

pyproject.toml

@@ -12,6 +12,7 @@ dependencies = [
     "cloudpickle >=2.0.0",
     "etelemetry >=0.2.2",
     "filelock >=3.0.0",
+    "importlib_resources >=5.7; python_version < '3.11'",
 ]
 license = {file = "LICENSE"}
 authors = [