Merge remote-tracking branch 'origin/master' into adi611-patch-testpsij-1

Author: adi611

.github/workflows/testslurm.yml

@@ -8,9 +8,13 @@ on:
 
 jobs:
   build:
+    strategy:
+      matrix:
+        python-version: [3.8.16, 3.9.16, 3.10.9, 3.11.5]
+      fail-fast: false
     runs-on: ubuntu-latest
     env:
-      DOCKER_IMAGE: giovtorres/docker-centos7-slurm:latest
+      DOCKER_IMAGE: adi611/docker-centos7-slurm:23.02.1
 
     steps:
     - name: Disable etelemetry
@@ -24,10 +28,7 @@ jobs:
     - name: Display previous jobs with sacct
       run: |
         echo "Allowing ports/daemons time to start" && sleep 10
-        docker exec slurm bash -c "sacctmgr -i add cluster name=linux \
-          && supervisorctl restart slurmdbd \
-          && supervisorctl restart slurmctld \
-          && sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'"
+        docker exec slurm bash -c "sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'"
         docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null
         if [ $? -ne 0 ]; then
             echo "Slurm docker image error"
@@ -38,12 +39,16 @@ jobs:
         docker exec slurm bash -c "echo $NO_ET"
         docker exec slurm bash -c "ls -la && echo list top level dir"
         docker exec slurm bash -c "ls -la /pydra && echo list pydra dir"
-        docker exec slurm bash -c "pip3.9 install --upgrade pip && pip3.9 install -e /pydra[test] && python3.9 -c 'import pydra; print(pydra.__version__)'"
+        if [[ "${{ matrix.python-version }}" == "3.11.5" ]]; then
+            docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5"
+        fi
+        docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}"
+        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'"
     - name: Run pytest
       run: |
-        docker exec slurm bash -c "pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'"
+        docker exec slurm bash -c "pytest --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'"
     - name: Upload to codecov
       run: |
-        docker exec slurm bash -c "pip3.9 install urllib3==1.26.6"
+        docker exec slurm bash -c "pip install urllib3==1.26.6"
         docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests"
         docker rm -f slurm

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: check-yaml
     -   id: check-added-large-files
 -   repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.9.1
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell

.zenodo.json

@@ -75,6 +75,11 @@
       "name": "Vaillant, Ghislain",
       "orcid": "0000-0003-0267-3033"
     },
+    {
+      "affiliation": "Indian Institute of Information Technology Kalyani",
+      "name": "Agarwal, Aditya",
+      "orcid": "0009-0008-2639-5334"
+    },
     {
       "affiliation": "MIT, HMS",
       "name": "Ghosh, Satrajit",

pydra/engine/tests/test_shelltask.py

@@ -2460,14 +2460,14 @@ def test_wf_shell_cmd_3a(plugin, tmp_path):
     assert res.output.cp_file.fspath.exists()
 
 
-def test_wf_shell_cmd_state_1(plugin):
+def test_wf_shell_cmd_state_1(plugin, tmp_path):
     """a workflow with 2 tasks and splitter on the wf level,
     first one has input with output_file_template (str, uses wf.lzin),
     that is passed to the second task
     """
-    wf = Workflow(name="wf", input_spec=["cmd1", "cmd2", "args"]).split(
-        "args", args=["newfile_1.txt", "newfile_2.txt"]
-    )
+    wf = Workflow(
+        name="wf", input_spec=["cmd1", "cmd2", "args"], cache_dir=tmp_path
+    ).split("args", args=["newfile_1.txt", "newfile_2.txt"])
 
     wf.inputs.cmd1 = "touch"
     wf.inputs.cmd2 = "cp"
@@ -2820,7 +2820,7 @@ def gather_output(field, output_dir):
 
 
 @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter])
-def test_shell_cmd_outputspec_5a(plugin, results_function):
+def test_shell_cmd_outputspec_5a(plugin, results_function, tmp_path):
     """
     customised output_spec, adding files to the output,
     using a function to collect output, the function is saved in the field metadata
@@ -2842,7 +2842,9 @@ def gather_output(executable, output_dir):
         ],
         bases=(ShellOutSpec,),
     )
-    shelly = ShellCommandTask(name="shelly", executable=cmd, output_spec=my_output_spec)
+    shelly = ShellCommandTask(
+        name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmp_path
+    )
 
     res = results_function(shelly, plugin)
     assert res.output.stdout == ""
@@ -2874,7 +2876,7 @@ def gather_output(executable, output_dir, ble):
 
 
 @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter])
-def test_shell_cmd_outputspec_5c(plugin, results_function):
+def test_shell_cmd_outputspec_5c(plugin, results_function, tmp_path):
     """
     Customised output spec defined as a class,
     using a static function to collect output files.
@@ -2893,6 +2895,7 @@ def gather_output(executable, output_dir):
         name="shelly",
         executable=["touch", "newfile_tmp1.txt", "newfile_tmp2.txt"],
         output_spec=SpecInfo(name="Output", bases=(MyOutputSpec,)),
+        cache_dir=tmp_path,
     )
 
     res = results_function(shelly, plugin)
@@ -3177,7 +3180,7 @@ def get_stderr(stderr):
     )
 
     shelly = ShellCommandTask(
-        name="shelly", executable=cmd, output_spec=my_output_spec
+        name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmp_path
     ).split("args", args=args)
 
     results = results_function(shelly, plugin)
@@ -3248,6 +3251,7 @@ def get_lowest_directory(directory_path):
         executable=cmd,
         output_spec=my_output_spec,
         resultsDir="outdir",
+        cache_dir=tmp_path,
     ).split("args", args=args)
 
     results_function(shelly, plugin)

pydra/utils/hash.py

@@ -3,9 +3,11 @@
 
 # import stat
 import struct
+import typing as ty
 from collections.abc import Mapping
 from functools import singledispatch
 from hashlib import blake2b
+import logging
 
 # from pathlib import Path
 from typing import (
@@ -14,10 +16,11 @@
     NewType,
     Sequence,
     Set,
-    _SpecialForm,
 )
 import attrs.exceptions
 
+logger = logging.getLogger("pydra")
+
 try:
     from typing import Protocol
 except ImportError:
@@ -88,7 +91,8 @@ def hash_single(obj: object, cache: Cache) -> Hash:
         h = blake2b(digest_size=16, person=b"pydra-hash")
         for chunk in bytes_repr(obj, cache):
             h.update(chunk)
-        cache[objid] = Hash(h.digest())
+        hsh = cache[objid] = Hash(h.digest())
+        logger.debug("Hash of %s object is %s", obj, hsh)
     return cache[objid]
 
 
@@ -102,15 +106,14 @@ def __bytes_repr__(self, cache: Cache) -> Iterator[bytes]:
 def bytes_repr(obj: object, cache: Cache) -> Iterator[bytes]:
     cls = obj.__class__
     yield f"{cls.__module__}.{cls.__name__}:{{".encode()
-    try:
+    dct: Dict[str, ty.Any]
+    if attrs.has(type(obj)):
+        # Drop any attributes that aren't used in comparisons by default
+        dct = attrs.asdict(obj, recurse=False, filter=lambda a, _: bool(a.eq))
+    elif hasattr(obj, "__slots__"):
+        dct = {attr: getattr(obj, attr) for attr in obj.__slots__}
+    else:
         dct = obj.__dict__
-    except AttributeError as e:
-        # Attrs creates slots classes by default, so we add this here to handle those
-        # cases
-        try:
-            dct = attrs.asdict(obj, recurse=False)  # type: ignore
-        except attrs.exceptions.NotAnAttrsClassError:
-            raise TypeError(f"Cannot hash {obj} as it is a slots class") from e
     yield from bytes_repr_mapping_contents(dct, cache)
     yield b"}"
 
@@ -224,10 +227,34 @@ def bytes_repr_dict(obj: dict, cache: Cache) -> Iterator[bytes]:
     yield b"}"
 
 
-@register_serializer(_SpecialForm)
+@register_serializer(ty._GenericAlias)
+@register_serializer(ty._SpecialForm)
 @register_serializer(type)
 def bytes_repr_type(klass: type, cache: Cache) -> Iterator[bytes]:
-    yield f"type:({klass.__module__}.{klass.__name__})".encode()
+    def type_name(tp):
+        try:
+            name = tp.__name__
+        except AttributeError:
+            name = tp._name
+        return name
+
+    yield b"type:("
+    origin = ty.get_origin(klass)
+    if origin:
+        yield f"{origin.__module__}.{type_name(origin)}[".encode()
+        for arg in ty.get_args(klass):
+            if isinstance(
+                arg, list
+            ):  # sometimes (e.g. Callable) the args of a type is a list
+                yield b"["
+                yield from (b for t in arg for b in bytes_repr_type(t, cache))
+                yield b"]"
+            else:
+                yield from bytes_repr_type(arg, cache)
+        yield b"]"
+    else:
+        yield f"{klass.__module__}.{type_name(klass)}".encode()
+    yield b")"
 
 
 @register_serializer(list)

pydra/utils/tests/test_hash.py

@@ -4,7 +4,8 @@
 
 import attrs
 import pytest
-
+import typing as ty
+from fileformats.application import Zip, Json
 from ..hash import Cache, UnhashableError, bytes_repr, hash_object, register_serializer
 
 
@@ -134,6 +135,17 @@ def __init__(self, x):
     assert re.match(rb".*\.MyClass:{str:1:x=.{16}}", obj_repr)
 
 
+def test_bytes_repr_slots_obj():
+    class MyClass:
+        __slots__ = ("x",)
+
+        def __init__(self, x):
+            self.x = x
+
+    obj_repr = join_bytes_repr(MyClass(1))
+    assert re.match(rb".*\.MyClass:{str:1:x=.{16}}", obj_repr)
+
+
 def test_bytes_repr_attrs_slots():
     @attrs.define
     class MyClass:
@@ -143,11 +155,67 @@ class MyClass:
     assert re.match(rb".*\.MyClass:{str:1:x=.{16}}", obj_repr)
 
 
-def test_bytes_repr_type():
+def test_bytes_repr_attrs_no_slots():
+    @attrs.define(slots=False)
+    class MyClass:
+        x: int
+
+    obj_repr = join_bytes_repr(MyClass(1))
+    assert re.match(rb".*\.MyClass:{str:1:x=.{16}}", obj_repr)
+
+
+def test_bytes_repr_type1():
     obj_repr = join_bytes_repr(Path)
     assert obj_repr == b"type:(pathlib.Path)"
 
 
+def test_bytes_repr_type1a():
+    obj_repr = join_bytes_repr(Zip[Json])
+    assert obj_repr == rb"type:(fileformats.application.archive.Json__Zip)"
+
+
+def test_bytes_repr_type2():
+    T = ty.TypeVar("T")
+
+    class MyClass(ty.Generic[T]):
+        pass
+
+    obj_repr = join_bytes_repr(MyClass[int])
+    assert (
+        obj_repr == b"type:(pydra.utils.tests.test_hash.MyClass[type:(builtins.int)])"
+    )
+
+
+def test_bytes_special_form1():
+    obj_repr = join_bytes_repr(ty.Union[int, float])
+    assert obj_repr == b"type:(typing.Union[type:(builtins.int)type:(builtins.float)])"
+
+
+def test_bytes_special_form2():
+    obj_repr = join_bytes_repr(ty.Any)
+    assert re.match(rb"type:\(typing.Any\)", obj_repr)
+
+
+def test_bytes_special_form3():
+    obj_repr = join_bytes_repr(ty.Optional[Path])
+    assert (
+        obj_repr == b"type:(typing.Union[type:(pathlib.Path)type:(builtins.NoneType)])"
+    )
+
+
+def test_bytes_special_form4():
+    obj_repr = join_bytes_repr(ty.Type[Path])
+    assert obj_repr == b"type:(builtins.type[type:(pathlib.Path)])"
+
+
+def test_bytes_special_form5():
+    obj_repr = join_bytes_repr(ty.Callable[[Path, int], ty.Tuple[float, str]])
+    assert obj_repr == (
+        b"type:(collections.abc.Callable[[type:(pathlib.Path)type:(builtins.int)]"
+        b"type:(builtins.tuple[type:(builtins.float)type:(builtins.str)])])"
+    )
+
+
 def test_recursive_object():
     a = []
     b = [a]

pydra/utils/tests/test_typing.py

@@ -8,7 +8,7 @@
 from ...engine.specs import File, LazyOutField
 from ..typing import TypeParser
 from pydra import Workflow
-from fileformats.serialization import Json
+from fileformats.application import Json
 from .utils import (
     generic_func_task,
     GenericShellTask,

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "flit_scm:buildapi"
 name = "pydra"
 description = "Pydra dataflow engine"
 readme = "README.rst"
-requires-python = ">=3.8"
+requires-python = ">=3.8, !=3.11.1"
 dependencies = [
     "attrs >=19.1.0",
     "cloudpickle >=2.0.0",