replace fronzensets with lists in task class serialization and add filter test

Author: tclose

pydra/compose/base/field.py

@@ -60,7 +60,7 @@ class Requirement:
 
     name: str
     allowed_values: list[str] | None = attrs.field(
-        default=None, converter=allowed_values_converter
+        factory=None, converter=allowed_values_converter
     )
 
     def satisfied(self, inputs: "Task") -> bool:
@@ -326,7 +326,7 @@ class Arg(Field):
         it is False
     """
 
-    allowed_values: frozenset = attrs.field(default=(), converter=frozenset)
+    allowed_values: frozenset = attrs.field(factory=frozenset, converter=frozenset)
     copy_mode: File.CopyMode = File.CopyMode.any
     copy_collation: File.CopyCollation = File.CopyCollation.any
     copy_ext_decomp: File.ExtensionDecomposition = File.ExtensionDecomposition.single

pydra/utils/general.py

@@ -4,6 +4,7 @@
 import inspect
 import sys
 import typing as ty
+from collections.abc import Mapping, Collection
 from copy import copy
 import re
 import attrs
@@ -624,20 +625,36 @@ def serialize_task_class(
     from pydra.compose.base import Out
 
     if filter is None:
-        filter = _filter_out_defaults
+        filter = filter_out_defaults
+
+    def full_val_serializer(
+        obj: ty.Any, field: attrs.Attribute, value: ty.Any
+    ) -> ty.Any:
+        """A wrapper for the value serializer to handle the case where it is None."""
+        if value_serializer is not None:
+            value = value_serializer(obj, field, value)
+        if isinstance(value, str):
+            return value
+        if isinstance(value, Mapping) and not isinstance(value, dict):
+            # If the value is a mapping, convert it to a dict
+            value = dict(value)
+        elif isinstance(value, Collection) and not isinstance(value, list):
+            # If the value is not a collection or string, convert it to a list
+            value = list(value)
+        return value
 
     input_fields = get_fields(task_class)
     executor = input_fields.pop(task_class._executor_name).default
     input_dicts = [
-        attrs.asdict(i, filter=filter, value_serializer=value_serializer, **kwargs)
+        attrs.asdict(i, filter=filter, value_serializer=full_val_serializer, **kwargs)
         for i in input_fields
         if (
             not isinstance(i, Out)  # filter out outarg fields
             and i.name not in task_class.BASE_ATTRS
         )
     ]
     output_dicts = [
-        attrs.asdict(o, filter=filter, value_serializer=value_serializer, **kwargs)
+        attrs.asdict(o, filter=filter, value_serializer=full_val_serializer, **kwargs)
         for o in get_fields(task_class.Outputs)
         if o.name not in task_class.Outputs.BASE_ATTRS
     ]
@@ -649,6 +666,8 @@ def serialize_task_class(
         "outputs": {d.pop("name"): d for d in output_dicts},
     }
     class_attrs = {a: getattr(task_class, "_" + a) for a in task_class.TASK_CLASS_ATTRS}
+    # Convert the frozensets to lists for serialization
+    class_attrs["xor"] = [list(x) for x in class_attrs.pop("xor")]
     if value_serializer:
         # We need to create a mock attrs object for the class attrs to apply the
         # value_serializer to it
@@ -658,7 +677,7 @@ def serialize_task_class(
         attrs_fields = {f.name: f for f in attrs.fields(mock_cls)}
         mock = mock_cls(**class_attrs)
         class_attrs = {
-            n: value_serializer(mock, attrs_fields[n], v)
+            n: full_val_serializer(mock, attrs_fields[n], v)
             for n, v in class_attrs.items()
         }
     dct.update(class_attrs)
@@ -685,7 +704,7 @@ def unserialize_task_class(task_class_dict: dict[str, ty.Any]) -> type["Task"]:
     return mod.define(dct.pop(mod.Task._executor_name), **dct)
 
 
-def _filter_out_defaults(atr: attrs.Attribute, value: ty.Any) -> bool:
+def filter_out_defaults(atr: attrs.Attribute, value: ty.Any) -> bool:
     """Filter out values that match the attributes default value."""
     if isinstance(atr.default, attrs.Factory) and atr.default.factory() == value:
         return False

pydra/utils/tests/test_general.py

@@ -1,15 +1,34 @@
 import typing as ty
 import attrs
+from collections.abc import Collection
 from pydra.compose import python, workflow, shell
 from fileformats.text import TextFile
 from pydra.utils.general import (
     serialize_task_class,
     unserialize_task_class,
     get_fields,
+    filter_out_defaults,
 )
 from pydra.utils.tests.utils import Concatenate
 
 
+def check_dict_fully_serialized(dct: dict):
+    """Checks if there are any Pydra objects or non list/dict containers in the dict."""
+    stack = [dct]
+    while stack:
+        item = stack.pop()
+        if isinstance(item, dict):
+            stack.extend(item.values())
+        elif isinstance(item, list):
+            stack.extend(item)
+        elif isinstance(item, str):
+            pass
+        elif isinstance(item, Collection):
+            raise ValueError(f"Unserializable container object {item} found in dict")
+        elif type(item).__module__.split(".")[0] == "pydra":
+            raise ValueError(f"Unserialized Pydra object {item} found in dict")
+
+
 @python.define(outputs=["out_int"], xor=["b", "c"])
 def Add(a: int, b: int | None = None, c: int | None = None) -> int:
     """
@@ -32,12 +51,15 @@ def Add(a: int, b: int | None = None, c: int | None = None) -> int:
 
 def test_python_serialize_task_class(tmp_path):
 
+    assert Add(a=1, b=2)(cache_root=tmp_path / "cache1").out_int == 3
+
     dct = serialize_task_class(Add)
+    assert isinstance(dct, dict)
+    check_dict_fully_serialized(dct)
     Reloaded = unserialize_task_class(dct)
     assert get_fields(Add) == get_fields(Reloaded)
 
-    add = Reloaded(a=1, b=2)
-    assert add(cache_root=tmp_path / "cache").out_int == 3
+    assert Reloaded(a=1, b=2)(cache_root=tmp_path / "cache2").out_int == 3
 
 
 def test_shell_serialize_task_class():
@@ -47,6 +69,8 @@ def test_shell_serialize_task_class():
     )
 
     dct = serialize_task_class(MyCmd)
+    assert isinstance(dct, dict)
+    check_dict_fully_serialized(dct)
     Reloaded = unserialize_task_class(dct)
     assert get_fields(MyCmd) == get_fields(Reloaded)
 
@@ -61,6 +85,8 @@ def AWorkflow(in_file: TextFile, a_param: int) -> TextFile:
         return concatenate.out_file
 
     dct = serialize_task_class(AWorkflow)
+    assert isinstance(dct, dict)
+    check_dict_fully_serialized(dct)
     Reloaded = unserialize_task_class(dct)
     assert get_fields(AWorkflow) == get_fields(Reloaded)
 
@@ -73,15 +99,57 @@ def AWorkflow(in_file: TextFile, a_param: int) -> TextFile:
 
 def test_serialize_task_class_with_value_serializer():
 
-    def frozen_set_to_list_serializer(
-        mock_class: ty.Any, atr: attrs.Attribute, value: ty.Any
+    @python.define
+    def Identity(a: int) -> int:
+        """
+        Parameters
+        ----------
+        a: int
+            the arg
+
+        Returns
+        -------
+        out : int
+            a returned as is
+        """
+        return a
+
+    def type_to_str_serializer(
+        klass: ty.Any, atr: attrs.Attribute, value: ty.Any
     ) -> ty.Any:
-        # This is just a dummy serializer
-        if isinstance(value, frozenset):
-            return list(
-                frozen_set_to_list_serializer(mock_class, atr, v) for v in value
-            )
+        if isinstance(value, type):
+            return value.__module__ + "." + value.__name__
         return value
 
-    dct = serialize_task_class(Add, value_serializer=frozen_set_to_list_serializer)
-    assert dct["xor"] == [["b", "c"]] or dct["xor"] == [["c", "b"]]
+    dct = serialize_task_class(Identity, value_serializer=type_to_str_serializer)
+    assert isinstance(dct, dict)
+    check_dict_fully_serialized(dct)
+    assert dct["inputs"] == {"a": {"type": "builtins.int", "help": "the arg"}}
+
+
+def test_serialize_task_class_with_filter():
+
+    @python.define
+    def Identity(a: int) -> int:
+        """
+        Parameters
+        ----------
+        a: int
+            the arg
+
+        Returns
+        -------
+        out : int
+            a returned as is
+        """
+        return a
+
+    def no_helps_filter(atr: attrs.Attribute, value: ty.Any) -> bool:
+        if atr.name == "help":
+            return False
+        return filter_out_defaults(atr, value)
+
+    dct = serialize_task_class(Identity, filter=no_helps_filter)
+    assert isinstance(dct, dict)
+    check_dict_fully_serialized(dct)
+    assert dct["inputs"] == {"a": {"type": int}}