handle Set and Map

Author: jaymccon

python/rpdk/python/resolver.py

@@ -36,7 +36,7 @@ def translate_type(resolved_type):
 
 
 def contains_model(resolved_type):
-    if resolved_type.container == ContainerType.LIST:
+    if resolved_type.container in [ContainerType.LIST, ContainerType.SET]:
         return contains_model(resolved_type.type)
     if resolved_type.container == ContainerType.MODEL:
         return True

src/cloudformation_cli_python_lib/interface.py

@@ -68,6 +68,8 @@ def _serialize(self) -> Mapping[str, Any]:
                 ser[k] = v._serialize()  # pylint: disable=protected-access
             elif v is None:
                 del_keys.append(k)
+            else:
+                ser[k] = v
         for k in del_keys:
             del ser[k]
         return ser

src/cloudformation_cli_python_lib/recast.py

@@ -1,5 +1,5 @@
 import typing
-from typing import Any, Dict, List, Mapping
+from typing import Any, Dict, List, Mapping, Set
 
 from .exceptions import InvalidRequest
 
@@ -11,12 +11,17 @@ def recast_object(
 ) -> None:
     if not isinstance(json_data, dict):
         raise InvalidRequest(f"Can only parse dict items, not {type(json_data)}")
+    # if type is Any, we leave it as is
+    if cls == typing.Any:
+        return
     for k, v in json_data.items():
         if isinstance(v, dict):
             child_cls = _field_to_type(cls.__dataclass_fields__[k].type, k, classes)
             recast_object(child_cls, v, classes)
         elif isinstance(v, list):
             json_data[k] = _recast_lists(cls, k, v, classes)
+        elif isinstance(v, set):
+            json_data[k] = _recast_sets(cls, k, v, classes)
         elif isinstance(v, str):
             dest_type = _field_to_type(cls.__dataclass_fields__[k].type, k, classes)
             json_data[k] = _recast_primitive(dest_type, k, v)
@@ -25,24 +30,46 @@ def recast_object(
 
 
 def _recast_lists(cls: Any, k: str, v: List[Any], classes: Dict[str, Any]) -> List[Any]:
+    # Leave as is if type is Any
+    if cls == typing.Any:
+        return v
     casted_list: List[Any] = []
-    if k in cls.__dataclass_fields__:
+    if "__dataclass_fields__" not in dir(cls):
+        pass
+    elif k in cls.__dataclass_fields__:
         cls = _field_to_type(cls.__dataclass_fields__[k].type, k, classes)
     for item in v:
-        if isinstance(item, str):
-            casted_item: Any = _recast_primitive(cls, k, item)
-        elif isinstance(item, list):
-            casted_item = _recast_lists(cls, k, item, classes)
-        elif isinstance(item, dict):
-            recast_object(cls, item, classes)
-            casted_item = item
-        else:
-            raise InvalidRequest(f"Unsupported type: {type(v)} for {k}")
-        casted_list.append(casted_item)
+        casted_list.append(cast_sequence_item(cls, k, item, classes))
     return casted_list
 
 
+def _recast_sets(cls: Any, k: str, v: Set[Any], classes: Dict[str, Any]) -> Set[Any]:
+    casted_set: Set[Any] = set()
+    if "__dataclass_fields__" in dir(cls):
+        cls = _field_to_type(cls.__dataclass_fields__[k].type, k, classes)
+    for item in v:
+        casted_set.add(cast_sequence_item(cls, k, item, classes))
+    return casted_set
+
+
+def cast_sequence_item(cls: Any, k: str, item: Any, classes: Dict[str, Any]) -> Any:
+    if isinstance(item, str):
+        return _recast_primitive(cls, k, item)
+    if isinstance(item, list):
+        return _recast_lists(cls, k, item, classes)
+    if isinstance(item, set):
+        return _recast_sets(cls, k, item, classes)
+    if isinstance(item, dict):
+        recast_object(cls, item, classes)
+        return item
+    raise InvalidRequest(f"Unsupported type: {type(item)} for {k}")
+
+
 def _recast_primitive(cls: Any, k: str, v: str) -> Any:
+    if cls == typing.Any:
+        # If the type is Any, we cannot guess what the original type was, so we leave
+        # it as a string
+        return v
     if cls == bool:
         if v.lower() == "true":
             return True
@@ -53,7 +80,7 @@ def _recast_primitive(cls: Any, k: str, v: str) -> Any:
 
 
 def _field_to_type(field: Any, key: str, classes: Dict[str, Any]) -> Any:
-    if field in [int, float, str, bool]:
+    if field in [int, float, str, bool, typing.Any]:
         return field
     # If it's a ForwardRef we need to find base type
     if isinstance(field, get_forward_ref_type()):
@@ -64,26 +91,31 @@ def _field_to_type(field: Any, key: str, classes: Dict[str, Any]) -> Any:
     try:
         possible_types = field.__args__
     except AttributeError:
-        raise InvalidRequest(f"Cannot process type {field.__repr__()} for field {key}")
+        raise InvalidRequest(f"Cannot process type {field} for field {key}")
     # Assuming that the union is generated from typing.Optional, so only
     # contains one type and None
     # pylint: disable=unidiomatic-typecheck
-    fields = [t for t in possible_types if type(None) != t]
+    fields = [t for t in possible_types if type(None) != t] if possible_types else []
     if len(fields) != 1:
-        raise InvalidRequest(f"Cannot process type {field.__repr__()} for field {key}")
+        raise InvalidRequest(f"Cannot process type {field} for field {key}")
     field = fields[0]
     # If it's a primitive we're done
-    if field in [int, float, str, bool]:
+    if field in [int, float, str, bool, typing.Any]:
         return field
     # If it's a ForwardRef we need to find base type
     if isinstance(field, get_forward_ref_type()):
         # Assuming codegen added an _ as a prefix, removing it and then getting the
         # class from model classes
         return classes[field.__forward_arg__[1:]]
-    # If it's not a type we don't know how to handle we bail
-    if not str(field).startswith("typing.Sequence"):
-        raise InvalidRequest(f"Cannot process type {field} for field {key}")
-    return _field_to_type(field.__args__[0], key, classes)
+    # reduce Sequence/AbstractSet to inner type
+    if str(field).startswith("typing.Sequence") or str(field).startswith(
+        "typing.AbstractSet"
+    ):
+        return _field_to_type(field.__args__[0], key, classes)
+    if str(field).startswith("typing.MutableMapping"):
+        return _field_to_type(field.__args__[1], key, classes)
+    # If it's a type we don't know how to handle, we bail
+    raise InvalidRequest(f"Cannot process type {field} for field {key}")
 
 
 # pylint: disable=protected-access,no-member

tests/lib/recast_test.py

@@ -1,5 +1,5 @@
 # pylint: disable=protected-access
-from typing import Any, Optional, Union
+from typing import Awaitable, Generic, Optional, Union
 from unittest.mock import patch
 
 import pytest
@@ -15,8 +15,14 @@
 from .sample_model import ResourceModel as ComplexResourceModel, SimpleResourceModel
 
 
-def test_recast_object_simple():
+def test_recast_complex_object():
     payload = {
+        "ListListAny": [[{"key": "val"}]],
+        "ListListInt": [["1", "2", "3"]],
+        "ListSetInt": [{"1", "2", "3"}],
+        "ASet": {"1", "2", "3"},
+        "AnotherSet": {"a", "b", "c"},
+        "AFreeformDict": {"somekey": "somevalue", "someotherkey": "1"},
         "AnInt": "1",
         "ABool": "true",
         "AList": [
@@ -42,6 +48,12 @@ def test_recast_object_simple():
         ],
     }
     expected = {
+        "ListSetInt": [{1, 2, 3}],
+        "ListListInt": [[1, 2, 3]],
+        "ListListAny": [[{"key": "val"}]],
+        "ASet": {"1", "2", "3"},
+        "AnotherSet": {"a", "b", "c"},
+        "AFreeformDict": {"somekey": "somevalue", "someotherkey": "1"},
         "AnInt": 1,
         "ABool": True,
         "AList": [
@@ -87,10 +99,10 @@ def test_recast_object_invalid_sub_type():
 
 def test_recast_list_invalid_sub_type():
     k = "key"
-    v = (1, 2)
+    v = [(1, 2)]
     with pytest.raises(InvalidRequest) as excinfo:
         _recast_lists(SimpleResourceModel, k, v, {})
-    assert str(excinfo.value) == f"Unsupported type: {type(v)} for {k}"
+    assert str(excinfo.value) == f"Unsupported type: {type(v[0])} for {k}"
 
 
 def test_recast_boolean_invalid_value():
@@ -103,7 +115,7 @@ def test_recast_boolean_invalid_value():
 
 def test_field_to_type_unhandled_types():
     k = "key"
-    for field in [Union[str, list], Any, Optional[Any]]:
+    for field in [Union[str, list], Generic, Optional[Awaitable]]:
         with pytest.raises(InvalidRequest) as excinfo:
             _field_to_type(field, k, {})
         assert str(excinfo.value).startswith("Cannot process type ")

tests/lib/sample_model.py

@@ -5,7 +5,16 @@
 import sys
 from dataclasses import dataclass
 from inspect import getmembers, isclass
-from typing import Any, Mapping, Optional, Sequence, Type, TypeVar
+from typing import (
+    AbstractSet,
+    Any,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Sequence,
+    Type,
+    TypeVar,
+)
 
 from cloudformation_cli_python_lib.interface import (
     BaseResourceHandlerRequest,
@@ -26,6 +35,12 @@ class ResourceHandlerRequest(BaseResourceHandlerRequest):
 
 @dataclass
 class ResourceModel(BaseResourceModel):
+    ListListAny: Optional[Sequence[Sequence[Any]]]
+    ListSetInt: Optional[Sequence[AbstractSet[int]]]
+    ListListInt: Optional[Sequence[Sequence[int]]]
+    ASet: Optional[AbstractSet[Any]]
+    AnotherSet: Optional[AbstractSet[str]]
+    AFreeformDict: Optional[MutableMapping[str, Any]]
     AnInt: Optional[int]
     ABool: Optional[bool]
     NestedList: Optional[Sequence[Sequence["_NestedList"]]]
@@ -47,6 +62,12 @@ def _deserialize(
         dataclasses = {n: o for n, o in getmembers(sys.modules[__name__]) if isclass(o)}
         recast_object(cls, json_data, dataclasses)
         return cls(
+            ListSetInt=json_data.get("ListSetInt"),
+            ListListInt=json_data.get("ListListInt"),
+            ListListAny=json_data.get("ListListAny"),
+            ASet=json_data.get("ASet"),
+            AnotherSet=json_data.get("AnotherSet"),
+            AFreeformDict=json_data.get("AFreeformDict"),
             AnInt=json_data.get("AnInt"),
             ABool=json_data.get("ABool"),
             NestedList=deserialize_list(json_data.get("NestedList"), NestedList),