handle schemas with nested arrays/objects. recast int/float/bool from strings

Author: jaymccon

.pylintrc

@@ -16,7 +16,7 @@ disable=
 
 [BASIC]
 
-good-names=e,ex,f,fp,i,j,k,n,_
+good-names=e,ex,f,fp,i,j,k,v,n,_
 
 [FORMAT]
 

python/rpdk/python/codegen.py

@@ -15,7 +15,7 @@
 from rpdk.core.jsonutils.resolver import ContainerType, resolve_models
 from rpdk.core.plugin_base import LanguagePlugin
 
-from .resolver import translate_type
+from .resolver import contains_model, translate_type
 
 LOG = logging.getLogger(__name__)
 
@@ -44,6 +44,7 @@ def __init__(self):
             trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True
         )
         self.env.filters["translate_type"] = translate_type
+        self.env.filters["contains_model"] = contains_model
         self.env.globals["ContainerType"] = ContainerType
         self.namespace = None
         self.package_name = None

python/rpdk/python/resolver.py

@@ -33,3 +33,11 @@ def translate_type(resolved_type):
         return f"AbstractSet[{item_type}]"
 
     raise ValueError(f"Unknown container type {resolved_type.container}")
+
+
+def contains_model(resolved_type):
+    if resolved_type.container == ContainerType.LIST:
+        return contains_model(resolved_type.type)
+    if resolved_type.container == ContainerType.MODEL:
+        return True
+    return False

python/rpdk/python/templates/models.py

@@ -1,5 +1,7 @@
 # DO NOT modify this file by hand, changes will be overwritten
+import sys
 from dataclasses import dataclass
+from inspect import getmembers, isclass
 from typing import (
     AbstractSet,
     Any,
@@ -16,6 +18,8 @@
     BaseResourceHandlerRequest,
     BaseResourceModel,
 )
+from cloudformation_cli_python_lib.recast import recast_object
+from cloudformation_cli_python_lib.utils import deserialize_list
 
 T = TypeVar("T")
 
@@ -35,7 +39,7 @@ class ResourceHandlerRequest(BaseResourceHandlerRequest):
 
 {% for model, properties in models.items() %}
 @dataclass
-class {{ model }}{% if model == "ResourceModel" %}(BaseResourceModel){% endif %}:
+class {{ model }}(BaseResourceModel):
     {% for name, type in properties.items() %}
     {{ name }}: Optional[{{ type|translate_type }}]
     {% endfor %}
@@ -47,12 +51,22 @@ def _deserialize(
     ) -> Optional["_{{ model }}"]:
         if not json_data:
             return None
+        {% if model == "ResourceModel" %}
+        dataclasses = {n: o for n, o in getmembers(sys.modules[__name__]) if isclass(o)}
+        recast_object(cls, json_data, dataclasses)
+        {% endif %}
         return cls(
             {% for name, type in properties.items() %}
             {% if type.container == ContainerType.MODEL %}
             {{ name }}={{ type.type }}._deserialize(json_data.get("{{ name }}")),
             {% elif type.container == ContainerType.SET %}
             {{ name }}=set_or_none(json_data.get("{{ name }}")),
+            {% elif type.container == ContainerType.LIST %}
+            {% if type | contains_model %}
+            {{name}}=deserialize_list(json_data.get("{{ name }}"), {{name}}),
+            {% else %}
+            {{ name }}=json_data.get("{{ name }}"),
+            {% endif %}
             {% else %}
             {{ name }}=json_data.get("{{ name }}"),
             {% endif %}

src/cloudformation_cli_python_lib/interface.py

@@ -59,7 +59,29 @@ class HandlerErrorCode(str, _AutoName):
 
 class BaseResourceModel:
     def _serialize(self) -> Mapping[str, Any]:
-        return self.__dict__
+        ser = self.__dict__
+        del_keys = []
+        for k, v in ser.items():
+            if isinstance(v, list):
+                ser[k] = self._serialize_list(v)
+            elif isinstance(v, BaseResourceModel):
+                ser[k] = v._serialize()  # pylint: disable=protected-access
+            elif v is None:
+                del_keys.append(k)
+        for k in del_keys:
+            del ser[k]
+        return ser
+
+    def _serialize_list(self, v: List[Any]) -> List[Any]:
+        ser: List[Any] = []
+        for i in v:
+            if isinstance(i, list):
+                ser.append(self._serialize_list(i))
+            elif isinstance(i, BaseResourceModel):
+                ser.append(i._serialize())  # pylint: disable=protected-access
+            else:
+                ser.append(i)
+        return ser
 
     @classmethod
     def _deserialize(

src/cloudformation_cli_python_lib/recast.py

@@ -0,0 +1,87 @@
+# ignoring mypy on the import as it catches ForwardRef as invalid, and we are using it
+# for introspection https://docs.python.org/3/library/typing.html#typing.ForwardRef
+from typing import Any, Dict, ForwardRef, List, Mapping  # type: ignore
+
+from .exceptions import InvalidRequest
+
+
+# CloudFormation recasts all primitive types as strings, this tries to set them back to
+# the types in the type hints
+def recast_object(
+    cls: Any, json_data: Mapping[str, Any], classes: Dict[str, Any]
+) -> None:
+    if not isinstance(json_data, dict):
+        raise InvalidRequest(f"Can only parse dict items, not {type(json_data)}")
+    for k, v in json_data.items():
+        if isinstance(v, dict):
+            child_cls = _field_to_type(cls.__dataclass_fields__[k].type, k, classes)
+            recast_object(child_cls, v, classes)
+        elif isinstance(v, list):
+            json_data[k] = _recast_lists(cls, k, v, classes)
+        elif isinstance(v, str):
+            dest_type = _field_to_type(cls.__dataclass_fields__[k].type, k, classes)
+            json_data[k] = _recast_primitive(dest_type, k, v)
+        else:
+            raise InvalidRequest(f"Unsupported type: {type(v)} for {k}")
+
+
+def _recast_lists(cls: Any, k: str, v: List[Any], classes: Dict[str, Any]) -> List[Any]:
+    casted_list: List[Any] = []
+    if k in cls.__dataclass_fields__:
+        cls = _field_to_type(cls.__dataclass_fields__[k].type, k, classes)
+    for item in v:
+        if isinstance(item, str):
+            casted_item: Any = _recast_primitive(cls, k, item)
+        elif isinstance(item, list):
+            casted_item = _recast_lists(cls, k, item, classes)
+        elif isinstance(item, dict):
+            recast_object(cls, item, classes)
+            casted_item = item
+        else:
+            raise InvalidRequest(f"Unsupported type: {type(v)} for {k}")
+        casted_list.append(casted_item)
+    return casted_list
+
+
+def _recast_primitive(cls: Any, k: str, v: str) -> Any:
+    if cls == bool:
+        if v.lower() == "true":
+            return True
+        if v.lower() == "false":
+            return False
+        raise InvalidRequest(f'value for {k} "{v}" is not boolean')
+    return cls(v)
+
+
+def _field_to_type(field: Any, key: str, classes: Dict[str, Any]) -> Any:
+    if field in [int, float, str, bool]:
+        return field
+    # If it's a ForwardRef we need to find base type
+    if isinstance(field, ForwardRef):
+        # Assuming codegen added an _ as a prefix, removing it and then gettting the
+        # class from model classes
+        return classes[field.__forward_arg__[1:]]
+    # Assuming this is a generic object created by typing.Union
+    try:
+        possible_types = field.__args__
+    except AttributeError:
+        raise InvalidRequest(f"Cannot process type {field.__repr__()} for field {key}")
+    # Assuming that the union is generated from typing.Optional, so only
+    # contains one type and None
+    # pylint: disable=unidiomatic-typecheck
+    fields = [t for t in possible_types if type(None) != t]
+    if len(fields) != 1:
+        raise InvalidRequest(f"Cannot process type {field.__repr__()} for field {key}")
+    field = fields[0]
+    # If it's a primitive we're done
+    if field in [int, float, str, bool]:
+        return field
+    # If it's a ForwardRef we need to find base type
+    if isinstance(field, ForwardRef):
+        # Assuming codegen added an _ as a prefix, removing it and then gettting the
+        # class from model classes
+        return classes[field.__forward_arg__[1:]]
+    # If it's not a type we don't know how to handle we bail
+    if field._name not in ["Sequence"]:  # pylint: disable=protected-access
+        raise InvalidRequest(f"Cannot process type {field.__repr__()} for field {key}")
+    return _field_to_type(field.__args__[0], key, classes)

src/cloudformation_cli_python_lib/utils.py

@@ -2,8 +2,19 @@
 import json
 from dataclasses import dataclass, field
 from datetime import date, datetime, time
-from typing import Any, Callable, Mapping, MutableMapping, Optional, Type
-
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Type,
+    Union,
+)
+
+from .exceptions import InvalidRequest
 from .interface import Action, BaseResourceHandlerRequest, BaseResourceModel
 
 
@@ -121,3 +132,20 @@ def to_modelled(
 class LambdaContext:
     get_remaining_time_in_millis: Callable[["LambdaContext"], int]
     invoked_function_arn: str
+
+
+def deserialize_list(
+    json_data: Union[List[Any], Dict[str, Any]], inner_dataclass: Any
+) -> Optional[List[Any]]:
+    if not json_data:
+        return None
+    output = []
+    for item in json_data:
+        if isinstance(item, list):
+            output.append(deserialize_list(item, inner_dataclass))
+        elif isinstance(item, dict):
+            # pylint: disable=protected-access
+            output.append(inner_dataclass._deserialize(item))
+        else:
+            raise InvalidRequest(f"cannot deserialize lists of {type(item)}")
+    return output

tests/lib/recast_test.py

@@ -0,0 +1,107 @@
+# pylint: disable=protected-access
+from typing import Any, Optional, Union
+
+import pytest
+from cloudformation_cli_python_lib.exceptions import InvalidRequest
+from cloudformation_cli_python_lib.recast import (
+    _field_to_type,
+    _recast_lists,
+    _recast_primitive,
+    recast_object,
+)
+
+from .sample_model import ResourceModel as ComplexResourceModel, SimpleResourceModel
+
+
+def test_recast_object_simple():
+    payload = {
+        "AnInt": "1",
+        "ABool": "true",
+        "AList": [
+            {
+                "DeeperBool": "false",
+                "DeeperList": ["1", "2", "3"],
+                "DeeperDictInList": {"DeepestBool": "true", "DeepestList": ["3", "4"]},
+            },
+            {"DeeperDictInList": {"DeepestBool": "false", "DeepestList": ["6", "7"]}},
+        ],
+        "ADict": {
+            "DeepBool": "true",
+            "DeepList": ["10", "11"],
+            "DeepDict": {
+                "DeeperBool": "false",
+                "DeeperList": ["1", "2", "3"],
+                "DeeperDict": {"DeepestBool": "true", "DeepestList": ["13", "17"]},
+            },
+        },
+        "NestedList": [
+            [{"NestedListInt": "true", "NestedListList": ["1", "2", "3"]}],
+            [{"NestedListInt": "false", "NestedListList": ["11", "12", "13"]}],
+        ],
+    }
+    expected = {
+        "AnInt": 1,
+        "ABool": True,
+        "AList": [
+            {
+                "DeeperBool": False,
+                "DeeperList": [1, 2, 3],
+                "DeeperDictInList": {"DeepestBool": True, "DeepestList": [3, 4]},
+            },
+            {"DeeperDictInList": {"DeepestBool": False, "DeepestList": [6, 7]}},
+        ],
+        "ADict": {
+            "DeepBool": True,
+            "DeepList": [10, 11],
+            "DeepDict": {
+                "DeeperBool": False,
+                "DeeperList": [1, 2, 3],
+                "DeeperDict": {"DeepestBool": True, "DeepestList": [13, 17]},
+            },
+        },
+        "NestedList": [
+            [{"NestedListInt": True, "NestedListList": [1.0, 2.0, 3.0]}],
+            [{"NestedListInt": False, "NestedListList": [11.0, 12.0, 13.0]}],
+        ],
+    }
+    model = ComplexResourceModel._deserialize(payload)
+    assert expected == payload
+    assert expected == model._serialize()
+
+
+def test_recast_object_invalid_json_type():
+    with pytest.raises(InvalidRequest) as excinfo:
+        recast_object(SimpleResourceModel, [], {})
+    assert str(excinfo.value) == f"Can only parse dict items, not {type([])}"
+
+
+def test_recast_object_invalid_sub_type():
+    k = "key"
+    v = (1, 2)
+    with pytest.raises(InvalidRequest) as excinfo:
+        recast_object(SimpleResourceModel, {k: v}, {})
+    assert str(excinfo.value) == f"Unsupported type: {type(v)} for {k}"
+
+
+def test_recast_list_invalid_sub_type():
+    k = "key"
+    v = (1, 2)
+    with pytest.raises(InvalidRequest) as excinfo:
+        _recast_lists(SimpleResourceModel, k, v, {})
+    assert str(excinfo.value) == f"Unsupported type: {type(v)} for {k}"
+
+
+def test_recast_boolean_invalid_value():
+    k = "key"
+    v = "not-a-bool"
+    with pytest.raises(InvalidRequest) as excinfo:
+        _recast_primitive(bool, k, v)
+    assert str(excinfo.value) == f'value for {k} "{v}" is not boolean'
+
+
+def test_field_to_type_unhandled_types():
+    k = "key"
+    for field in [Union[str, list], Any, Optional[Any]]:
+        with pytest.raises(InvalidRequest) as excinfo:
+            _field_to_type(field, k, {})
+        assert str(excinfo.value).startswith("Cannot process type ")