Merge pull request #12 from RSS-Engineering/submodel-deserialization-fixes

Submodel Deserialization Fixes

Author: explorigin

pydanticrud/backends/dynamodb.py

@@ -1,8 +1,7 @@
-from typing import Optional, Set
+from typing import Optional, Set, Union
 import logging
 import json
 from datetime import datetime
-from base64 import b64encode, b64decode
 
 import boto3
 from boto3.dynamodb.conditions import Key, Attr
@@ -80,25 +79,19 @@ def _to_epoch_float(dt):
 
 SERIALIZE_MAP = {
     "number": str,  # float or decimal
-    "string": lambda d: d.isoformat() if isinstance(d, datetime) else d,  # string, datetime
+    "string": str,
+    "string:date-time": lambda d: d.isoformat(),
     "boolean": lambda d: 1 if d else 0,
     "object": json.dumps,
     "array": json.dumps,
-    "anyOf": str,  # FIXME - this could be more complicated. This is a hacky fix.
 }
 
 
-def do_nothing(x):
-    return x
-
-
 DESERIALIZE_MAP = {
     "number": float,
-    "string": do_nothing,
     "boolean": bool,
     "object": json.loads,
     "array": json.loads,
-    "anyOf": do_nothing,  # FIXME - this could be more complicated. This is a hacky fix.
 }
 
 
@@ -120,22 +113,48 @@ def index_definition(index_name, keys, gsi=False):
 
 class DynamoSerializer:
     def __init__(self, schema):
-        self.schema = schema
+        self.properties = schema["properties"]
+        self.definitions = schema.get("definitions")
+
+    def _get_type_possibilities(self, field_name) -> Set[tuple]:
+        field_properties = self.properties[field_name]
+
+        possible_types = []
+        if "anyOf" in field_properties:
+            possible_types.extend([r.get("$ref", r) for r in field_properties["anyOf"]])
+        else:
+            possible_types.append(field_properties.get("$ref", field_properties))
+
+        def type_from_definition(definition_signature: Union[str, dict]) -> dict:
+            if isinstance(definition_signature, str):
+                t = definition_signature.split('/')[-1]
+                return self.definitions[t]
+            return definition_signature
+
+        type_dicts = [
+            type_from_definition(t)
+            for t in possible_types
+        ]
+
+        return set([
+            (t['type'], t.get('format', ''))
+            for t in type_dicts
+        ])
 
     def _serialize_field(self, field_name, value):
-        definition = self.schema.get("definitions")
-        schema = self.schema["properties"]
-        if definition:
-            for k, v in definition.items():
-                schema[k.lower()] = v
-        schema = self.schema["properties"]
-        field_type = schema[field_name].get("type", "anyOf")
-        try:
-            if any([field_name in self.schema['required'], value is not None]):
-                return SERIALIZE_MAP[field_type](value)
-        except KeyError:
-            log.debug(f"No serializer for field_type {field_type}")
-            return value  # do nothing but log it.
+        field_types = self._get_type_possibilities(field_name)
+        if value is not None:
+            for t in field_types:
+                try:
+                    type_signature = ":".join(t).rstrip(':')
+                    try:
+                        return SERIALIZE_MAP[type_signature](value)
+                    except KeyError:
+                        return SERIALIZE_MAP[t[0]](value)
+                except (ValueError, TypeError, KeyError):
+                    pass
+
+        return value
 
     def serialize_record(self, data_dict) -> dict:
         """
@@ -147,18 +166,19 @@ def serialize_record(self, data_dict) -> dict:
         }
 
     def _deserialize_field(self, field_name, value):
-        definition = self.schema.get("definitions")
-        schema = self.schema["properties"]
-        if definition:
-            for k, v in definition.items():
-                schema[k.lower()] = v
-        field_type = schema[field_name].get("type", "anyOf")
-        try:
-            if any([field_name in self.schema['required'], value is not None]):
-                return DESERIALIZE_MAP[field_type](value)
-        except KeyError:
-            log.debug(f"No deserializer for field_type {field_type}")
-            return value  # do nothing but log it.
+        field_types = self._get_type_possibilities(field_name)
+        if value is not None:
+            for t in field_types:
+                try:
+                    type_signature = ":".join(t).rstrip(':')
+                    try:
+                        return DESERIALIZE_MAP[type_signature](value)
+                    except KeyError:
+                        return DESERIALIZE_MAP[t[0]](value)
+                except (ValueError, TypeError, KeyError):
+                    pass
+
+        return value
 
     def deserialize_record(self, data_dict) -> dict:
         """
@@ -213,7 +233,7 @@ def _key_param_to_dict(self, key):
         }
         if self.range_key:
             if not isinstance(key, tuple) or not len(key) == 2:
-                raise ValueError(f"{self.table_name} needs both a hash_key and a range_key to delete a record.")
+                raise ValueError(f"{self.table_name} needs both a hash_key and a range_key.")
             _key = {
                 self.hash_key: key[0],
                 self.range_key: key[1]
@@ -256,7 +276,7 @@ def initialize(self):
                 {
                     "AttributeName": attr,
                     "AttributeType": DYNAMO_TYPE_MAP.get(
-                        schema["properties"][attr].get("type", "anyOf"), "S"
+                        schema["properties"][attr].get("type"), "S"
                     ),
                 }
                 for attr in self.possible_keys

pydanticrud/main.py

@@ -1,3 +1,4 @@
+import pydantic.error_wrappers
 from pydantic import BaseModel as PydanticBaseModel
 from pydantic.main import ModelMetaclass
 from rule_engine import Rule
@@ -25,6 +26,9 @@ def __len__(self):
     def __iter__(self):
         return self
 
+    def __getitem__(self, indices):
+        return self.records.__getitem__(indices)
+
     def __next__(self):
         try:
             member = self.records[self._current_index]

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pydanticrud"
-version = "0.1.0"
+version = "0.2.0"
 description = "Supercharge your Pydantic models with CRUD methods and a pluggable backend"
 authors = ["Timothy Farrell <[email protected]>"]
 license = "MIT"

tests/test_dynamodb.py

@@ -1,9 +1,7 @@
-from typing import Dict, List, Optional
-from base64 import b64decode, b64encode
+from typing import Dict, List, Optional, Union
 from decimal import Decimal
 from datetime import datetime
-import json
-from uuid import uuid4
+from uuid import uuid4, UUID
 import random
 
 import docker
@@ -32,6 +30,7 @@ class SimpleKeyModel(BaseModel):
     enabled: bool
     data: Dict[int, int] = None
     items: List[int]
+    hash: UUID
 
     class Config:
         title = "ModelTitle123"
@@ -80,11 +79,17 @@ class Ticket(PydanticBaseModel):
     number: str
 
 
+class SomethingElse(PydanticBaseModel):
+    herp: bool
+    derp: int
+
+
 class NestedModel(BaseModel):
     account: str
     sort_date_key: str
     expires: str
     ticket: Optional[Ticket]
+    other: Union[Ticket, SomethingElse]
 
     class Config:
         title = "NestedModelTitle123"
@@ -116,6 +121,7 @@ def simple_model_data_generator(**kwargs):
         enabled=random.choice((True, False)),
         data={random.randint(0, 1000): random.randint(0, 1000)},
         items=[random.randint(0, 100000), random.randint(0, 100000), random.randint(0, 100000)],
+        hash=uuid4()
     )
     data.update(kwargs)
     return data
@@ -141,9 +147,20 @@ def nested_model_data_generator(include_ticket=True, **kwargs):
         expires=future_datetime(days=1, hours=random.randint(1, 12), minutes=random.randint(1, 58)).isoformat(),
         ticket={
             'created_time': random_datetime().isoformat(),
-            'number': random.randint(0, 1000)
+            'number': str(random.randint(0, 1000))
+
+        } if include_ticket else None,
+        other=random.choice([
+            {
+                'created_time': random_datetime().isoformat(),
+                'number': str(random.randint(0, 1000))
+
+            }, {
+                'herp': random.choice([True, False]),
+                'derp': random.randint(0, 1000)
 
-        } if include_ticket else None
+            }
+        ])
     )
     data.update(kwargs)
     return data
@@ -243,7 +260,7 @@ def alias_query_data(alias_table):
             AliasKeyModel.delete(datum["name"])
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def nested_query_data(nested_table):
     presets = [dict()] * 5
     data = [datum for datum in [nested_model_data_generator(**i) for i in presets]]
@@ -257,7 +274,7 @@ def nested_query_data(nested_table):
             NestedModel.delete((datum[NestedModel.Config.hash_key], datum[NestedModel.Config.range_key]))
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def nested_query_data_empty_ticket(nested_table):
     presets = [dict()] * 5
     data = [datum for datum in [nested_model_data_generator(include_ticket=False, **i) for i in presets]]
@@ -460,14 +477,16 @@ def test_query_scan_complex(dynamo, complex_query_data):
 
 def test_query_with_nested_model(dynamo, nested_query_data):
     res = NestedModel.query()
-    res_data = [m.ticket for m in res]
-    assert any(elem is not None for elem in res_data)
+    for m in res:
+        assert isinstance(m.ticket, Ticket)
+        assert m.ticket.created_time is not None
+        assert m.ticket.number is not None
+        assert isinstance(m.other, (Ticket, SomethingElse))
 
 
 def test_query_with_nested_model_optional(dynamo, nested_query_data_empty_ticket):
     res = NestedModel.query()
-    res_data = [m.ticket for m in res]
-    assert any(elem is None for elem in res_data)
+    assert all([m.ticket is None for m in res])
 
 
 def test_query_alias_save(dynamo):
@@ -485,5 +504,3 @@ def test_get_alias_model_data(dynamo, alias_query_data):
     data = alias_model_data_generator()
     res = AliasKeyModel.get(alias_query_data[0]['name'])
     assert res.dict(by_alias=True) == alias_query_data[0]
-
-

tests/test_model.py

@@ -11,6 +11,10 @@ def __init__(self, cfg):
     def get(cls, id):
         pass
 
+    @classmethod
+    def query(cls, id):
+        pass
+
 
 class Model(BaseModel):
     id: int
@@ -38,5 +42,19 @@ def test_model_backend_get():
         assert m.total == 3.0
 
 
+def test_model_backend_query():
+    with patch.object(
+        FalseBackend, "query", return_value=[dict(id=1, name="two", total=3.0)]
+    ) as mock_query:
+        m = Model.query(2)
+
+        mock_query.assert_called_with(2)
+        assert m.count is None  # In this case the backend did not provide a total count.
+        assert len(m[:]) == 1  # .. but we can cast to a list and get that length.
+        assert m[0].id == 1
+        assert m[0].name == "two"
+        assert m[0].total == 3.0
+
+
 def test_model_table_name_from_title():
     assert Model.get_table_name() == Model.Config.title.lower()