EmbeddedReference base class + SQLAEmbeddedReference (#28)

Author: wojcikstefan

.gitignore

@@ -6,6 +6,7 @@ venv
 # Packages
 *.egg
 *.egg-info
+.eggs
 dist
 build
 eggs
@@ -19,9 +20,10 @@ develop-eggs
 # Installer logs
 pip-log.txt
 
-# Unit test / coverage reports
+# Unit test / coverage reports and cache
 .coverage
 .tox
+.pytest_cache/
 
 #Translations
 *.mo

cleancat/__init__.py

@@ -1,11 +1,12 @@
 from cleancat.base import (
-    Bool, Choices, DateTime, Dict, Email, Embedded, Enum, Field, Integer,
-    List, Regex, RelaxedURL, Schema, SortedSet, StopValidation, String,
-    TrimmedString, URL, ValidationError
+    Bool, Choices, DateTime, Dict, Email, Embedded, EmbeddedReference, Enum,
+    Field, Integer, List, Regex, RelaxedURL, Schema, SortedSet, StopValidation,
+    String, TrimmedString, URL, ValidationError
 )
 
 __all__ = [
-    'Bool', 'Choices', 'DateTime', 'Dict', 'Email', 'Embedded', 'Enum',
-    'Field', 'Integer', 'List', 'Regex', 'RelaxedURL', 'Schema', 'SortedSet',
-    'StopValidation', 'String', 'TrimmedString', 'URL', 'ValidationError'
+    'Bool', 'Choices', 'DateTime', 'Dict', 'Email', 'Embedded',
+    'EmbeddedReference', 'Enum', 'Field', 'Integer', 'List', 'Regex',
+    'RelaxedURL', 'Schema', 'SortedSet', 'StopValidation', 'String',
+    'TrimmedString', 'URL', 'ValidationError'
 ]

cleancat/base.py

@@ -7,7 +7,9 @@
 
 
 if sys.version_info[0] == 3:
-    basestring = str
+    str_type = str
+else:
+    str_type = basestring
 
 
 class ValidationError(Exception):
@@ -67,7 +69,7 @@ def serialize(self, value):
 
 
 class String(Field):
-    base_type = basestring
+    base_type = str_type
     blank_value = ''
     min_length = None
     max_length = None
@@ -96,7 +98,7 @@ def has_value(self, value):
 
 
 class TrimmedString(String):
-    base_type = basestring
+    base_type = str_type
     blank_value = ''
 
     def clean(self, value):
@@ -184,7 +186,7 @@ class Email(Regex):
 
     def clean(self, value):
         # trim any leading/trailing whitespace before validating the email
-        if isinstance(value, basestring):
+        if isinstance(value, str_type):
             value = value.strip()
         return super(Email, self).clean(value)
 
@@ -345,6 +347,96 @@ def serialize(self, value):
         return self.schema_class(data=value).serialize()
 
 
+class ReferenceNotFoundError(Exception):
+    """Exception to be raised when a referenced object isn't found."""
+
+
+class EmbeddedReference(Dict):
+    """Represents an object which can be referenced by its ID.
+
+    This field allows one to submit a dict of values which will then create
+    a new instance of the object, or it will update fields of an existing
+    object if a field representing its ID (called `pk_field`) was included
+    in the submitted dict.
+    """
+
+    def __init__(self, object_class, schema_class, pk_field='id', **kwargs):
+        self.object_class = object_class
+        self.schema_class = schema_class
+        self.pk_field = pk_field
+        super(EmbeddedReference, self).__init__(schema_class, **kwargs)
+
+    def clean(self, value):
+        # Clean the dict first.
+        value = super(EmbeddedReference, self).clean(value)
+
+        # Then, depending on whether `pk_field` is in the dict of submitted
+        # values or not, update an existing object or create a new one.
+        if value and self.pk_field in value:
+            return self.clean_existing(value)
+        return self.clean_new(value)
+
+    def serialize(self, obj):
+        obj_data = self.get_orig_data_from_existing(obj)
+        serialized = self.schema_class(data=obj_data).serialize()
+        serialized[self.pk_field] = getattr(obj, self.pk_field)
+        return serialized
+
+    def clean_new(self, value):
+        """Return a new object instantiated with cleaned data."""
+        value = self.schema_class(value).full_clean()
+        return self.object_class(**value)
+
+    def clean_existing(self, value):
+        """Clean the data and return an existing document with its fields
+        updated based on the cleaned values.
+        """
+        existing_pk = value[self.pk_field]
+        try:
+            obj = self.fetch_existing(existing_pk)
+        except ReferenceNotFoundError:
+            raise ValidationError('Object does not exist.')
+        orig_data = self.get_orig_data_from_existing(obj)
+
+        # Clean the data (passing the new data dict and the original data to
+        # the schema).
+        value = self.schema_class(value, orig_data).full_clean()
+
+        # Set cleaned data on the object (except for the pk_field).
+        for field_name, field_value in value.items():
+            if field_name != self.pk_field:
+                setattr(obj, field_name, field_value)
+
+        return obj
+
+    def fetch_existing(self, pk):
+        """Fetch an existing object that corresponds to a given ID.
+
+        This needs to be subclassed since, depending on the object class,
+        the fetching mechanism might be different. See implementations of
+        SQLAEmbeddedResource and MongoEmbeddedResource for a concrete example
+        of fetching objects from a relational and non-relational database.
+
+        :param str pk: ID of the object that's supposed to exist.
+        :returns: an instance of the object class.
+        :raises: ReferenceNotFoundError if the object doesn't exist.
+        """
+        raise NotImplementedError  # should be subclassed
+
+    def get_orig_data_from_existing(self, obj):
+        """Return a dictionary of field names and values for a given object.
+
+        The values in the dictionary should be in their "cleaned" state (as
+        in, exactly as they were set on the object, without any serialization).
+
+        :param object obj: existing object for which new data is currently
+            being cleaned.
+        :returns: dict of fields and values that are currently set on the
+            object (before the new cleaned data is applied).
+        """
+        raise NotImplementedError  # should be subclassed
+
+
 class Choices(Field):
     """
     A field that accepts the given choices.
@@ -366,7 +458,7 @@ def clean(self, value):
         if self.case_insensitive:
             choices = {choice.lower(): choice for choice in choices}
 
-            if not isinstance(value, basestring):
+            if not isinstance(value, str_type):
                 raise ValidationError(u'Value needs to be a string.')
 
             if value.lower() not in choices:

cleancat/mongo.py

@@ -6,7 +6,10 @@
 
 from mongoengine import ValidationError as MongoValidationError
 
-from .base import Dict, Embedded, Field, ValidationError
+from .base import (
+    Embedded, EmbeddedReference, Field, ReferenceNotFoundError,
+    ValidationError, str_type
+)
 
 
 class MongoEmbedded(Embedded):
@@ -26,71 +29,34 @@ def clean(self, value):
         return self.document_class(**value)
 
 
-class MongoEmbeddedReference(MongoEmbedded):
+class MongoEmbeddedReference(EmbeddedReference):
     """
-    Represents a MongoEngine document, which can be created or updated based
-    on a provided dict of values.
-
-    The name of the field that acts as the primary key of the document can be
-    specified using pk_field (it's 'id' by default). If the passed document
-    contains the pk_field, we check if a document with that PK exists and then
-    we update its fields (or fail if it can't be found). If the input dict
-    does not contain the pk_field, it is assumed that a new document should be
-    created.
-
-    Examples:
-    {'id': 'existing_id', 'foo': 'bar'} -> valid (updates an existing document)
-    {'id': 'non-existing_id', 'foo': 'bar'} -> invalid
-    {'foo': 'bar'} -> valid (creates a new document)
-    """
-
-    def __init__(self, *args, **kwargs):
-        self.pk_field = kwargs.pop('pk_field', 'id')
-        super(MongoEmbeddedReference, self).__init__(*args, **kwargs)
+    Represents an embedded reference where the object class is a MongoEngine
+    document.
 
-    def clean(self, value):
-        value = Dict.clean(self, value)
-        if value and self.pk_field in value:
-            return self.clean_existing(value)
-        return self.clean_new(value)
-
-    def clean_new(self, value):
-        """Return a new document instantiated with cleaned data."""
-        value = self.schema_class(value).full_clean()
-        return self.document_class(**value)
+    Examples of passed data and how it's handled:
+    {'id': 'existing_id', 'foo': 'bar'} -> updates an existing document
+    {'id': 'non-existing_id', 'foo': 'bar'} -> fails bcos the PK is not valid
+    {'foo': 'bar'} -> creates a new document instance
+    """
 
-    def clean_existing(self, value):
-        """Clean the data and return an existing document with its fields
-        updated based on the cleaned values.
-        """
-        existing_pk = value[self.pk_field]
+    def fetch_existing(self, pk):
+        doc_cls = self.object_class
         try:
-            document = self.document_class.objects.get(pk=existing_pk)
-        except self.document_class.DoesNotExist:
-            raise ValidationError('Object does not exist.')
+            return doc_cls.objects.get(pk=pk)
+        except doc_cls.DoesNotExist:
+            raise ReferenceNotFoundError
         except MongoValidationError as e:
             raise ValidationError(str(e))
 
+    def get_orig_data_from_existing(self, obj):
         # Get a dict of existing document's field names and values.
-        if hasattr(document, 'to_dict'):
+        if hasattr(obj, 'to_dict'):
             # MongoMallard
-            document_data = document.to_dict()
+            return obj.to_dict()
         else:
             # Upstream MongoEngine
-            document_data = dict(document._data)
-        if None in document_data:
-            del document_data[None]
-
-        # Clean the data (passing the new data dict and the original data to
-        # the schema).
-        value = self.schema_class(value, document_data).full_clean()
-
-        # Set cleaned data on the document (except for the pk_field).
-        for field_name, field_value in value.items():
-            if field_name != self.pk_field:
-                setattr(document, field_name, field_value)
-
-        return document
+            return dict(obj._data)
 
 
 class MongoReference(Field):
@@ -99,7 +65,7 @@ class MongoReference(Field):
     Example document: ReferenceField(Doc)
     """
 
-    base_type = basestring
+    base_type = str_type
 
     def __init__(self, document_class=None, **kwargs):
         self.document_class = document_class

cleancat/sqla.py

@@ -0,0 +1,40 @@
+"""
+This module contains CleanCat fields specific to SQLAlchemy. SQLA is not
+a required dependency, hence to use these fields, you'll have to import
+them via `from cleancat.sqla import ...`.
+"""
+
+from sqlalchemy import inspect
+
+from .base import EmbeddedReference, ReferenceNotFoundError
+
+
+def object_as_dict(obj):
+    """Turn an SQLAlchemy model into a dict of field names and values.
+
+    Based on https://stackoverflow.com/a/37350445/1579058
+    """
+    return {c.key: getattr(obj, c.key)
+            for c in inspect(obj).mapper.column_attrs}
+
+
+class SQLAEmbeddedReference(EmbeddedReference):
+    """
+    Represents an embedded reference where the object class is an SQLAlchemy
+    model.
+
+    Examples of passed data and how it's handled:
+    {'id': 'existing_id', 'foo': 'bar'} -> updates an existing model
+    {'id': 'non-existing_id', 'foo': 'bar'} -> fails bcos the ID is not valid
+    {'foo': 'bar'} -> creates a new model instance
+    """
+
+    def fetch_existing(self, pk):
+        model_cls = self.object_class
+        model = model_cls.query.get(pk)
+        if not model:
+            raise ReferenceNotFoundError
+        return model
+
+    def get_orig_data_from_existing(self, obj):
+        return object_as_dict(obj)

requirements.txt

@@ -1,3 +1,6 @@
 flake8
+mongoengine
 python-dateutil
+pytest
 pytz
+sqlalchemy

setup.cfg

@@ -1,3 +1,10 @@
+[aliases]
+test=pytest
+
+[tool:pytest]
+python_files=tests/*.py
+testpaths=tests
+
 [flake8]
 ignore=E501
 exclude=venv,.tox,.eggs,build

setup.py

@@ -1,8 +1,14 @@
 from setuptools import setup
 
-test_requirements = [
-    'nose',
+install_requirements = [
+    'python-dateutil',
+    'pytz',
+]
+test_requirements = install_requirements + [
+    'pytest',
     'coverage',
+    'mongoengine',
+    'sqlalchemy'
 ]
 
 setup(
@@ -19,12 +25,11 @@
     packages=[
         'cleancat',
     ],
-    test_suite='nose.collector',
     zip_safe=False,
     platforms='any',
-    install_requires=[
-        'python-dateutil',
-    ],
+    install_requires=install_requirements,
+    setup_requires=['pytest-runner'],
+    test_suite='tests',
     tests_require=test_requirements,
     extras_require={'test': test_requirements},
     classifiers=[