Improve health of this package (#1409)

* added flake8 and flake8-import-order to travis for py27

* fixed a test that fails from time to time depending on an order of a dict

* flake8 tweaks for the entire codebase excluding tests

Author: wojcikstefan

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+
 python:
 - '2.6'
 - '2.7'
@@ -7,35 +8,51 @@ python:
 - '3.5'
 - pypy
 - pypy3
+
 env:
 - PYMONGO=2.7
 - PYMONGO=2.8
 - PYMONGO=3.0
 - PYMONGO=dev
+
 matrix:
   fast_finish: true
+
 before_install:
 - travis_retry sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
 - echo 'deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen' |
   sudo tee /etc/apt/sources.list.d/mongodb.list
 - travis_retry sudo apt-get update
 - travis_retry sudo apt-get install mongodb-org-server
+
 install:
 - sudo apt-get install python-dev python3-dev libopenjpeg-dev zlib1g-dev libjpeg-turbo8-dev
   libtiff4-dev libjpeg8-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev
   python-tk
-# virtualenv>=14.0.0 has dropped Python 3.2 support
-- travis_retry pip install "virtualenv<14.0.0" "tox>=1.9" coveralls
+- travis_retry pip install --upgrade pip
+- travis_retry pip install coveralls
+- travis_retry pip install flake8
+- travis_retry pip install tox>=1.9
+- travis_retry pip install "virtualenv<14.0.0"  # virtualenv>=14.0.0 has dropped Python 3.2 support (and pypy3 is based on py32)
 - travis_retry tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -e test
+
+# Run flake8 for py27
+before_script:
+- if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then tox -e flake8; fi
+
 script:
 - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- --with-coverage
+
 after_script: coveralls --verbose
+
 notifications:
   irc: irc.freenode.org#mongoengine
+
 branches:
   only:
   - master
   - /^v.*$/
+
 deploy:
   provider: pypi
   user: the_drow

mongoengine/__init__.py

@@ -1,15 +1,15 @@
+import connection
+from connection import *
 import document
 from document import *
+import errors
+from errors import *
 import fields
 from fields import *
-import connection
-from connection import *
 import queryset
 from queryset import *
 import signals
 from signals import *
-from errors import *
-import errors
 
 __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ +
            list(queryset.__all__) + signals.__all__ + list(errors.__all__))
@@ -22,4 +22,5 @@ def get_version():
         return '.'.join(map(str, VERSION[:-1])) + VERSION[-1]
     return '.'.join(map(str, VERSION))
 
+
 __version__ = get_version()

mongoengine/base/datastructures.py

@@ -1,5 +1,5 @@
-import weakref
 import itertools
+import weakref
 
 from mongoengine.common import _import_class
 from mongoengine.errors import DoesNotExist, MultipleObjectsReturned
@@ -199,8 +199,9 @@ def sort(self, *args, **kwargs):
     def _mark_as_changed(self, key=None):
         if hasattr(self._instance, '_mark_as_changed'):
             if key:
-                self._instance._mark_as_changed('%s.%s' % (self._name, 
-                    key % len(self)))
+                self._instance._mark_as_changed(
+                    '%s.%s' % (self._name, key % len(self))
+                )
             else:
                 self._instance._mark_as_changed(self._name)
 

mongoengine/base/document.py

@@ -1,28 +1,28 @@
 import copy
-import operator
 import numbers
+import operator
 from collections import Hashable
 from functools import partial
 
-import pymongo
-from bson import json_util, ObjectId
+from bson import ObjectId, json_util
 from bson.dbref import DBRef
 from bson.son import SON
+import pymongo
 
 from mongoengine import signals
-from mongoengine.common import _import_class
-from mongoengine.errors import (ValidationError, InvalidDocumentError,
-                                LookUpError, FieldDoesNotExist)
-from mongoengine.python_support import PY3, txt_type
-from mongoengine.base.common import get_document, ALLOW_INHERITANCE
+from mongoengine.base.common import ALLOW_INHERITANCE, get_document
 from mongoengine.base.datastructures import (
     BaseDict,
     BaseList,
     EmbeddedDocumentList,
-    StrictDict,
-    SemiStrictDict
+    SemiStrictDict,
+    StrictDict
 )
 from mongoengine.base.fields import ComplexBaseField
+from mongoengine.common import _import_class
+from mongoengine.errors import (FieldDoesNotExist, InvalidDocumentError,
+                                LookUpError, ValidationError)
+from mongoengine.python_support import PY3, txt_type
 
 __all__ = ('BaseDocument', 'NON_FIELD_ERRORS')
 
@@ -73,7 +73,7 @@ def __init__(self, *args, **values):
         # if so raise an Exception.
         if not self._dynamic and (self._meta.get('strict', True) or _created):
             _undefined_fields = set(values.keys()) - set(
-              self._fields.keys() + ['id', 'pk', '_cls', '_text_score'])
+                self._fields.keys() + ['id', 'pk', '_cls', '_text_score'])
             if _undefined_fields:
                 msg = (
                     "The fields '{0}' do not exist on the document '{1}'"
@@ -310,7 +310,7 @@ def to_mongo(self, use_db_field=True, fields=None):
         data = SON()
         data["_id"] = None
         data['_cls'] = self._class_name
-        
+
         # only root fields ['test1.a', 'test2'] => ['test1', 'test2']
         root_fields = set([f.split('.')[0] for f in fields])
 
@@ -333,11 +333,11 @@ def to_mongo(self, use_db_field=True, fields=None):
                         i.replace(key, '') for i in fields
                         if i.startswith(key)]
 
-                    ex_vars['fields'] = embedded_fields 
+                    ex_vars['fields'] = embedded_fields
 
                 if 'use_db_field' in f_inputs:
                     ex_vars['use_db_field'] = use_db_field
-                     
+
                 value = field.to_mongo(value, **ex_vars)
 
             # Handle self generating fields
@@ -566,8 +566,10 @@ def _get_changed_fields(self, inspected=None):
                     continue
             if isinstance(field, ReferenceField):
                 continue
-            elif (isinstance(data, (EmbeddedDocument, DynamicEmbeddedDocument))
-                  and db_field_name not in changed_fields):
+            elif (
+                isinstance(data, (EmbeddedDocument, DynamicEmbeddedDocument)) and
+                db_field_name not in changed_fields
+            ):
                 # Find all embedded fields that have been changed
                 changed = data._get_changed_fields(inspected)
                 changed_fields += ["%s%s" % (key, k) for k in changed if k]
@@ -608,7 +610,7 @@ def _delta(self):
                         break
                     elif isinstance(d, list) and p.lstrip('-').isdigit():
                         if p[0] == '-':
-                            p = str(len(d)+int(p))
+                            p = str(len(d) + int(p))
                         try:
                             d = d[int(p)]
                         except IndexError:
@@ -644,7 +646,7 @@ def _delta(self):
                 for p in parts:
                     if isinstance(d, list) and p.lstrip('-').isdigit():
                         if p[0] == '-':
-                            p = str(len(d)+int(p))
+                            p = str(len(d) + int(p))
                         d = d[int(p)]
                     elif (hasattr(d, '__getattribute__') and
                           not isinstance(d, dict)):
@@ -775,8 +777,12 @@ def _build_index_spec(cls, spec):
         # Check to see if we need to include _cls
         allow_inheritance = cls._meta.get('allow_inheritance',
                                           ALLOW_INHERITANCE)
-        include_cls = (allow_inheritance and not spec.get('sparse', False) and
-                       spec.get('cls',  True) and '_cls' not in spec['fields'])
+        include_cls = (
+            allow_inheritance and
+            not spec.get('sparse', False) and
+            spec.get('cls', True) and
+            '_cls' not in spec['fields']
+        )
 
         # 733: don't include cls if index_cls is False unless there is an explicit cls with the index
         include_cls = include_cls and (spec.get('cls', False) or cls._meta.get('index_cls', True))

mongoengine/base/fields.py

@@ -5,12 +5,12 @@
 from bson import DBRef, ObjectId, SON
 import pymongo
 
-from mongoengine.common import _import_class
-from mongoengine.errors import ValidationError
 from mongoengine.base.common import ALLOW_INHERITANCE
 from mongoengine.base.datastructures import (
     BaseDict, BaseList, EmbeddedDocumentList
 )
+from mongoengine.common import _import_class
+from mongoengine.errors import ValidationError
 
 __all__ = ("BaseField", "ComplexBaseField",
            "ObjectIdField", "GeoJsonBaseField")
@@ -85,13 +85,13 @@ def __init__(self, db_field=None, name=None, required=False, default=None,
         self.null = null
         self.sparse = sparse
         self._owner_document = None
-        
+
         # Detect and report conflicts between metadata and base properties.
         conflicts = set(dir(self)) & set(kwargs)
         if conflicts:
             raise TypeError("%s already has attribute(s): %s" % (
-                self.__class__.__name__, ', '.join(conflicts) ))
-        
+                self.__class__.__name__, ', '.join(conflicts)))
+
         # Assign metadata to the instance
         # This efficient method is available because no __slots__ are defined.
         self.__dict__.update(kwargs)
@@ -169,11 +169,11 @@ def _to_mongo_safe_call(self, value, use_db_field=True, fields=None):
         f_inputs = self.to_mongo.__code__.co_varnames
         ex_vars = {}
         if 'fields' in f_inputs:
-            ex_vars['fields'] = fields 
+            ex_vars['fields'] = fields
 
         if 'use_db_field' in f_inputs:
             ex_vars['use_db_field'] = use_db_field
-             
+
         return self.to_mongo(value, **ex_vars)
 
     def prepare_query_value(self, op, value):
@@ -206,7 +206,6 @@ def _validate_choices(self, value):
         elif value not in choice_list:
             self.error('Value must be one of %s' % unicode(choice_list))
 
-
     def _validate(self, value, **kwargs):
         # Check the Choices Constraint
         if self.choices:

mongoengine/base/metaclasses.py

@@ -1,22 +1,22 @@
 import warnings
 
+from mongoengine.base.common import ALLOW_INHERITANCE, _document_registry
+from mongoengine.base.fields import BaseField, ComplexBaseField, ObjectIdField
 from mongoengine.common import _import_class
 from mongoengine.errors import InvalidDocumentError
 from mongoengine.python_support import PY3
 from mongoengine.queryset import (DO_NOTHING, DoesNotExist,
                                   MultipleObjectsReturned,
                                   QuerySetManager)
 
-from mongoengine.base.common import _document_registry, ALLOW_INHERITANCE
-from mongoengine.base.fields import BaseField, ComplexBaseField, ObjectIdField
 
 __all__ = ('DocumentMetaclass', 'TopLevelDocumentMetaclass')
 
 
 class DocumentMetaclass(type):
-    """Metaclass for all documents.
-    """
+    """Metaclass for all documents."""
 
+    # TODO lower complexity of this method
     def __new__(cls, name, bases, attrs):
         flattened_bases = cls._get_bases(bases)
         super_new = super(DocumentMetaclass, cls).__new__
@@ -162,7 +162,7 @@ def __new__(cls, name, bases, attrs):
         # copies __func__ into im_func and __self__ into im_self for
         # classmethod objects in Document derived classes.
         if PY3:
-            for key, val in new_class.__dict__.items():
+            for val in new_class.__dict__.values():
                 if isinstance(val, classmethod):
                     f = val.__get__(new_class)
                     if hasattr(f, '__func__') and not hasattr(f, 'im_func'):

mongoengine/dereference.py

@@ -1,15 +1,14 @@
 from bson import DBRef, SON
 
-from mongoengine.python_support import txt_type
-
-from base import (
+from .base import (
     BaseDict, BaseList, EmbeddedDocumentList,
     TopLevelDocumentMetaclass, get_document
 )
-from fields import (ReferenceField, ListField, DictField, MapField)
-from connection import get_db
-from queryset import QuerySet
-from document import Document, EmbeddedDocument
+from .connection import get_db
+from .document import Document, EmbeddedDocument
+from .fields import DictField, ListField, MapField, ReferenceField
+from .python_support import txt_type
+from .queryset import QuerySet
 
 
 class DeReference(object):