Merge branch 'master' of github.com:MongoEngine/mongoengine into improve_doc_upd_agg_pipeline

Author: bagerard

.github/workflows/github-actions.yml

@@ -21,6 +21,7 @@ env:
   PYMONGO_3_9: 3.9
   PYMONGO_3_11: 3.11
   PYMONGO_3_12: 3.12
+  PYMONGO_4_0: 4.0
 
   MAIN_PYTHON_VERSION: 3.7
 
@@ -61,6 +62,9 @@ jobs:
           - python-version: 3.7
             MONGODB: $MONGODB_4_4
             PYMONGO: $PYMONGO_3_12
+          - python-version: 3.9
+            MONGODB: $MONGODB_4_4
+            PYMONGO: $PYMONGO_4_0
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

AUTHORS

@@ -263,3 +263,4 @@ that much better:
  * Timothé Perez (https://github.com/AchilleAsh)
  * oleksandr-l5 (https://github.com/oleksandr-l5)
  * Ido Shraga (https://github.com/idoshr)
+ * Terence Honles (https://github.com/terencehonles)

docs/changelog.rst

@@ -7,12 +7,42 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
-- EnumField improvements: now `choices` limits the values of an enum to allow
+- EnumField improvements: now ``choices`` limits the values of an enum to allow
 - Fix bug that prevented instance queryset from using custom queryset_class #2589
 - Fix deepcopy of EmbeddedDocument #2202
 - Fix error when using precision=0 with DecimalField #2535
 - Add support for regex and whole word text search query #2568
 - Add support for update aggregation pipeline #2578
+- BREAKING CHANGE: Updates to support pymongo 4.0. Where possible deprecated
+  functionality has been migrated, but additional care should be taken when
+  migrating to pymongo 4.0 as existing code may have been using deprecated
+  features which have now been removed #2614.
+
+  For the pymongo migration guide see:
+  https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html.
+
+  In addition to the changes in the migration guide, the following is a high
+  level overview of the changes made to MongoEngine when using pymongo 4.0:
+
+  - limited support of geohaystack indexes has been removed
+  - ``QuerySet.map_reduce`` has been migrated from ``Collection.map_reduce``
+    and ``Collection.inline_map_reduce`` to use
+    ``db.command({mapReduce: ..., ...})`` and support between the two may need
+    additional verification.
+  - UUIDs are encoded with the ``pythonLegacy`` encoding by default instead of
+    the newer and cross platform ``standard`` encoding. Existing UUIDs will
+    need to be migrated before changing the encoding, and this should be done
+    explicitly by the user rather than switching to a new default by
+    MongoEngine. This default will change at a later date, but to allow
+    specifying and then migrating to the new format a default ``json_options``
+    has been provided.
+  - ``Queryset.count`` has been using ``Collection.count_documents`` and
+    transparently falling back to ``Collection.count`` when using features that
+    are not supported by ``Collection.count_documents``. ``Collection.count``
+    has been removed and no automatic fallback is possible. The migration guide
+    documents the extended functionality which is no longer supported. Rewrite
+    the unsupported queries or fetch the whole result set and perform the count
+    locally.
 
 Changes in 0.23.1
 ===========

mongoengine/base/document.py

@@ -1,5 +1,6 @@
 import copy
 import numbers
+import warnings
 from functools import partial
 
 import pymongo
@@ -23,11 +24,17 @@
     OperationError,
     ValidationError,
 )
+from mongoengine.pymongo_support import LEGACY_JSON_OPTIONS
 
 __all__ = ("BaseDocument", "NON_FIELD_ERRORS")
 
 NON_FIELD_ERRORS = "__all__"
 
+try:
+    GEOHAYSTACK = pymongo.GEOHAYSTACK
+except AttributeError:
+    GEOHAYSTACK = None
+
 
 class BaseDocument:
     # TODO simplify how `_changed_fields` is used.
@@ -439,10 +446,19 @@ def to_json(self, *args, **kwargs):
             Defaults to True.
         """
         use_db_field = kwargs.pop("use_db_field", True)
+        if "json_options" not in kwargs:
+            warnings.warn(
+                "No 'json_options' are specified! Falling back to "
+                "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. "
+                "For use with other MongoDB drivers specify the UUID "
+                "representation to use.",
+                DeprecationWarning,
+            )
+            kwargs["json_options"] = LEGACY_JSON_OPTIONS
         return json_util.dumps(self.to_mongo(use_db_field), *args, **kwargs)
 
     @classmethod
-    def from_json(cls, json_data, created=False):
+    def from_json(cls, json_data, created=False, **kwargs):
         """Converts json data to a Document instance
 
         :param str json_data: The json data to load into the Document
@@ -460,7 +476,16 @@ def from_json(cls, json_data, created=False):
         # TODO should `created` default to False? If the object already exists
         # in the DB, you would likely retrieve it from MongoDB itself through
         # a query, not load it from JSON data.
-        return cls._from_son(json_util.loads(json_data), created=created)
+        if "json_options" not in kwargs:
+            warnings.warn(
+                "No 'json_options' are specified! Falling back to "
+                "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. "
+                "For use with other MongoDB drivers specify the UUID "
+                "representation to use.",
+                DeprecationWarning,
+            )
+            kwargs["json_options"] = LEGACY_JSON_OPTIONS
+        return cls._from_son(json_util.loads(json_data, **kwargs), created=created)
 
     def __expand_dynamic_values(self, name, value):
         """Expand any dynamic values to their correct types / values."""
@@ -898,7 +923,10 @@ def _build_index_spec(cls, spec):
             elif key.startswith("("):
                 direction = pymongo.GEOSPHERE
             elif key.startswith(")"):
-                direction = pymongo.GEOHAYSTACK
+                try:
+                    direction = pymongo.GEOHAYSTACK
+                except AttributeError:
+                    raise NotImplementedError
             elif key.startswith("*"):
                 direction = pymongo.GEO2D
             if key.startswith(("+", "-", "*", "$", "#", "(", ")")):
@@ -923,10 +951,10 @@ def _build_index_spec(cls, spec):
             index_list.append((key, direction))
 
         # Don't add cls to a geo index
-        if include_cls and direction not in (
-            pymongo.GEO2D,
-            pymongo.GEOHAYSTACK,
-            pymongo.GEOSPHERE,
+        if (
+            include_cls
+            and direction not in (pymongo.GEO2D, pymongo.GEOSPHERE)
+            and (GEOHAYSTACK is None or direction != GEOHAYSTACK)
         ):
             index_list.insert(0, ("_cls", 1))
 

mongoengine/connection.py

@@ -1,6 +1,10 @@
+import warnings
+
 from pymongo import MongoClient, ReadPreference, uri_parser
 from pymongo.database import _check_name
 
+from mongoengine.pymongo_support import PYMONGO_VERSION
+
 __all__ = [
     "DEFAULT_CONNECTION_NAME",
     "DEFAULT_DATABASE_NAME",
@@ -162,6 +166,18 @@ def _get_connection_settings(
     kwargs.pop("slaves", None)
     kwargs.pop("is_slave", None)
 
+    if "uuidRepresentation" not in kwargs:
+        warnings.warn(
+            "No uuidRepresentation is specified! Falling back to "
+            "'pythonLegacy' which is the default for pymongo 3.x. "
+            "For compatibility with other MongoDB drivers this should be "
+            "specified as 'standard' or '{java,csharp}Legacy' to work with "
+            "older drivers in those languages. This will be changed to "
+            "'standard' in a future release.",
+            DeprecationWarning,
+        )
+        kwargs["uuidRepresentation"] = "pythonLegacy"
+
     conn_settings.update(kwargs)
     return conn_settings
 
@@ -263,15 +279,25 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
         raise ConnectionFailure(msg)
 
     def _clean_settings(settings_dict):
-        irrelevant_fields_set = {
-            "name",
-            "username",
-            "password",
-            "authentication_source",
-            "authentication_mechanism",
-        }
+        if PYMONGO_VERSION < (4,):
+            irrelevant_fields_set = {
+                "name",
+                "username",
+                "password",
+                "authentication_source",
+                "authentication_mechanism",
+            }
+            rename_fields = {}
+        else:
+            irrelevant_fields_set = {"name"}
+            rename_fields = {
+                "authentication_source": "authSource",
+                "authentication_mechanism": "authMechanism",
+            }
         return {
-            k: v for k, v in settings_dict.items() if k not in irrelevant_fields_set
+            rename_fields.get(k, k): v
+            for k, v in settings_dict.items()
+            if k not in irrelevant_fields_set and v is not None
         }
 
     raw_conn_settings = _connection_settings[alias].copy()
@@ -351,14 +377,18 @@ def get_db(alias=DEFAULT_CONNECTION_NAME, reconnect=False):
         conn = get_connection(alias)
         conn_settings = _connection_settings[alias]
         db = conn[conn_settings["name"]]
-        auth_kwargs = {"source": conn_settings["authentication_source"]}
-        if conn_settings["authentication_mechanism"] is not None:
-            auth_kwargs["mechanism"] = conn_settings["authentication_mechanism"]
         # Authenticate if necessary
-        if conn_settings["username"] and (
-            conn_settings["password"]
-            or conn_settings["authentication_mechanism"] == "MONGODB-X509"
+        if (
+            PYMONGO_VERSION < (4,)
+            and conn_settings["username"]
+            and (
+                conn_settings["password"]
+                or conn_settings["authentication_mechanism"] == "MONGODB-X509"
+            )
         ):
+            auth_kwargs = {"source": conn_settings["authentication_source"]}
+            if conn_settings["authentication_mechanism"] is not None:
+                auth_kwargs["mechanism"] = conn_settings["authentication_mechanism"]
             db.authenticate(
                 conn_settings["username"], conn_settings["password"], **auth_kwargs
             )

mongoengine/context_managers.py

@@ -210,13 +210,14 @@ def __init__(self, alias=DEFAULT_CONNECTION_NAME):
         }
 
     def _turn_on_profiling(self):
-        self.initial_profiling_level = self.db.profiling_level()
-        self.db.set_profiling_level(0)
+        profile_update_res = self.db.command({"profile": 0})
+        self.initial_profiling_level = profile_update_res["was"]
+
         self.db.system.profile.drop()
-        self.db.set_profiling_level(2)
+        self.db.command({"profile": 2})
 
     def _resets_profiling(self):
-        self.db.set_profiling_level(self.initial_profiling_level)
+        self.db.command({"profile": self.initial_profiling_level})
 
     def __enter__(self):
         self._turn_on_profiling()

mongoengine/document.py

@@ -888,10 +888,6 @@ def ensure_indexes(cls):
         index_cls = cls._meta.get("index_cls", True)
 
         collection = cls._get_collection()
-        # 746: when connection is via mongos, the read preference is not necessarily an indication that
-        # this code runs on a secondary
-        if not collection.is_mongos and collection.read_preference > 1:
-            return
 
         # determine if an index which we are creating includes
         # _cls as its first field; if so, we can avoid creating

mongoengine/pymongo_support.py

@@ -1,14 +1,18 @@
 """
-Helper functions, constants, and types to aid with PyMongo v2.7 - v3.x support.
+Helper functions, constants, and types to aid with PyMongo support.
 """
 import pymongo
+from bson import binary, json_util
 from pymongo.errors import OperationFailure
 
-_PYMONGO_37 = (3, 7)
-
 PYMONGO_VERSION = tuple(pymongo.version_tuple[:2])
 
-IS_PYMONGO_GTE_37 = PYMONGO_VERSION >= _PYMONGO_37
+if PYMONGO_VERSION >= (4,):
+    LEGACY_JSON_OPTIONS = json_util.LEGACY_JSON_OPTIONS.with_options(
+        uuid_representation=binary.UuidRepresentation.PYTHON_LEGACY,
+    )
+else:
+    LEGACY_JSON_OPTIONS = json_util.DEFAULT_JSON_OPTIONS
 
 
 def count_documents(
@@ -29,15 +33,28 @@ def count_documents(
         kwargs["collation"] = collation
 
     # count_documents appeared in pymongo 3.7
-    if IS_PYMONGO_GTE_37:
+    if PYMONGO_VERSION >= (3, 7):
         try:
             return collection.count_documents(filter=filter, **kwargs)
-        except OperationFailure:
+        except OperationFailure as err:
+            if PYMONGO_VERSION >= (4,):
+                raise
+
             # OperationFailure - accounts for some operators that used to work
             # with .count but are no longer working with count_documents (i.e $geoNear, $near, and $nearSphere)
             # fallback to deprecated Cursor.count
             # Keeping this should be reevaluated the day pymongo removes .count entirely
-            pass
+            message = str(err)
+            if not (
+                "not allowed in this context" in message
+                and (
+                    "$where" in message
+                    or "$geoNear" in message
+                    or "$near" in message
+                    or "$nearSphere" in message
+                )
+            ):
+                raise
 
     cursor = collection.find(filter)
     for option, option_value in kwargs.items():
@@ -49,7 +66,7 @@ def count_documents(
 
 def list_collection_names(db, include_system_collections=False):
     """Pymongo>3.7 deprecates collection_names in favour of list_collection_names"""
-    if IS_PYMONGO_GTE_37:
+    if PYMONGO_VERSION >= (3, 7):
         collections = db.list_collection_names()
     else:
         collections = db.collection_names()