Merge pull request #2483 from bagerard/mongodb44_ci

Add Mongodb 44 ci and fix incompat

Author: bagerard

.github/workflows/github-actions.yml

@@ -11,9 +11,10 @@ on:
     tags:
       - 'v[0-9]+\.[0-9]+\.[0-9]+*'
 env:
-  MONGODB_3_4: 3.4.19
-  MONGODB_3_6: 3.6.13
-  MONGODB_4_0: 4.0.13
+  MONGODB_3_6: 3.6.14
+  MONGODB_4_0: 4.0.23
+  MONGODB_4_2: 4.2
+  MONGODB_4_4: 4.4
 
   PYMONGO_3_4: 3.4
   PYMONGO_3_6: 3.6
@@ -47,14 +48,14 @@ jobs:
         MONGODB: [$MONGODB_4_0]
         PYMONGO: [$PYMONGO_3_11]
         include:
-          - python-version: 3.7
-            MONGODB: $MONGODB_3_4
-            PYMONGO: $PYMONGO_3_6
           - python-version: 3.7
             MONGODB: $MONGODB_3_6
             PYMONGO: $PYMONGO_3_9
           - python-version: 3.7
-            MONGODB: $MONGODB_3_6
+            MONGODB: $MONGODB_4_2
+            PYMONGO: $PYMONGO_3_6
+          - python-version: 3.7
+            MONGODB: $MONGODB_4_4
             PYMONGO: $PYMONGO_3_11
     steps:
     - uses: actions/checkout@v2

.github/workflows/install_mongo.sh

@@ -2,7 +2,17 @@
 
 MONGODB=$1
 
+# Mongo > 4.0 follows different name convention for download links
 mongo_build=mongodb-linux-x86_64-${MONGODB}
+
+if [[ "$MONGODB" == *"4.2"* ]]; then
+  mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest
+elif [[ "$MONGODB" == *"4.4"* ]]; then
+  mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest
+fi
+
 wget http://fastdl.mongodb.org/linux/$mongo_build.tgz
 tar xzf $mongo_build.tgz
-${PWD}/$mongo_build/bin/mongod --version
+
+mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*")
+$mongodb_dir/bin/mongod --version

.github/workflows/start_mongo.sh

@@ -2,7 +2,8 @@
 
 MONGODB=$1
 
-mongodb_dir=${PWD}/mongodb-linux-x86_64-${MONGODB}
+mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*")
+
 mkdir $mongodb_dir/data
 $mongodb_dir/bin/mongod --dbpath $mongodb_dir/data --logpath $mongodb_dir/mongodb.log --fork
 mongo --eval 'db.version();'    # Make sure mongo is awake

docs/changelog.rst

@@ -7,6 +7,7 @@ Development
 ===========
 - (Fill this out as you fix issues and develop your features).
 - Bugfix: manually setting SequenceField in DynamicDocument doesn't increment the counter #2471
+- Add MongoDB 4.2 and 4.4 to CI
 
 Changes in 0.22.1
 =================

mongoengine/mongodb_support.py

@@ -8,6 +8,8 @@
 # get_mongodb_version()
 MONGODB_34 = (3, 4)
 MONGODB_36 = (3, 6)
+MONGODB_42 = (4, 2)
+MONGODB_44 = (4, 4)
 
 
 def get_mongodb_version():

mongoengine/queryset/base.py

@@ -1355,21 +1355,18 @@ def map_reduce(
 
         MapReduceDocument = _import_class("MapReduceDocument")
 
-        if not hasattr(self._collection, "map_reduce"):
-            raise NotImplementedError("Requires MongoDB >= 1.7.1")
-
         map_f_scope = {}
         if isinstance(map_f, Code):
             map_f_scope = map_f.scope
             map_f = str(map_f)
-        map_f = Code(queryset._sub_js_fields(map_f), map_f_scope)
+        map_f = Code(queryset._sub_js_fields(map_f), map_f_scope or None)
 
         reduce_f_scope = {}
         if isinstance(reduce_f, Code):
             reduce_f_scope = reduce_f.scope
             reduce_f = str(reduce_f)
         reduce_f_code = queryset._sub_js_fields(reduce_f)
-        reduce_f = Code(reduce_f_code, reduce_f_scope)
+        reduce_f = Code(reduce_f_code, reduce_f_scope or None)
 
         mr_args = {"query": queryset._query}
 
@@ -1379,7 +1376,7 @@ def map_reduce(
                 finalize_f_scope = finalize_f.scope
                 finalize_f = str(finalize_f)
             finalize_f_code = queryset._sub_js_fields(finalize_f)
-            finalize_f = Code(finalize_f_code, finalize_f_scope)
+            finalize_f = Code(finalize_f_code, finalize_f_scope or None)
             mr_args["finalize"] = finalize_f
 
         if scope:

tests/document/test_indexes.py

@@ -7,6 +7,7 @@
 
 from mongoengine import *
 from mongoengine.connection import get_db
+from mongoengine.mongodb_support import MONGODB_42, get_mongodb_version
 
 
 class TestIndexes(unittest.TestCase):
@@ -452,9 +453,11 @@ class Test(Document):
             .get("stage")
             == "IXSCAN"
         )
+        mongo_db = get_mongodb_version()
+        PROJECTION_STR = "PROJECTION" if mongo_db < MONGODB_42 else "PROJECTION_COVERED"
         assert (
             query_plan.get("queryPlanner").get("winningPlan").get("stage")
-            == "PROJECTION"
+            == PROJECTION_STR
         )
 
         query_plan = Test.objects(a=1).explain()

tests/queryset/test_queryset.py

@@ -21,7 +21,10 @@
     QuerySetManager,
     queryset_manager,
 )
-from tests.utils import requires_mongodb_gte_44
+from tests.utils import (
+    requires_mongodb_gte_44,
+    requires_mongodb_lt_42,
+)
 
 
 class db_ops_tracker(query_counter):
@@ -1490,6 +1493,7 @@ class BlogPost(Document):
 
         BlogPost.drop_collection()
 
+    @requires_mongodb_lt_42
     def test_exec_js_query(self):
         """Ensure that queries are properly formed for use in exec_js."""
 
@@ -1527,6 +1531,7 @@ class BlogPost(Document):
 
         BlogPost.drop_collection()
 
+    @requires_mongodb_lt_42
     def test_exec_js_field_sub(self):
         """Ensure that field substitutions occur properly in exec_js functions."""
 
@@ -2660,6 +2665,8 @@ class BlogPost(Document):
             title = StringField(primary_key=True)
             tags = ListField(StringField())
 
+        BlogPost.drop_collection()
+
         post1 = BlogPost(title="Post #1", tags=["mongodb", "mongoengine"])
         post2 = BlogPost(title="Post #2", tags=["django", "mongodb"])
         post3 = BlogPost(title="Post #3", tags=["hitchcock films"])
@@ -2688,20 +2695,22 @@ class BlogPost(Document):
             }
         """
 
-        results = BlogPost.objects.map_reduce(map_f, reduce_f, "myresults")
+        results = BlogPost.objects.order_by("_id").map_reduce(
+            map_f, reduce_f, "myresults2"
+        )
         results = list(results)
 
-        assert results[0].object == post1
-        assert results[1].object == post2
-        assert results[2].object == post3
+        assert len(results) == 3
+        assert results[0].object.id == post1.id
+        assert results[1].object.id == post2.id
+        assert results[2].object.id == post3.id
 
         BlogPost.drop_collection()
 
     def test_map_reduce_custom_output(self):
         """
         Test map/reduce custom output
         """
-        register_connection("test2", "mongoenginetest2")
 
         class Family(Document):
             id = IntField(primary_key=True)
@@ -2774,6 +2783,7 @@ class Person(Document):
                             family.persons.push(person);
                             family.totalAge += person.age;
                         });
+                        family.persons.sort((a, b) => (a.age > b.age))
                     }
                 });
 
@@ -2802,10 +2812,10 @@ class Person(Document):
             "_id": 1,
             "value": {
                 "persons": [
+                    {"age": 17, "name": "Tayza Mariana"},
                     {"age": 21, "name": "Wilson Jr"},
-                    {"age": 45, "name": "Wilson Father"},
                     {"age": 40, "name": "Eliana Costa"},
-                    {"age": 17, "name": "Tayza Mariana"},
+                    {"age": 45, "name": "Wilson Father"},
                 ],
                 "totalAge": 123,
             },
@@ -2815,9 +2825,9 @@ class Person(Document):
             "_id": 2,
             "value": {
                 "persons": [
+                    {"age": 10, "name": "Igor Gabriel"},
                     {"age": 16, "name": "Isabella Luanna"},
                     {"age": 36, "name": "Sandra Mara"},
-                    {"age": 10, "name": "Igor Gabriel"},
                 ],
                 "totalAge": 62,
             },
@@ -2827,8 +2837,8 @@ class Person(Document):
             "_id": 3,
             "value": {
                 "persons": [
-                    {"age": 30, "name": "Arthur WA"},
                     {"age": 25, "name": "Paula Leonel"},
+                    {"age": 30, "name": "Arthur WA"},
                 ],
                 "totalAge": 55,
             },
@@ -3109,6 +3119,7 @@ class Person(Document):
         freq = Person.objects.item_frequencies("city", normalize=True, map_reduce=True)
         assert freq == {"CRB": 0.5, None: 0.5}
 
+    @requires_mongodb_lt_42
     def test_item_frequencies_with_null_embedded(self):
         class Data(EmbeddedDocument):
             name = StringField()
@@ -3137,6 +3148,7 @@ class Person(Document):
         ot = Person.objects.item_frequencies("extra.tag", map_reduce=True)
         assert ot == {None: 1.0, "friend": 1.0}
 
+    @requires_mongodb_lt_42
     def test_item_frequencies_with_0_values(self):
         class Test(Document):
             val = IntField()
@@ -3151,6 +3163,7 @@ class Test(Document):
         ot = Test.objects.item_frequencies("val", map_reduce=False)
         assert ot == {0: 1}
 
+    @requires_mongodb_lt_42
     def test_item_frequencies_with_False_values(self):
         class Test(Document):
             val = BooleanField()
@@ -3165,6 +3178,7 @@ class Test(Document):
         ot = Test.objects.item_frequencies("val", map_reduce=False)
         assert ot == {False: 1}
 
+    @requires_mongodb_lt_42
     def test_item_frequencies_normalize(self):
         class Test(Document):
             val = IntField()
@@ -3551,7 +3565,8 @@ class Book(Document):
         Book.objects.create(title="The Stories", authors=[mark_twain, john_tolkien])
 
         authors = Book.objects.distinct("authors")
-        assert authors == [mark_twain, john_tolkien]
+        authors_names = {author.name for author in authors}
+        assert authors_names == {mark_twain.name, john_tolkien.name}
 
     def test_distinct_ListField_EmbeddedDocumentField_EmbeddedDocumentField(self):
         class Continent(EmbeddedDocument):
@@ -3588,7 +3603,8 @@ class Book(Document):
         assert country_list == [scotland, tibet]
 
         continent_list = Book.objects.distinct("authors.country.continent")
-        assert continent_list == [europe, asia]
+        continent_list_names = {c.continent_name for c in continent_list}
+        assert continent_list_names == {europe.continent_name, asia.continent_name}
 
     def test_distinct_ListField_ReferenceField(self):
         class Bar(Document):

tests/utils.py

@@ -34,6 +34,10 @@ def get_as_pymongo(doc):
     return doc.__class__.objects.as_pymongo().get(id=doc.id)
 
 
+def requires_mongodb_lt_42(func):
+    return _decorated_with_ver_requirement(func, (4, 2), oper=operator.lt)
+
+
 def requires_mongodb_gte_44(func):
     return _decorated_with_ver_requirement(func, (4, 4), oper=operator.ge)