Code review fixes (2/x)

Author: aclark4life

django_mongodb_backend/base.py

@@ -250,9 +250,9 @@ def cursor(self):
 
     def get_database_version(self):
         """Return a tuple of the database's version."""
-        # Avoid using PyMongo to check the database version or require
-        # pymongocrypt>=1.14.2 which will contain a fix for the `buildInfo`
-        # command. https://jira.mongodb.org/browse/PYTHON-5429
+        # This can use PyMongo's
+        # `tuple(self.connection.server_info()["versionArray"])` on
+        # pymongocrypt>=1.14.2. See: https://jira.mongodb.org/browse/PYTHON-5429
         return tuple(self.connection.admin.command("buildInfo")["versionArray"])
 
     ## Transaction API for django_mongodb_backend.transaction.atomic()

django_mongodb_backend/management/commands/showencryptedfieldsmap.py

@@ -26,16 +26,16 @@ def add_arguments(self, parser):
 
     def handle(self, *args, **options):
         db = options["database"]
-        create_data_keys = options.get("create_new_keys", False)
+        create_new_keys = options.get("`create_new_keys`", False)
         connection = connections[db]
         client = connection.connection
         encrypted_fields_map = {}
-        auto_encryption_opts = getattr(client._options, "auto_encryption_opts", None)
-        for app_config in apps.get_app_configs():
-            for model in router.get_migratable_models(app_config, db):
-                if model_has_encrypted_fields(model):
-                    fields = connection.schema_editor()._get_encrypted_fields_map(
-                        model, client, auto_encryption_opts, create_data_keys=create_data_keys
-                    )
-                    encrypted_fields_map[model._meta.db_table] = fields
+        with connection.schema_editor() as editor:
+            for app_config in apps.get_app_configs():
+                for model in router.get_migratable_models(app_config, db):
+                    if model_has_encrypted_fields(model):
+                        fields = editor._get_encrypted_fields_map(
+                            model, client, create_new_keys=create_new_keys
+                        )
+                        encrypted_fields_map[model._meta.db_table] = fields
         self.stdout.write(json_util.dumps(encrypted_fields_map, indent=2))

django_mongodb_backend/schema.py

@@ -450,16 +450,16 @@ def _create_collection(self, model):
             else:
                 # If the encrypted fields map is provided, get the map for the
                 # specific collection.
-                encrypted_fields_map = encrypted_fields_map.get(db_table, None)
+                encrypted_fields_map = encrypted_fields_map.get(db_table)
             db.create_collection(db_table, encryptedFields=encrypted_fields_map)
         else:
             db.create_collection(db_table)
 
-    def _get_encrypted_fields_map(
-        self, model, client, auto_encryption_opts, create_data_keys=False
-    ):
+    def _get_encrypted_fields_map(self, model, client, create_new_keys=False):
         connection = self.connection
         fields = model._meta.fields
+        options = client._options
+        auto_encryption_opts = options.auto_encryption_opts
         kms_provider = router.kms_provider(model)
         master_key = self.connection.settings_dict.get("KMS_CREDENTIALS").get(kms_provider)
         client_encryption = ClientEncryption(
@@ -475,7 +475,7 @@ def _get_encrypted_fields_map(
         for field in fields:
             if getattr(field, "encrypted", False):
                 key_alt_name = f"{db_table}_{field.column}"
-                if not create_data_keys:
+                if not create_new_keys:
                     key_doc = key_vault_collection.find_one({"keyAltNames": key_alt_name})
                     if not key_doc:
                         raise ValueError(f"No key found in keyvault for keyAltName={key_alt_name}")

docs/source/howto/queryable-encryption.rst

@@ -122,10 +122,11 @@ In addition to the :ref:`settings described in the how-to guide
 <server-side-queryable-encryption-settings>` you will need to provide a
 ``encrypted_fields_map`` to the ``AutoEncryptionOpts``.
 
-You can use the ``showencryptedfieldsmap`` management command to generate the
-schema map for your encrypted fields and then use the results in your settings::
+You can use the :djadmin:`showencryptedfieldsmap` management command to generate
+the schema map for your encrypted fields and then use the results in your
+settings::
 
-    python manage.py showencryptedfieldsmap
+    python manage.py showencryptedfieldsmap --database=encrypted
 
 .. admonition:: Didn't work?
 

docs/source/topics/queryable-encryption.rst

@@ -1,3 +1,4 @@
+====================
 Queryable Encryption
 ====================
 
@@ -10,7 +11,7 @@ data in MongoDB.
 .. _encrypted-field-example:
 
 The basics
-----------
+==========
 
 Let's consider this example::
 
@@ -28,7 +29,7 @@ Let's consider this example::
             return self.ssn
 
 Querying encrypted fields
--------------------------
+=========================
 
 The ``ssn`` field is only visible from an encrypted client connection. From an
 unencrypted client connection, the patient data looks like this:
@@ -40,8 +41,8 @@ unencrypted client connection, the patient data looks like this:
       ssn: Binary.createFromBase64('DkrbD67ejkt2u…', 6),
     }
 
-You can query encrypted fields using a
-:ref:`manual:qe-supported-query-operators` which must be specified in the
+You can query encrypted fields using the :ref:`supported query type options
+<manual:qe-fundamentals-encrypt-query>` which must be specified in the
 field definition. For example, to query the ``ssn`` field for equality, you
 can use the ``{"queryType": "equality"}`` operator as shown in the example
 above.

tests/encryption_/models.py

@@ -22,7 +22,6 @@
 
 EQUALITY_QUERY = {"queryType": "equality"}
 RANGE_QUERY = {"queryType": "range"}
-RANGE_QUERY_MIN_MAX = {"queryType": "range", "min": 0, "max": 100}
 
 
 class Appointment(models.Model):
@@ -53,7 +52,9 @@ class PatientRecord(models.Model):
     ssn = EncryptedCharField(max_length=11, queries=EQUALITY_QUERY)
     birth_date = EncryptedDateField(queries=RANGE_QUERY)
     profile_picture = EncryptedBinaryField(queries=EQUALITY_QUERY)
-    patient_age = EncryptedIntegerField("patient_age", queries=RANGE_QUERY_MIN_MAX)
+    patient_age = EncryptedIntegerField(
+        "patient_age", queries={**RANGE_QUERY, "min": 0, "max": 100}
+    )
     weight = EncryptedFloatField(queries=RANGE_QUERY)
 
     # TODO: Embed Billing model

tests/encryption_/routers.py

@@ -8,11 +8,6 @@ class TestEncryptedRouter:
     DATABASE_ROUTERS=[TestEncryptedRouter()]) takes effect.
     """
 
-    def allow_migrate(self, db, app_label, model_name=None, model=None, **hints):
-        if model:
-            return db == ("encrypted" if model_has_encrypted_fields(model) else "default")
-        return db == "default"
-
     def db_for_read(self, model, **hints):
         if model_has_encrypted_fields(model):
             return "encrypted"

tests/encryption_/test_base.py

@@ -1,40 +1,19 @@
-from datetime import datetime, time
+from datetime import datetime
 
-import pymongo
-from bson.binary import Binary
-from django.conf import settings
-from django.db import connections, models
 from django.test import TransactionTestCase, override_settings
 
-from django_mongodb_backend.fields import EncryptedFieldMixin
-
 from .models import (
     Appointment,
     Billing,
-    EncryptedNumbers,
     Patient,
     PatientPortalUser,
     PatientRecord,
 )
 from .routers import TestEncryptedRouter
 
 
-class EncryptedDurationField(EncryptedFieldMixin, models.DurationField):
-    """
-    Unsupported by MongoDB when used with Queryable Encryption.
-    Included in tests until fix or wontfix.
-    """
-
-
-class EncryptedSlugField(EncryptedFieldMixin, models.SlugField):
-    """
-    Unsupported by MongoDB when used with Queryable Encryption.
-    Included in tests until fix or wontfix.
-    """
-
-
 @override_settings(DATABASE_ROUTERS=[TestEncryptedRouter()])
-class EncryptedFieldTests(TransactionTestCase):
+class QueryableEncryptionTestCase(TransactionTestCase):
     databases = {"default", "encrypted"}
     available_apps = ["encryption_"]
 
@@ -65,142 +44,3 @@ def setUp(self):
 
         # TODO: Embed billing and patient_record models in patient model
         # then add tests
-
-    def test_get_encrypted_fields_map(self):
-        """Test class method called by schema editor
-        and management command to get encrypted fields map for
-        `create_encrypted_collection` and `auto_encryption_opts` respectively.
-        There are no data keys in the results.
-
-        Data keys for the schema editor are created by
-        `create_encrypted_collection` and data keys for the
-        management command are created by the management command
-        using code similar to the code in `create_encrypted_collection`
-        in Pymongo.
-        """
-        expected_encrypted_fields_map = {
-            "fields": [
-                {
-                    "bsonType": "int",
-                    "path": "patient_id",
-                    "queries": {"queryType": "equality"},
-                },
-                {
-                    "bsonType": "string",
-                    "path": "patient_name",
-                },
-                {
-                    "bsonType": "string",
-                    "path": "patient_notes",
-                    "queries": {"queryType": "equality"},
-                },
-                {
-                    "bsonType": "date",
-                    "path": "registration_date",
-                    "queries": {"queryType": "equality"},
-                },
-                {
-                    "bsonType": "bool",
-                    "path": "is_active",
-                    "queries": {"queryType": "equality"},
-                },
-                {
-                    "bsonType": "string",
-                    "path": "email",
-                    "queries": {"queryType": "equality"},
-                },
-            ]
-        }
-        connection = connections["encrypted"]
-        auto_encryption_opts = getattr(connection.connection._options, "auto_encryption_opts", None)
-        with connection.schema_editor() as editor:
-            client = connection.connection
-            encrypted_fields_map = editor._get_encrypted_fields_map(
-                self.patient, client, auto_encryption_opts
-            )
-            for field in encrypted_fields_map["fields"]:
-                # Remove data keys from the output; they are expected to differ
-                field.pop("keyId", None)
-            self.assertEqual(
-                encrypted_fields_map,
-                expected_encrypted_fields_map,
-            )
-
-    def test_appointment(self):
-        self.assertEqual(Appointment.objects.get(time="8:00").time, time(8, 0))
-
-    def test_billing(self):
-        self.assertEqual(
-            Billing.objects.get(cc_number=1234567890123456).cc_number, 1234567890123456
-        )
-        self.assertEqual(Billing.objects.get(cc_type="Visa").cc_type, "Visa")
-        self.assertTrue(Billing.objects.filter(account_balance__gte=100.0).exists())
-
-    def test_patientportaluser(self):
-        self.assertEqual(
-            PatientPortalUser.objects.get(ip_address="127.0.0.1").ip_address, "127.0.0.1"
-        )
-
-    def test_patientrecord(self):
-        self.assertEqual(PatientRecord.objects.get(ssn="123-45-6789").ssn, "123-45-6789")
-        with self.assertRaises(PatientRecord.DoesNotExist):
-            PatientRecord.objects.get(ssn="000-00-0000")
-        self.assertTrue(PatientRecord.objects.filter(birth_date__gte="1969-01-01").exists())
-        self.assertEqual(
-            PatientRecord.objects.get(ssn="123-45-6789").profile_picture, b"image data"
-        )
-        self.assertTrue(PatientRecord.objects.filter(patient_age__gte=40).exists())
-        self.assertFalse(PatientRecord.objects.filter(patient_age__gte=80).exists())
-        self.assertTrue(PatientRecord.objects.filter(weight__gte=175.0).exists())
-
-        # Test encrypted patient record in unencrypted database.
-        conn_params = connections["encrypted"].get_connection_params()
-        db_name = settings.DATABASES["encrypted"]["NAME"]
-        if conn_params.pop("auto_encryption_opts", False):
-            # Call MongoClient instead of get_new_connection because
-            # get_new_connection will return the encrypted connection
-            # from the connection pool.
-            with pymongo.MongoClient(**conn_params) as new_connection:
-                patientrecords = new_connection[db_name].encryption__patientrecord.find()
-                ssn = patientrecords[0]["ssn"]
-                self.assertTrue(isinstance(ssn, Binary))
-
-    def test_patient(self):
-        self.assertEqual(
-            Patient.objects.get(patient_notes="patient notes " * 25).patient_notes,
-            "patient notes " * 25,
-        )
-        self.assertEqual(
-            Patient.objects.get(
-                registration_date=datetime(2023, 10, 1, 12, 0, 0)
-            ).registration_date,
-            datetime(2023, 10, 1, 12, 0, 0),
-        )
-        self.assertTrue(Patient.objects.get(patient_id=1).is_active)
-        self.assertEqual(
-            Patient.objects.get(email="[email protected]").email, "[email protected]"
-        )
-
-        # Test decrypted patient record in encrypted database.
-        patients = connections["encrypted"].database.encryption__patient.find()
-        self.assertEqual(len(list(patients)), 1)
-        records = connections["encrypted"].database.encryption__patientrecord.find()
-        self.assertTrue("__safeContent__" in records[0])
-
-    def test_numeric_fields(self):
-        """
-        Fields that have not been tested elsewhere.
-        """
-        EncryptedNumbers.objects.create(
-            pos_bigint=1000000,
-            pos_smallint=12345,
-            smallint=-12345,
-        )
-
-        obj = EncryptedNumbers.objects.get(pos_bigint=1000000)
-        obj = EncryptedNumbers.objects.get(pos_smallint=12345)
-        obj = EncryptedNumbers.objects.get(smallint=-12345)
-
-        self.assertEqual(obj.pos_bigint, 1000000)
-        self.assertEqual(obj.pos_smallint, 12345)
-        self.assertEqual(obj.smallint, -12345)