Merge branch 'main' into DRIVERS-2917

Author: aclark4life

.evergreen/config.yml

@@ -14,6 +14,46 @@ post_timeout_secs: 1800 # 5 minutes
 include:
   - filename: .evergreen/functions.yml
 
+  # Encryption-specific functions
+  "start csfle servers":
+    - command: ec2.assume_role
+      params:
+        role_arn: ${aws_test_secrets_role}
+    - command: subprocess.exec
+      params:
+        binary: bash
+        include_expansions_in_env: [
+            "AWS_SECRET_ACCESS_KEY",
+            "AWS_ACCESS_KEY_ID",
+            "AWS_SESSION_TOKEN",
+        ]
+        args:
+          - ${DRIVERS_TOOLS}/.evergreen/csfle/setup.sh
+
+  "teardown csfle":
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh
+
+  "run encryption tests":
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        working_dir: "src"
+        include_expansions_in_env: [
+            "AWS_KMS_ARN",
+            "DRIVERS_TOOLS",
+            "MONGODB_URI",
+            "DJANGO_SETTINGS_MODULE",
+            "CRYPT_SHARED_LIB_PATH",
+        ]
+        args:
+          - ./.evergreen/run-tests.sh
+          - encryption
+
 pre:
   - func: setup
   - func: bootstrap mongo-orchestration
@@ -31,24 +71,29 @@ tasks:
       - func: "run performance tests"
       - func: "attach benchmark test results"
       - func: "send dashboard data"
+  - name: run-encryption-tests
+    commands:
+      - func: "start csfle servers"
+      - func: "run encryption tests"
+      - func: "teardown csfle"
 
 buildvariants:
-  - name: tests-6-noauth-nossl
-    display_name: Run Tests 6.0 NoAuth NoSSL
+  - name: tests-7-noauth-nossl
+    display_name: Run Tests 7.0 NoAuth NoSSL
     run_on: rhel87-small
     expansions:
-      MONGODB_VERSION: "6.0"
+      MONGODB_VERSION: "7.0"
       TOPOLOGY: server
       AUTH: "noauth"
       SSL: "nossl"
     tasks:
       - name: run-tests
 
-  - name: tests-6-auth-ssl
-    display_name: Run Tests 6.0 Auth SSL
+  - name: tests-7-auth-ssl
+    display_name: Run Tests 7.0 Auth SSL
     run_on: rhel87-small
     expansions:
-      MONGODB_VERSION: "6.0"
+      MONGODB_VERSION: "7.0"
       TOPOLOGY: server
       AUTH: "auth"
       SSL: "ssl"
@@ -84,3 +129,22 @@ buildvariants:
     batchtime: 1440
     tasks:
       - name: perf-tests
+  - name: tests-8-qe-local
+    display_name: Run Tests 8.2 QE local KMS
+    run_on: rhel87-small
+    expansions:
+      MONGODB_VERSION: "8.2"
+      TOPOLOGY: replica_set
+      DJANGO_SETTINGS_MODULE: "encrypted_settings"
+    tasks:
+      - name: run-encryption-tests
+
+  - name: tests-8-qe-aws
+    display_name: Run Tests 8.2 QE AWS KMS
+    run_on: rhel87-small
+    expansions:
+      MONGODB_VERSION: "8.2"
+      TOPOLOGY: replica_set
+      DJANGO_SETTINGS_MODULE: "encrypted_aws_settings"
+    tasks:
+      - name: run-encryption-tests

.evergreen/run-tests.sh

@@ -2,14 +2,27 @@
 
 set -eux
 
-# Install django-mongodb-backend
-/opt/python/3.10/bin/python3 -m venv venv
+# Export secrets as environment variables
+# https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/README.md#usage
+if [[ "${1:-}" == "encryption" ]]; then
+    . ../secrets-export.sh
+fi
+
+# Set up virtual environment
+/opt/python/3.12/bin/python3 -m venv venv
 . venv/bin/activate
 python -m pip install -U pip
-pip install -e .
+
+# Install django-mongodb-backend
+if [[ "${1:-}" == "encryption" ]]; then
+    # Install encryption dependencies for the Queryable Encryption build
+    pip install -e '.[encryption]'
+else
+    pip install -e .
+fi
 
 # Install django and test dependencies
-git clone --branch mongodb-5.2.x https://github.com/mongodb-forks/django django_repo
+git clone --branch mongodb-6.0.x https://github.com/mongodb-forks/django django_repo
 pushd django_repo/tests/
 pip install -e ..
 pip install -r requirements/py3.txt

.github/dependabot.yml

@@ -5,6 +5,8 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    cooldown:
+      default-days: 7
     groups:
       actions:
         patterns:

.github/workflows/codeql.yml

@@ -56,7 +56,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@cf1bb45a277cb3c205638b2cd5c984db1c46a412 # v4.31.7
+      uses: github/codeql-action/init@cdefb33c0f6224e58673d9004f47f7cb3e328b89 # v4.31.10
       with:
         languages: ${{ matrix.language }}
         build-mode: none
@@ -72,6 +72,6 @@ jobs:
         pip install -e .
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@cf1bb45a277cb3c205638b2cd5c984db1c46a412 # v4.31.7
+      uses: github/codeql-action/analyze@cdefb33c0f6224e58673d9004f47f7cb3e328b89 # v4.31.10
       with:
         category: "/language:${{ matrix.language }}"

.github/workflows/dist.yml

@@ -32,7 +32,7 @@ jobs:
     - name: Create packages
       run: python -m build .
     - name: Store package artifacts
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
           name: all-dist-${{ github.run_id }}
           path: "dist/*"

.github/workflows/encrypted_aws_settings.py

@@ -0,0 +1,29 @@
+# Settings for django_mongodb_backend/tests with AWS Key Management System.
+import os
+
+from encrypted_settings import *  # noqa: F403
+from pymongo.encryption import AutoEncryptionOpts
+
+DATABASES["encrypted"] = {  # noqa: F405
+    "ENGINE": "django_mongodb_backend",
+    "NAME": "djangotests_encrypted",
+    "OPTIONS": {
+        "auto_encryption_opts": AutoEncryptionOpts(
+            key_vault_namespace="djangotests_encrypted.__keyVault",
+            kms_providers={
+                "aws": {
+                    "accessKeyId": os.environ["AWS_ACCESS_KEY_ID"],
+                    "secretAccessKey": os.environ["AWS_SECRET_ACCESS_KEY"],
+                }
+            },
+            crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"],
+            crypt_shared_lib_required=True,
+        ),
+    },
+    "KMS_CREDENTIALS": {
+        "aws": {
+            "key": os.environ["AWS_KMS_ARN"],
+            "region": os.environ["AWS_DEFAULT_REGION"],
+        }
+    },
+}

.github/workflows/encrypted_settings.py

@@ -0,0 +1,41 @@
+# Settings for django_mongodb_backend/tests when encryption is supported.
+import os
+
+from mongodb_settings import *  # noqa: F403
+from pymongo.encryption import AutoEncryptionOpts
+
+DATABASES["encrypted"] = {  # noqa: F405
+    "ENGINE": "django_mongodb_backend",
+    "NAME": "djangotests_encrypted",
+    "OPTIONS": {
+        "auto_encryption_opts": AutoEncryptionOpts(
+            key_vault_namespace="djangotests_encrypted.__keyVault",
+            kms_providers={"local": {"key": os.urandom(96)}},
+            crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"],
+            crypt_shared_lib_required=True,
+        ),
+        "directConnection": True,
+    },
+}
+
+
+class EncryptedRouter:
+    def db_for_read(self, model, **hints):
+        # All models in the encryption_ app use the encrypted database.
+        if model._meta.app_label == "encryption_":
+            return "encrypted"
+        return None
+
+    db_for_write = db_for_read
+
+    def allow_migrate(self, db, app_label, model_name=None, **hints):
+        # Create the encryption_ app's models only in the encrypted database.
+        if app_label == "encryption_":
+            return db == "encrypted"
+        # Don't create other apps' models in the encrypted database.
+        if db == "encrypted":
+            return False
+        return None
+
+
+DATABASE_ROUTERS.append(EncryptedRouter())  # noqa: F405

.github/workflows/linters.yml

@@ -17,7 +17,7 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-python@v6
         with:
-          python-version: '3.10'
+          python-version: '3.12'
           cache: 'pip'
           cache-dependency-path: 'pyproject.toml'
       - name: Install Python dependencies
@@ -39,7 +39,7 @@ jobs:
         with:
           cache: 'pip'
           cache-dependency-path: 'pyproject.toml'
-          python-version: '3.10'
+          python-version: '3.12'
       - name: Install dependencies
         run: |
           pip install -U pip

.github/workflows/mongodb_settings.py

@@ -1,4 +1,5 @@
-# Settings for django_mongodb_backend/tests.
+# Settings for django_mongodb_backend/tests when encryption isn't supported.
 from django_settings import *  # noqa: F403
 
+DATABASES["encrypted"] = {}  # noqa: F405
 DATABASE_ROUTERS = ["django_mongodb_backend.routers.MongoRouter"]

.github/workflows/release-python.yml

@@ -75,7 +75,7 @@ jobs:
       id-token: write
     steps:
       - name: Download all the dists
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v7
         with:
           name: all-dist-${{ github.run_id }}
           path: dist/