Add Queryable Encryption builds on evergreen

Author: aclark4life

.evergreen/config.yml

@@ -12,6 +12,17 @@ post_error_fails_task: true
 post_timeout_secs: 1800 # 5 minutes
 
 functions:
+  "start csfle servers":
+    - command: ec2.assume_role
+      params:
+        role_arn: ${aws_test_secrets_role}
+    - command: subprocess.exec
+      params:
+        binary: bash
+        include_expansions_in_env: ["AWS_SECRET_ACCESS_KEY", "AWS_ACCESS_KEY_ID", "AWS_SESSION_TOKEN"]
+        args:
+          - ${DRIVERS_TOOLS}/.evergreen/csfle/setup.sh
+
   "setup":
     - command: git.get_project
       params:
@@ -48,15 +59,28 @@ functions:
         args:
           - ./.evergreen/run-tests.sh
 
+  "run encryption tests":
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        working_dir: "src"
+        include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI", "DJANGO_SETTINGS_MODULE", "CRYPT_SHARED_LIB_PATH"]
+        args:
+          - ./.evergreen/run-tests.sh
+          - encryption
+
   "teardown":
     - command: subprocess.exec
       params:
         binary: bash
         args:
           - ${DRIVERS_TOOLS}/.evergreen/teardown.sh
+          - ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh
 
 pre:
   - func: setup
+  - func: start csfle servers
   - func: bootstrap mongo-orchestration
 
 post:
@@ -67,6 +91,10 @@ tasks:
     commands:
       - func: "run unit tests"
 
+  - name: run-encryption-tests
+    commands:
+      - func: "run encryption tests"
+
 buildvariants:
   - name: tests-7-noauth-nossl
     display_name: Run Tests 7.0 NoAuth NoSSL
@@ -111,3 +139,23 @@ buildvariants:
       SSL: "ssl"
     tasks:
       - name: run-tests
+
+  - name: tests-8-qe-local
+    display_name: Run Tests 8.2 QE local KMS
+    run_on: rhel87-small
+    expansions:
+      MONGODB_VERSION: "8.2"
+      TOPOLOGY: replica_set
+      DJANGO_SETTINGS_MODULE: "encrypted_settings"
+    tasks:
+      - name: run-encryption-tests
+
+  - name: tests-8-qe-aws
+    display_name: Run Tests 8.2 QE AWS KMS
+    run_on: rhel87-small
+    expansions:
+      MONGODB_VERSION: "8.2"
+      TOPOLOGY: replica_set
+      DJANGO_SETTINGS_MODULE: "encrypted_aws_settings"
+    tasks:
+      - name: run-encryption-tests

.evergreen/run-tests.sh

@@ -2,11 +2,24 @@
 
 set -eux
 
-# Install django-mongodb-backend
+# Export secrets as environment variables
+# https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/csfle/README.md#usage
+if [[ "${1:-}" == "encryption" ]]; then
+    . ../secrets-export.sh
+fi
+
+# Set up virtual environment
 /opt/python/3.12/bin/python3 -m venv venv
 . venv/bin/activate
 python -m pip install -U pip
-pip install -e .
+
+# Install django-mongodb-backend
+if [[ "${1:-}" == "encryption" ]]; then
+    # Install encryption dependencies for the Queryable Encryption build
+    pip install -e '.[encryption]'
+else
+    pip install -e .
+fi
 
 # Install django and test dependencies
 git clone --branch mongodb-6.0.x https://github.com/mongodb-forks/django django_repo

.github/workflows/encrypted_aws_settings.py

@@ -0,0 +1,28 @@
+import os
+
+from encrypted_settings import *  # noqa: F403
+from pymongo.encryption import AutoEncryptionOpts
+
+DATABASES["encrypted"] = {  # noqa: F405
+    "ENGINE": "django_mongodb_backend",
+    "NAME": "djangotests_encrypted",
+    "OPTIONS": {
+        "auto_encryption_opts": AutoEncryptionOpts(
+            key_vault_namespace="djangotests_encrypted.__keyVault",
+            kms_providers={
+                "aws": {
+                    "accessKeyId": os.environ["AWS_ACCESS_KEY_ID"],
+                    "secretAccessKey": os.environ["AWS_SECRET_ACCESS_KEY"],
+                }
+            },
+            crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"],
+            crypt_shared_lib_required=True,
+        ),
+    },
+    "KMS_CREDENTIALS": {
+        "aws": {
+            "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
+            "region": os.environ["AWS_DEFAULT_REGION"],
+        }
+    },
+}

.github/workflows/encrypted_settings.py

@@ -11,7 +11,7 @@
         "auto_encryption_opts": AutoEncryptionOpts(
             key_vault_namespace="djangotests_encrypted.__keyVault",
             kms_providers={"local": {"key": os.urandom(96)}},
-            crypt_shared_lib_path=os.environ["GITHUB_WORKSPACE"] + "/lib/mongo_crypt_v1.so",
+            crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"],
         ),
         "directConnection": True,
     },

.github/workflows/test-python-encryption.yml

@@ -62,3 +62,4 @@ jobs:
         contents: read
     env:
       DJANGO_SETTINGS_MODULE: "encrypted_settings"
+      CRYPT_SHARED_LIB_PATH: "${{ github.workspace }}/lib/mongo_crypt_v1.so"