From 8025271a7df1b027bf24a39fed7458064109dba8 Mon Sep 17 00:00:00 2001
From: Steven Silvester <steve.silvester@mongodb.com>
Date: Fri, 14 Mar 2025 07:57:39 -0500
Subject: [PATCH 01/12] add files

---
 .evergreen/config.yml                       | 80 +--------------------
 .evergreen/generated_configs/tasks.yml      | 25 +++++++
 .evergreen/generated_configs/variants.yml   | 10 ++-
 .evergreen/run-deployed-lambda-aws-tests.sh | 10 ---
 .evergreen/scripts/generate_config.py       | 32 ++++++++-
 .evergreen/scripts/run_server.py            | 20 +++---
 .evergreen/scripts/run_tests.py             | 32 ++++++++-
 .evergreen/scripts/setup_tests.py           | 12 +++-
 .evergreen/scripts/teardown_tests.py        |  4 ++
 .evergreen/scripts/utils.py                 |  5 +-
 CONTRIBUTING.md                             | 11 +++
 test/lambda/build.sh                        |  2 +
 12 files changed, 139 insertions(+), 104 deletions(-)
 delete mode 100755 .evergreen/run-deployed-lambda-aws-tests.sh

diff --git a/.evergreen/config.yml b/.evergreen/config.yml
index 24f08d67af..420aba25fd 100644
--- a/.evergreen/config.yml
+++ b/.evergreen/config.yml
@@ -293,27 +293,7 @@ functions:
     - command: ec2.assume_role
       params:
         role_arn: ${aws_test_secrets_role}
-
-  "setup atlas":
-    - command: subprocess.exec
-      params:
-        binary: bash
-        include_expansions_in_env: ["task_id", "execution"]
-        env:
-          MONGODB_VERSION: "7.0"
-          LAMBDA_STACK_NAME: dbx-python-lambda
-        args:
-          - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh
-    - command: expansions.update
-      params:
-        file: atlas-expansion.yml
-
-  "teardown atlas":
-    - command: subprocess.exec
-      params:
-        binary: bash
-        args:
-          - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh
+        duration_seconds: 3600
 
   "run perf tests":
     - command: subprocess.exec
@@ -350,31 +330,6 @@ post:
   - func: "upload test results"
   - func: "cleanup"
 
-task_groups:
-  - name: test_aws_lambda_task_group
-    setup_group:
-      - func: fetch source
-      - func: setup system
-      - func: setup atlas
-    teardown_task:
-      - func: teardown atlas
-    setup_group_can_fail_task: true
-    setup_group_timeout_secs: 1800
-    tasks:
-      - test-aws-lambda-deployed
-
-  - name: test_atlas_task_group_search_indexes
-    setup_group:
-      - func: fetch source
-      - func: setup system
-      - func: setup atlas
-    teardown_task:
-      - func: teardown atlas
-    setup_group_can_fail_task: true
-    setup_group_timeout_secs: 1800
-    tasks:
-      - test-search-index-helpers
-
 tasks:
     # Wildcard task. Do you need to find out what tools are available and where?
     # Throw it here, and execute this task on all buildvariants
@@ -401,17 +356,6 @@ tasks:
         - func: "run server"
         - func: "run doctests"
 
-    - name: "test-search-index-helpers"
-      commands:
-        - func: "run server"
-          vars:
-            VERSION: "6.0"
-            TOPOLOGY: "replica_set"
-        - func: "run tests"
-          vars:
-            TEST_NAME: index_management
-            AUTH: "auth"
-
     - name: "mod-wsgi-standalone"
       tags: ["mod_wsgi"]
       commands:
@@ -467,22 +411,6 @@ tasks:
           vars:
             TEST_NAME: "data_lake"
 
-    - name: "test-aws-lambda-deployed"
-      commands:
-        - command: ec2.assume_role
-          params:
-            role_arn: ${LAMBDA_AWS_ROLE_ARN}
-            duration_seconds: 3600
-        - command: subprocess.exec
-          params:
-            working_dir: src
-            binary: bash
-            add_expansions_to_env: true
-            args:
-              - .evergreen/run-deployed-lambda-aws-tests.sh
-            env:
-              TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/test/lambda
-
 # }}}
     - name: "coverage-report"
       tags: ["coverage"]
@@ -637,12 +565,6 @@ buildvariants:
       batchtime: 10080  # 7 days
     - name: test-azurekms-fail
 
-- name: rhel8-test-lambda
-  display_name: FaaS Lambda
-  run_on: rhel87-small
-  tasks:
-    - name: test_aws_lambda_task_group
-
 - name: rhel8-import-time
   display_name: Import Time
   run_on: rhel87-small
diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml
index 0b0f09329a..5a44e0b59a 100644
--- a/.evergreen/generated_configs/tasks.yml
+++ b/.evergreen/generated_configs/tasks.yml
@@ -8,6 +8,16 @@ tasks:
           TEST_NAME: atlas_connect
     tags: [atlas_connect]
 
+  # Aws lambda tests
+  - name: test-aws-lambda-deployed
+    commands:
+      - func: assume ec2 role
+      - func: setup atlas
+      - func: run tests
+        vars:
+          TEST_NAME: aws_lambda
+    tags: [aws_lambda]
+
   # Aws tests
   - name: test-auth-aws-4.4-regular
     commands:
@@ -696,11 +706,13 @@ tasks:
         vars:
           TEST_NAME: enterprise_auth
           AUTH: auth
+          FOO: ${THIS THING}
       - func: assume ec2 role
       - func: run tests
         vars:
           TEST_NAME: enterprise_auth
           AUTH: auth
+          FOO: ${THIS THING}
     tags: [enterprise_auth]
 
   # Kms tests
@@ -1109,6 +1121,19 @@ tasks:
           SUB_TEST_NAME: gke
     tags: [auth_oidc, auth_oidc_remote]
 
+  # Search index tests
+  - name: test-search-index-helpers
+    commands:
+      - func: assume ec2 role
+      - func: setup atlas
+      - func: run server
+        vars:
+          TEST_NAME: index_management
+      - func: run tests
+        vars:
+          TEST_NAME: index_management
+    tags: [index_managment]
+
   # Server tests
   - name: test-4.0-standalone-auth-ssl-sync
     commands:
diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml
index 4c54abf4b9..605465345c 100644
--- a/.evergreen/generated_configs/variants.yml
+++ b/.evergreen/generated_configs/variants.yml
@@ -154,6 +154,14 @@ buildvariants:
     expansions:
       PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3
 
+  # Aws lambda tests
+  - name: faas-lambda
+    tasks:
+      - name: .aws_lambda
+    display_name: FaaS Lambda
+    run_on:
+      - rhel87-small
+
   # Compression tests
   - name: compression-snappy-rhel8-python3.9-no-c
     tasks:
@@ -1005,7 +1013,7 @@ buildvariants:
   # Search index tests
   - name: search-index-helpers-rhel8-python3.9
     tasks:
-      - name: test_atlas_task_group_search_indexes
+      - name: .index_management
     display_name: Search Index Helpers RHEL8 Python3.9
     run_on:
       - rhel87-small
diff --git a/.evergreen/run-deployed-lambda-aws-tests.sh b/.evergreen/run-deployed-lambda-aws-tests.sh
deleted file mode 100755
index aa16d62650..0000000000
--- a/.evergreen/run-deployed-lambda-aws-tests.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-set -o errexit  # Exit the script with error if any of the commands fail
-
-export PATH="/opt/python/3.9/bin:${PATH}"
-python --version
-pushd ./test/lambda
-
-. build.sh
-popd
-. ${DRIVERS_TOOLS}/.evergreen/aws_lambda/run-deployed-lambda-aws-tests.sh
diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py
index d91e0e6ded..5dfe37a2b9 100644
--- a/.evergreen/scripts/generate_config.py
+++ b/.evergreen/scripts/generate_config.py
@@ -680,7 +680,7 @@ def create_search_index_variants():
     python = CPYTHONS[0]
     return [
         create_variant(
-            ["test_atlas_task_group_search_indexes"],
+            [".index_management"],
             get_display_name("Search Index Helpers", host, python=python),
             python=python,
             host=host,
@@ -780,6 +780,11 @@ def create_alternative_hosts_variants():
     return variants
 
 
+def create_aws_lambda_variants():
+    host = HOSTS["rhel8"]
+    return [create_variant([".aws_lambda"], display_name="FaaS Lambda", host=host)]
+
+
 ##############
 # Tasks
 ##############
@@ -910,6 +915,29 @@ def _create_ocsp_task(algo, variant, server_type, base_task_name):
     return EvgTask(name=task_name, tags=tags, commands=commands)
 
 
+def create_aws_lambda_tasks():
+    assume_func = FunctionCall(func="assume ec2 role")
+    atlas_func = FunctionCall(func="setup atlas")
+    vars = dict(TEST_NAME="aws_lambda")
+    test_func = FunctionCall(func="run tests", vars=vars)
+    task_name = "test-aws-lambda-deployed"
+    tags = ["aws_lambda"]
+    commands = [assume_func, atlas_func, test_func]
+    return [EvgTask(name=task_name, tags=tags, commands=commands)]
+
+
+def create_search_index_tasks():
+    assume_func = FunctionCall(func="assume ec2 role")
+    atlas_func = FunctionCall(func="setup atlas")
+    server_func = FunctionCall(func="run server", vars=dict(TEST_NAME="index_management"))
+    vars = dict(TEST_NAME="index_management")
+    test_func = FunctionCall(func="run tests", vars=vars)
+    task_name = "test-search-index-helpers"
+    tags = ["index_managment"]
+    commands = [assume_func, atlas_func, server_func, test_func]
+    return [EvgTask(name=task_name, tags=tags, commands=commands)]
+
+
 def create_atlas_connect_tasks():
     vars = dict(TEST_NAME="atlas_connect")
     assume_func = FunctionCall(func="assume ec2 role")
@@ -920,7 +948,7 @@ def create_atlas_connect_tasks():
 
 
 def create_enterprise_auth_tasks():
-    vars = dict(TEST_NAME="enterprise_auth", AUTH="auth")
+    vars = dict(TEST_NAME="enterprise_auth", AUTH="auth", FOO="${THIS THING}")
     server_func = FunctionCall(func="run server", vars=vars)
     assume_func = FunctionCall(func="assume ec2 role")
     test_func = FunctionCall(func="run tests", vars=vars)
diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py
index f43ada4bbb..6b2e1d96ef 100644
--- a/.evergreen/scripts/run_server.py
+++ b/.evergreen/scripts/run_server.py
@@ -22,14 +22,6 @@ def start_server():
     if "VERSION" in os.environ:
         os.environ["MONGODB_VERSION"] = os.environ["VERSION"]
 
-    if opts.auth:
-        extra_opts.append("--auth")
-
-    if opts.verbose:
-        extra_opts.append("-v")
-    elif opts.quiet:
-        extra_opts.append("-q")
-
     if test_name == "auth_aws":
         set_env("AUTH_AWS")
 
@@ -51,6 +43,10 @@ def start_server():
             if not found:
                 raise ValueError("Please provide an orchestration file")
 
+    elif test_name == "index_management":
+        os.environ["TOPOLOGY"] = "replica_set"
+        os.environ["MONGODB_VERSION"] = "6.0"
+
     if not os.environ.get("TEST_CRYPT_SHARED"):
         set_env("SKIP_CRYPT_SHARED")
 
@@ -62,6 +58,14 @@ def start_server():
             set_env("TLS_PEM_KEY_FILE", certs / "server.pem")
             set_env("TLS_CA_FILE", certs / "ca.pem")
 
+    if opts.auth:
+        extra_opts.append("--auth")
+
+    if opts.verbose:
+        extra_opts.append("-v")
+    elif opts.quiet:
+        extra_opts.append("-q")
+
     cmd = ["bash", f"{DRIVERS_TOOLS}/.evergreen/run-orchestration.sh", *extra_opts]
     run_command(cmd, cwd=DRIVERS_TOOLS)
 
diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py
index 38fd3c67cb..3ff3ba488b 100644
--- a/.evergreen/scripts/run_tests.py
+++ b/.evergreen/scripts/run_tests.py
@@ -6,10 +6,13 @@
 import platform
 import shutil
 import sys
+import tarfile
+import tempfile
 from datetime import datetime
+from shutil import which
 
 import pytest
-from utils import DRIVERS_TOOLS, LOGGER, ROOT, run_command
+from utils import DRIVERS_TOOLS, LOGGER, ROOT, create_archive, run_command
 
 AUTH = os.environ.get("AUTH", "noauth")
 SSL = os.environ.get("SSL", "nossl")
@@ -82,6 +85,28 @@ def handle_pymongocrypt() -> None:
     LOGGER.info(f"libmongocrypt version: {pymongocrypt.libmongocrypt_version()})")
 
 
+def handle_aws_lambda() -> None:
+    env = os.environ.copy()
+    target_dir = ROOT / "test/lambda"
+    env["TEST_LAMBDA_DIRECTORY"] = str(target_dir)
+    archive = create_archive()
+    with tempfile.TemporaryDirectory() as td:
+        # Unpack the archive
+        fid = tarfile.open(archive)
+        fid.extractall(td)
+        fid.close()
+        # Build the c extensions.
+        docker = which("docker") or which("podman")
+        if not docker:
+            raise ValueError("Could not find docker!")
+        image = "quay.io/pypa/manylinux2014_x86_64:latest"
+        run_command(f'{docker} run --rm -v "{td}:/src" {image} /src/test/lambda/build_internal.sh')
+        shutil.copytree(td, ROOT / "test/lambda/mongodb")
+
+    script_name = "run-deployed-lambda-aws-tests.sh"
+    run_command(f"bash {DRIVERS_TOOLS}/.evergreen/aws_lambda/{script_name}", env=env)
+
+
 def run() -> None:
     # Handle green framework first so they can patch modules.
     if GREEN_FRAMEWORK:
@@ -123,6 +148,11 @@ def run() -> None:
         test_oidc_send_to_remote(SUB_TEST_NAME)
         return
 
+    # Run deployed aws lambda tests.
+    if TEST_NAME == "aws_lambda":
+        handle_aws_lambda()
+        return
+
     if os.environ.get("DEBUG_LOG"):
         TEST_ARGS.extend(f"-o log_cli_level={logging.DEBUG} -o log_cli=1".split())
 
diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py
index 53e3a568ba..011f83af99 100644
--- a/.evergreen/scripts/setup_tests.py
+++ b/.evergreen/scripts/setup_tests.py
@@ -172,6 +172,16 @@ def handle_test_env() -> None:
         if not config:
             AUTH = "noauth"
 
+    if test_name in ["aws_lambda", "index_management"]:
+        env = os.environ.copy()
+        env["MONGODB_VERSION"] = "7.0"
+        env["LAMBDA_STACK_NAME"] = "dbx-python-lambda"
+        write_env("LAMBDA_STACK_NAME", env["LAMBDA_STACK_NAME"])
+    #    run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh", env=env, cwd=DRIVERS_TOOLS)
+
+    if test_name == "index_management":
+        AUTH = "auth"
+
     if AUTH != "noauth":
         if test_name == "data_lake":
             config = read_env(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/secrets-export.sh")
@@ -381,7 +391,7 @@ def handle_test_env() -> None:
         # Use --capture=tee-sys so pytest prints test output inline:
         # https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html
         TEST_ARGS = f"-v --capture=tee-sys --durations=5 {TEST_ARGS}"
-        TEST_SUITE = TEST_SUITE_MAP[test_name]
+        TEST_SUITE = TEST_SUITE_MAP.get(test_name)
         if TEST_SUITE:
             TEST_ARGS = f"-m {TEST_SUITE} {TEST_ARGS}"
 
diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py
index 3920180422..a562ecf478 100644
--- a/.evergreen/scripts/teardown_tests.py
+++ b/.evergreen/scripts/teardown_tests.py
@@ -38,6 +38,10 @@
 elif TEST_NAME == "serverless":
     run_command(f"bash {DRIVERS_TOOLS}/.evergreen/serverless/teardown.sh")
 
+# Tear down atlas cluster if applicable.
+if TEST_NAME in ["aws_lambda", "index_management"]:
+    run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh")
+
 # Tear down auth_aws if applicable.
 # We do not run web-identity hosts on macos, because the hosts lack permissions,
 # so there is no reason to run the teardown, which would error with a 401.
diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py
index 08d376461e..a8a49c8dcd 100644
--- a/.evergreen/scripts/utils.py
+++ b/.evergreen/scripts/utils.py
@@ -62,7 +62,7 @@ def get_test_options(
     if require_sub_test_name:
         parser.add_argument(
             "test_name",
-            choices=sorted(TEST_SUITE_MAP),
+            choices=sorted(TEST_SUITE_MAP) + ["aws_lambda", "index_management"],  # noqa: RUF005
             nargs="?",
             default="default",
             help="The optional name of the test suite to set up, typically the same name as a pytest marker.",
@@ -146,7 +146,8 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None:
     LOGGER.info("Running command '%s'... done.", cmd)
 
 
-def create_archive() -> None:
+def create_archive() -> str:
     run_command("git add .", cwd=ROOT)
     run_command('git commit -m "add files"', check=False, cwd=ROOT)
     run_command(f"git archive -o {TMP_DRIVER_FILE} HEAD", cwd=ROOT)
+    return TMP_DRIVER_FILE
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8844565d31..9576014ce0 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -275,6 +275,17 @@ Note: these tests can only be run from an Evergreen host.
 - Run `just setup-tests atlas_connect`.
 - Run `just run-tests`.
 
+### Search Index tests
+
+- Run `just run-server index_management`.
+- Run `just setup-tests index_management`.
+- Run `just run-tests`.
+
+### AWS Lambda tests
+
+- Run `just setup-tests aws_lambda`.
+- Run `just run-tests`.
+
 ### OCSP tests
 
   - Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`.
diff --git a/test/lambda/build.sh b/test/lambda/build.sh
index c7cc24eab2..6a2412238f 100755
--- a/test/lambda/build.sh
+++ b/test/lambda/build.sh
@@ -2,6 +2,8 @@
 set -o errexit  # Exit the script with error if any of the commands fail
 set -o xtrace
 
+# TODO: create a git archive, build the files, and then move to the location
+# This will be done in python
 rm -rf mongodb/pymongo
 rm -rf mongodb/gridfs
 rm -rf mongodb/bson

From 2a1affef0cb641d7b43bd327ce056052c7ff8e31 Mon Sep 17 00:00:00 2001
From: Steven Silvester <steve.silvester@mongodb.com>
Date: Fri, 14 Mar 2025 10:41:07 -0500
Subject: [PATCH 02/12] add more files

---
 .evergreen/scripts/run_tests.py   | 45 ++++++++++++++++++++-----------
 .evergreen/scripts/setup_tests.py |  3 +++
 .gitignore                        |  1 +
 test/lambda/build.sh              | 30 ---------------------
 4 files changed, 33 insertions(+), 46 deletions(-)
 delete mode 100755 test/lambda/build.sh

diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py
index 3ff3ba488b..c07dfc3224 100644
--- a/.evergreen/scripts/run_tests.py
+++ b/.evergreen/scripts/run_tests.py
@@ -6,13 +6,12 @@
 import platform
 import shutil
 import sys
-import tarfile
-import tempfile
 from datetime import datetime
+from pathlib import Path
 from shutil import which
 
 import pytest
-from utils import DRIVERS_TOOLS, LOGGER, ROOT, create_archive, run_command
+from utils import DRIVERS_TOOLS, LOGGER, ROOT, run_command
 
 AUTH = os.environ.get("AUTH", "noauth")
 SSL = os.environ.get("SSL", "nossl")
@@ -89,19 +88,33 @@ def handle_aws_lambda() -> None:
     env = os.environ.copy()
     target_dir = ROOT / "test/lambda"
     env["TEST_LAMBDA_DIRECTORY"] = str(target_dir)
-    archive = create_archive()
-    with tempfile.TemporaryDirectory() as td:
-        # Unpack the archive
-        fid = tarfile.open(archive)
-        fid.extractall(td)
-        fid.close()
-        # Build the c extensions.
-        docker = which("docker") or which("podman")
-        if not docker:
-            raise ValueError("Could not find docker!")
-        image = "quay.io/pypa/manylinux2014_x86_64:latest"
-        run_command(f'{docker} run --rm -v "{td}:/src" {image} /src/test/lambda/build_internal.sh')
-        shutil.copytree(td, ROOT / "test/lambda/mongodb")
+    env.setdefault("AWS_REGION", "us-east-1")
+    dirs = ["pymongo", "gridfs", "bson"]
+    # Store the original .so files.
+    before_sos = []
+    for dname in dirs:
+        before_sos.extend(f"{f.parent.name}/{f.name}" for f in (ROOT / dname).glob("*.so"))
+    # Build the c extensions.
+    docker = which("docker") or which("podman")
+    if not docker:
+        raise ValueError("Could not find docker!")
+    image = "quay.io/pypa/manylinux2014_x86_64:latest"
+    run_command(
+        f'{docker} run --rm -v "{ROOT}:/src" --platform linux/amd64 {image} /src/test/lambda/build_internal.sh'
+    )
+    for dname in dirs:
+        target = ROOT / "test/lambda/mongodb" / dname
+        shutil.rmtree(target, ignore_errors=True)
+        shutil.copytree(ROOT / dname, target)
+    # Remove the original so files from the lambda directory.
+    for so_path in before_sos:
+        (ROOT / "test/lambda/mongodb" / so_path).unlink()
+    # Remove the new so files from the ROOT directory.
+    for dname in dirs:
+        so_paths = [f"{f.parent.name}/{f.name}" for f in (ROOT / dname).glob("*.so")]
+        for so_path in list(so_paths):
+            if so_path not in before_sos:
+                Path(so_path).unlink()
 
     script_name = "run-deployed-lambda-aws-tests.sh"
     run_command(f"bash {DRIVERS_TOOLS}/.evergreen/aws_lambda/{script_name}", env=env)
diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py
index 011f83af99..adb6ae0f5a 100644
--- a/.evergreen/scripts/setup_tests.py
+++ b/.evergreen/scripts/setup_tests.py
@@ -182,6 +182,9 @@ def handle_test_env() -> None:
     if test_name == "index_management":
         AUTH = "auth"
 
+    if test_name == "aws_lambda":
+        UV_ARGS.append("--with pip")
+
     if AUTH != "noauth":
         if test_name == "data_lake":
             config = read_env(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/secrets-export.sh")
diff --git a/.gitignore b/.gitignore
index 8c095c2157..b06a08b2cb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,7 @@ mongocryptd.pid
 .idea/
 .vscode/
 .nova/
+.temp/
 venv/
 secrets-export.sh
 libmongocrypt.tar.gz
diff --git a/test/lambda/build.sh b/test/lambda/build.sh
deleted file mode 100755
index 6a2412238f..0000000000
--- a/test/lambda/build.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-set -o errexit  # Exit the script with error if any of the commands fail
-set -o xtrace
-
-# TODO: create a git archive, build the files, and then move to the location
-# This will be done in python
-rm -rf mongodb/pymongo
-rm -rf mongodb/gridfs
-rm -rf mongodb/bson
-
-pushd ../..
-rm -f pymongo/*.so
-rm -f bson/*.so
-image="quay.io/pypa/manylinux2014_x86_64:latest"
-
-DOCKER=$(command -v docker) || true
-if [ -z "$DOCKER" ]; then
-    PODMAN=$(command -v podman) || true
-    if [ -z "$PODMAN" ]; then
-        echo "docker or podman are required!"
-        exit 1
-    fi
-    DOCKER=podman
-fi
-
-$DOCKER run --rm -v "`pwd`:/src" $image /src/test/lambda/build_internal.sh
-cp -r pymongo ./test/lambda/mongodb/pymongo
-cp -r bson ./test/lambda/mongodb/bson
-cp -r gridfs ./test/lambda/mongodb/gridfs
-popd

From 7770cac944b8e1156b7cb1ebe8378224493f7942 Mon Sep 17 00:00:00 2001
From: Steven Silvester <steve.silvester@mongodb.com>
Date: Fri, 14 Mar 2025 13:36:41 -0500
Subject: [PATCH 03/12] finish logic

---
 .evergreen/scripts/setup_tests.py | 6 +++++-
 .gitignore                        | 2 +-
 CONTRIBUTING.md                   | 3 +++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py
index 13e1d0630b..3ad1187525 100644
--- a/.evergreen/scripts/setup_tests.py
+++ b/.evergreen/scripts/setup_tests.py
@@ -177,7 +177,11 @@ def handle_test_env() -> None:
         env["MONGODB_VERSION"] = "7.0"
         env["LAMBDA_STACK_NAME"] = "dbx-python-lambda"
         write_env("LAMBDA_STACK_NAME", env["LAMBDA_STACK_NAME"])
-    #    run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh", env=env, cwd=DRIVERS_TOOLS)
+        run_command(
+            f"bash {DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh",
+            env=env,
+            cwd=DRIVERS_TOOLS,
+        )
 
     if test_name == "index_management":
         AUTH = "auth"
diff --git a/.gitignore b/.gitignore
index b06a08b2cb..9220bab109 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,10 +31,10 @@ expansion.yml
 
 # Lambda temp files
 test/lambda/.aws-sam
-test/lambda/env.json
 test/lambda/mongodb/pymongo/*
 test/lambda/mongodb/gridfs/*
 test/lambda/mongodb/bson/*
+test/lambda/*.json
 
 # test results and logs
 xunit-results/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 008cc192ef..66e0b382f4 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -248,6 +248,7 @@ the pages will re-render and the browser will automatically refresh.
 - Run the tests with `just run-tests`.
 
 The supported types are [`default`, `azure`, `gcp`, `eks`, `aks`, and `gke`].
+For the `eks` test, you will need to set up access to the `drivers-test-secrets-role`, see the [Wiki](https://wiki.corp.mongodb.com/spaces/DRIVERS/pages/239737385/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets).
 
 ### KMS tests
 
@@ -283,6 +284,8 @@ Note: these tests can only be run from an Evergreen host.
 
 ### AWS Lambda tests
 
+You will need to set up access to the `drivers-test-secrets-role`, see the [Wiki](https://wiki.corp.mongodb.com/spaces/DRIVERS/pages/239737385/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets).
+
 - Run `just setup-tests aws_lambda`.
 - Run `just run-tests`.
 

From 705d9c2f6a236da40bedfb154e44b22c23b54f86 Mon Sep 17 00:00:00 2001
From: Steven Silvester <steve.silvester@mongodb.com>
Date: Fri, 14 Mar 2025 13:37:28 -0500
Subject: [PATCH 04/12] fix tasks

---
 .evergreen/generated_configs/tasks.yml | 2 +-
 .evergreen/scripts/generate_config.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml
index cd99d6869e..5f3bd30494 100644
--- a/.evergreen/generated_configs/tasks.yml
+++ b/.evergreen/generated_configs/tasks.yml
@@ -1174,7 +1174,7 @@ tasks:
       - func: run tests
         vars:
           TEST_NAME: index_management
-    tags: [index_managment]
+    tags: [index_management]
 
   # Server tests
   - name: test-4.0-standalone-auth-ssl-sync
diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py
index b3fc6b2ed4..621adb90b9 100644
--- a/.evergreen/scripts/generate_config.py
+++ b/.evergreen/scripts/generate_config.py
@@ -946,7 +946,7 @@ def create_search_index_tasks():
     vars = dict(TEST_NAME="index_management")
     test_func = FunctionCall(func="run tests", vars=vars)
     task_name = "test-search-index-helpers"
-    tags = ["index_managment"]
+    tags = ["index_management"]
     commands = [assume_func, atlas_func, server_func, test_func]
     return [EvgTask(name=task_name, tags=tags, commands=commands)]
 

From a09d0d404f9dfaff1bf35246a9b833731e2bc118 Mon Sep 17 00:00:00 2001
From: Steven Silvester <steve.silvester@mongodb.com>
Date: Fri, 14 Mar 2025 13:37:57 -0500
Subject: [PATCH 05/12] fix tasks

---
 .evergreen/generated_configs/tasks.yml | 2 --
 .evergreen/scripts/generate_config.py  | 6 ++----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml
index 5f3bd30494..48f127eef4 100644
--- a/.evergreen/generated_configs/tasks.yml
+++ b/.evergreen/generated_configs/tasks.yml
@@ -12,7 +12,6 @@ tasks:
   - name: test-aws-lambda-deployed
     commands:
       - func: assume ec2 role
-      - func: setup atlas
       - func: run tests
         vars:
           TEST_NAME: aws_lambda
@@ -1167,7 +1166,6 @@ tasks:
   - name: test-search-index-helpers
     commands:
       - func: assume ec2 role
-      - func: setup atlas
       - func: run server
         vars:
           TEST_NAME: index_management
diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py
index 621adb90b9..1314c4196e 100644
--- a/.evergreen/scripts/generate_config.py
+++ b/.evergreen/scripts/generate_config.py
@@ -930,24 +930,22 @@ def _create_ocsp_task(algo, variant, server_type, base_task_name):
 
 def create_aws_lambda_tasks():
     assume_func = FunctionCall(func="assume ec2 role")
-    atlas_func = FunctionCall(func="setup atlas")
     vars = dict(TEST_NAME="aws_lambda")
     test_func = FunctionCall(func="run tests", vars=vars)
     task_name = "test-aws-lambda-deployed"
     tags = ["aws_lambda"]
-    commands = [assume_func, atlas_func, test_func]
+    commands = [assume_func, test_func]
     return [EvgTask(name=task_name, tags=tags, commands=commands)]
 
 
 def create_search_index_tasks():
     assume_func = FunctionCall(func="assume ec2 role")
-    atlas_func = FunctionCall(func="setup atlas")
     server_func = FunctionCall(func="run server", vars=dict(TEST_NAME="index_management"))
     vars = dict(TEST_NAME="index_management")
     test_func = FunctionCall(func="run tests", vars=vars)
     task_name = "test-search-index-helpers"
     tags = ["index_management"]
-    commands = [assume_func, atlas_func, server_func, test_func]
+    commands = [assume_func, server_func, test_func]
     return [EvgTask(name=task_name, tags=tags, commands=commands)]
 
 

From e54f8c6ff22b22524c2f674b25e1c85f5ca1cb3c Mon Sep 17 00:00:00 2001
From: Steven Silvester <steve.silvester@mongodb.com>
Date: Fri, 14 Mar 2025 14:02:51 -0500
Subject: [PATCH 06/12] fix aws creds handling

---
 .evergreen/scripts/setup_tests.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py
index 3ad1187525..1f50db526f 100644
--- a/.evergreen/scripts/setup_tests.py
+++ b/.evergreen/scripts/setup_tests.py
@@ -188,6 +188,11 @@ def handle_test_env() -> None:
 
     if test_name == "aws_lambda":
         UV_ARGS.append("--with pip")
+        # Store AWS creds if they were given.
+        if "AWS_ACCESS_KEY_ID" in os.environ:
+            for key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]:
+                if key in os.environ:
+                    write_env(key, os.environ[key])
 
     if AUTH != "noauth":
         if test_name == "data_lake":

From 40211c76b5fb8113bc8a0edd85b2fdafdd5fdc48 Mon Sep 17 00:00:00 2001
From: Steven Silvester <steve.silvester@mongodb.com>
Date: Fri, 14 Mar 2025 15:53:35 -0500
Subject: [PATCH 07/12] cleanup

---
 .evergreen/generated_configs/tasks.yml | 2 --
 .evergreen/scripts/generate_config.py  | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml
index 48f127eef4..bba982fa75 100644
--- a/.evergreen/generated_configs/tasks.yml
+++ b/.evergreen/generated_configs/tasks.yml
@@ -705,13 +705,11 @@ tasks:
         vars:
           TEST_NAME: enterprise_auth
           AUTH: auth
-          FOO: ${THIS THING}
       - func: assume ec2 role
       - func: run tests
         vars:
           TEST_NAME: enterprise_auth
           AUTH: auth
-          FOO: ${THIS THING}
     tags: [enterprise_auth]
 
   # Kms tests
diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py
index 1314c4196e..b97cf78b57 100644
--- a/.evergreen/scripts/generate_config.py
+++ b/.evergreen/scripts/generate_config.py
@@ -959,7 +959,7 @@ def create_atlas_connect_tasks():
 
 
 def create_enterprise_auth_tasks():
-    vars = dict(TEST_NAME="enterprise_auth", AUTH="auth", FOO="${THIS THING}")
+    vars = dict(TEST_NAME="enterprise_auth", AUTH="auth")
     server_func = FunctionCall(func="run server", vars=vars)
     assume_func = FunctionCall(func="assume ec2 role")
     test_func = FunctionCall(func="run tests", vars=vars)

From 1df7d48ecf947434afc7f4ed23456d9893f00b01 Mon Sep 17 00:00:00 2001
From: Steven Silvester <steve.silvester@mongodb.com>
Date: Wed, 19 Mar 2025 11:06:51 -0500
Subject: [PATCH 08/12] address review

---
 .evergreen/generated_configs/tasks.yml    | 6 +++---
 .evergreen/generated_configs/variants.yml | 2 +-
 .evergreen/scripts/generate_config.py     | 8 ++++----
 .evergreen/scripts/run_server.py          | 4 ++--
 .evergreen/scripts/setup_tests.py         | 6 +++---
 .evergreen/scripts/utils.py               | 2 +-
 6 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml
index 33924c05a4..cd30978f36 100644
--- a/.evergreen/generated_configs/tasks.yml
+++ b/.evergreen/generated_configs/tasks.yml
@@ -1235,11 +1235,11 @@ tasks:
       - func: assume ec2 role
       - func: run server
         vars:
-          TEST_NAME: index_management
+          TEST_NAME: search_index
       - func: run tests
         vars:
-          TEST_NAME: index_management
-    tags: [index_management]
+          TEST_NAME: search_index
+    tags: [search-index]
 
   # Server tests
   - name: test-4.0-standalone-auth-ssl-sync
diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml
index 88b886b862..361a0c9dde 100644
--- a/.evergreen/generated_configs/variants.yml
+++ b/.evergreen/generated_configs/variants.yml
@@ -994,7 +994,7 @@ buildvariants:
   # Search index tests
   - name: search-index-helpers-rhel8-python3.9
     tasks:
-      - name: .index_management
+      - name: .search_index
     display_name: Search Index Helpers RHEL8 Python3.9
     run_on:
       - rhel87-small
diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py
index 81ba7482fa..ef93dc48aa 100644
--- a/.evergreen/scripts/generate_config.py
+++ b/.evergreen/scripts/generate_config.py
@@ -672,7 +672,7 @@ def create_search_index_variants():
     python = CPYTHONS[0]
     return [
         create_variant(
-            [".index_management"],
+            [".search_index"],
             get_display_name("Search Index Helpers", host, python=python),
             python=python,
             host=host,
@@ -944,11 +944,11 @@ def create_aws_lambda_tasks():
 
 def create_search_index_tasks():
     assume_func = FunctionCall(func="assume ec2 role")
-    server_func = FunctionCall(func="run server", vars=dict(TEST_NAME="index_management"))
-    vars = dict(TEST_NAME="index_management")
+    server_func = FunctionCall(func="run server", vars=dict(TEST_NAME="search_index"))
+    vars = dict(TEST_NAME="search_index")
     test_func = FunctionCall(func="run tests", vars=vars)
     task_name = "test-search-index-helpers"
-    tags = ["index_management"]
+    tags = ["search-index"]
     commands = [assume_func, server_func, test_func]
     return [EvgTask(name=task_name, tags=tags, commands=commands)]
 
diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py
index 6b2e1d96ef..f85207daa4 100644
--- a/.evergreen/scripts/run_server.py
+++ b/.evergreen/scripts/run_server.py
@@ -43,9 +43,9 @@ def start_server():
             if not found:
                 raise ValueError("Please provide an orchestration file")
 
-    elif test_name == "index_management":
+    elif test_name == "search_index":
         os.environ["TOPOLOGY"] = "replica_set"
-        os.environ["MONGODB_VERSION"] = "6.0"
+        os.environ["MONGODB_VERSION"] = "7.0"
 
     if not os.environ.get("TEST_CRYPT_SHARED"):
         set_env("SKIP_CRYPT_SHARED")
diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py
index 0760631c5b..ae7fde5efc 100644
--- a/.evergreen/scripts/setup_tests.py
+++ b/.evergreen/scripts/setup_tests.py
@@ -175,7 +175,7 @@ def handle_test_env() -> None:
         if not config:
             AUTH = "noauth"
 
-    if test_name in ["aws_lambda", "index_management"]:
+    if test_name in ["aws_lambda", "search_index"]:
         env = os.environ.copy()
         env["MONGODB_VERSION"] = "7.0"
         env["LAMBDA_STACK_NAME"] = "dbx-python-lambda"
@@ -186,7 +186,7 @@ def handle_test_env() -> None:
             cwd=DRIVERS_TOOLS,
         )
 
-    if test_name == "index_management":
+    if test_name == "search_index":
         AUTH = "auth"
 
     if test_name == "aws_lambda":
@@ -219,7 +219,7 @@ def handle_test_env() -> None:
         elif test_name == "auth_oidc":
             DB_USER = config["OIDC_ADMIN_USER"]
             DB_PASSWORD = config["OIDC_ADMIN_PWD"]
-        elif test_name == "index_management":
+        elif test_name == "search_index":
             config = read_env(f"{DRIVERS_TOOLS}/.evergreen/atlas/secrets-export.sh")
             DB_USER = config["DRIVERS_ATLAS_LAMBDA_USER"]
             DB_PASSWORD = config["DRIVERS_ATLAS_LAMBDA_PASSWORD"]
diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py
index b218545841..44bfecd011 100644
--- a/.evergreen/scripts/utils.py
+++ b/.evergreen/scripts/utils.py
@@ -39,7 +39,7 @@ class Distro:
     "default_sync": "default",
     "encryption": "encryption",
     "enterprise_auth": "auth",
-    "index_management": "index_management",
+    "search_index": "search_index",
     "kms": "kms",
     "load_balancer": "load_balancer",
     "mockupdb": "mockupdb",

From 71e7230a55e2411b0e956598bc18b480c15e0031 Mon Sep 17 00:00:00 2001
From: Steven Silvester <steve.silvester@mongodb.com>
Date: Wed, 19 Mar 2025 11:07:11 -0500
Subject: [PATCH 09/12] update docs

---
 CONTRIBUTING.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index e72d43fb9f..5f55e56684 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -278,8 +278,8 @@ Note: these tests can only be run from an Evergreen host.
 
 ### Search Index tests
 
-- Run `just run-server index_management`.
-- Run `just setup-tests index_management`.
+- Run `just run-server search_index`.
+- Run `just setup-tests search_index`.
 - Run `just run-tests`.
 
 ### AWS Lambda tests

From 12ecd07e957bb7529d9c478d08e9caed06fddca5 Mon Sep 17 00:00:00 2001
From: Steven Silvester <steve.silvester@mongodb.com>
Date: Wed, 19 Mar 2025 11:08:22 -0500
Subject: [PATCH 10/12] update tag

---
 .evergreen/generated_configs/tasks.yml | 2 +-
 .evergreen/scripts/generate_config.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml
index cd30978f36..c692ec31db 100644
--- a/.evergreen/generated_configs/tasks.yml
+++ b/.evergreen/generated_configs/tasks.yml
@@ -1239,7 +1239,7 @@ tasks:
       - func: run tests
         vars:
           TEST_NAME: search_index
-    tags: [search-index]
+    tags: [search_index]
 
   # Server tests
   - name: test-4.0-standalone-auth-ssl-sync
diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py
index ef93dc48aa..dad4f27bcb 100644
--- a/.evergreen/scripts/generate_config.py
+++ b/.evergreen/scripts/generate_config.py
@@ -948,7 +948,7 @@ def create_search_index_tasks():
     vars = dict(TEST_NAME="search_index")
     test_func = FunctionCall(func="run tests", vars=vars)
     task_name = "test-search-index-helpers"
-    tags = ["search-index"]
+    tags = ["search_index"]
     commands = [assume_func, server_func, test_func]
     return [EvgTask(name=task_name, tags=tags, commands=commands)]
 

From 8c593f91024e0d87af433a46e5f67d1970033c46 Mon Sep 17 00:00:00 2001
From: Steven Silvester <steve.silvester@mongodb.com>
Date: Wed, 19 Mar 2025 11:23:34 -0500
Subject: [PATCH 11/12] update markers

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 993b3e5aee..f8c25ed602 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -128,7 +128,7 @@ markers = [
     "atlas_connect: tests that rely on an atlas connection",
     "data_lake: tests that rely on atlas data lake",
     "perf: benchmark tests",
-    "index_management: index management tests",
+    "search_index: search index helper tests",
     "kms: client-side field-level encryption tests using kms",
     "encryption: encryption tests",
     "load_balancer: load balancer tests",

From 54553683b16096d75f85b2ad1bd430686b4e1b32 Mon Sep 17 00:00:00 2001
From: Steven Silvester <steve.silvester@mongodb.com>
Date: Wed, 19 Mar 2025 11:26:46 -0500
Subject: [PATCH 12/12] more fixups

---
 .evergreen/scripts/teardown_tests.py       | 2 +-
 .evergreen/scripts/utils.py                | 2 +-
 test/asynchronous/test_index_management.py | 2 +-
 test/test_index_management.py              | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py
index efce4b381f..390e0a68eb 100644
--- a/.evergreen/scripts/teardown_tests.py
+++ b/.evergreen/scripts/teardown_tests.py
@@ -41,7 +41,7 @@
     run_command(f"bash {DRIVERS_TOOLS}/.evergreen/serverless/teardown.sh")
 
 # Tear down atlas cluster if applicable.
-if TEST_NAME in ["aws_lambda", "index_management"]:
+if TEST_NAME in ["aws_lambda", "search_index"]:
     run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh")
 
 # Tear down auth_aws if applicable.
diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py
index 44bfecd011..0ff3b76a5f 100644
--- a/.evergreen/scripts/utils.py
+++ b/.evergreen/scripts/utils.py
@@ -52,7 +52,7 @@ class Distro:
 # Tests that require a sub test suite.
 SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"]
 
-EXTRA_TESTS = ["mod_wsgi", "aws_lambda", "index_management"]
+EXTRA_TESTS = ["mod_wsgi", "aws_lambda", "search_index"]
 
 
 def get_test_options(
diff --git a/test/asynchronous/test_index_management.py b/test/asynchronous/test_index_management.py
index 4b218de130..890788fc56 100644
--- a/test/asynchronous/test_index_management.py
+++ b/test/asynchronous/test_index_management.py
@@ -38,7 +38,7 @@
 
 _IS_SYNC = False
 
-pytestmark = pytest.mark.index_management
+pytestmark = pytest.mark.search_index
 
 # Location of JSON test specifications.
 if _IS_SYNC:
diff --git a/test/test_index_management.py b/test/test_index_management.py
index 3a2b17cd3d..dea8c0e2be 100644
--- a/test/test_index_management.py
+++ b/test/test_index_management.py
@@ -38,7 +38,7 @@
 
 _IS_SYNC = True
 
-pytestmark = pytest.mark.index_management
+pytestmark = pytest.mark.search_index
 
 # Location of JSON test specifications.
 if _IS_SYNC: