diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ee29d65f2b..15bdac315d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -213,20 +213,6 @@ functions: params: file: ${DRIVERS_TOOLS}/mo-expansion.yml - "bootstrap data lake": - - command: subprocess.exec - type: setup - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh - - command: subprocess.exec - type: setup - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh - "run doctests": - command: subprocess.exec type: test @@ -404,13 +390,6 @@ tasks: TOPOLOGY: "replica_set" - func: "run tests" - - name: atlas-data-lake-tests - commands: - - func: "bootstrap data lake" - - func: "run tests" - vars: - TEST_NAME: "data_lake" - - name: "test-aws-lambda-deployed" commands: - command: ec2.assume_role diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 5b5cf92d68..7d03822a6f 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -8,6 +8,21 @@ tasks: TEST_NAME: atlas_connect tags: [atlas_connect] + # Atlas data lake tests + - name: test-atlas-data-lake-with_ext + commands: + - func: run tests + vars: + TEST_NAME: data_lake + NO_EXT: "1" + tags: [atlas_data_lake] + - name: test-atlas-data-lake-without_ext + commands: + - func: run tests + vars: + TEST_NAME: data_lake + tags: [atlas_data_lake] + # Aws tests - name: test-auth-aws-4.4-regular commands: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 864b061a15..78f181fc64 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -65,43 +65,21 @@ buildvariants: PYTHON_BINARY: /opt/python/3.13/bin/python3 # Atlas data lake tests - - name: atlas-data-lake-ubuntu-22-python3.9-auth-no-c + - name: atlas-data-lake-ubuntu-22-python3.9 tasks: - - name: atlas-data-lake-tests - display_name: Atlas Data Lake Ubuntu-22 Python3.9 Auth No C + - name: .atlas_data_lake + display_name: Atlas Data Lake Ubuntu-22 Python3.9 run_on: - ubuntu2204-small expansions: - AUTH: auth - NO_EXT: "1" - PYTHON_BINARY: /opt/python/3.9/bin/python3 - - name: atlas-data-lake-ubuntu-22-python3.9-auth - tasks: - - name: atlas-data-lake-tests - display_name: Atlas Data Lake Ubuntu-22 Python3.9 Auth - run_on: - - ubuntu2204-small - expansions: - AUTH: auth PYTHON_BINARY: /opt/python/3.9/bin/python3 - - name: atlas-data-lake-ubuntu-22-python3.13-auth-no-c + - name: atlas-data-lake-ubuntu-22-python3.13 tasks: - - name: atlas-data-lake-tests - display_name: Atlas Data Lake Ubuntu-22 Python3.13 Auth No C + - name: .atlas_data_lake + display_name: Atlas Data Lake Ubuntu-22 Python3.13 run_on: - ubuntu2204-small expansions: - AUTH: auth - NO_EXT: "1" - PYTHON_BINARY: /opt/python/3.13/bin/python3 - - name: atlas-data-lake-ubuntu-22-python3.13-auth - tasks: - - name: atlas-data-lake-tests - display_name: Atlas Data Lake Ubuntu-22 Python3.13 Auth - run_on: - - ubuntu2204-small - expansions: - AUTH: auth PYTHON_BINARY: /opt/python/3.13/bin/python3 # Aws auth tests diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index b9f9377066..ebb4b5d31c 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -213,7 +213,7 @@ def zip_cycle(*iterables, empty_default=None): yield tuple(next(i, empty_default) for i in cycles) -def handle_c_ext(c_ext, expansions): +def handle_c_ext(c_ext, expansions) -> None: """Handle c extension option.""" if c_ext == C_EXTS[0]: expansions["NO_EXT"] = "1" @@ -600,14 +600,10 @@ def create_no_c_ext_variants(): def create_atlas_data_lake_variants(): variants = [] host = HOSTS["ubuntu22"] - for python, c_ext in product(MIN_MAX_PYTHON, C_EXTS): - tasks = ["atlas-data-lake-tests"] - expansions = dict(AUTH="auth") - handle_c_ext(c_ext, expansions) - display_name = get_display_name("Atlas Data Lake", host, python=python, **expansions) - variant = create_variant( - tasks, display_name, host=host, python=python, expansions=expansions - ) + for python in MIN_MAX_PYTHON: + tasks = [".atlas_data_lake"] + display_name = get_display_name("Atlas Data Lake", host, python=python) + variant = create_variant(tasks, display_name, host=host, python=python) variants.append(variant) return variants @@ -970,6 +966,18 @@ def create_perf_tasks(): return tasks +def create_atlas_data_lake_tasks(): + tags = ["atlas_data_lake"] + tasks = [] + for c_ext in C_EXTS: + vars = dict(TEST_NAME="data_lake") + handle_c_ext(c_ext, vars) + test_func = FunctionCall(func="run tests", vars=vars) + task_name = f"test-atlas-data-lake-{c_ext}" + tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) + return tasks + + def create_ocsp_tasks(): tasks = [] tests = [ diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 6d7c8037c7..56011341a3 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -175,6 +175,12 @@ def handle_test_env() -> None: if not config: AUTH = "noauth" + if test_name == "data_lake": + # Stop any running mongo-orchestration which might be using the port. + run_command(f"bash {DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh") + run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas_data_lake/setup.sh") + AUTH = "auth" + if AUTH != "noauth": if test_name == "data_lake": config = read_env(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/secrets-export.sh") diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py index b081478423..d27cb8682e 100644 --- a/.evergreen/scripts/teardown_tests.py +++ b/.evergreen/scripts/teardown_tests.py @@ -57,6 +57,10 @@ teardown_mod_wsgi() +# Tear down data_lake if applicable. +elif TEST_NAME == "data_lake": + run_command(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/teardown.sh") + # Tear down coverage if applicable. if os.environ.get("COVERAGE"): shutil.rmtree(".pytest_cache", ignore_errors=True) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 47eb01dbf0..e466c6847e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -286,16 +286,23 @@ Note: these tests can only be run from an Evergreen Linux host that has the Pyth The `mode` can be `standalone` or `embedded`. For the `replica_set` version of the tests, use `TOPOLOGY=replica_set just run-server`. +### Atlas Data Lake tests. + +You must have `docker` or `podman` installed locally. + +- Run `just setup-tests data_lake`. +- Run `just run-tests`. + ### OCSP tests - - Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`. - This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`. - MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA. - - Run `just run-server ocsp`. - - Run `just setup-tests ocsp ` (options are "valid", "revoked", "valid-delegate", "revoked-delegate"). - - Run `just run-tests` +- Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`. +This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`. +MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA. +- Run `just run-server ocsp`. +- Run `just setup-tests ocsp ` (options are "valid", "revoked", "valid-delegate", "revoked-delegate"). +- Run `just run-tests` - If you are running one of the `no-responder` tests, omit the `run-server` step. +If you are running one of the `no-responder` tests, omit the `run-server` step. ### Perf Tests