diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 6f53ced61c..04dd16d34f 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -24,12 +24,6 @@ else exit 1 fi -# Source the local secrets export file if available. -if [ -f "./secrets-export.sh" ]; then - echo "Sourcing local secrets file" - . "./secrets-export.sh" -fi - # List the packages. uv sync ${UV_ARGS} --reinstall uv pip list diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py index f85207daa4..5d9aa54e11 100644 --- a/.evergreen/scripts/run_server.py +++ b/.evergreen/scripts/run_server.py @@ -28,11 +28,6 @@ def start_server(): elif test_name == "load_balancer": set_env("LOAD_BALANCER") - elif test_name == "auth_oidc": - raise ValueError( - "OIDC auth does not use run-orchestration directly, do not use run-server!" - ) - elif test_name == "ocsp": opts.ssl = True if "ORCHESTRATION_FILE" not in os.environ: diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 17f9de1a71..74971bca76 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -115,8 +115,17 @@ def setup_libmongocrypt(): run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll") -def get_secrets(name: str) -> None: - run_command(f"bash {DRIVERS_TOOLS}/.evergreen/secrets_handling/setup-secrets.sh {name}") +def load_config_from_file(path: str | Path) -> dict[str, str]: + config = read_env(path) + for key, value in config.items(): + write_env(key, value) + return config + + +def get_secrets(name: str) -> dict[str, str]: + secrets_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/secrets_handling") + run_command(f"bash {secrets_dir}/setup-secrets.sh {name}", cwd=secrets_dir) + return load_config_from_file(secrets_dir / "secrets-export.sh") def handle_test_env() -> None: @@ -158,7 +167,7 @@ def handle_test_env() -> None: # Handle pass through env vars. for var in PASS_THROUGH_ENV: - if is_set(var): + if is_set(var) or getattr(opts, var.lower()): write_env(var, os.environ[var]) if extra := EXTRAS_MAP.get(test_name, ""): @@ -233,12 +242,11 @@ def handle_test_env() -> None: if is_set("MONGODB_URI"): write_env("PYMONGO_MUST_CONNECT", "true") - if is_set("DISABLE_TEST_COMMANDS"): + if is_set("DISABLE_TEST_COMMANDS") or opts.disable_test_commands: write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1") if test_name == "enterprise_auth": - get_secrets("drivers/enterprise_auth") - config = read_env(f"{ROOT}/secrets-export.sh") + config = get_secrets("drivers/enterprise_auth") if PLATFORM == "windows": LOGGER.info("Setting GSSAPI_PASS") write_env("GSSAPI_PASS", config["SASL_PASS"]) @@ -316,7 +324,7 @@ def handle_test_env() -> None: write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem") write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem") - compressors = os.environ.get("COMPRESSORS") + compressors = os.environ.get("COMPRESSORS") or opts.compressor if compressors == "snappy": UV_ARGS.append("--extra snappy") elif compressors == "zstd": @@ -349,13 +357,15 @@ def handle_test_env() -> None: if test_name == "encryption": if not DRIVERS_TOOLS: raise RuntimeError("Missing DRIVERS_TOOLS") - run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh") - run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh") + csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle") + run_command(f"bash {csfle_dir}/setup-secrets.sh", cwd=csfle_dir) + load_config_from_file(csfle_dir / "secrets-export.sh") + run_command(f"bash {csfle_dir}/start-servers.sh") if sub_test_name == "pyopenssl": UV_ARGS.append("--extra ocsp") - if is_set("TEST_CRYPT_SHARED"): + if is_set("TEST_CRYPT_SHARED") or opts.crypt_shared: config = read_env(f"{DRIVERS_TOOLS}/mo-expansion.sh") CRYPT_SHARED_DIR = Path(config["CRYPT_SHARED_LIB_PATH"]).parent.as_posix() LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR) @@ -414,15 +424,15 @@ def handle_test_env() -> None: # Add coverage if requested. # Only cover CPython. PyPy reports suspiciously low coverage. - if is_set("COVERAGE") and platform.python_implementation() == "CPython": + if (is_set("COVERAGE") or opts.cov) and platform.python_implementation() == "CPython": # Keep in sync with combine-coverage.sh. # coverage >=5 is needed for relative_files=true. UV_ARGS.append("--group coverage") TEST_ARGS = f"{TEST_ARGS} --cov" write_env("COVERAGE") - if is_set("GREEN_FRAMEWORK"): - framework = os.environ["GREEN_FRAMEWORK"] + if is_set("GREEN_FRAMEWORK") or opts.green_framework: + framework = opts.green_framework or os.environ["GREEN_FRAMEWORK"] UV_ARGS.append(f"--group {framework}") else: diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 0ff3b76a5f..3eb44f2ab9 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -52,7 +52,10 @@ class Distro: # Tests that require a sub test suite. SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"] -EXTRA_TESTS = ["mod_wsgi", "aws_lambda", "search_index"] +EXTRA_TESTS = ["mod_wsgi", "aws_lambda"] + +# Tests that do not use run-orchestration. +NO_RUN_ORCHESTRATION = ["auth_oidc", "atlas_connect", "data_lake", "mockupdb", "serverless"] def get_test_options( @@ -75,19 +78,47 @@ def get_test_options( else: parser.add_argument( "test_name", - choices=sorted(TEST_SUITE_MAP), + choices=set(TEST_SUITE_MAP) - set(NO_RUN_ORCHESTRATION), nargs="?", default="default", help="The optional name of the test suite to be run, which informs the server configuration.", ) parser.add_argument( - "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level" + "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level." ) parser.add_argument( - "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level" + "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level." ) - parser.add_argument("--auth", action="store_true", help="Whether to add authentication") - parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration") + parser.add_argument("--auth", action="store_true", help="Whether to add authentication.") + parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration.") + + # Add the test modifiers. + if require_sub_test_name: + parser.add_argument( + "--debug-log", action="store_true", help="Enable pymongo standard logging." + ) + parser.add_argument("--cov", action="store_true", help="Add test coverage.") + parser.add_argument( + "--green-framework", + nargs=1, + choices=["eventlet", "gevent"], + help="Optional green framework to test against.", + ) + parser.add_argument( + "--compressor", + nargs=1, + choices=["zlib", "zstd", "snappy"], + help="Optional compression algorithm.", + ) + parser.add_argument("--crypt-shared", action="store_true", help="Test with crypt_shared.") + parser.add_argument("--no-ext", action="store_true", help="Run without c extensions.") + parser.add_argument( + "--mongodb-api-version", choices=["1"], help="MongoDB stable API version to use." + ) + parser.add_argument( + "--disable-test-commands", action="store_true", help="Disable test commands." + ) + # Get the options. if not allow_extra_opts: opts, extra_opts = parser.parse_args(), [] @@ -113,7 +144,7 @@ def get_test_options( return opts, extra_opts -def read_env(path: Path | str) -> dict[str, Any]: +def read_env(path: Path | str) -> dict[str, str]: config = dict() with Path(path).open() as fid: for line in fid.readlines(): diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e75510171e..d8cc8c8bcd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -217,9 +217,11 @@ the pages will re-render and the browser will automatically refresh. ### Usage -- Run `just run-server` with optional args to set up the server. - All given flags will be passed to `run-orchestration.sh` in `$DRIVERS_TOOLS`. +- Run `just run-server` with optional args to set up the server. All given options will be passed to + `run-orchestration.sh` in `$DRIVERS_TOOLS`. See `$DRIVERS_TOOLS/evergreen/run-orchestration.sh -h` + for a full list of options. - Run `just setup-tests` with optional args to set up the test environment, secrets, etc. + See `just setup-tests -h` for a full list of available options. - Run `just run-tests` to run the tests in an appropriate Python environment. - When done, run `just teardown-tests` to clean up and `just stop-server` to stop the server. @@ -346,11 +348,28 @@ If you are running one of the `no-responder` tests, omit the `run-server` step. - Run the tests: `just run-tests`. ## Enable Debug Logs + - Use `-o log_cli_level="DEBUG" -o log_cli=1` with `just test` or `pytest`. - Add `log_cli_level = "DEBUG` and `log_cli = 1` to the `tool.pytest.ini_options` section in `pyproject.toml` for Evergreen patches or to enable debug logs by default on your machine. - You can also set `DEBUG_LOG=1` and run either `just setup-tests` or `just-test`. +- Finally, you can use `just setup-tests --debug-log`. - For evergreen patch builds, you can use `evergreen patch --param DEBUG_LOG=1` to enable debug logs for the patch. +## Adding a new test suite + +- If adding new tests files that should only be run for that test suite, add a pytest marker to the file and add + to the list of pytest markers in `pyproject.toml`. Then add the test suite to the `TEST_SUITE_MAP` in `.evergreen/scripts/utils.py`. If for some reason it is not a pytest-runnable test, add it to the list of `EXTRA_TESTS` instead. +- If the test uses Atlas or otherwise doesn't use `run-orchestration.sh`, add it to the `NO_RUN_ORCHESTRATION` list in + `.evergreen/scripts/utils.py`. +- If there is something special required to run the local server or there is an extra flag that should always be set + like `AUTH`, add that logic to `.evergreen/scripts/run_server.py`. +- The bulk of the logic will typically be in `.evergreen/scripts/setup_tests.py`. This is where you should fetch secrets and make them available using `write_env`, start services, and write other env vars needed using `write_env`. +- If there are any special test considerations, including not running `pytest` at all, handle it in `.evergreen/scripts/run_tests.py`. +- If there are any services or atlas clusters to teardown, handle them in `.evergreen/scripts/teardown_tests.py`. +- Add functions to generate the test variant(s) and task(s) to the `.evergreen/scripts/generate_config.py`. +- Regenerate the test variants and tasks using the instructions in `.evergreen/scripts/generate_config.py`. +- Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. + ## Re-sync Spec Tests If you would like to re-sync the copy of the specification tests in the