PYTHON-5231 Finish up test scripts and add documentation for creating a new test suite (#2224)

Author: blink1073

.evergreen/run-tests.sh

@@ -24,12 +24,6 @@ else
   exit 1
 fi
 
-# Source the local secrets export file if available.
-if [ -f "./secrets-export.sh" ]; then
-  echo "Sourcing local secrets file"
-  . "./secrets-export.sh"
-fi
-
 # List the packages.
 uv sync ${UV_ARGS} --reinstall
 uv pip list

.evergreen/scripts/run_server.py

@@ -28,11 +28,6 @@ def start_server():
     elif test_name == "load_balancer":
         set_env("LOAD_BALANCER")
 
-    elif test_name == "auth_oidc":
-        raise ValueError(
-            "OIDC auth does not use run-orchestration directly, do not use run-server!"
-        )
-
     elif test_name == "ocsp":
         opts.ssl = True
         if "ORCHESTRATION_FILE" not in os.environ:

.evergreen/scripts/setup_tests.py

@@ -115,8 +115,17 @@ def setup_libmongocrypt():
         run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll")
 
 
-def get_secrets(name: str) -> None:
-    run_command(f"bash {DRIVERS_TOOLS}/.evergreen/secrets_handling/setup-secrets.sh {name}")
+def load_config_from_file(path: str | Path) -> dict[str, str]:
+    config = read_env(path)
+    for key, value in config.items():
+        write_env(key, value)
+    return config
+
+
+def get_secrets(name: str) -> dict[str, str]:
+    secrets_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/secrets_handling")
+    run_command(f"bash {secrets_dir}/setup-secrets.sh {name}", cwd=secrets_dir)
+    return load_config_from_file(secrets_dir / "secrets-export.sh")
 
 
 def handle_test_env() -> None:
@@ -158,7 +167,7 @@ def handle_test_env() -> None:
 
     # Handle pass through env vars.
     for var in PASS_THROUGH_ENV:
-        if is_set(var):
+        if is_set(var) or getattr(opts, var.lower()):
             write_env(var, os.environ[var])
 
     if extra := EXTRAS_MAP.get(test_name, ""):
@@ -233,12 +242,11 @@ def handle_test_env() -> None:
     if is_set("MONGODB_URI"):
         write_env("PYMONGO_MUST_CONNECT", "true")
 
-    if is_set("DISABLE_TEST_COMMANDS"):
+    if is_set("DISABLE_TEST_COMMANDS") or opts.disable_test_commands:
         write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1")
 
     if test_name == "enterprise_auth":
-        get_secrets("drivers/enterprise_auth")
-        config = read_env(f"{ROOT}/secrets-export.sh")
+        config = get_secrets("drivers/enterprise_auth")
         if PLATFORM == "windows":
             LOGGER.info("Setting GSSAPI_PASS")
             write_env("GSSAPI_PASS", config["SASL_PASS"])
@@ -316,7 +324,7 @@ def handle_test_env() -> None:
         write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem")
         write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem")
 
-    compressors = os.environ.get("COMPRESSORS")
+    compressors = os.environ.get("COMPRESSORS") or opts.compressor
     if compressors == "snappy":
         UV_ARGS.append("--extra snappy")
     elif compressors == "zstd":
@@ -349,13 +357,15 @@ def handle_test_env() -> None:
     if test_name == "encryption":
         if not DRIVERS_TOOLS:
             raise RuntimeError("Missing DRIVERS_TOOLS")
-        run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh")
-        run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh")
+        csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle")
+        run_command(f"bash {csfle_dir}/setup-secrets.sh", cwd=csfle_dir)
+        load_config_from_file(csfle_dir / "secrets-export.sh")
+        run_command(f"bash {csfle_dir}/start-servers.sh")
 
         if sub_test_name == "pyopenssl":
             UV_ARGS.append("--extra ocsp")
 
-    if is_set("TEST_CRYPT_SHARED"):
+    if is_set("TEST_CRYPT_SHARED") or opts.crypt_shared:
         config = read_env(f"{DRIVERS_TOOLS}/mo-expansion.sh")
         CRYPT_SHARED_DIR = Path(config["CRYPT_SHARED_LIB_PATH"]).parent.as_posix()
         LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR)
@@ -414,15 +424,15 @@ def handle_test_env() -> None:
 
     # Add coverage if requested.
     # Only cover CPython. PyPy reports suspiciously low coverage.
-    if is_set("COVERAGE") and platform.python_implementation() == "CPython":
+    if (is_set("COVERAGE") or opts.cov) and platform.python_implementation() == "CPython":
         # Keep in sync with combine-coverage.sh.
         # coverage >=5 is needed for relative_files=true.
         UV_ARGS.append("--group coverage")
         TEST_ARGS = f"{TEST_ARGS} --cov"
         write_env("COVERAGE")
 
-    if is_set("GREEN_FRAMEWORK"):
-        framework = os.environ["GREEN_FRAMEWORK"]
+    if is_set("GREEN_FRAMEWORK") or opts.green_framework:
+        framework = opts.green_framework or os.environ["GREEN_FRAMEWORK"]
         UV_ARGS.append(f"--group {framework}")
 
     else:

.evergreen/scripts/utils.py

@@ -52,7 +52,10 @@ class Distro:
 # Tests that require a sub test suite.
 SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"]
 
-EXTRA_TESTS = ["mod_wsgi", "aws_lambda", "search_index"]
+EXTRA_TESTS = ["mod_wsgi", "aws_lambda"]
+
+# Tests that do not use run-orchestration.
+NO_RUN_ORCHESTRATION = ["auth_oidc", "atlas_connect", "data_lake", "mockupdb", "serverless"]
 
 
 def get_test_options(
@@ -75,19 +78,47 @@ def get_test_options(
     else:
         parser.add_argument(
             "test_name",
-            choices=sorted(TEST_SUITE_MAP),
+            choices=set(TEST_SUITE_MAP) - set(NO_RUN_ORCHESTRATION),
             nargs="?",
             default="default",
             help="The optional name of the test suite to be run, which informs the server configuration.",
         )
     parser.add_argument(
-        "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level"
+        "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level."
     )
     parser.add_argument(
-        "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level"
+        "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level."
     )
-    parser.add_argument("--auth", action="store_true", help="Whether to add authentication")
-    parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration")
+    parser.add_argument("--auth", action="store_true", help="Whether to add authentication.")
+    parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration.")
+
+    # Add the test modifiers.
+    if require_sub_test_name:
+        parser.add_argument(
+            "--debug-log", action="store_true", help="Enable pymongo standard logging."
+        )
+        parser.add_argument("--cov", action="store_true", help="Add test coverage.")
+        parser.add_argument(
+            "--green-framework",
+            nargs=1,
+            choices=["eventlet", "gevent"],
+            help="Optional green framework to test against.",
+        )
+        parser.add_argument(
+            "--compressor",
+            nargs=1,
+            choices=["zlib", "zstd", "snappy"],
+            help="Optional compression algorithm.",
+        )
+        parser.add_argument("--crypt-shared", action="store_true", help="Test with crypt_shared.")
+        parser.add_argument("--no-ext", action="store_true", help="Run without c extensions.")
+        parser.add_argument(
+            "--mongodb-api-version", choices=["1"], help="MongoDB stable API version to use."
+        )
+        parser.add_argument(
+            "--disable-test-commands", action="store_true", help="Disable test commands."
+        )
+
     # Get the options.
     if not allow_extra_opts:
         opts, extra_opts = parser.parse_args(), []
@@ -113,7 +144,7 @@ def get_test_options(
     return opts, extra_opts
 
 
-def read_env(path: Path | str) -> dict[str, Any]:
+def read_env(path: Path | str) -> dict[str, str]:
     config = dict()
     with Path(path).open() as fid:
         for line in fid.readlines():

CONTRIBUTING.md

@@ -217,9 +217,11 @@ the pages will re-render and the browser will automatically refresh.
 
 ### Usage
 
-- Run `just run-server` with optional args to set up the server.
-  All given flags will be passed to `run-orchestration.sh` in `$DRIVERS_TOOLS`.
+- Run `just run-server` with optional args to set up the server.  All given options will be passed to
+  `run-orchestration.sh` in `$DRIVERS_TOOLS`.  See `$DRIVERS_TOOLS/evergreen/run-orchestration.sh -h`
+  for a full list of options.
 - Run `just setup-tests` with optional args to set up the test environment, secrets, etc.
+  See `just setup-tests -h` for a full list of available options.
 - Run `just run-tests` to run the tests in an appropriate Python environment.
 - When done, run `just teardown-tests` to clean up and `just stop-server` to stop the server.
 
@@ -346,11 +348,28 @@ If you are running one of the `no-responder` tests, omit the `run-server` step.
 - Run the tests: `just run-tests`.
 
 ## Enable Debug Logs
+
 - Use `-o log_cli_level="DEBUG" -o log_cli=1` with `just test` or `pytest`.
 - Add `log_cli_level = "DEBUG` and `log_cli = 1` to the `tool.pytest.ini_options` section in `pyproject.toml` for Evergreen patches or to enable debug logs by default on your machine.
 - You can also set `DEBUG_LOG=1` and run either `just setup-tests` or `just-test`.
+- Finally, you can use `just setup-tests --debug-log`.
 - For evergreen patch builds, you can use `evergreen patch --param DEBUG_LOG=1` to enable debug logs for the patch.
 
+## Adding a new test suite
+
+- If adding new tests files that should only be run for that test suite, add a pytest marker to the file and add
+  to the list of pytest markers in `pyproject.toml`.  Then add the test suite to the `TEST_SUITE_MAP` in `.evergreen/scripts/utils.py`.  If for some reason it is not a pytest-runnable test, add it to the list of `EXTRA_TESTS` instead.
+- If the test uses Atlas or otherwise doesn't use `run-orchestration.sh`, add it to the `NO_RUN_ORCHESTRATION` list in
+  `.evergreen/scripts/utils.py`.
+- If there is something special required to run the local server or there is an extra flag that should always be set
+  like `AUTH`, add that logic to `.evergreen/scripts/run_server.py`.
+- The bulk of the logic will typically be in `.evergreen/scripts/setup_tests.py`.  This is where you should fetch secrets and make them available using `write_env`, start services, and write other env vars needed using `write_env`.
+- If there are any special test considerations, including not running `pytest` at all, handle it in `.evergreen/scripts/run_tests.py`.
+- If there are any services or atlas clusters to teardown, handle them in `.evergreen/scripts/teardown_tests.py`.
+- Add functions to generate the test variant(s) and task(s) to the `.evergreen/scripts/generate_config.py`.
+- Regenerate the test variants and tasks using the instructions in `.evergreen/scripts/generate_config.py`.
+- Make sure to add instructions for running the test suite to `CONTRIBUTING.md`.
+
 ## Re-sync Spec Tests
 
 If you would like to re-sync the copy of the specification tests in the