Merge branch 'master' into PYTHON-4946

Author: NoahStapp

.evergreen/generated_configs/variants.yml

@@ -45,6 +45,17 @@ buildvariants:
     batchtime: 10080
     expansions:
       NO_EXT: "1"
+  - name: other-hosts-amazon2023
+    tasks:
+      - name: .latest !.sync_async .sharded_cluster .auth .ssl
+      - name: .latest !.sync_async .replica_set .noauth .ssl
+      - name: .latest !.sync_async .standalone .noauth .nossl
+    display_name: Other hosts Amazon2023
+    run_on:
+      - amazon2023-arm64-latest-large-m8g
+    batchtime: 10080
+    expansions:
+      NO_EXT: "1"
 
   # Atlas connect tests
   - name: atlas-connect-rhel8-python3.9
@@ -1246,12 +1257,12 @@ buildvariants:
   # Stable api tests
   - name: stable-api-require-v1-rhel8-python3.9-auth
     tasks:
-      - name: .standalone .5.0 .noauth .nossl .sync_async
-      - name: .standalone .6.0 .noauth .nossl .sync_async
-      - name: .standalone .7.0 .noauth .nossl .sync_async
-      - name: .standalone .8.0 .noauth .nossl .sync_async
-      - name: .standalone .rapid .noauth .nossl .sync_async
-      - name: .standalone .latest .noauth .nossl .sync_async
+      - name: "!.replica_set .5.0 .noauth .nossl .sync_async"
+      - name: "!.replica_set .6.0 .noauth .nossl .sync_async"
+      - name: "!.replica_set .7.0 .noauth .nossl .sync_async"
+      - name: "!.replica_set .8.0 .noauth .nossl .sync_async"
+      - name: "!.replica_set .rapid .noauth .nossl .sync_async"
+      - name: "!.replica_set .latest .noauth .nossl .sync_async"
     display_name: Stable API require v1 RHEL8 Python3.9 Auth
     run_on:
       - rhel87-small
@@ -1279,12 +1290,12 @@ buildvariants:
     tags: [versionedApi_tag]
   - name: stable-api-require-v1-rhel8-python3.13-auth
     tasks:
-      - name: .standalone .5.0 .noauth .nossl .sync_async
-      - name: .standalone .6.0 .noauth .nossl .sync_async
-      - name: .standalone .7.0 .noauth .nossl .sync_async
-      - name: .standalone .8.0 .noauth .nossl .sync_async
-      - name: .standalone .rapid .noauth .nossl .sync_async
-      - name: .standalone .latest .noauth .nossl .sync_async
+      - name: "!.replica_set .5.0 .noauth .nossl .sync_async"
+      - name: "!.replica_set .6.0 .noauth .nossl .sync_async"
+      - name: "!.replica_set .7.0 .noauth .nossl .sync_async"
+      - name: "!.replica_set .8.0 .noauth .nossl .sync_async"
+      - name: "!.replica_set .rapid .noauth .nossl .sync_async"
+      - name: "!.replica_set .latest .noauth .nossl .sync_async"
     display_name: Stable API require v1 RHEL8 Python3.13 Auth
     run_on:
       - rhel87-small

.evergreen/run-tests.sh

@@ -24,12 +24,6 @@ else
   exit 1
 fi
 
-# Source the local secrets export file if available.
-if [ -f "./secrets-export.sh" ]; then
-  echo "Sourcing local secrets file"
-  . "./secrets-export.sh"
-fi
-
 # List the packages.
 uv sync ${UV_ARGS} --reinstall
 uv pip list

.evergreen/scripts/generate_config.py

@@ -73,9 +73,16 @@ class Host:
 DEFAULT_HOST = HOSTS["rhel8"]
 
 # Other hosts
-OTHER_HOSTS = ["RHEL9-FIPS", "RHEL8-zseries", "RHEL8-POWER8", "RHEL8-arm64"]
+OTHER_HOSTS = ["RHEL9-FIPS", "RHEL8-zseries", "RHEL8-POWER8", "RHEL8-arm64", "Amazon2023"]
 for name, run_on in zip(
-    OTHER_HOSTS, ["rhel92-fips", "rhel8-zseries-small", "rhel8-power-small", "rhel82-arm64-small"]
+    OTHER_HOSTS,
+    [
+        "rhel92-fips",
+        "rhel8-zseries-small",
+        "rhel8-power-small",
+        "rhel82-arm64-small",
+        "amazon2023-arm64-latest-large-m8g",
+    ],
 ):
     HOSTS[name] = Host(name, run_on, name, dict())
 
@@ -539,7 +546,6 @@ def create_storage_engine_variants():
 def create_stable_api_variants():
     host = DEFAULT_HOST
     tags = ["versionedApi_tag"]
-    tasks = [f".standalone .{v} .noauth .nossl .sync_async" for v in get_versions_from("5.0")]
     variants = []
     types = ["require v1", "accept v2"]
 
@@ -553,11 +559,17 @@ def create_stable_api_variants():
             expansions["REQUIRE_API_VERSION"] = "1"
             # MONGODB_API_VERSION is the apiVersion to use in the test suite.
             expansions["MONGODB_API_VERSION"] = "1"
+            tasks = [
+                f"!.replica_set .{v} .noauth .nossl .sync_async" for v in get_versions_from("5.0")
+            ]
         else:
             # Test against a cluster with acceptApiVersion2 but without
             # requireApiVersion, and don't automatically add apiVersion to
             # clients created in the test suite.
             expansions["ORCHESTRATION_FILE"] = "versioned-api-testing.json"
+            tasks = [
+                f".standalone .{v} .noauth .nossl .sync_async" for v in get_versions_from("5.0")
+            ]
         base_display_name = f"Stable API {test_type}"
         display_name = get_display_name(base_display_name, host, python=python, **expansions)
         variant = create_variant(
@@ -772,9 +784,12 @@ def create_alternative_hosts_variants():
     handle_c_ext(C_EXTS[0], expansions)
     for host_name in OTHER_HOSTS:
         host = HOSTS[host_name]
+        tags = [".6.0 .standalone !.sync_async"]
+        if host_name == "Amazon2023":
+            tags = [f".latest !.sync_async {t}" for t in SUB_TASKS]
         variants.append(
             create_variant(
-                [".6.0 .standalone !.sync_async"],
+                tags,
                 display_name=get_display_name("Other hosts", host),
                 batchtime=batchtime,
                 host=host,

.evergreen/scripts/run_server.py

@@ -28,11 +28,6 @@ def start_server():
     elif test_name == "load_balancer":
         set_env("LOAD_BALANCER")
 
-    elif test_name == "auth_oidc":
-        raise ValueError(
-            "OIDC auth does not use run-orchestration directly, do not use run-server!"
-        )
-
     elif test_name == "ocsp":
         opts.ssl = True
         if "ORCHESTRATION_FILE" not in os.environ:

.evergreen/scripts/setup_tests.py

@@ -115,8 +115,17 @@ def setup_libmongocrypt():
         run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll")
 
 
-def get_secrets(name: str) -> None:
-    run_command(f"bash {DRIVERS_TOOLS}/.evergreen/secrets_handling/setup-secrets.sh {name}")
+def load_config_from_file(path: str | Path) -> dict[str, str]:
+    config = read_env(path)
+    for key, value in config.items():
+        write_env(key, value)
+    return config
+
+
+def get_secrets(name: str) -> dict[str, str]:
+    secrets_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/secrets_handling")
+    run_command(f"bash {secrets_dir}/setup-secrets.sh {name}", cwd=secrets_dir)
+    return load_config_from_file(secrets_dir / "secrets-export.sh")
 
 
 def handle_test_env() -> None:
@@ -158,7 +167,7 @@ def handle_test_env() -> None:
 
     # Handle pass through env vars.
     for var in PASS_THROUGH_ENV:
-        if is_set(var):
+        if is_set(var) or getattr(opts, var.lower()):
             write_env(var, os.environ[var])
 
     if extra := EXTRAS_MAP.get(test_name, ""):
@@ -189,7 +198,7 @@ def handle_test_env() -> None:
         AUTH = "auth"
 
     if test_name == "aws_lambda":
-        UV_ARGS.append("--with pip")
+        UV_ARGS.append("--group pip")
         # Store AWS creds if they were given.
         if "AWS_ACCESS_KEY_ID" in os.environ:
             for key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]:
@@ -233,12 +242,11 @@ def handle_test_env() -> None:
     if is_set("MONGODB_URI"):
         write_env("PYMONGO_MUST_CONNECT", "true")
 
-    if is_set("DISABLE_TEST_COMMANDS"):
+    if is_set("DISABLE_TEST_COMMANDS") or opts.disable_test_commands:
         write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1")
 
     if test_name == "enterprise_auth":
-        get_secrets("drivers/enterprise_auth")
-        config = read_env(f"{ROOT}/secrets-export.sh")
+        config = get_secrets("drivers/enterprise_auth")
         if PLATFORM == "windows":
             LOGGER.info("Setting GSSAPI_PASS")
             write_env("GSSAPI_PASS", config["SASL_PASS"])
@@ -316,7 +324,7 @@ def handle_test_env() -> None:
         write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem")
         write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem")
 
-    compressors = os.environ.get("COMPRESSORS")
+    compressors = os.environ.get("COMPRESSORS") or opts.compressor
     if compressors == "snappy":
         UV_ARGS.append("--extra snappy")
     elif compressors == "zstd":
@@ -349,13 +357,15 @@ def handle_test_env() -> None:
     if test_name == "encryption":
         if not DRIVERS_TOOLS:
             raise RuntimeError("Missing DRIVERS_TOOLS")
-        run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh")
-        run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh")
+        csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle")
+        run_command(f"bash {csfle_dir}/setup-secrets.sh", cwd=csfle_dir)
+        load_config_from_file(csfle_dir / "secrets-export.sh")
+        run_command(f"bash {csfle_dir}/start-servers.sh")
 
         if sub_test_name == "pyopenssl":
             UV_ARGS.append("--extra ocsp")
 
-    if is_set("TEST_CRYPT_SHARED"):
+    if is_set("TEST_CRYPT_SHARED") or opts.crypt_shared:
         config = read_env(f"{DRIVERS_TOOLS}/mo-expansion.sh")
         CRYPT_SHARED_DIR = Path(config["CRYPT_SHARED_LIB_PATH"]).parent.as_posix()
         LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR)
@@ -414,15 +424,15 @@ def handle_test_env() -> None:
 
     # Add coverage if requested.
     # Only cover CPython. PyPy reports suspiciously low coverage.
-    if is_set("COVERAGE") and platform.python_implementation() == "CPython":
+    if (is_set("COVERAGE") or opts.cov) and platform.python_implementation() == "CPython":
         # Keep in sync with combine-coverage.sh.
         # coverage >=5 is needed for relative_files=true.
         UV_ARGS.append("--group coverage")
         TEST_ARGS = f"{TEST_ARGS} --cov"
         write_env("COVERAGE")
 
-    if is_set("GREEN_FRAMEWORK"):
-        framework = os.environ["GREEN_FRAMEWORK"]
+    if is_set("GREEN_FRAMEWORK") or opts.green_framework:
+        framework = opts.green_framework or os.environ["GREEN_FRAMEWORK"]
         UV_ARGS.append(f"--group {framework}")
 
     else:

.evergreen/scripts/utils.py

@@ -52,7 +52,10 @@ class Distro:
 # Tests that require a sub test suite.
 SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"]
 
-EXTRA_TESTS = ["mod_wsgi", "aws_lambda", "search_index"]
+EXTRA_TESTS = ["mod_wsgi", "aws_lambda"]
+
+# Tests that do not use run-orchestration.
+NO_RUN_ORCHESTRATION = ["auth_oidc", "atlas_connect", "data_lake", "mockupdb", "serverless"]
 
 
 def get_test_options(
@@ -75,19 +78,47 @@ def get_test_options(
     else:
         parser.add_argument(
             "test_name",
-            choices=sorted(TEST_SUITE_MAP),
+            choices=set(TEST_SUITE_MAP) - set(NO_RUN_ORCHESTRATION),
             nargs="?",
             default="default",
             help="The optional name of the test suite to be run, which informs the server configuration.",
         )
     parser.add_argument(
-        "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level"
+        "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level."
     )
     parser.add_argument(
-        "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level"
+        "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level."
     )
-    parser.add_argument("--auth", action="store_true", help="Whether to add authentication")
-    parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration")
+    parser.add_argument("--auth", action="store_true", help="Whether to add authentication.")
+    parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration.")
+
+    # Add the test modifiers.
+    if require_sub_test_name:
+        parser.add_argument(
+            "--debug-log", action="store_true", help="Enable pymongo standard logging."
+        )
+        parser.add_argument("--cov", action="store_true", help="Add test coverage.")
+        parser.add_argument(
+            "--green-framework",
+            nargs=1,
+            choices=["eventlet", "gevent"],
+            help="Optional green framework to test against.",
+        )
+        parser.add_argument(
+            "--compressor",
+            nargs=1,
+            choices=["zlib", "zstd", "snappy"],
+            help="Optional compression algorithm.",
+        )
+        parser.add_argument("--crypt-shared", action="store_true", help="Test with crypt_shared.")
+        parser.add_argument("--no-ext", action="store_true", help="Run without c extensions.")
+        parser.add_argument(
+            "--mongodb-api-version", choices=["1"], help="MongoDB stable API version to use."
+        )
+        parser.add_argument(
+            "--disable-test-commands", action="store_true", help="Disable test commands."
+        )
+
     # Get the options.
     if not allow_extra_opts:
         opts, extra_opts = parser.parse_args(), []
@@ -113,7 +144,7 @@ def get_test_options(
     return opts, extra_opts
 
 
-def read_env(path: Path | str) -> dict[str, Any]:
+def read_env(path: Path | str) -> dict[str, str]:
     config = dict()
     with Path(path).open() as fid:
         for line in fid.readlines():

CONTRIBUTING.md

@@ -217,9 +217,11 @@ the pages will re-render and the browser will automatically refresh.
 
 ### Usage
 
-- Run `just run-server` with optional args to set up the server.
-  All given flags will be passed to `run-orchestration.sh` in `$DRIVERS_TOOLS`.
+- Run `just run-server` with optional args to set up the server.  All given options will be passed to
+  `run-orchestration.sh` in `$DRIVERS_TOOLS`.  See `$DRIVERS_TOOLS/evergreen/run-orchestration.sh -h`
+  for a full list of options.
 - Run `just setup-tests` with optional args to set up the test environment, secrets, etc.
+  See `just setup-tests -h` for a full list of available options.
 - Run `just run-tests` to run the tests in an appropriate Python environment.
 - When done, run `just teardown-tests` to clean up and `just stop-server` to stop the server.
 
@@ -346,11 +348,28 @@ If you are running one of the `no-responder` tests, omit the `run-server` step.
 - Run the tests: `just run-tests`.
 
 ## Enable Debug Logs
+
 - Use `-o log_cli_level="DEBUG" -o log_cli=1` with `just test` or `pytest`.
 - Add `log_cli_level = "DEBUG` and `log_cli = 1` to the `tool.pytest.ini_options` section in `pyproject.toml` for Evergreen patches or to enable debug logs by default on your machine.
 - You can also set `DEBUG_LOG=1` and run either `just setup-tests` or `just-test`.
+- Finally, you can use `just setup-tests --debug-log`.
 - For evergreen patch builds, you can use `evergreen patch --param DEBUG_LOG=1` to enable debug logs for the patch.
 
+## Adding a new test suite
+
+- If adding new tests files that should only be run for that test suite, add a pytest marker to the file and add
+  to the list of pytest markers in `pyproject.toml`.  Then add the test suite to the `TEST_SUITE_MAP` in `.evergreen/scripts/utils.py`.  If for some reason it is not a pytest-runnable test, add it to the list of `EXTRA_TESTS` instead.
+- If the test uses Atlas or otherwise doesn't use `run-orchestration.sh`, add it to the `NO_RUN_ORCHESTRATION` list in
+  `.evergreen/scripts/utils.py`.
+- If there is something special required to run the local server or there is an extra flag that should always be set
+  like `AUTH`, add that logic to `.evergreen/scripts/run_server.py`.
+- The bulk of the logic will typically be in `.evergreen/scripts/setup_tests.py`.  This is where you should fetch secrets and make them available using `write_env`, start services, and write other env vars needed using `write_env`.
+- If there are any special test considerations, including not running `pytest` at all, handle it in `.evergreen/scripts/run_tests.py`.
+- If there are any services or atlas clusters to teardown, handle them in `.evergreen/scripts/teardown_tests.py`.
+- Add functions to generate the test variant(s) and task(s) to the `.evergreen/scripts/generate_config.py`.
+- Regenerate the test variants and tasks using the instructions in `.evergreen/scripts/generate_config.py`.
+- Make sure to add instructions for running the test suite to `CONTRIBUTING.md`.
+
 ## Re-sync Spec Tests
 
 If you would like to re-sync the copy of the specification tests in the

doc/changelog.rst

@@ -11,9 +11,16 @@ PyMongo 4.12 brings a number of changes including:
 - Support for $lookup in CSFLE and QE supported on MongoDB 8.1+.
 - Added :meth:`gridfs.asynchronous.grid_file.AsyncGridFSBucket.rename_by_name` and :meth:`gridfs.grid_file.GridFSBucket.rename_by_name`
   for more performant renaming of file revisions.
+- AsyncMongoClient no longer performs DNS resolution for "mongodb+srv://" connection strings on creation.
+  To avoid blocking the asyncio loop, the resolution is now deferred until the client is first connected.
 - Added index hinting support to the
   :meth:`~pymongo.asynchronous.collection.AsyncCollection.distinct` and
   :meth:`~pymongo.collection.Collection.distinct` commands.
+- Deprecated the ``hedge`` parameter for
+  :class:`~pymongo.read_preferences.PrimaryPreferred`,
+  :class:`~pymongo.read_preferences.Secondary`,
+  :class:`~pymongo.read_preferences.SecondaryPreferred`,
+  :class:`~pymongo.read_preferences.Nearest`. Support for ``hedge`` will be removed in PyMongo 5.0.
 
 Issues Resolved
 ...............

pymongo/asynchronous/encryption.py

@@ -87,7 +87,7 @@
 from pymongo.results import BulkWriteResult, DeleteResult
 from pymongo.ssl_support import get_ssl_context
 from pymongo.typings import _DocumentType, _DocumentTypeArg
-from pymongo.uri_parser import parse_host
+from pymongo.uri_parser_shared import parse_host
 from pymongo.write_concern import WriteConcern
 
 if TYPE_CHECKING: