Merge branch 'master' of github.com:mongodb/mongo-python-driver

Author: blink1073

.evergreen/config.yml

@@ -432,9 +432,6 @@ functions:
           if [ -n "${test_serverless}" ]; then
             export TEST_SERVERLESS=1
           fi
-          if [ -n "${IS_SERVERLESS_PROXY}" ]; then
-            export IS_SERVERLESS_PROXY=1
-          fi
           if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then
             export TEST_INDEX_MANAGEMENT=1
           fi
@@ -1121,7 +1118,7 @@ tasks:
 
     - name: "release-mac"
       tags: ["release_tag"]
-      run_on: macos-1100
+      run_on: macos-11
       commands:
         - func: "build release"
           vars:
@@ -2219,16 +2216,16 @@ axes:
   - id: platform
     display_name: OS
     values:
-      - id: macos-1100
-        display_name: "macOS 11.00"
-        run_on: macos-1100
+      - id: macos
+        display_name: "macOS"
+        run_on: macos-14
         variables:
           skip_EC2_auth_test: true
           skip_ECS_auth_test: true
           skip_web_identity_auth_test: true
-      - id: macos-1100-arm64
-        display_name: "macOS 11.00 Arm64"
-        run_on: macos-1100-arm64
+      - id: macos-arm64
+        display_name: "macOS Arm64"
+        run_on: macos-14-arm64
         variables:
           skip_EC2_auth_test: true
           skip_ECS_auth_test: true
@@ -2600,18 +2597,11 @@ axes:
   - id: serverless
     display_name: "Serverless"
     values:
-      - id: "original"
+      - id: "enabled"
         display_name: "Serverless"
         variables:
           test_serverless: true
         batchtime: 10080  # 7 days
-      - id: "proxy"
-        display_name: "Serverless Proxy"
-        variables:
-          test_serverless: true
-          VAULT_NAME: "serverless_next"
-          IS_SERVERLESS_PROXY: true
-        batchtime: 10080  # 7 days
 
 buildvariants:
 - matrix_name: "tests-fips"
@@ -2629,12 +2619,12 @@ buildvariants:
     platform:
       # MacOS introduced SSL support with MongoDB >= 3.2.
       # Older server versions (2.6, 3.0) are supported without SSL.
-      - macos-1100
+      - macos
     auth: "*"
     ssl: "*"
   exclude_spec:
     # No point testing with SSL without auth.
-    - platform: macos-1100
+    - platform: macos
       auth: "noauth"
       ssl: "ssl"
   display_name: "${platform} ${auth} ${ssl}"
@@ -2652,7 +2642,7 @@ buildvariants:
 - matrix_name: "test-macos-arm64"
   matrix_spec:
     platform:
-      - macos-1100-arm64
+      - macos-arm64
     auth-ssl: "*"
   display_name: "${platform} ${auth-ssl}"
   tasks:
@@ -2666,7 +2656,7 @@ buildvariants:
 - matrix_name: "test-macos-encryption"
   matrix_spec:
     platform:
-      - macos-1100
+      - macos
     auth: "auth"
     ssl: "nossl"
     encryption: "*"
@@ -2675,7 +2665,7 @@ buildvariants:
   rules:
     - if:
         encryption: ["encryption", "encryption_crypt_shared"]
-        platform: macos-1100
+        platform: macos
         auth: "auth"
         ssl: "nossl"
       then:
@@ -2743,7 +2733,7 @@ buildvariants:
 
 - matrix_name: "tests-pyopenssl-macOS"
   matrix_spec:
-    platform: macos-1100
+    platform: macos
     auth: "auth"
     ssl: "ssl"
     pyopenssl: "*"
@@ -3012,17 +3002,7 @@ buildvariants:
     platform: rhel8
     python-version: "*"
     auth-ssl: auth-ssl
-    serverless: "original"
-  display_name: "${serverless} ${python-version} ${platform}"
-  tasks:
-    - "serverless_task_group"
-
-- matrix_name: "serverless_proxy"
-  matrix_spec:
-    platform: rhel8
-    python-version: ["3.8", "3.10"]
-    auth-ssl: auth-ssl
-    serverless: "proxy"
+    serverless: "enabled"
   display_name: "${serverless} ${python-version} ${platform}"
   tasks:
     - "serverless_task_group"
@@ -3077,7 +3057,7 @@ buildvariants:
 
 - matrix_name: "ocsp-test-macos"
   matrix_spec:
-    platform: macos-1100
+    platform: macos
     mongodb-version: ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"]
     auth: "noauth"
     ssl: "ssl"
@@ -3089,7 +3069,7 @@ buildvariants:
 
 - matrix_name: "oidc-auth-test"
   matrix_spec:
-    platform: [ rhel8, macos-1100, windows-64-vsMulti-small ]
+    platform: [ rhel8, macos, windows-64-vsMulti-small ]
   display_name: "OIDC Auth ${platform}"
   tasks:
     - name: testoidc_task_group
@@ -3125,7 +3105,7 @@ buildvariants:
 
 - matrix_name: "aws-auth-test-mac"
   matrix_spec:
-    platform: [macos-1100]
+    platform: [macos]
   display_name: "MONGODB-AWS Auth ${platform} ${python-version-mac}"
   tasks:
     - name: "aws-auth-test-4.4"

.evergreen/run-tests.sh

@@ -30,7 +30,7 @@ set -o xtrace
 
 AUTH=${AUTH:-noauth}
 SSL=${SSL:-nossl}
-TEST_SUITES="test/ test/asynchronous/"
+TEST_SUITES=""
 TEST_ARGS="${*:1}"
 
 export PIP_QUIET=1  # Quiet by default
@@ -96,7 +96,7 @@ if [ -n "$TEST_LOADBALANCER" ]; then
     export LOAD_BALANCER=1
     export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI:-mongodb://127.0.0.1:8000/?loadBalanced=true}"
     export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI:-mongodb://127.0.0.1:8001/?loadBalanced=true}"
-    export TEST_SUITES="test/test_load_balancer.py"
+    export TEST_SUITES="load_balancer"
 fi
 
 if [ "$SSL" != "nossl" ]; then
@@ -172,7 +172,7 @@ if [ -n "$TEST_ENCRYPTION" ]; then
         export PATH=$CRYPT_SHARED_DIR:$PATH
     fi
     # Only run the encryption tests.
-    TEST_SUITES="test/test_encryption.py"
+    TEST_SUITES="encryption"
 fi
 
 if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
@@ -185,8 +185,7 @@ if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
       echo "MONGODB_URI unexpectedly contains user credentials in FLE test!";
       exit 1
     fi
-
-    TEST_SUITES="test/test_on_demand_csfle.py"
+    TEST_SUITES="csfle"
 fi
 
 if [ -n "$TEST_INDEX_MANAGEMENT" ]; then
@@ -195,36 +194,36 @@ if [ -n "$TEST_INDEX_MANAGEMENT" ]; then
     set +x
     export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}"
     set -x
-    TEST_SUITES="test/test_index_management.py"
+    TEST_SUITES="index_management"
 fi
 
 if [ -n "$TEST_DATA_LAKE" ] && [ -z "$TEST_ARGS" ]; then
-    TEST_SUITES="test/test_data_lake.py"
+    TEST_SUITES="data_lake"
 fi
 
 if [ -n "$TEST_ATLAS" ]; then
-    TEST_SUITES="test/atlas/test_connection.py"
+    TEST_SUITES="atlas"
 fi
 
 if [ -n "$TEST_OCSP" ]; then
     python -m pip install ".[ocsp]"
-    TEST_SUITES="test/ocsp/test_ocsp.py"
+    TEST_SUITES="ocsp"
 fi
 
 if [ -n "$TEST_AUTH_AWS" ]; then
     python -m pip install ".[aws]"
-    TEST_SUITES="test/auth_aws/test_auth_aws.py"
+    TEST_SUITES="auth_aws"
 fi
 
 if [ -n "$TEST_AUTH_OIDC" ]; then
     python -m pip install ".[aws]"
-    TEST_SUITES="test/auth_oidc/test_auth_oidc.py $TEST_ARGS"
+    TEST_SUITES="auth_oidc"
 fi
 
 if [ -n "$PERF_TEST" ]; then
     python -m pip install simplejson
     start_time=$(date +%s)
-    TEST_SUITES="test/performance/perf_test.py"
+    TEST_SUITES="perf"
 fi
 
 echo "Running $AUTH tests over $SSL with python $(which python)"
@@ -254,7 +253,11 @@ PIP_QUIET=0 python -m pip list
 if [ -z "$GREEN_FRAMEWORK" ]; then
     # Use --capture=tee-sys so pytest prints test output inline:
     # https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html
-    python -m pytest -v --capture=tee-sys --durations=5 --maxfail=10 $TEST_SUITES $TEST_ARGS
+    if [ -z "$TEST_SUITES" ]; then
+      python -m pytest -v --capture=tee-sys --durations=5 --maxfail=10 $TEST_ARGS
+    else
+      python -m pytest -v --capture=tee-sys --durations=5 --maxfail=10 -m $TEST_SUITES $TEST_ARGS
+    fi
 else
     python green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS
 fi

hatch.toml

@@ -42,7 +42,7 @@ features = ["test"]
 test = "pytest -v --durations=5 --maxfail=10 {args}"
 test-eg = "bash ./.evergreen/run-tests.sh {args}"
 test-async = "test test/asynchronous/ {args}"
-test-mockupdb = ["pip install -U git+https://github.com/ajdavis/mongo-mockup-db@master", "test ./test/mockupdb"]
+test-mockupdb = ["pip install -U git+https://github.com/ajdavis/mongo-mockup-db@master", "test -m mockupdb"]
 
 [envs.encryption]
 skip-install = true

pymongo/pyopenssl_context.py

@@ -292,7 +292,7 @@ def load_cert_chain(
         # Password callback MUST be set first or it will be ignored.
         if password:
 
-            def _pwcb(_max_length: int, _prompt_twice: bool, _user_data: bytes) -> bytes:
+            def _pwcb(_max_length: int, _prompt_twice: bool, _user_data: Optional[bytes]) -> bytes:
                 # XXX:We could check the password length against what OpenSSL
                 # tells us is the max, but we can't raise an exception, so...
                 # warn?
@@ -332,6 +332,7 @@ def _load_certifi(self) -> None:
     def _load_wincerts(self, store: str) -> None:
         """Attempt to load CA certs from Windows trust store."""
         cert_store = self._ctx.get_cert_store()
+        assert cert_store is not None
         oid = _stdlibssl.Purpose.SERVER_AUTH.oid
 
         for cert, encoding, trust in _stdlibssl.enum_certificates(store):  # type: ignore

pyproject.toml

@@ -70,10 +70,9 @@ zstd = ["requirements/zstd.txt"]
 
 [tool.pytest.ini_options]
 minversion = "7"
-addopts = ["-ra", "--strict-config", "--strict-markers", "--junitxml=xunit-results/TEST-results.xml"]
+addopts = ["-ra", "--strict-config", "--strict-markers", "--junitxml=xunit-results/TEST-results.xml", "-m default"]
 testpaths = ["test"]
 log_cli_level = "INFO"
-norecursedirs = ["test/*"]
 faulthandler_timeout = 1500
 xfail_strict = true
 filterwarnings = [
@@ -96,6 +95,20 @@ filterwarnings = [
     # https://github.com/dateutil/dateutil/issues/1314
     "module:datetime.datetime.utc:DeprecationWarning:dateutil",
 ]
+markers = [
+    "auth_aws: tests that rely on pymongo-auth-aws",
+    "auth_oidc: tests that rely on oidc auth",
+    "ocsp: tests that rely on ocsp",
+    "atlas: tests that rely on atlas",
+    "data_lake: tests that rely on atlas data lake",
+    "perf: benchmark tests",
+    "index_management: index management tests",
+    "csfle: client-side field-level encryption tests",
+    "encryption: encryption tests",
+    "load_balancer: load balancer tests",
+    "mockupdb: tests that rely on mockupdb",
+    "default: default test suite",
+]
 
 [tool.mypy]
 strict = true

test/asynchronous/conftest.py

@@ -12,3 +12,9 @@ async def test_setup_and_teardown():
     await async_setup()
     yield
     await async_teardown()
+
+
+def pytest_collection_modifyitems(items, config):
+    for item in items:
+        if not any(item.iter_markers()):
+            item.add_marker("default")

test/atlas/test_connection.py

@@ -20,11 +20,16 @@
 import unittest
 from collections import defaultdict
 
+import pytest
+
 sys.path[0:0] = [""]
 
 import pymongo
 from pymongo.ssl_support import HAS_SNI
 
+pytestmark = pytest.mark.atlas
+
+
 URIS = {
     "ATLAS_REPL": os.environ.get("ATLAS_REPL"),
     "ATLAS_SHRD": os.environ.get("ATLAS_SHRD"),

test/auth_aws/test_auth_aws.py

@@ -20,14 +20,21 @@
 import unittest
 from unittest.mock import patch
 
+import pytest
+
 sys.path[0:0] = [""]
 
-from pymongo_auth_aws import AwsCredential, auth
+try:
+    from pymongo_auth_aws import AwsCredential, auth
+except ImportError:
+    pass
 
 from pymongo import MongoClient
 from pymongo.errors import OperationFailure
 from pymongo.uri_parser import parse_uri
 
+pytestmark = pytest.mark.auth_aws
+
 
 class TestAuthAWS(unittest.TestCase):
     uri: str

test/auth_oidc/test_auth_oidc.py

@@ -25,6 +25,8 @@
 from pathlib import Path
 from typing import Dict
 
+import pytest
+
 sys.path[0:0] = [""]
 
 from test.unified_format import generate_test_classes
@@ -55,6 +57,8 @@
 # Generate unified tests.
 globals().update(generate_test_classes(str(TEST_PATH), module=__name__))
 
+pytestmark = pytest.mark.auth_oidc
+
 
 class OIDCTestBase(unittest.TestCase):
     @classmethod
@@ -96,6 +100,7 @@ def fail_point(self, command_args):
             client.admin.command("configureFailPoint", cmd_on["configureFailPoint"], mode="off")
 
 
+@pytest.mark.auth_oidc
 class TestAuthOIDCHuman(OIDCTestBase):
     uri: str
 

test/conftest.py

@@ -12,3 +12,9 @@ def test_setup_and_teardown():
     setup()
     yield
     teardown()
+
+
+def pytest_collection_modifyitems(items, config):
+    for item in items:
+        if not any(item.iter_markers()):
+            item.add_marker("default")