diff --git a/.evergreen/config.yml b/.evergreen/config.yml index dee4b608ec..9083da145b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -409,6 +409,7 @@ functions: AUTH=${AUTH} \ SSL=${SSL} \ TEST_DATA_LAKE=${TEST_DATA_LAKE} \ + TEST_SUITES=${TEST_SUITES} \ MONGODB_API_VERSION=${MONGODB_API_VERSION} \ SKIP_HATCH=${SKIP_HATCH} \ bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg @@ -2111,23 +2112,6 @@ axes: AUTH: "noauth" SSL: "nossl" - # Choice of wire protocol compression support - - id: compression - display_name: Compression - values: - - id: snappy - display_name: snappy compression - variables: - COMPRESSORS: "snappy" - - id: zlib - display_name: zlib compression - variables: - COMPRESSORS: "zlib" - - id: zstd - display_name: zstd compression - variables: - COMPRESSORS: "zstd" - # Choice of MongoDB server version - id: mongodb-version display_name: "MongoDB" @@ -2321,42 +2305,6 @@ axes: variables: COVERAGE: "coverage" - # Run encryption tests? - - id: encryption - display_name: "Encryption" - values: - - id: "encryption" - display_name: "Encryption" - tags: ["encryption_tag"] - variables: - test_encryption: true - batchtime: 10080 # 7 days - - id: "encryption_pyopenssl" - display_name: "Encryption PyOpenSSL" - tags: ["encryption_tag"] - variables: - test_encryption: true - test_encryption_pyopenssl: true - batchtime: 10080 # 7 days - # The path to crypt_shared is stored in the $CRYPT_SHARED_LIB_PATH expansion. - - id: "encryption_crypt_shared" - display_name: "Encryption shared lib" - tags: ["encryption_tag"] - variables: - test_encryption: true - test_crypt_shared: true - batchtime: 10080 # 7 days - - # Run pyopenssl tests? - - id: pyopenssl - display_name: "PyOpenSSL" - values: - - id: "enabled" - display_name: "PyOpenSSL" - variables: - test_pyopenssl: true - batchtime: 10080 # 7 days - - id: versionedApi display_name: "versionedApi" values: @@ -2379,16 +2327,6 @@ axes: variables: ORCHESTRATION_FILE: "versioned-api-testing.json" - # Run load balancer tests? - - id: loadbalancer - display_name: "Load Balancer" - values: - - id: "enabled" - display_name: "Load Balancer" - variables: - test_loadbalancer: true - batchtime: 10080 # 7 days - - id: serverless display_name: "Serverless" values: @@ -2399,80 +2337,1044 @@ axes: batchtime: 10080 # 7 days buildvariants: -- matrix_name: "tests-fips" - matrix_spec: - platform: - - rhel9-fips - auth: "auth" - ssl: "ssl" - display_name: "${platform} ${auth} ${ssl}" +# Server Tests for RHEL8. +- name: test-rhel8-py3.9-auth-ssl-cov tasks: - - "test-fips-standalone" + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Test RHEL8 py3.9 Auth SSL cov + run_on: + - rhel87-small + expansions: + AUTH: auth + SSL: ssl + COVERAGE: coverage + PYTHON_BINARY: /opt/python/3.9/bin/python3 + tags: [coverage_tag] +- name: test-rhel8-py3.9-noauth-ssl-cov + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Test RHEL8 py3.9 NoAuth SSL cov + run_on: + - rhel87-small + expansions: + AUTH: noauth + SSL: ssl + COVERAGE: coverage + PYTHON_BINARY: /opt/python/3.9/bin/python3 + tags: [coverage_tag] +- name: test-rhel8-py3.9-noauth-nossl-cov + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Test RHEL8 py3.9 NoAuth NoSSL cov + run_on: + - rhel87-small + expansions: + AUTH: noauth + SSL: nossl + COVERAGE: coverage + PYTHON_BINARY: /opt/python/3.9/bin/python3 + tags: [coverage_tag] +- name: test-rhel8-py3.13-auth-ssl-cov + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Test RHEL8 py3.13 Auth SSL cov + run_on: + - rhel87-small + expansions: + AUTH: auth + SSL: ssl + COVERAGE: coverage + PYTHON_BINARY: /opt/python/3.13/bin/python3 + tags: [coverage_tag] +- name: test-rhel8-py3.13-noauth-ssl-cov + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Test RHEL8 py3.13 NoAuth SSL cov + run_on: + - rhel87-small + expansions: + AUTH: noauth + SSL: ssl + COVERAGE: coverage + PYTHON_BINARY: /opt/python/3.13/bin/python3 + tags: [coverage_tag] +- name: test-rhel8-py3.13-noauth-nossl-cov + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Test RHEL8 py3.13 NoAuth NoSSL cov + run_on: + - rhel87-small + expansions: + AUTH: noauth + SSL: nossl + COVERAGE: coverage + PYTHON_BINARY: /opt/python/3.13/bin/python3 + tags: [coverage_tag] +- name: test-rhel8-pypy3.10-auth-ssl-cov + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Test RHEL8 pypy3.10 Auth SSL cov + run_on: + - rhel87-small + expansions: + AUTH: auth + SSL: ssl + COVERAGE: coverage + PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 + tags: [coverage_tag] +- name: test-rhel8-pypy3.10-noauth-ssl-cov + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Test RHEL8 pypy3.10 NoAuth SSL cov + run_on: + - rhel87-small + expansions: + AUTH: noauth + SSL: ssl + COVERAGE: coverage + PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 + tags: [coverage_tag] +- name: test-rhel8-pypy3.10-noauth-nossl-cov + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Test RHEL8 pypy3.10 NoAuth NoSSL cov + run_on: + - rhel87-small + expansions: + AUTH: noauth + SSL: nossl + COVERAGE: coverage + PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 + tags: [coverage_tag] +- name: test-rhel8-py3.10-auth-ssl + tasks: + - name: .standalone + display_name: Test RHEL8 py3.10 Auth SSL + run_on: + - rhel87-small + expansions: + AUTH: auth + SSL: ssl + PYTHON_BINARY: /opt/python/3.10/bin/python3 +- name: test-rhel8-py3.11-noauth-ssl + tasks: + - name: .replica_set + display_name: Test RHEL8 py3.11 NoAuth SSL + run_on: + - rhel87-small + expansions: + AUTH: noauth + SSL: ssl + PYTHON_BINARY: /opt/python/3.11/bin/python3 +- name: test-rhel8-py3.12-noauth-nossl + tasks: + - name: .sharded_cluster + display_name: Test RHEL8 py3.12 NoAuth NoSSL + run_on: + - rhel87-small + expansions: + AUTH: noauth + SSL: nossl + PYTHON_BINARY: /opt/python/3.12/bin/python3 +- name: test-rhel8-pypy3.9-auth-ssl + tasks: + - name: .standalone + display_name: Test RHEL8 pypy3.9 Auth SSL + run_on: + - rhel87-small + expansions: + AUTH: auth + SSL: ssl + PYTHON_BINARY: /opt/python/pypy3.9/bin/python3 -- matrix_name: "test-macos" - matrix_spec: - platform: - # MacOS introduced SSL support with MongoDB >= 3.2. - # Older server versions (2.6, 3.0) are supported without SSL. - - macos - auth: "*" - ssl: "*" - exclude_spec: - # No point testing with SSL without auth. - - platform: macos - auth: "noauth" - ssl: "ssl" - display_name: "${platform} ${auth} ${ssl}" +# Server tests for MacOS. +- name: test-macos-py3.9-auth-ssl-sync tasks: - - ".latest" - - ".8.0" - - ".7.0" - - ".6.0" - - ".5.0" - - ".4.4" - - ".4.2" - - ".4.0" + - name: .standalone + display_name: Test macOS py3.9 Auth SSL Sync + run_on: + - macos-14 + expansions: + AUTH: auth + SSL: ssl + TEST_SUITES: default + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 + SKIP_CSOT_TESTS: "true" +- name: test-macos-py3.9-auth-ssl-async + tasks: + - name: .standalone + display_name: Test macOS py3.9 Auth SSL Async + run_on: + - macos-14 + expansions: + AUTH: auth + SSL: ssl + TEST_SUITES: default_async + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 + SKIP_CSOT_TESTS: "true" +- name: test-macos-py3.13-noauth-ssl-sync + tasks: + - name: .replica_set + display_name: Test macOS py3.13 NoAuth SSL Sync + run_on: + - macos-14 + expansions: + AUTH: noauth + SSL: ssl + TEST_SUITES: default + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 + SKIP_CSOT_TESTS: "true" +- name: test-macos-py3.13-noauth-ssl-async + tasks: + - name: .replica_set + display_name: Test macOS py3.13 NoAuth SSL Async + run_on: + - macos-14 + expansions: + AUTH: noauth + SSL: ssl + TEST_SUITES: default_async + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 + SKIP_CSOT_TESTS: "true" +- name: test-macos-py3.9-noauth-nossl-sync + tasks: + - name: .sharded_cluster + display_name: Test macOS py3.9 NoAuth NoSSL Sync + run_on: + - macos-14 + expansions: + AUTH: noauth + SSL: nossl + TEST_SUITES: default + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 + SKIP_CSOT_TESTS: "true" +- name: test-macos-py3.9-noauth-nossl-async + tasks: + - name: .sharded_cluster + display_name: Test macOS py3.9 NoAuth NoSSL Async + run_on: + - macos-14 + expansions: + AUTH: noauth + SSL: nossl + TEST_SUITES: default_async + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 + SKIP_CSOT_TESTS: "true" -- matrix_name: "test-macos-arm64" - matrix_spec: - platform: - - macos-arm64 - auth-ssl: "*" - display_name: "${platform} ${auth-ssl}" +# Server tests for macOS Arm64. +- name: test-macos-arm64-py3.9-auth-ssl-sync + tasks: + - name: .standalone .6.0 + - name: .standalone .7.0 + - name: .standalone .8.0 + - name: .standalone .rapid + - name: .standalone .latest + display_name: Test macOS Arm64 py3.9 Auth SSL Sync + run_on: + - macos-14-arm64 + expansions: + AUTH: auth + SSL: ssl + TEST_SUITES: default + SKIP_CSOT_TESTS: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 +- name: test-macos-arm64-py3.9-auth-ssl-async + tasks: + - name: .standalone .6.0 + - name: .standalone .7.0 + - name: .standalone .8.0 + - name: .standalone .rapid + - name: .standalone .latest + display_name: Test macOS Arm64 py3.9 Auth SSL Async + run_on: + - macos-14-arm64 + expansions: + AUTH: auth + SSL: ssl + TEST_SUITES: default_async + SKIP_CSOT_TESTS: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 +- name: test-macos-arm64-py3.13-noauth-ssl-sync + tasks: + - name: .replica_set .6.0 + - name: .replica_set .7.0 + - name: .replica_set .8.0 + - name: .replica_set .rapid + - name: .replica_set .latest + display_name: Test macOS Arm64 py3.13 NoAuth SSL Sync + run_on: + - macos-14-arm64 + expansions: + AUTH: noauth + SSL: ssl + TEST_SUITES: default + SKIP_CSOT_TESTS: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 +- name: test-macos-arm64-py3.13-noauth-ssl-async + tasks: + - name: .replica_set .6.0 + - name: .replica_set .7.0 + - name: .replica_set .8.0 + - name: .replica_set .rapid + - name: .replica_set .latest + display_name: Test macOS Arm64 py3.13 NoAuth SSL Async + run_on: + - macos-14-arm64 + expansions: + AUTH: noauth + SSL: ssl + TEST_SUITES: default_async + SKIP_CSOT_TESTS: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 +- name: test-macos-arm64-py3.9-noauth-nossl-sync + tasks: + - name: .sharded_cluster .6.0 + - name: .sharded_cluster .7.0 + - name: .sharded_cluster .8.0 + - name: .sharded_cluster .rapid + - name: .sharded_cluster .latest + display_name: Test macOS Arm64 py3.9 NoAuth NoSSL Sync + run_on: + - macos-14-arm64 + expansions: + AUTH: noauth + SSL: nossl + TEST_SUITES: default + SKIP_CSOT_TESTS: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 +- name: test-macos-arm64-py3.9-noauth-nossl-async + tasks: + - name: .sharded_cluster .6.0 + - name: .sharded_cluster .7.0 + - name: .sharded_cluster .8.0 + - name: .sharded_cluster .rapid + - name: .sharded_cluster .latest + display_name: Test macOS Arm64 py3.9 NoAuth NoSSL Async + run_on: + - macos-14-arm64 + expansions: + AUTH: noauth + SSL: nossl + TEST_SUITES: default_async + SKIP_CSOT_TESTS: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 + +# Server tests for Windows. +- name: test-win64-py3.9-auth-ssl-sync + tasks: + - name: .standalone + display_name: Test Win64 py3.9 Auth SSL Sync + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: auth + SSL: ssl + TEST_SUITES: default + PYTHON_BINARY: C:/python/Python39/python.exe + SKIP_CSOT_TESTS: "true" +- name: test-win64-py3.9-auth-ssl-async + tasks: + - name: .standalone + display_name: Test Win64 py3.9 Auth SSL Async + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: auth + SSL: ssl + TEST_SUITES: default_async + PYTHON_BINARY: C:/python/Python39/python.exe + SKIP_CSOT_TESTS: "true" +- name: test-win64-py3.13-noauth-ssl-sync + tasks: + - name: .replica_set + display_name: Test Win64 py3.13 NoAuth SSL Sync + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: noauth + SSL: ssl + TEST_SUITES: default + PYTHON_BINARY: C:/python/Python313/python.exe + SKIP_CSOT_TESTS: "true" +- name: test-win64-py3.13-noauth-ssl-async + tasks: + - name: .replica_set + display_name: Test Win64 py3.13 NoAuth SSL Async + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: noauth + SSL: ssl + TEST_SUITES: default_async + PYTHON_BINARY: C:/python/Python313/python.exe + SKIP_CSOT_TESTS: "true" +- name: test-win64-py3.9-noauth-nossl-sync + tasks: + - name: .sharded_cluster + display_name: Test Win64 py3.9 NoAuth NoSSL Sync + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: noauth + SSL: nossl + TEST_SUITES: default + PYTHON_BINARY: C:/python/Python39/python.exe + SKIP_CSOT_TESTS: "true" +- name: test-win64-py3.9-noauth-nossl-async + tasks: + - name: .sharded_cluster + display_name: Test Win64 py3.9 NoAuth NoSSL Async + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: noauth + SSL: nossl + TEST_SUITES: default_async + PYTHON_BINARY: C:/python/Python39/python.exe + SKIP_CSOT_TESTS: "true" +- name: test-win32-py3.9-auth-ssl-sync + tasks: + - name: .standalone + display_name: Test Win32 py3.9 Auth SSL Sync + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: auth + SSL: ssl + TEST_SUITES: default + PYTHON_BINARY: C:/python/32/Python39/python.exe + SKIP_CSOT_TESTS: "true" + +# Server tests for Win32. +- name: test-win32-py3.9-auth-ssl-async + tasks: + - name: .standalone + display_name: Test Win32 py3.9 Auth SSL Async + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: auth + SSL: ssl + TEST_SUITES: default_async + PYTHON_BINARY: C:/python/32/Python39/python.exe + SKIP_CSOT_TESTS: "true" +- name: test-win32-py3.13-noauth-ssl-sync + tasks: + - name: .replica_set + display_name: Test Win32 py3.13 NoAuth SSL Sync + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: noauth + SSL: ssl + TEST_SUITES: default + PYTHON_BINARY: C:/python/32/Python313/python.exe + SKIP_CSOT_TESTS: "true" +- name: test-win32-py3.13-noauth-ssl-async + tasks: + - name: .replica_set + display_name: Test Win32 py3.13 NoAuth SSL Async + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: noauth + SSL: ssl + TEST_SUITES: default_async + PYTHON_BINARY: C:/python/32/Python313/python.exe + SKIP_CSOT_TESTS: "true" +- name: test-win32-py3.9-noauth-nossl-sync + tasks: + - name: .sharded_cluster + display_name: Test Win32 py3.9 NoAuth NoSSL Sync + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: noauth + SSL: nossl + TEST_SUITES: default + PYTHON_BINARY: C:/python/32/Python39/python.exe + SKIP_CSOT_TESTS: "true" +- name: test-win32-py3.9-noauth-nossl-async + tasks: + - name: .sharded_cluster + display_name: Test Win32 py3.9 NoAuth NoSSL Async + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: noauth + SSL: nossl + TEST_SUITES: default_async + PYTHON_BINARY: C:/python/32/Python39/python.exe + SKIP_CSOT_TESTS: "true" + +# Encryption tests. +- name: encryption-rhel8-py3.9-auth-ssl + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Encryption RHEL8 py3.9 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + PYTHON_BINARY: /opt/python/3.9/bin/python3 + tags: [encryption_tag] +- name: encryption-rhel8-py3.13-auth-ssl + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Encryption RHEL8 py3.13 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + PYTHON_BINARY: /opt/python/3.13/bin/python3 + tags: [encryption_tag] +- name: encryption-rhel8-pypy3.10-auth-ssl + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Encryption RHEL8 pypy3.10 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 + tags: [encryption_tag] +- name: encryption-crypt_shared-rhel8-py3.9-auth-ssl + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Encryption crypt_shared RHEL8 py3.9 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + test_crypt_shared: "true" + PYTHON_BINARY: /opt/python/3.9/bin/python3 + tags: [encryption_tag] +- name: encryption-crypt_shared-rhel8-py3.13-auth-ssl + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Encryption crypt_shared RHEL8 py3.13 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + test_crypt_shared: "true" + PYTHON_BINARY: /opt/python/3.13/bin/python3 + tags: [encryption_tag] +- name: encryption-crypt_shared-rhel8-pypy3.10-auth-ssl + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Encryption crypt_shared RHEL8 pypy3.10 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + test_crypt_shared: "true" + PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 + tags: [encryption_tag] +- name: encryption-pyopenssl-rhel8-py3.9-auth-ssl + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Encryption PyOpenSSL RHEL8 py3.9 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + test_encryption_pyopenssl: "true" + PYTHON_BINARY: /opt/python/3.9/bin/python3 + tags: [encryption_tag] +- name: encryption-pyopenssl-rhel8-py3.13-auth-ssl + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Encryption PyOpenSSL RHEL8 py3.13 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + test_encryption_pyopenssl: "true" + PYTHON_BINARY: /opt/python/3.13/bin/python3 + tags: [encryption_tag] +- name: encryption-pyopenssl-rhel8-pypy3.10-auth-ssl + tasks: + - name: .standalone + - name: .replica_set + - name: .sharded_cluster + display_name: Encryption PyOpenSSL RHEL8 pypy3.10 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + test_encryption_pyopenssl: "true" + PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 + tags: [encryption_tag] +- name: encryption-rhel8-py3.10-auth-ssl + tasks: + - name: .replica_set + display_name: Encryption RHEL8 py3.10 Auth SSL + run_on: + - rhel87-small + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + PYTHON_BINARY: /opt/python/3.10/bin/python3 +- name: encryption-crypt_shared-rhel8-py3.11-auth-nossl + tasks: + - name: .replica_set + display_name: Encryption crypt_shared RHEL8 py3.11 Auth NoSSL + run_on: + - rhel87-small + expansions: + AUTH: auth + SSL: nossl + test_encryption: "true" + test_crypt_shared: "true" + PYTHON_BINARY: /opt/python/3.11/bin/python3 +- name: encryption-pyopenssl-rhel8-py3.12-auth-ssl + tasks: + - name: .replica_set + display_name: Encryption PyOpenSSL RHEL8 py3.12 Auth SSL + run_on: + - rhel87-small + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + TEST_ENCRYPTION_PYOPENSSL: "true" + PYTHON_BINARY: /opt/python/3.12/bin/python3 +- name: encryption-rhel8-pypy3.9-auth-nossl + tasks: + - name: .replica_set + display_name: Encryption RHEL8 pypy3.9 Auth NoSSL + run_on: + - rhel87-small + expansions: + AUTH: auth + SSL: nossl + test_encryption: "true" + PYTHON_BINARY: /opt/python/pypy3.9/bin/python3 +- name: encryption-macos-py3.9-auth-ssl + tasks: + - name: .latest .replica_set + display_name: Encryption macOS py3.9 Auth SSL + run_on: + - macos-14 + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 + tags: [encryption_tag] +- name: encryption-macos-py3.13-auth-nossl + tasks: + - name: .latest .replica_set + display_name: Encryption macOS py3.13 Auth NoSSL + run_on: + - macos-14 + batchtime: 10080 + expansions: + AUTH: auth + SSL: nossl + test_encryption: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 + tags: [encryption_tag] +- name: encryption-crypt_shared-macos-py3.9-auth-ssl + tasks: + - name: .latest .replica_set + display_name: Encryption crypt_shared macOS py3.9 Auth SSL + run_on: + - macos-14 + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + test_crypt_shared: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 + tags: [encryption_tag] +- name: encryption-crypt_shared-macos-py3.13-auth-nossl + tasks: + - name: .latest .replica_set + display_name: Encryption crypt_shared macOS py3.13 Auth NoSSL + run_on: + - macos-14 + batchtime: 10080 + expansions: + AUTH: auth + SSL: nossl + test_encryption: "true" + test_crypt_shared: "true" + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 + tags: [encryption_tag] +- name: encryption-win64-py3.9-auth-ssl + tasks: + - name: .latest .replica_set + display_name: Encryption Win64 py3.9 Auth SSL + run_on: + - windows-64-vsMulti-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + PYTHON_BINARY: C:/python/Python39/python.exe + tags: [encryption_tag] +- name: encryption-win64-py3.13-auth-nossl + tasks: + - name: .latest .replica_set + display_name: Encryption Win64 py3.13 Auth NoSSL + run_on: + - windows-64-vsMulti-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: nossl + test_encryption: "true" + PYTHON_BINARY: C:/python/Python313/python.exe + tags: [encryption_tag] +- name: encryption-crypt_shared-win64-py3.9-auth-ssl + tasks: + - name: .latest .replica_set + display_name: Encryption crypt_shared Win64 py3.9 Auth SSL + run_on: + - windows-64-vsMulti-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: ssl + test_encryption: "true" + test_crypt_shared: "true" + PYTHON_BINARY: C:/python/Python39/python.exe + tags: [encryption_tag] +- name: encryption-crypt_shared-win64-py3.13-auth-nossl + tasks: + - name: .latest .replica_set + display_name: Encryption crypt_shared Win64 py3.13 Auth NoSSL + run_on: + - windows-64-vsMulti-small + batchtime: 10080 + expansions: + AUTH: auth + SSL: nossl + test_encryption: "true" + test_crypt_shared: "true" + PYTHON_BINARY: C:/python/Python313/python.exe + tags: [encryption_tag] + +# Compressor tests. +- name: snappy-compression-rhel8-py3.9-no-c + tasks: + - name: .standalone + display_name: snappy compression RHEL8 py3.9 No C + run_on: + - rhel87-small + expansions: + COMPRESSORS: snappy + NO_EXT: "1" + PYTHON_BINARY: /opt/python/3.9/bin/python3 +- name: snappy-compression-rhel8-py3.10 + tasks: + - name: .standalone + display_name: snappy compression RHEL8 py3.10 + run_on: + - rhel87-small + expansions: + COMPRESSORS: snappy + PYTHON_BINARY: /opt/python/3.10/bin/python3 +- name: zlib-compression-rhel8-py3.11-no-c + tasks: + - name: .standalone + display_name: zlib compression RHEL8 py3.11 No C + run_on: + - rhel87-small + expansions: + COMPRESSORS: zlib + NO_EXT: "1" + PYTHON_BINARY: /opt/python/3.11/bin/python3 +- name: zlib-compression-rhel8-py3.12 + tasks: + - name: .standalone + display_name: zlib compression RHEL8 py3.12 + run_on: + - rhel87-small + expansions: + COMPRESSORS: zlib + PYTHON_BINARY: /opt/python/3.12/bin/python3 +- name: zstd-compression-rhel8-py3.13-no-c + tasks: + - name: .standalone !.4.0 + display_name: zstd compression RHEL8 py3.13 No C + run_on: + - rhel87-small + expansions: + COMPRESSORS: zstd + NO_EXT: "1" + PYTHON_BINARY: /opt/python/3.13/bin/python3 +- name: zstd-compression-rhel8-py3.9 + tasks: + - name: .standalone !.4.0 + display_name: zstd compression RHEL8 py3.9 + run_on: + - rhel87-small + expansions: + COMPRESSORS: zstd + PYTHON_BINARY: /opt/python/3.9/bin/python3 +- name: snappy-compression-rhel8-pypy3.9 + tasks: + - name: .standalone + display_name: snappy compression RHEL8 pypy3.9 + run_on: + - rhel87-small + expansions: + COMPRESSORS: snappy + PYTHON_BINARY: /opt/python/pypy3.9/bin/python3 +- name: zlib-compression-rhel8-pypy3.10 + tasks: + - name: .standalone + display_name: zlib compression RHEL8 pypy3.10 + run_on: + - rhel87-small + expansions: + COMPRESSORS: zlib + PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 +- name: zstd-compression-rhel8-pypy3.9 + tasks: + - name: .standalone !.4.0 + display_name: zstd compression RHEL8 pypy3.9 + run_on: + - rhel87-small + expansions: + COMPRESSORS: zstd + PYTHON_BINARY: /opt/python/pypy3.9/bin/python3 + +# Enterprise auth tests. +- name: enterprise-auth-macos-py3.9-auth + tasks: + - name: test-enterprise-auth + display_name: Enterprise Auth macOS py3.9 Auth + run_on: + - macos-14 + expansions: + AUTH: auth + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 +- name: enterprise-auth-rhel8-py3.10-auth + tasks: + - name: test-enterprise-auth + display_name: Enterprise Auth RHEL8 py3.10 Auth + run_on: + - rhel87-small + expansions: + AUTH: auth + PYTHON_BINARY: /opt/python/3.10/bin/python3 +- name: enterprise-auth-rhel8-py3.11-auth + tasks: + - name: test-enterprise-auth + display_name: Enterprise Auth RHEL8 py3.11 Auth + run_on: + - rhel87-small + expansions: + AUTH: auth + PYTHON_BINARY: /opt/python/3.11/bin/python3 +- name: enterprise-auth-rhel8-py3.12-auth + tasks: + - name: test-enterprise-auth + display_name: Enterprise Auth RHEL8 py3.12 Auth + run_on: + - rhel87-small + expansions: + AUTH: auth + PYTHON_BINARY: /opt/python/3.12/bin/python3 +- name: enterprise-auth-win64-py3.13-auth + tasks: + - name: test-enterprise-auth + display_name: Enterprise Auth Win64 py3.13 Auth + run_on: + - windows-64-vsMulti-small + expansions: + AUTH: auth + PYTHON_BINARY: C:/python/Python313/python.exe +- name: enterprise-auth-rhel8-pypy3.9-auth + tasks: + - name: test-enterprise-auth + display_name: Enterprise Auth RHEL8 pypy3.9 Auth + run_on: + - rhel87-small + expansions: + AUTH: auth + PYTHON_BINARY: /opt/python/pypy3.9/bin/python3 +- name: enterprise-auth-rhel8-pypy3.10-auth + tasks: + - name: test-enterprise-auth + display_name: Enterprise Auth RHEL8 pypy3.10 Auth + run_on: + - rhel87-small + expansions: + AUTH: auth + PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 + +# PyOpenSSL tests. +- name: pyopenssl-macos-py3.9 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL macOS py3.9 + run_on: + - macos-14 + batchtime: 10080 + expansions: + AUTH: noauth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3 +- name: pyopenssl-rhel8-py3.10 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL RHEL8 py3.10 + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: /opt/python/3.10/bin/python3 +- name: pyopenssl-rhel8-py3.11 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL RHEL8 py3.11 + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: /opt/python/3.11/bin/python3 +- name: pyopenssl-rhel8-py3.12 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL RHEL8 py3.12 + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: /opt/python/3.12/bin/python3 +- name: pyopenssl-win64-py3.13 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL Win64 py3.13 + run_on: + - windows-64-vsMulti-small + batchtime: 10080 + expansions: + AUTH: auth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: C:/python/Python313/python.exe +- name: pyopenssl-rhel8-pypy3.9 + tasks: + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL RHEL8 pypy3.9 + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: /opt/python/pypy3.9/bin/python3 +- name: pyopenssl-rhel8-pypy3.10 tasks: - - ".latest" - - ".8.0" - - ".7.0" - - ".6.0" - - ".5.0" - - ".4.4" + - name: .replica_set + - name: .7.0 + display_name: PyOpenSSL RHEL8 pypy3.10 + run_on: + - rhel87-small + batchtime: 10080 + expansions: + AUTH: auth + test_pyopenssl: "true" + SSL: ssl + PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 -- matrix_name: "test-macos-encryption" +- matrix_name: "tests-fips" matrix_spec: platform: - - macos + - rhel9-fips auth: "auth" - ssl: "nossl" - encryption: "*" - display_name: "${encryption} ${platform} ${auth} ${ssl}" - tasks: "test-latest-replica_set" - rules: - - if: - encryption: ["encryption", "encryption_crypt_shared"] - platform: macos - auth: "auth" - ssl: "nossl" - then: - add_tasks: &encryption-server-versions - - ".rapid" - - ".latest" - - ".8.0" - - ".7.0" - - ".6.0" - - ".5.0" - - ".4.4" - - ".4.2" - - ".4.0" + ssl: "ssl" + display_name: "${platform} ${auth} ${ssl}" + tasks: + - "test-fips-standalone" # Test one server version with zSeries, POWER8, and ARM. - matrix_name: "test-different-cpu-architectures" @@ -2486,85 +3388,6 @@ buildvariants: tasks: - ".6.0" -- matrix_name: "tests-python-version-rhel8-test-ssl" - matrix_spec: - platform: rhel8 - python-version: "*" - auth-ssl: "*" - coverage: "*" - display_name: "${python-version} ${platform} ${auth-ssl} ${coverage}" - tasks: &all-server-versions - - ".rapid" - - ".latest" - - ".8.0" - - ".7.0" - - ".6.0" - - ".5.0" - - ".4.4" - - ".4.2" - - ".4.0" - -- matrix_name: "tests-pyopenssl" - matrix_spec: - platform: rhel8 - python-version: "*" - auth: "*" - ssl: "ssl" - pyopenssl: "*" - # Only test "noauth" with Python 3.9. - exclude_spec: - platform: rhel8 - python-version: ["3.10", "3.11", "3.12", "3.13", "pypy3.9", "pypy3.10"] - auth: "noauth" - ssl: "ssl" - pyopenssl: "*" - display_name: "PyOpenSSL ${platform} ${python-version} ${auth}" - tasks: - - '.replica_set' - # Test standalone and sharded only on 7.0. - - '.7.0' - -- matrix_name: "tests-pyopenssl-macOS" - matrix_spec: - platform: macos - auth: "auth" - ssl: "ssl" - pyopenssl: "*" - display_name: "PyOpenSSL ${platform} ${auth}" - tasks: - - '.replica_set' - -- matrix_name: "tests-pyopenssl-windows" - matrix_spec: - platform: windows - python-version-windows: "*" - auth: "auth" - ssl: "ssl" - pyopenssl: "*" - display_name: "PyOpenSSL ${platform} ${python-version-windows} ${auth}" - tasks: - - '.replica_set' - -- matrix_name: "tests-python-version-rhel8-test-encryption" - matrix_spec: - platform: rhel8 - python-version: "*" - auth-ssl: noauth-nossl -# TODO: dependency error for 'coverage-report' task: -# dependency tests-python-version-rhel62-test-encryption_.../test-2.6-standalone is not present in the project config -# coverage: "*" - encryption: "*" - display_name: "${encryption} ${python-version} ${platform} ${auth-ssl}" - tasks: "test-latest-replica_set" - rules: - - if: - encryption: ["encryption", "encryption_crypt_shared"] - platform: rhel8 - auth-ssl: noauth-nossl - python-version: "*" - then: - add_tasks: *encryption-server-versions - - matrix_name: "tests-python-version-rhel8-without-c-extensions" matrix_spec: platform: rhel8 @@ -2580,39 +3403,16 @@ buildvariants: auth-ssl: "*" coverage: "*" display_name: "${c-extensions} ${python-version} ${platform} ${auth} ${ssl} ${coverage}" - tasks: *all-server-versions - -- matrix_name: "tests-python-version-rhel8-compression" - matrix_spec: - platform: rhel8 - python-version: "*" - c-extensions: "*" - compression: "*" - exclude_spec: - # These interpreters are always tested without extensions. - - platform: rhel8 - python-version: ["pypy3.9", "pypy3.10"] - c-extensions: "with-c-extensions" - compression: "*" - display_name: "${compression} ${c-extensions} ${python-version} ${platform}" - tasks: - - "test-latest-standalone" - - "test-8.0-standalone" - - "test-7.0-standalone" - - "test-6.0-standalone" - - "test-5.0-standalone" - - "test-4.4-standalone" - - "test-4.2-standalone" - - "test-4.0-standalone" - rules: - # Server version 4.0 supports snappy and zlib but not zstd. - - if: - python-version: "*" - c-extensions: "*" - compression: ["zstd"] - then: - remove_tasks: - - "test-4.0-standalone" + tasks: &all-server-versions + - ".rapid" + - ".latest" + - ".8.0" + - ".7.0" + - ".6.0" + - ".5.0" + - ".4.4" + - ".4.2" + - ".4.0" - matrix_name: "tests-python-version-green-framework-rhel8" matrix_spec: @@ -2629,22 +3429,6 @@ buildvariants: display_name: "${green-framework} ${python-version} ${platform} ${auth-ssl}" tasks: *all-server-versions -- matrix_name: "tests-windows-python-version" - matrix_spec: - platform: windows - python-version-windows: "*" - auth-ssl: "*" - display_name: "${platform} ${python-version-windows} ${auth-ssl}" - tasks: *all-server-versions - -- matrix_name: "tests-windows-python-version-32-bit" - matrix_spec: - platform: windows - python-version-windows-32: "*" - auth-ssl: "*" - display_name: "${platform} ${python-version-windows-32} ${auth-ssl}" - tasks: *all-server-versions - - matrix_name: "tests-python-version-supports-openssl-102-test-ssl" matrix_spec: platform: rhel7 @@ -2655,23 +3439,6 @@ buildvariants: tasks: - ".5.0" -- matrix_name: "tests-windows-encryption" - matrix_spec: - platform: windows - python-version-windows: "*" - auth-ssl: "*" - encryption: "*" - display_name: "${encryption} ${platform} ${python-version-windows} ${auth-ssl}" - tasks: "test-latest-replica_set" - rules: - - if: - encryption: ["encryption", "encryption_crypt_shared"] - platform: windows - python-version-windows: "*" - auth-ssl: "*" - then: - add_tasks: *encryption-server-versions - # Storage engine tests on RHEL 8.4 (x86_64) with Python 3.9. - matrix_name: "tests-storage-engines" matrix_spec: @@ -2714,24 +3481,6 @@ buildvariants: tasks: - ".latest" -- matrix_name: "test-linux-enterprise-auth" - matrix_spec: - platform: rhel8 - python-version: "*" - auth: "auth" - display_name: "Enterprise ${auth} ${platform} ${python-version}" - tasks: - - name: "test-enterprise-auth" - -- matrix_name: "tests-windows-enterprise-auth" - matrix_spec: - platform: windows - python-version-windows: "*" - auth: "auth" - display_name: "Enterprise ${auth} ${platform} ${python-version-windows}" - tasks: - - name: "test-enterprise-auth" - - matrix_name: "test-search-index-helpers" matrix_spec: platform: rhel8 @@ -2971,6 +3720,203 @@ buildvariants: VERSION: "8.0" PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3 +# Load balancer tests +- name: load-balancer-rhel8-v6.0-py3.9-auth-ssl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 v6.0 py3.9 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: "6.0" + AUTH: auth + SSL: ssl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/3.9/bin/python3 +- name: load-balancer-rhel8-v6.0-py3.10-noauth-ssl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 v6.0 py3.10 NoAuth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: "6.0" + AUTH: noauth + SSL: ssl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/3.10/bin/python3 +- name: load-balancer-rhel8-v6.0-py3.11-noauth-nossl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 v6.0 py3.11 NoAuth NoSSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: "6.0" + AUTH: noauth + SSL: nossl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/3.11/bin/python3 +- name: load-balancer-rhel8-v7.0-py3.12-auth-ssl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 v7.0 py3.12 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: "7.0" + AUTH: auth + SSL: ssl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/3.12/bin/python3 +- name: load-balancer-rhel8-v7.0-py3.13-noauth-ssl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 v7.0 py3.13 NoAuth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: "7.0" + AUTH: noauth + SSL: ssl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/3.13/bin/python3 +- name: load-balancer-rhel8-v7.0-pypy3.9-noauth-nossl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 v7.0 pypy3.9 NoAuth NoSSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: "7.0" + AUTH: noauth + SSL: nossl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/pypy3.9/bin/python3 +- name: load-balancer-rhel8-v8.0-pypy3.10-auth-ssl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 v8.0 pypy3.10 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: "8.0" + AUTH: auth + SSL: ssl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 +- name: load-balancer-rhel8-v8.0-py3.9-noauth-ssl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 v8.0 py3.9 NoAuth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: "8.0" + AUTH: noauth + SSL: ssl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/3.9/bin/python3 +- name: load-balancer-rhel8-v8.0-py3.10-noauth-nossl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 v8.0 py3.10 NoAuth NoSSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: "8.0" + AUTH: noauth + SSL: nossl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/3.10/bin/python3 +- name: load-balancer-rhel8-latest-py3.11-auth-ssl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 latest py3.11 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: latest + AUTH: auth + SSL: ssl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/3.11/bin/python3 +- name: load-balancer-rhel8-latest-py3.12-noauth-ssl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 latest py3.12 NoAuth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: latest + AUTH: noauth + SSL: ssl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/3.12/bin/python3 +- name: load-balancer-rhel8-latest-py3.13-noauth-nossl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 latest py3.13 NoAuth NoSSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: latest + AUTH: noauth + SSL: nossl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/3.13/bin/python3 +- name: load-balancer-rhel8-rapid-pypy3.9-auth-ssl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 rapid pypy3.9 Auth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: rapid + AUTH: auth + SSL: ssl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/pypy3.9/bin/python3 +- name: load-balancer-rhel8-rapid-pypy3.10-noauth-ssl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 rapid pypy3.10 NoAuth SSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: rapid + AUTH: noauth + SSL: ssl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/pypy3.10/bin/python3 +- name: load-balancer-rhel8-rapid-py3.9-noauth-nossl + tasks: + - name: load-balancer-test + display_name: Load Balancer RHEL8 rapid py3.9 NoAuth NoSSL + run_on: + - rhel87-small + batchtime: 10080 + expansions: + VERSION: rapid + AUTH: noauth + SSL: nossl + test_loadbalancer: "true" + PYTHON_BINARY: /opt/python/3.9/bin/python3 + - matrix_name: "oidc-auth-test" matrix_spec: platform: [ rhel8, macos, windows ] @@ -3034,17 +3980,6 @@ buildvariants: - name: "aws-auth-test-rapid" - name: "aws-auth-test-latest" -- matrix_name: "load-balancer" - matrix_spec: - platform: rhel8 - mongodb-version: ["6.0", "7.0", "8.0", "rapid", "latest"] - auth-ssl: "*" - python-version: "*" - loadbalancer: "*" - display_name: "Load Balancer ${platform} ${python-version} ${mongodb-version} ${auth-ssl}" - tasks: - - name: "load-balancer-test" - - name: testgcpkms-variant display_name: "GCP KMS" run_on: diff --git a/.evergreen/hatch.sh b/.evergreen/hatch.sh index 6f3d36b389..45d5113cd6 100644 --- a/.evergreen/hatch.sh +++ b/.evergreen/hatch.sh @@ -34,8 +34,8 @@ else # Set up virtualenv before installing hatch fi export HATCH_CONFIG hatch config restore - hatch config set dirs.data ".hatch/data" - hatch config set dirs.cache ".hatch/cache" + hatch config set dirs.data "$(pwd)/.hatch/data" + hatch config set dirs.cache "$(pwd)/.hatch/cache" run_hatch() { python -m hatch run "$@" diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 5e8429dd28..36fa76e317 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -30,7 +30,7 @@ set -o xtrace AUTH=${AUTH:-noauth} SSL=${SSL:-nossl} -TEST_SUITES="" +TEST_SUITES=${TEST_SUITES:-} TEST_ARGS="${*:1}" export PIP_QUIET=1 # Quiet by default @@ -90,6 +90,8 @@ if [ -n "$TEST_ENTERPRISE_AUTH" ]; then export GSSAPI_HOST=${SASL_HOST} export GSSAPI_PORT=${SASL_PORT} export GSSAPI_PRINCIPAL=${PRINCIPAL} + + export TEST_SUITES="auth" fi if [ -n "$TEST_LOADBALANCER" ]; then diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index e98e527b72..6d614a9afe 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -23,10 +23,23 @@ ############## ALL_VERSIONS = ["4.0", "4.4", "5.0", "6.0", "7.0", "8.0", "rapid", "latest"] +VERSIONS_6_0_PLUS = ["6.0", "7.0", "8.0", "rapid", "latest"] CPYTHONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] PYPYS = ["pypy3.9", "pypy3.10"] ALL_PYTHONS = CPYTHONS + PYPYS +MIN_MAX_PYTHON = [CPYTHONS[0], CPYTHONS[-1]] BATCHTIME_WEEK = 10080 +AUTH_SSLS = [("auth", "ssl"), ("noauth", "ssl"), ("noauth", "nossl")] +TOPOLOGIES = ["standalone", "replica_set", "sharded_cluster"] +C_EXTS = ["with_ext", "without_ext"] +SYNCS = ["sync", "async"] +DISPLAY_LOOKUP = dict( + ssl=dict(ssl="SSL", nossl="NoSSL"), + auth=dict(auth="Auth", noauth="NoAuth"), + test_suites=dict(default="Sync", default_async="Async"), + coverage=dict(coverage="cov"), + no_ext={"1": "No C"}, +) HOSTS = dict() @@ -39,7 +52,9 @@ class Host: HOSTS["rhel8"] = Host("rhel8", "rhel87-small", "RHEL8") HOSTS["win64"] = Host("win64", "windows-64-vsMulti-small", "Win64") +HOSTS["win32"] = Host("win32", "windows-64-vsMulti-small", "Win32") HOSTS["macos"] = Host("macos", "macos-14", "macOS") +HOSTS["macos-arm64"] = Host("macos-arm64", "macos-14-arm64", "macOS Arm64") ############## @@ -80,10 +95,8 @@ def create_variant( def get_python_binary(python: str, host: str) -> str: """Get the appropriate python binary given a python version and host.""" - if host == "win64": - is_32 = python.startswith("32-bit") - if is_32: - _, python = python.split() + if host in ["win64", "win32"]: + if host == "win32": base = "C:/python/32" else: base = "C:/python" @@ -93,19 +106,31 @@ def get_python_binary(python: str, host: str) -> str: if host == "rhel8": return f"/opt/python/{python}/bin/python3" - if host == "macos": + if host in ["macos", "macos-arm64"]: return f"/Library/Frameworks/Python.Framework/Versions/{python}/bin/python3" raise ValueError(f"no match found for python {python} on {host}") -def get_display_name(base: str, host: str, version: str, python: str) -> str: +def get_display_name(base: str, host: str, **kwargs) -> str: """Get the display name of a variant.""" - if version not in ["rapid", "latest"]: - version = f"v{version}" - if not python.startswith("pypy"): - python = f"py{python}" - return f"{base} {HOSTS[host].display_name} {version} {python}" + display_name = f"{base} {HOSTS[host].display_name}" + version = kwargs.pop("VERSION", None) + if version: + if version not in ["rapid", "latest"]: + version = f"v{version}" + display_name = f"{display_name} {version}" + for key, value in kwargs.items(): + name = value + if key.lower() == "python": + if not value.startswith("pypy"): + name = f"py{value}" + elif key.lower() in DISPLAY_LOOKUP: + name = DISPLAY_LOOKUP[key.lower()][value] + else: + continue + display_name = f"{display_name} {name}" + return display_name def zip_cycle(*iterables, empty_default=None): @@ -115,6 +140,21 @@ def zip_cycle(*iterables, empty_default=None): yield tuple(next(i, empty_default) for i in cycles) +def handle_c_ext(c_ext, expansions): + """Handle c extension option.""" + if c_ext == C_EXTS[0]: + expansions["NO_EXT"] = "1" + + +def generate_yaml(tasks=None, variants=None): + """Generate the yaml for a given set of tasks and variants.""" + project = EvgProject(tasks=tasks, buildvariants=variants) + out = ShrubService.generate_yaml(project) + # Dedent by two spaces to match what we use in config.yml + lines = [line[2:] for line in out.splitlines()] + print("\n".join(lines)) # noqa: T201 + + ############## # Variants ############## @@ -159,9 +199,253 @@ def create_ocsp_variants() -> list[BuildVariant]: return variants +def create_server_variants() -> list[BuildVariant]: + variants = [] + + # Run the full matrix on linux with min and max CPython, and latest pypy. + host = "rhel8" + for python, (auth, ssl) in product([*MIN_MAX_PYTHON, PYPYS[-1]], AUTH_SSLS): + display_name = f"Test {host}" + expansions = dict(AUTH=auth, SSL=ssl, COVERAGE="coverage") + display_name = get_display_name("Test", host, python=python, **expansions) + variant = create_variant( + [f".{t}" for t in TOPOLOGIES], + display_name, + python=python, + host=host, + tags=["coverage_tag"], + expansions=expansions, + ) + variants.append(variant) + + # Test the rest of the pythons on linux. + for python, (auth, ssl), topology in zip_cycle( + CPYTHONS[1:-1] + PYPYS[:-1], AUTH_SSLS, TOPOLOGIES + ): + display_name = f"Test {host}" + expansions = dict(AUTH=auth, SSL=ssl) + display_name = get_display_name("Test", host, python=python, **expansions) + variant = create_variant( + [f".{topology}"], + display_name, + python=python, + host=host, + expansions=expansions, + ) + variants.append(variant) + + # Test a subset on each of the other platforms. + for host in ("macos", "macos-arm64", "win64", "win32"): + for (python, (auth, ssl), topology), sync in product( + zip_cycle(MIN_MAX_PYTHON, AUTH_SSLS, TOPOLOGIES), SYNCS + ): + test_suite = "default" if sync == "sync" else "default_async" + tasks = [f".{topology}"] + # MacOS arm64 only works on server versions 6.0+ + if host == "macos-arm64": + tasks = [f".{topology} .{version}" for version in VERSIONS_6_0_PLUS] + expansions = dict(AUTH=auth, SSL=ssl, TEST_SUITES=test_suite, SKIP_CSOT_TESTS="true") + display_name = get_display_name("Test", host, python=python, **expansions) + variant = create_variant( + tasks, + display_name, + python=python, + host=host, + expansions=expansions, + ) + variants.append(variant) + + return variants + + +def create_encryption_variants() -> list[BuildVariant]: + variants = [] + tags = ["encryption_tag"] + batchtime = BATCHTIME_WEEK + + def get_encryption_expansions(encryption, ssl="ssl"): + expansions = dict(AUTH="auth", SSL=ssl, test_encryption="true") + if "crypt_shared" in encryption: + expansions["test_crypt_shared"] = "true" + if "PyOpenSSL" in encryption: + expansions["test_encryption_pyopenssl"] = "true" + return expansions + + host = "rhel8" + + # Test against all server versions and topolgies for the three main python versions. + encryptions = ["Encryption", "Encryption crypt_shared", "Encryption PyOpenSSL"] + for encryption, python in product(encryptions, [*MIN_MAX_PYTHON, PYPYS[-1]]): + expansions = get_encryption_expansions(encryption) + display_name = get_display_name(encryption, host, python=python, **expansions) + variant = create_variant( + [f".{t}" for t in TOPOLOGIES], + display_name, + python=python, + host=host, + expansions=expansions, + batchtime=batchtime, + tags=tags, + ) + variants.append(variant) + + # Test the rest of the pythons on linux for all server versions. + for encryption, python, ssl in zip_cycle( + encryptions, CPYTHONS[1:-1] + PYPYS[:-1], ["ssl", "nossl"] + ): + expansions = get_encryption_expansions(encryption, ssl) + display_name = get_display_name(encryption, host, python=python, **expansions) + variant = create_variant( + [".replica_set"], + display_name, + python=python, + host=host, + expansions=expansions, + ) + variants.append(variant) + + # Test on macos and linux on one server version and topology for min and max python. + encryptions = ["Encryption", "Encryption crypt_shared"] + task_names = [".latest .replica_set"] + for host, encryption, python in product(["macos", "win64"], encryptions, MIN_MAX_PYTHON): + ssl = "ssl" if python == CPYTHONS[0] else "nossl" + expansions = get_encryption_expansions(encryption, ssl) + display_name = get_display_name(encryption, host, python=python, **expansions) + variant = create_variant( + task_names, + display_name, + python=python, + host=host, + expansions=expansions, + batchtime=batchtime, + tags=tags, + ) + variants.append(variant) + return variants + + +def create_load_balancer_variants(): + # Load balancer tests - run all supported versions for all combinations of auth and ssl and system python. + host = "rhel8" + task_names = ["load-balancer-test"] + batchtime = BATCHTIME_WEEK + expansions_base = dict(test_loadbalancer="true") + versions = ["6.0", "7.0", "8.0", "latest", "rapid"] + variants = [] + pythons = CPYTHONS + PYPYS + for ind, (version, (auth, ssl)) in enumerate(product(versions, AUTH_SSLS)): + expansions = dict(VERSION=version, AUTH=auth, SSL=ssl) + expansions.update(expansions_base) + python = pythons[ind % len(pythons)] + display_name = get_display_name("Load Balancer", host, python=python, **expansions) + variant = create_variant( + task_names, + display_name, + python=python, + host=host, + expansions=expansions, + batchtime=batchtime, + ) + variants.append(variant) + return variants + + +def create_compression_variants(): + # Compression tests - standalone versions of each server, across python versions, with and without c extensions. + # PyPy interpreters are always tested without extensions. + host = "rhel8" + task_names = dict(snappy=[".standalone"], zlib=[".standalone"], zstd=[".standalone !.4.0"]) + variants = [] + for ind, (compressor, c_ext) in enumerate(product(["snappy", "zlib", "zstd"], C_EXTS)): + expansions = dict(COMPRESSORS=compressor) + handle_c_ext(c_ext, expansions) + base_name = f"{compressor} compression" + python = CPYTHONS[ind % len(CPYTHONS)] + display_name = get_display_name(base_name, host, python=python, **expansions) + variant = create_variant( + task_names[compressor], + display_name, + python=python, + host=host, + expansions=expansions, + ) + variants.append(variant) + + other_pythons = PYPYS + CPYTHONS[ind:] + for compressor, python in zip_cycle(["snappy", "zlib", "zstd"], other_pythons): + expansions = dict(COMPRESSORS=compressor) + handle_c_ext(c_ext, expansions) + base_name = f"{compressor} compression" + display_name = get_display_name(base_name, host, python=python, **expansions) + variant = create_variant( + task_names[compressor], + display_name, + python=python, + host=host, + expansions=expansions, + ) + variants.append(variant) + + return variants + + +def create_enterprise_auth_variants(): + expansions = dict(AUTH="auth") + variants = [] + + # All python versions across platforms. + for python in ALL_PYTHONS: + if python == CPYTHONS[0]: + host = "macos" + elif python == CPYTHONS[-1]: + host = "win64" + else: + host = "rhel8" + display_name = get_display_name("Enterprise Auth", host, python=python, **expansions) + variant = create_variant( + ["test-enterprise-auth"], display_name, host=host, python=python, expansions=expansions + ) + variants.append(variant) + + return variants + + +def create_pyopenssl_variants(): + base_name = "PyOpenSSL" + batchtime = BATCHTIME_WEEK + base_expansions = dict(test_pyopenssl="true", SSL="ssl") + variants = [] + + for python in ALL_PYTHONS: + # Only test "noauth" with min python. + auth = "noauth" if python == CPYTHONS[0] else "auth" + if python == CPYTHONS[0]: + host = "macos" + elif python == CPYTHONS[-1]: + host = "win64" + else: + host = "rhel8" + expansions = dict(AUTH=auth) + expansions.update(base_expansions) + + display_name = get_display_name(base_name, host, python=python) + variant = create_variant( + [".replica_set", ".7.0"], + display_name, + python=python, + host=host, + expansions=expansions, + batchtime=batchtime, + ) + variants.append(variant) + + return variants + + ################## # Generate Config ################## -project = EvgProject(tasks=None, buildvariants=create_ocsp_variants()) -print(ShrubService.generate_yaml(project)) # noqa: T201 +variants = create_pyopenssl_variants() +# print(len(variants)) +generate_yaml(variants=variants) diff --git a/doc/changelog.rst b/doc/changelog.rst index e7b160b176..4c1955d19d 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -14,6 +14,15 @@ PyMongo 4.11 brings a number of changes including: - Dropped support for MongoDB 3.6. - Added support for free-threaded Python with the GIL disabled. For more information see: `Free-threaded CPython `_. +- :attr:`~pymongo.asynchronous.mongo_client.AsyncMongoClient.address` and + :attr:`~pymongo.mongo_client.MongoClient.address` now correctly block when called on unconnected clients + until either connection succeeds or a server selection timeout error is raised. +- Added :func:`repr` support to :class:`pymongo.operations.IndexModel`. +- Added :func:`repr` support to :class:`pymongo.operations.SearchIndexModel`. +- Added ``sort`` parameter to + :meth:`~pymongo.collection.Collection.update_one`, :meth:`~pymongo.collection.Collection.replace_one`, + :class:`~pymongo.operations.UpdateOne`, and + :class:`~pymongo.operations.UpdateMany`, Issues Resolved ............... diff --git a/pymongo/asynchronous/bulk.py b/pymongo/asynchronous/bulk.py index 9d33a990ed..e6cfe5b36e 100644 --- a/pymongo/asynchronous/bulk.py +++ b/pymongo/asynchronous/bulk.py @@ -109,6 +109,7 @@ def __init__( self.uses_array_filters = False self.uses_hint_update = False self.uses_hint_delete = False + self.uses_sort = False self.is_retryable = True self.retrying = False self.started_retryable_write = False @@ -144,6 +145,7 @@ def add_update( collation: Optional[Mapping[str, Any]] = None, array_filters: Optional[list[Mapping[str, Any]]] = None, hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, ) -> None: """Create an update document and add it to the list of ops.""" validate_ok_for_update(update) @@ -159,6 +161,9 @@ def add_update( if hint is not None: self.uses_hint_update = True cmd["hint"] = hint + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort if multi: # A bulk_write containing an update_many is not retryable. self.is_retryable = False @@ -171,6 +176,7 @@ def add_replace( upsert: bool = False, collation: Optional[Mapping[str, Any]] = None, hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, ) -> None: """Create a replace document and add it to the list of ops.""" validate_ok_for_replace(replacement) @@ -181,6 +187,9 @@ def add_replace( if hint is not None: self.uses_hint_update = True cmd["hint"] = hint + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort self.ops.append((_UPDATE, cmd)) def add_delete( @@ -699,6 +708,10 @@ async def execute_no_results( raise ConfigurationError( "Must be connected to MongoDB 4.2+ to use hint on unacknowledged update commands." ) + if unack and self.uses_sort and conn.max_wire_version < 25: + raise ConfigurationError( + "Must be connected to MongoDB 8.0+ to use sort on unacknowledged update commands." + ) # Cannot have both unacknowledged writes and bypass document validation. if self.bypass_doc_val: raise OperationFailure( diff --git a/pymongo/asynchronous/client_bulk.py b/pymongo/asynchronous/client_bulk.py index dc800c9549..96571c21eb 100644 --- a/pymongo/asynchronous/client_bulk.py +++ b/pymongo/asynchronous/client_bulk.py @@ -118,6 +118,7 @@ def __init__( self.uses_array_filters = False self.uses_hint_update = False self.uses_hint_delete = False + self.uses_sort = False self.is_retryable = self.client.options.retry_writes self.retrying = False @@ -148,6 +149,7 @@ def add_update( collation: Optional[Mapping[str, Any]] = None, array_filters: Optional[list[Mapping[str, Any]]] = None, hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, ) -> None: """Create an update document and add it to the list of ops.""" validate_ok_for_update(update) @@ -169,6 +171,9 @@ def add_update( if collation is not None: self.uses_collation = True cmd["collation"] = collation + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort if multi: # A bulk_write containing an update_many is not retryable. self.is_retryable = False @@ -184,6 +189,7 @@ def add_replace( upsert: Optional[bool] = None, collation: Optional[Mapping[str, Any]] = None, hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, ) -> None: """Create a replace document and add it to the list of ops.""" validate_ok_for_replace(replacement) @@ -202,6 +208,9 @@ def add_replace( if collation is not None: self.uses_collation = True cmd["collation"] = collation + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort self.ops.append(("replace", cmd)) self.namespaces.append(namespace) self.total_ops += 1 diff --git a/pymongo/asynchronous/collection.py b/pymongo/asynchronous/collection.py index 4ddcbab4d2..9b73423627 100644 --- a/pymongo/asynchronous/collection.py +++ b/pymongo/asynchronous/collection.py @@ -993,6 +993,7 @@ async def _update( session: Optional[AsyncClientSession] = None, retryable_write: bool = False, let: Optional[Mapping[str, Any]] = None, + sort: Optional[Mapping[str, Any]] = None, comment: Optional[Any] = None, ) -> Optional[Mapping[str, Any]]: """Internal update / replace helper.""" @@ -1024,6 +1025,14 @@ async def _update( if not isinstance(hint, str): hint = helpers_shared._index_document(hint) update_doc["hint"] = hint + if sort is not None: + if not acknowledged and conn.max_wire_version < 25: + raise ConfigurationError( + "Must be connected to MongoDB 8.0+ to use sort on unacknowledged update commands." + ) + common.validate_is_mapping("sort", sort) + update_doc["sort"] = sort + command = {"update": self.name, "ordered": ordered, "updates": [update_doc]} if let is not None: common.validate_is_mapping("let", let) @@ -1079,6 +1088,7 @@ async def _update_retryable( hint: Optional[_IndexKeyHint] = None, session: Optional[AsyncClientSession] = None, let: Optional[Mapping[str, Any]] = None, + sort: Optional[Mapping[str, Any]] = None, comment: Optional[Any] = None, ) -> Optional[Mapping[str, Any]]: """Internal update / replace helper.""" @@ -1102,6 +1112,7 @@ async def _update( session=session, retryable_write=retryable_write, let=let, + sort=sort, comment=comment, ) @@ -1122,6 +1133,7 @@ async def replace_one( hint: Optional[_IndexKeyHint] = None, session: Optional[AsyncClientSession] = None, let: Optional[Mapping[str, Any]] = None, + sort: Optional[Mapping[str, Any]] = None, comment: Optional[Any] = None, ) -> UpdateResult: """Replace a single document matching the filter. @@ -1176,8 +1188,13 @@ async def replace_one( aggregate expression context (e.g. "$$var"). :param comment: A user-provided comment to attach to this command. + :param sort: Specify which document the operation updates if the query matches + multiple documents. The first document matched by the sort order will be updated. + This option is only supported on MongoDB 8.0 and above. :return: - An instance of :class:`~pymongo.results.UpdateResult`. + .. versionchanged:: 4.11 + Added ``sort`` parameter. .. versionchanged:: 4.1 Added ``let`` parameter. Added ``comment`` parameter. @@ -1209,6 +1226,7 @@ async def replace_one( hint=hint, session=session, let=let, + sort=sort, comment=comment, ), write_concern.acknowledged, @@ -1225,6 +1243,7 @@ async def update_one( hint: Optional[_IndexKeyHint] = None, session: Optional[AsyncClientSession] = None, let: Optional[Mapping[str, Any]] = None, + sort: Optional[Mapping[str, Any]] = None, comment: Optional[Any] = None, ) -> UpdateResult: """Update a single document matching the filter. @@ -1283,11 +1302,16 @@ async def update_one( constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an aggregate expression context (e.g. "$$var"). + :param sort: Specify which document the operation updates if the query matches + multiple documents. The first document matched by the sort order will be updated. + This option is only supported on MongoDB 8.0 and above. :param comment: A user-provided comment to attach to this command. :return: - An instance of :class:`~pymongo.results.UpdateResult`. + .. versionchanged:: 4.11 + Added ``sort`` parameter. .. versionchanged:: 4.1 Added ``let`` parameter. Added ``comment`` parameter. @@ -1322,6 +1346,7 @@ async def update_one( hint=hint, session=session, let=let, + sort=sort, comment=comment, ), write_concern.acknowledged, diff --git a/pymongo/network_layer.py b/pymongo/network_layer.py index d14a21f41d..aa16e85a07 100644 --- a/pymongo/network_layer.py +++ b/pymongo/network_layer.py @@ -205,7 +205,7 @@ async def _async_sendall_ssl( total_sent += sent async def _async_receive_ssl( - conn: _sslConn, length: int, dummy: AbstractEventLoop + conn: _sslConn, length: int, dummy: AbstractEventLoop, once: Optional[bool] = False ) -> memoryview: mv = memoryview(bytearray(length)) total_read = 0 @@ -217,6 +217,9 @@ async def _async_receive_ssl( read = conn.recv_into(mv[total_read:]) if read == 0: raise OSError("connection closed") + # KMS responses update their expected size after the first batch, stop reading after one loop + if once: + return mv[:read] except BLOCKING_IO_ERRORS: await asyncio.sleep(backoff) read = 0 diff --git a/pymongo/operations.py b/pymongo/operations.py index d2e1feba69..8905048c4e 100644 --- a/pymongo/operations.py +++ b/pymongo/operations.py @@ -325,6 +325,7 @@ class ReplaceOne(Generic[_DocumentType]): "_collation", "_hint", "_namespace", + "_sort", ) def __init__( @@ -335,6 +336,7 @@ def __init__( collation: Optional[_CollationIn] = None, hint: Optional[_IndexKeyHint] = None, namespace: Optional[str] = None, + sort: Optional[Mapping[str, Any]] = None, ) -> None: """Create a ReplaceOne instance. @@ -353,8 +355,12 @@ def __init__( :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` or :meth:`~pymongo.collection.Collection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.2 and above. + :param sort: Specify which document the operation updates if the query matches + multiple documents. The first document matched by the sort order will be updated. :param namespace: (optional) The namespace in which to replace a document. + .. versionchanged:: 4.10 + Added ``sort`` option. .. versionchanged:: 4.9 Added the `namespace` option to support `MongoClient.bulk_write`. .. versionchanged:: 3.11 @@ -371,6 +377,7 @@ def __init__( else: self._hint = hint + self._sort = sort self._filter = filter self._doc = replacement self._upsert = upsert @@ -385,6 +392,7 @@ def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None: self._upsert, collation=validate_collation_or_none(self._collation), hint=self._hint, + sort=self._sort, ) def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None: @@ -400,6 +408,7 @@ def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None: self._upsert, collation=validate_collation_or_none(self._collation), hint=self._hint, + sort=self._sort, ) def __eq__(self, other: Any) -> bool: @@ -411,13 +420,15 @@ def __eq__(self, other: Any) -> bool: other._collation, other._hint, other._namespace, + other._sort, ) == ( self._filter, self._doc, self._upsert, self._collation, - other._hint, + self._hint, self._namespace, + self._sort, ) return NotImplemented @@ -426,7 +437,7 @@ def __ne__(self, other: Any) -> bool: def __repr__(self) -> str: if self._namespace: - return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format( + return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format( self.__class__.__name__, self._filter, self._doc, @@ -434,14 +445,16 @@ def __repr__(self) -> str: self._collation, self._hint, self._namespace, + self._sort, ) - return "{}({!r}, {!r}, {!r}, {!r}, {!r})".format( + return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format( self.__class__.__name__, self._filter, self._doc, self._upsert, self._collation, self._hint, + self._sort, ) @@ -456,6 +469,7 @@ class _UpdateOp: "_array_filters", "_hint", "_namespace", + "_sort", ) def __init__( @@ -467,6 +481,7 @@ def __init__( array_filters: Optional[list[Mapping[str, Any]]], hint: Optional[_IndexKeyHint], namespace: Optional[str], + sort: Optional[Mapping[str, Any]], ): if filter is not None: validate_is_mapping("filter", filter) @@ -478,13 +493,13 @@ def __init__( self._hint: Union[str, dict[str, Any], None] = helpers_shared._index_document(hint) else: self._hint = hint - self._filter = filter self._doc = doc self._upsert = upsert self._collation = collation self._array_filters = array_filters self._namespace = namespace + self._sort = sort def __eq__(self, other: object) -> bool: if isinstance(other, type(self)): @@ -496,6 +511,7 @@ def __eq__(self, other: object) -> bool: other._array_filters, other._hint, other._namespace, + other._sort, ) == ( self._filter, self._doc, @@ -504,6 +520,7 @@ def __eq__(self, other: object) -> bool: self._array_filters, self._hint, self._namespace, + self._sort, ) return NotImplemented @@ -512,7 +529,7 @@ def __ne__(self, other: Any) -> bool: def __repr__(self) -> str: if self._namespace: - return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format( + return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format( self.__class__.__name__, self._filter, self._doc, @@ -521,8 +538,9 @@ def __repr__(self) -> str: self._array_filters, self._hint, self._namespace, + self._sort, ) - return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format( + return "{}({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format( self.__class__.__name__, self._filter, self._doc, @@ -530,6 +548,7 @@ def __repr__(self) -> str: self._collation, self._array_filters, self._hint, + self._sort, ) @@ -547,6 +566,7 @@ def __init__( array_filters: Optional[list[Mapping[str, Any]]] = None, hint: Optional[_IndexKeyHint] = None, namespace: Optional[str] = None, + sort: Optional[Mapping[str, Any]] = None, ) -> None: """Represents an update_one operation. @@ -567,8 +587,12 @@ def __init__( :meth:`~pymongo.asynchronous.collection.AsyncCollection.create_index` or :meth:`~pymongo.collection.Collection.create_index` (e.g. ``[('field', ASCENDING)]``). This option is only supported on MongoDB 4.2 and above. - :param namespace: (optional) The namespace in which to update a document. + :param namespace: The namespace in which to update a document. + :param sort: Specify which document the operation updates if the query matches + multiple documents. The first document matched by the sort order will be updated. + .. versionchanged:: 4.10 + Added ``sort`` option. .. versionchanged:: 4.9 Added the `namespace` option to support `MongoClient.bulk_write`. .. versionchanged:: 3.11 @@ -580,7 +604,7 @@ def __init__( .. versionchanged:: 3.5 Added the `collation` option. """ - super().__init__(filter, update, upsert, collation, array_filters, hint, namespace) + super().__init__(filter, update, upsert, collation, array_filters, hint, namespace, sort) def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None: """Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`.""" @@ -592,6 +616,7 @@ def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None: collation=validate_collation_or_none(self._collation), array_filters=self._array_filters, hint=self._hint, + sort=self._sort, ) def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None: @@ -609,6 +634,7 @@ def _add_to_client_bulk(self, bulkobj: _AgnosticClientBulk) -> None: collation=validate_collation_or_none(self._collation), array_filters=self._array_filters, hint=self._hint, + sort=self._sort, ) @@ -659,7 +685,7 @@ def __init__( .. versionchanged:: 3.5 Added the `collation` option. """ - super().__init__(filter, update, upsert, collation, array_filters, hint, namespace) + super().__init__(filter, update, upsert, collation, array_filters, hint, namespace, None) def _add_to_bulk(self, bulkobj: _AgnosticBulk) -> None: """Add this operation to the _AsyncBulk/_Bulk instance `bulkobj`.""" @@ -773,6 +799,13 @@ def document(self) -> dict[str, Any]: """ return self.__document + def __repr__(self) -> str: + return "{}({}{})".format( + self.__class__.__name__, + self.document["key"], + "".join([f", {key}={value!r}" for key, value in self.document.items() if key != "key"]), + ) + class SearchIndexModel: """Represents a search index to create.""" @@ -812,3 +845,9 @@ def __init__( def document(self) -> Mapping[str, Any]: """The document for this index.""" return self.__document + + def __repr__(self) -> str: + return "{}({})".format( + self.__class__.__name__, + ", ".join([f"{key}={value!r}" for key, value in self.document.items()]), + ) diff --git a/pymongo/synchronous/bulk.py b/pymongo/synchronous/bulk.py index c658157ea1..7fb29a977f 100644 --- a/pymongo/synchronous/bulk.py +++ b/pymongo/synchronous/bulk.py @@ -109,6 +109,7 @@ def __init__( self.uses_array_filters = False self.uses_hint_update = False self.uses_hint_delete = False + self.uses_sort = False self.is_retryable = True self.retrying = False self.started_retryable_write = False @@ -144,6 +145,7 @@ def add_update( collation: Optional[Mapping[str, Any]] = None, array_filters: Optional[list[Mapping[str, Any]]] = None, hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, ) -> None: """Create an update document and add it to the list of ops.""" validate_ok_for_update(update) @@ -159,6 +161,9 @@ def add_update( if hint is not None: self.uses_hint_update = True cmd["hint"] = hint + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort if multi: # A bulk_write containing an update_many is not retryable. self.is_retryable = False @@ -171,6 +176,7 @@ def add_replace( upsert: bool = False, collation: Optional[Mapping[str, Any]] = None, hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, ) -> None: """Create a replace document and add it to the list of ops.""" validate_ok_for_replace(replacement) @@ -181,6 +187,9 @@ def add_replace( if hint is not None: self.uses_hint_update = True cmd["hint"] = hint + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort self.ops.append((_UPDATE, cmd)) def add_delete( @@ -697,6 +706,10 @@ def execute_no_results( raise ConfigurationError( "Must be connected to MongoDB 4.2+ to use hint on unacknowledged update commands." ) + if unack and self.uses_sort and conn.max_wire_version < 25: + raise ConfigurationError( + "Must be connected to MongoDB 8.0+ to use sort on unacknowledged update commands." + ) # Cannot have both unacknowledged writes and bypass document validation. if self.bypass_doc_val: raise OperationFailure( diff --git a/pymongo/synchronous/client_bulk.py b/pymongo/synchronous/client_bulk.py index f41f0203f2..2c38b1d76c 100644 --- a/pymongo/synchronous/client_bulk.py +++ b/pymongo/synchronous/client_bulk.py @@ -118,6 +118,7 @@ def __init__( self.uses_array_filters = False self.uses_hint_update = False self.uses_hint_delete = False + self.uses_sort = False self.is_retryable = self.client.options.retry_writes self.retrying = False @@ -148,6 +149,7 @@ def add_update( collation: Optional[Mapping[str, Any]] = None, array_filters: Optional[list[Mapping[str, Any]]] = None, hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, ) -> None: """Create an update document and add it to the list of ops.""" validate_ok_for_update(update) @@ -169,6 +171,9 @@ def add_update( if collation is not None: self.uses_collation = True cmd["collation"] = collation + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort if multi: # A bulk_write containing an update_many is not retryable. self.is_retryable = False @@ -184,6 +189,7 @@ def add_replace( upsert: Optional[bool] = None, collation: Optional[Mapping[str, Any]] = None, hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, ) -> None: """Create a replace document and add it to the list of ops.""" validate_ok_for_replace(replacement) @@ -202,6 +208,9 @@ def add_replace( if collation is not None: self.uses_collation = True cmd["collation"] = collation + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort self.ops.append(("replace", cmd)) self.namespaces.append(namespace) self.total_ops += 1 diff --git a/pymongo/synchronous/collection.py b/pymongo/synchronous/collection.py index 6fd2ac82dd..6edfddc9a9 100644 --- a/pymongo/synchronous/collection.py +++ b/pymongo/synchronous/collection.py @@ -992,6 +992,7 @@ def _update( session: Optional[ClientSession] = None, retryable_write: bool = False, let: Optional[Mapping[str, Any]] = None, + sort: Optional[Mapping[str, Any]] = None, comment: Optional[Any] = None, ) -> Optional[Mapping[str, Any]]: """Internal update / replace helper.""" @@ -1023,6 +1024,14 @@ def _update( if not isinstance(hint, str): hint = helpers_shared._index_document(hint) update_doc["hint"] = hint + if sort is not None: + if not acknowledged and conn.max_wire_version < 25: + raise ConfigurationError( + "Must be connected to MongoDB 8.0+ to use sort on unacknowledged update commands." + ) + common.validate_is_mapping("sort", sort) + update_doc["sort"] = sort + command = {"update": self.name, "ordered": ordered, "updates": [update_doc]} if let is not None: common.validate_is_mapping("let", let) @@ -1078,6 +1087,7 @@ def _update_retryable( hint: Optional[_IndexKeyHint] = None, session: Optional[ClientSession] = None, let: Optional[Mapping[str, Any]] = None, + sort: Optional[Mapping[str, Any]] = None, comment: Optional[Any] = None, ) -> Optional[Mapping[str, Any]]: """Internal update / replace helper.""" @@ -1101,6 +1111,7 @@ def _update( session=session, retryable_write=retryable_write, let=let, + sort=sort, comment=comment, ) @@ -1121,6 +1132,7 @@ def replace_one( hint: Optional[_IndexKeyHint] = None, session: Optional[ClientSession] = None, let: Optional[Mapping[str, Any]] = None, + sort: Optional[Mapping[str, Any]] = None, comment: Optional[Any] = None, ) -> UpdateResult: """Replace a single document matching the filter. @@ -1175,8 +1187,13 @@ def replace_one( aggregate expression context (e.g. "$$var"). :param comment: A user-provided comment to attach to this command. + :param sort: Specify which document the operation updates if the query matches + multiple documents. The first document matched by the sort order will be updated. + This option is only supported on MongoDB 8.0 and above. :return: - An instance of :class:`~pymongo.results.UpdateResult`. + .. versionchanged:: 4.11 + Added ``sort`` parameter. .. versionchanged:: 4.1 Added ``let`` parameter. Added ``comment`` parameter. @@ -1208,6 +1225,7 @@ def replace_one( hint=hint, session=session, let=let, + sort=sort, comment=comment, ), write_concern.acknowledged, @@ -1224,6 +1242,7 @@ def update_one( hint: Optional[_IndexKeyHint] = None, session: Optional[ClientSession] = None, let: Optional[Mapping[str, Any]] = None, + sort: Optional[Mapping[str, Any]] = None, comment: Optional[Any] = None, ) -> UpdateResult: """Update a single document matching the filter. @@ -1282,11 +1301,16 @@ def update_one( constant or closed expressions that do not reference document fields. Parameters can then be accessed as variables in an aggregate expression context (e.g. "$$var"). + :param sort: Specify which document the operation updates if the query matches + multiple documents. The first document matched by the sort order will be updated. + This option is only supported on MongoDB 8.0 and above. :param comment: A user-provided comment to attach to this command. :return: - An instance of :class:`~pymongo.results.UpdateResult`. + .. versionchanged:: 4.11 + Added ``sort`` parameter. .. versionchanged:: 4.1 Added ``let`` parameter. Added ``comment`` parameter. @@ -1321,6 +1345,7 @@ def update_one( hint=hint, session=session, let=let, + sort=sort, comment=comment, ), write_concern.acknowledged, diff --git a/pyproject.toml b/pyproject.toml index b4f59f67d5..9a29a777fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,6 +99,7 @@ filterwarnings = [ markers = [ "auth_aws: tests that rely on pymongo-auth-aws", "auth_oidc: tests that rely on oidc auth", + "auth: tests that rely on authentication", "ocsp: tests that rely on ocsp", "atlas: tests that rely on atlas", "data_lake: tests that rely on atlas data lake", diff --git a/requirements/typing.txt b/requirements/typing.txt index 06c33c6db6..2c23212da7 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -1,5 +1,5 @@ mypy==1.11.2 -pyright==1.1.383 +pyright==1.1.384 typing_extensions -r ./encryption.txt -r ./ocsp.txt diff --git a/test/asynchronous/test_auth.py b/test/asynchronous/test_auth.py index fbaca41f09..9262714374 100644 --- a/test/asynchronous/test_auth.py +++ b/test/asynchronous/test_auth.py @@ -32,6 +32,8 @@ ) from test.utils import AllowListEventListener, delay, ignore_deprecations +import pytest + from pymongo import AsyncMongoClient, monitoring from pymongo.asynchronous.auth import HAVE_KERBEROS from pymongo.auth_shared import _build_credentials_tuple @@ -42,6 +44,8 @@ _IS_SYNC = False +pytestmark = pytest.mark.auth + # YOU MUST RUN KINIT BEFORE RUNNING GSSAPI TESTS ON UNIX. GSSAPI_HOST = os.environ.get("GSSAPI_HOST") GSSAPI_PORT = int(os.environ.get("GSSAPI_PORT", "27017")) diff --git a/test/asynchronous/test_bulk.py b/test/asynchronous/test_bulk.py index e01dd53d7e..7191a412c1 100644 --- a/test/asynchronous/test_bulk.py +++ b/test/asynchronous/test_bulk.py @@ -961,6 +961,9 @@ async def cause_wtimeout(self, requests, ordered): @async_client_context.require_replica_set @async_client_context.require_secondaries_count(1) async def test_write_concern_failure_ordered(self): + self.skipTest("Skipping until PYTHON-4865 is resolved.") + details = None + # Ensure we don't raise on wnote. coll_ww = self.coll.with_options(write_concern=WriteConcern(w=self.w)) result = await coll_ww.bulk_write([DeleteOne({"something": "that does no exist"})]) @@ -1041,6 +1044,9 @@ async def test_write_concern_failure_ordered(self): @async_client_context.require_replica_set @async_client_context.require_secondaries_count(1) async def test_write_concern_failure_unordered(self): + self.skipTest("Skipping until PYTHON-4865 is resolved.") + details = None + # Ensure we don't raise on wnote. coll_ww = self.coll.with_options(write_concern=WriteConcern(w=self.w)) result = await coll_ww.bulk_write( diff --git a/test/asynchronous/test_change_stream.py b/test/asynchronous/test_change_stream.py index db8a74f55a..873631bbe5 100644 --- a/test/asynchronous/test_change_stream.py +++ b/test/asynchronous/test_change_stream.py @@ -39,6 +39,7 @@ from test.utils import ( AllowListEventListener, EventListener, + OvertCommandListener, async_wait_until, ) @@ -179,7 +180,7 @@ async def _wait_until(): @no_type_check async def test_try_next_runs_one_getmore(self): - listener = EventListener() + listener = OvertCommandListener() client = await self.async_rs_or_single_client(event_listeners=[listener]) # Connect to the cluster. await client.admin.command("ping") @@ -237,7 +238,7 @@ async def _wait_until(): @no_type_check async def test_batch_size_is_honored(self): - listener = EventListener() + listener = OvertCommandListener() client = await self.async_rs_or_single_client(event_listeners=[listener]) # Connect to the cluster. await client.admin.command("ping") diff --git a/test/asynchronous/test_collation.py b/test/asynchronous/test_collation.py index abbca1aff9..d7fd85b168 100644 --- a/test/asynchronous/test_collation.py +++ b/test/asynchronous/test_collation.py @@ -18,7 +18,7 @@ import functools import warnings from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest -from test.utils import EventListener +from test.utils import EventListener, OvertCommandListener from typing import Any from pymongo.asynchronous.helpers import anext @@ -100,13 +100,12 @@ class TestCollation(AsyncIntegrationTest): @async_client_context.require_connection async def asyncSetUp(self) -> None: await super().asyncSetUp() - self.listener = EventListener() + self.listener = OvertCommandListener() self.client = await self.async_rs_or_single_client(event_listeners=[self.listener]) self.db = self.client.pymongo_test self.collation = Collation("en_US") self.warn_context = warnings.catch_warnings() self.warn_context.__enter__() - warnings.simplefilter("ignore", DeprecationWarning) async def asyncTearDown(self) -> None: self.warn_context.__exit__() diff --git a/test/asynchronous/test_collection.py b/test/asynchronous/test_collection.py index a2ed4de388..528919f63c 100644 --- a/test/asynchronous/test_collection.py +++ b/test/asynchronous/test_collection.py @@ -36,6 +36,7 @@ from test.utils import ( IMPOSSIBLE_WRITE_CONCERN, EventListener, + OvertCommandListener, async_get_pool, async_is_mongos, async_wait_until, @@ -2101,7 +2102,7 @@ async def test_find_one_and(self): self.assertEqual(4, (await c.find_one_and_update({}, {"$inc": {"i": 1}}, sort=sort))["j"]) async def test_find_one_and_write_concern(self): - listener = EventListener() + listener = OvertCommandListener() db = (await self.async_single_client(event_listeners=[listener]))[self.db.name] # non-default WriteConcern. c_w0 = db.get_collection("test", write_concern=WriteConcern(w=0)) diff --git a/test/asynchronous/test_collection_management.py b/test/asynchronous/test_collection_management.py new file mode 100644 index 0000000000..c0edf91581 --- /dev/null +++ b/test/asynchronous/test_collection_management.py @@ -0,0 +1,41 @@ +# Copyright 2021-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test the collection management unified spec tests.""" +from __future__ import annotations + +import os +import pathlib +import sys + +sys.path[0:0] = [""] + +from test import unittest +from test.asynchronous.unified_format import generate_test_classes + +_IS_SYNC = False + +# Location of JSON test specifications. +if _IS_SYNC: + _TEST_PATH = os.path.join(pathlib.Path(__file__).resolve().parent, "collection_management") +else: + _TEST_PATH = os.path.join( + pathlib.Path(__file__).resolve().parent.parent, "collection_management" + ) + +# Generate unified tests. +globals().update(generate_test_classes(_TEST_PATH, module=__name__)) + +if __name__ == "__main__": + unittest.main() diff --git a/test/asynchronous/test_create_entities.py b/test/asynchronous/test_create_entities.py new file mode 100644 index 0000000000..cb2ec63f4c --- /dev/null +++ b/test/asynchronous/test_create_entities.py @@ -0,0 +1,128 @@ +# Copyright 2021-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import sys +import unittest + +sys.path[0:0] = [""] + +from test.asynchronous import AsyncIntegrationTest +from test.asynchronous.unified_format import UnifiedSpecTestMixinV1 + +_IS_SYNC = False + + +class TestCreateEntities(AsyncIntegrationTest): + async def test_store_events_as_entities(self): + self.scenario_runner = UnifiedSpecTestMixinV1() + spec = { + "description": "blank", + "schemaVersion": "1.2", + "createEntities": [ + { + "client": { + "id": "client0", + "storeEventsAsEntities": [ + { + "id": "events1", + "events": [ + "PoolCreatedEvent", + ], + } + ], + } + }, + ], + "tests": [{"description": "foo", "operations": []}], + } + self.scenario_runner.TEST_SPEC = spec + await self.scenario_runner.asyncSetUp() + await self.scenario_runner.run_scenario(spec["tests"][0]) + await self.scenario_runner.entity_map["client0"].close() + final_entity_map = self.scenario_runner.entity_map + self.assertIn("events1", final_entity_map) + self.assertGreater(len(final_entity_map["events1"]), 0) + for event in final_entity_map["events1"]: + self.assertIn("PoolCreatedEvent", event["name"]) + + async def test_store_all_others_as_entities(self): + self.scenario_runner = UnifiedSpecTestMixinV1() + spec = { + "description": "Find", + "schemaVersion": "1.2", + "createEntities": [ + { + "client": { + "id": "client0", + "uriOptions": {"retryReads": True}, + } + }, + {"database": {"id": "database0", "client": "client0", "databaseName": "dat"}}, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "dat", + } + }, + ], + "tests": [ + { + "description": "test loops", + "operations": [ + { + "name": "loop", + "object": "testRunner", + "arguments": { + "storeIterationsAsEntity": "iterations", + "storeSuccessesAsEntity": "successes", + "storeFailuresAsEntity": "failures", + "storeErrorsAsEntity": "errors", + "numIterations": 5, + "operations": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": {"document": {"_id": 1, "x": 44}}, + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": {"document": {"_id": 2, "x": 44}}, + }, + ], + }, + } + ], + } + ], + } + + await self.client.dat.dat.delete_many({}) + self.scenario_runner.TEST_SPEC = spec + await self.scenario_runner.asyncSetUp() + await self.scenario_runner.run_scenario(spec["tests"][0]) + await self.scenario_runner.entity_map["client0"].close() + entity_map = self.scenario_runner.entity_map + self.assertEqual(len(entity_map["errors"]), 4) + for error in entity_map["errors"]: + self.assertEqual(error["type"], "DuplicateKeyError") + self.assertEqual(entity_map["failures"], []) + self.assertEqual(entity_map["successes"], 2) + self.assertEqual(entity_map["iterations"], 5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/asynchronous/test_cursor.py b/test/asynchronous/test_cursor.py index 09955ca66f..d216479451 100644 --- a/test/asynchronous/test_cursor.py +++ b/test/asynchronous/test_cursor.py @@ -1601,7 +1601,7 @@ async def test_read_concern(self): await anext(c.find_raw_batches()) async def test_monitoring(self): - listener = EventListener() + listener = OvertCommandListener() client = await self.async_rs_or_single_client(event_listeners=[listener]) c = client.pymongo_test.test await c.drop() @@ -1764,7 +1764,7 @@ async def test_collation(self): await anext(await self.db.test.aggregate_raw_batches([], collation=Collation("en_US"))) async def test_monitoring(self): - listener = EventListener() + listener = OvertCommandListener() client = await self.async_rs_or_single_client(event_listeners=[listener]) c = client.pymongo_test.test await c.drop() diff --git a/test/asynchronous/test_grid_file.py b/test/asynchronous/test_grid_file.py index 14446106e0..affdacde91 100644 --- a/test/asynchronous/test_grid_file.py +++ b/test/asynchronous/test_grid_file.py @@ -33,7 +33,7 @@ sys.path[0:0] = [""] -from test.utils import EventListener +from test.utils import OvertCommandListener from bson.objectid import ObjectId from gridfs.asynchronous.grid_file import ( @@ -811,7 +811,7 @@ async def test_survive_cursor_not_found(self): # Use 102 batches to cause a single getMore. chunk_size = 1024 data = b"d" * (102 * chunk_size) - listener = EventListener() + listener = OvertCommandListener() client = await self.async_rs_or_single_client(event_listeners=[listener]) db = client.pymongo_test async with AsyncGridIn(db.fs, chunk_size=chunk_size) as infile: diff --git a/test/asynchronous/test_monitoring.py b/test/asynchronous/test_monitoring.py index a5f991b2f0..eaad60beac 100644 --- a/test/asynchronous/test_monitoring.py +++ b/test/asynchronous/test_monitoring.py @@ -31,6 +31,7 @@ ) from test.utils import ( EventListener, + OvertCommandListener, async_wait_until, ) @@ -52,7 +53,7 @@ class AsyncTestCommandMonitoring(AsyncIntegrationTest): @classmethod def setUpClass(cls) -> None: - cls.listener = EventListener() + cls.listener = OvertCommandListener() @async_client_context.require_connection async def asyncSetUp(self) -> None: @@ -1094,11 +1095,13 @@ async def test_first_batch_helper(self): @async_client_context.require_version_max(6, 1, 99) async def test_sensitive_commands(self): - listeners = self.client._event_listeners + listener = EventListener() + client = await self.async_rs_or_single_client(event_listeners=[listener]) + listeners = client._event_listeners - self.listener.reset() + listener.reset() cmd = SON([("getnonce", 1)]) - listeners.publish_command_start(cmd, "pymongo_test", 12345, await self.client.address, None) # type: ignore[arg-type] + listeners.publish_command_start(cmd, "pymongo_test", 12345, await client.address, None) # type: ignore[arg-type] delta = datetime.timedelta(milliseconds=100) listeners.publish_command_success( delta, @@ -1109,15 +1112,15 @@ async def test_sensitive_commands(self): None, database_name="pymongo_test", ) - started = self.listener.started_events[0] - succeeded = self.listener.succeeded_events[0] - self.assertEqual(0, len(self.listener.failed_events)) + started = listener.started_events[0] + succeeded = listener.succeeded_events[0] + self.assertEqual(0, len(listener.failed_events)) self.assertIsInstance(started, monitoring.CommandStartedEvent) self.assertEqual({}, started.command) self.assertEqual("pymongo_test", started.database_name) self.assertEqual("getnonce", started.command_name) self.assertIsInstance(started.request_id, int) - self.assertEqual(await self.client.address, started.connection_id) + self.assertEqual(await client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertEqual(succeeded.duration_micros, 100000) self.assertEqual(started.command_name, succeeded.command_name) @@ -1132,7 +1135,7 @@ class AsyncTestGlobalListener(AsyncIntegrationTest): @classmethod def setUpClass(cls) -> None: - cls.listener = EventListener() + cls.listener = OvertCommandListener() # We plan to call register(), which internally modifies _LISTENERS. cls.saved_listeners = copy.deepcopy(monitoring._LISTENERS) monitoring.register(cls.listener) @@ -1140,17 +1143,11 @@ def setUpClass(cls) -> None: @async_client_context.require_connection async def asyncSetUp(self): await super().asyncSetUp() - self.listener = EventListener() - # We plan to call register(), which internally modifies _LISTENERS. - self.saved_listeners = copy.deepcopy(monitoring._LISTENERS) - monitoring.register(self.listener) + self.listener.reset() self.client = await self.async_single_client() # Get one (authenticated) socket in the pool. await self.client.pymongo_test.command("ping") - async def asyncTearDown(self) -> None: - self.listener.reset() - @classmethod def tearDownClass(cls): monitoring._LISTENERS = cls.saved_listeners diff --git a/test/asynchronous/test_session.py b/test/asynchronous/test_session.py index e424796ce0..42bc253b56 100644 --- a/test/asynchronous/test_session.py +++ b/test/asynchronous/test_session.py @@ -36,6 +36,7 @@ from test.utils import ( EventListener, ExceptionCatchingThread, + OvertCommandListener, async_wait_until, ) @@ -191,7 +192,7 @@ def test_implicit_sessions_checkout(self): lsid_set = set() failures = 0 for _ in range(5): - listener = EventListener() + listener = OvertCommandListener() client = self.async_rs_or_single_client(event_listeners=[listener], maxPoolSize=1) cursor = client.db.test.find({}) ops: List[Tuple[Callable, List[Any]]] = [ diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index f25e96e04d..11b124a124 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -772,7 +772,7 @@ async def _databaseOperation_listCollections(self, target, *args, **kwargs): if "batch_size" in kwargs: kwargs["cursor"] = {"batchSize": kwargs.pop("batch_size")} cursor = await target.list_collections(*args, **kwargs) - return list(cursor) + return await cursor.to_list() async def _databaseOperation_createCollection(self, target, *args, **kwargs): # PYTHON-1936 Ignore the listCollections event from create_collection. diff --git a/test/auth_oidc/test_auth_oidc.py b/test/auth_oidc/test_auth_oidc.py index 6d31f3db4e..6526391daf 100644 --- a/test/auth_oidc/test_auth_oidc.py +++ b/test/auth_oidc/test_auth_oidc.py @@ -31,7 +31,7 @@ sys.path[0:0] = [""] from test.unified_format import generate_test_classes -from test.utils import EventListener +from test.utils import EventListener, OvertCommandListener from bson import SON from pymongo import MongoClient @@ -348,7 +348,7 @@ def test_4_1_reauthenticate_succeeds(self): # Create a default OIDC client and add an event listener. # The following assumes that the driver does not emit saslStart or saslContinue events. # If the driver does emit those events, ignore/filter them for the purposes of this test. - listener = EventListener() + listener = OvertCommandListener() client = self.create_client(event_listeners=[listener]) # Perform a find operation that succeeds. @@ -1021,7 +1021,7 @@ def fetch(self, _): def test_4_4_speculative_authentication_should_be_ignored_on_reauthentication(self): # Create an OIDC configured client that can listen for `SaslStart` commands. - listener = EventListener() + listener = OvertCommandListener() client = self.create_client(event_listeners=[listener]) # Preload the *Client Cache* with a valid access token to enforce Speculative Authentication. diff --git a/test/crud/unified/aggregate-write-readPreference.json b/test/crud/unified/aggregate-write-readPreference.json index bc887e83cb..c1fa3b4574 100644 --- a/test/crud/unified/aggregate-write-readPreference.json +++ b/test/crud/unified/aggregate-write-readPreference.json @@ -78,11 +78,6 @@ "x": 33 } ] - }, - { - "collectionName": "coll1", - "databaseName": "db0", - "documents": [] } ], "tests": [ @@ -159,22 +154,6 @@ } ] } - ], - "outcome": [ - { - "collectionName": "coll1", - "databaseName": "db0", - "documents": [ - { - "_id": 2, - "x": 22 - }, - { - "_id": 3, - "x": 33 - } - ] - } ] }, { @@ -250,22 +229,6 @@ } ] } - ], - "outcome": [ - { - "collectionName": "coll1", - "databaseName": "db0", - "documents": [ - { - "_id": 2, - "x": 22 - }, - { - "_id": 3, - "x": 33 - } - ] - } ] }, { @@ -344,22 +307,6 @@ } ] } - ], - "outcome": [ - { - "collectionName": "coll1", - "databaseName": "db0", - "documents": [ - { - "_id": 2, - "x": 22 - }, - { - "_id": 3, - "x": 33 - } - ] - } ] }, { @@ -438,22 +385,6 @@ } ] } - ], - "outcome": [ - { - "collectionName": "coll1", - "databaseName": "db0", - "documents": [ - { - "_id": 2, - "x": 22 - }, - { - "_id": 3, - "x": 33 - } - ] - } ] } ] diff --git a/test/crud/unified/bulkWrite-replaceOne-sort.json b/test/crud/unified/bulkWrite-replaceOne-sort.json new file mode 100644 index 0000000000..c0bd383514 --- /dev/null +++ b/test/crud/unified/bulkWrite-replaceOne-sort.json @@ -0,0 +1,239 @@ +{ + "description": "BulkWrite replaceOne-sort", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "BulkWrite replaceOne with sort option", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "collection0", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "replaceOne": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "_id": -1 + }, + "replacement": { + "x": 1 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "coll0", + "updates": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "u": { + "x": 1 + }, + "sort": { + "_id": -1 + }, + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } + } + ] + } + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1 + }, + "commandName": "update" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 1 + } + ] + } + ] + }, + { + "description": "BulkWrite replaceOne with sort option unsupported (server-side error)", + "runOnRequirements": [ + { + "maxServerVersion": "7.99" + } + ], + "operations": [ + { + "object": "collection0", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "replaceOne": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "_id": -1 + }, + "replacement": { + "x": 1 + } + } + } + ] + }, + "expectError": { + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "coll0", + "updates": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "u": { + "x": 1 + }, + "sort": { + "_id": -1 + }, + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + } + ] +} diff --git a/test/crud/unified/bulkWrite-updateOne-sort.json b/test/crud/unified/bulkWrite-updateOne-sort.json new file mode 100644 index 0000000000..f78bd3bf3e --- /dev/null +++ b/test/crud/unified/bulkWrite-updateOne-sort.json @@ -0,0 +1,255 @@ +{ + "description": "BulkWrite updateOne-sort", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "BulkWrite updateOne with sort option", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "object": "collection0", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "updateOne": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "_id": -1 + }, + "update": [ + { + "$set": { + "x": 1 + } + } + ] + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "coll0", + "updates": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "u": [ + { + "$set": { + "x": 1 + } + } + ], + "sort": { + "_id": -1 + }, + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } + } + ] + } + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1 + }, + "commandName": "update" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 1 + } + ] + } + ] + }, + { + "description": "BulkWrite updateOne with sort option unsupported (server-side error)", + "runOnRequirements": [ + { + "maxServerVersion": "7.99" + } + ], + "operations": [ + { + "object": "collection0", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "updateOne": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "_id": -1 + }, + "update": [ + { + "$set": { + "x": 1 + } + } + ] + } + } + ] + }, + "expectError": { + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "coll0", + "updates": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "u": [ + { + "$set": { + "x": 1 + } + } + ], + "sort": { + "_id": -1 + }, + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + } + ] +} diff --git a/test/crud/unified/client-bulkWrite-partialResults.json b/test/crud/unified/client-bulkWrite-partialResults.json new file mode 100644 index 0000000000..b35e94a2ea --- /dev/null +++ b/test/crud/unified/client-bulkWrite-partialResults.json @@ -0,0 +1,540 @@ +{ + "description": "client bulkWrite partial results", + "schemaVersion": "1.4", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "serverless": "forbid" + } + ], + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0", + "newDocument": { + "_id": 2, + "x": 22 + } + }, + "tests": [ + { + "description": "partialResult is unset when first operation fails during an ordered bulk write (verbose)", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + }, + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "ordered": true, + "verboseResults": true + }, + "expectError": { + "expectResult": { + "$$unsetOrMatches": { + "insertedCount": { + "$$exists": false + }, + "upsertedCount": { + "$$exists": false + }, + "matchedCount": { + "$$exists": false + }, + "modifiedCount": { + "$$exists": false + }, + "deletedCount": { + "$$exists": false + }, + "insertResults": { + "$$exists": false + }, + "updateResults": { + "$$exists": false + }, + "deleteResults": { + "$$exists": false + } + } + } + } + } + ] + }, + { + "description": "partialResult is unset when first operation fails during an ordered bulk write (summary)", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + }, + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "ordered": true, + "verboseResults": false + }, + "expectError": { + "expectResult": { + "$$unsetOrMatches": { + "insertedCount": { + "$$exists": false + }, + "upsertedCount": { + "$$exists": false + }, + "matchedCount": { + "$$exists": false + }, + "modifiedCount": { + "$$exists": false + }, + "deletedCount": { + "$$exists": false + }, + "insertResults": { + "$$exists": false + }, + "updateResults": { + "$$exists": false + }, + "deleteResults": { + "$$exists": false + } + } + } + } + } + ] + }, + { + "description": "partialResult is set when second operation fails during an ordered bulk write (verbose)", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 2, + "x": 22 + } + } + }, + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + } + ], + "ordered": true, + "verboseResults": true + }, + "expectError": { + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 2 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + } + ] + }, + { + "description": "partialResult is set when second operation fails during an ordered bulk write (summary)", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 2, + "x": 22 + } + } + }, + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + } + ], + "ordered": true, + "verboseResults": false + }, + "expectError": { + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + } + ] + }, + { + "description": "partialResult is unset when all operations fail during an unordered bulk write", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + }, + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + } + ], + "ordered": false + }, + "expectError": { + "expectResult": { + "$$unsetOrMatches": { + "insertedCount": { + "$$exists": false + }, + "upsertedCount": { + "$$exists": false + }, + "matchedCount": { + "$$exists": false + }, + "modifiedCount": { + "$$exists": false + }, + "deletedCount": { + "$$exists": false + }, + "insertResults": { + "$$exists": false + }, + "updateResults": { + "$$exists": false + }, + "deleteResults": { + "$$exists": false + } + } + } + } + } + ] + }, + { + "description": "partialResult is set when first operation fails during an unordered bulk write (verbose)", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + }, + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "ordered": false, + "verboseResults": true + }, + "expectError": { + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "1": { + "insertedId": 2 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + } + ] + }, + { + "description": "partialResult is set when first operation fails during an unordered bulk write (summary)", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + }, + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "ordered": false, + "verboseResults": false + }, + "expectError": { + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + } + ] + }, + { + "description": "partialResult is set when second operation fails during an unordered bulk write (verbose)", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 2, + "x": 22 + } + } + }, + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + } + ], + "ordered": false, + "verboseResults": true + }, + "expectError": { + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "0": { + "insertedId": 2 + } + }, + "updateResults": {}, + "deleteResults": {} + } + } + } + ] + }, + { + "description": "partialResult is set when first operation fails during an unordered bulk write (summary)", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 2, + "x": 22 + } + } + }, + { + "insertOne": { + "namespace": "crud-tests.coll0", + "document": { + "_id": 1, + "x": 11 + } + } + } + ], + "ordered": false, + "verboseResults": false + }, + "expectError": { + "expectResult": { + "insertedCount": 1, + "upsertedCount": 0, + "matchedCount": 0, + "modifiedCount": 0, + "deletedCount": 0, + "insertResults": { + "$$unsetOrMatches": {} + }, + "updateResults": { + "$$unsetOrMatches": {} + }, + "deleteResults": { + "$$unsetOrMatches": {} + } + } + } + } + ] + } + ] +} diff --git a/test/crud/unified/client-bulkWrite-replaceOne-sort.json b/test/crud/unified/client-bulkWrite-replaceOne-sort.json new file mode 100644 index 0000000000..53218c1f48 --- /dev/null +++ b/test/crud/unified/client-bulkWrite-replaceOne-sort.json @@ -0,0 +1,162 @@ +{ + "description": "client bulkWrite updateOne-sort", + "schemaVersion": "1.4", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite replaceOne with sort option", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "_id": -1 + }, + "replacement": { + "x": 1 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "ops": [ + { + "update": 0, + "filter": { + "_id": { + "$gt": 1 + } + }, + "updateMods": { + "x": 1 + }, + "sort": { + "_id": -1 + }, + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "nErrors": 0, + "nMatched": 1, + "nModified": 1 + }, + "commandName": "bulkWrite" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 1 + } + ] + } + ] + } + ] +} diff --git a/test/crud/unified/client-bulkWrite-updateOne-sort.json b/test/crud/unified/client-bulkWrite-updateOne-sort.json new file mode 100644 index 0000000000..4a07b8b97c --- /dev/null +++ b/test/crud/unified/client-bulkWrite-updateOne-sort.json @@ -0,0 +1,166 @@ +{ + "description": "client bulkWrite updateOne-sort", + "schemaVersion": "1.4", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite updateOne with sort option", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "updateOne": { + "namespace": "crud-tests.coll0", + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "_id": -1 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite", + "databaseName": "admin", + "command": { + "bulkWrite": 1, + "ops": [ + { + "update": 0, + "filter": { + "_id": { + "$gt": 1 + } + }, + "updateMods": { + "$inc": { + "x": 1 + } + }, + "sort": { + "_id": -1 + }, + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } + } + ], + "nsInfo": [ + { + "ns": "crud-tests.coll0" + } + ] + } + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "nErrors": 0, + "nMatched": 1, + "nModified": 1 + }, + "commandName": "bulkWrite" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 34 + } + ] + } + ] + } + ] +} diff --git a/test/crud/unified/db-aggregate-write-readPreference.json b/test/crud/unified/db-aggregate-write-readPreference.json index 2a81282de8..b6460f001f 100644 --- a/test/crud/unified/db-aggregate-write-readPreference.json +++ b/test/crud/unified/db-aggregate-write-readPreference.json @@ -52,13 +52,6 @@ } } ], - "initialData": [ - { - "collectionName": "coll0", - "databaseName": "db0", - "documents": [] - } - ], "tests": [ { "description": "Database-level aggregate with $out includes read preference for 5.0+ server", @@ -141,17 +134,6 @@ } ] } - ], - "outcome": [ - { - "collectionName": "coll0", - "databaseName": "db0", - "documents": [ - { - "_id": 1 - } - ] - } ] }, { @@ -235,17 +217,6 @@ } ] } - ], - "outcome": [ - { - "collectionName": "coll0", - "databaseName": "db0", - "documents": [ - { - "_id": 1 - } - ] - } ] }, { @@ -332,17 +303,6 @@ } ] } - ], - "outcome": [ - { - "collectionName": "coll0", - "databaseName": "db0", - "documents": [ - { - "_id": 1 - } - ] - } ] }, { @@ -429,17 +389,6 @@ } ] } - ], - "outcome": [ - { - "collectionName": "coll0", - "databaseName": "db0", - "documents": [ - { - "_id": 1 - } - ] - } ] } ] diff --git a/test/crud/unified/replaceOne-sort.json b/test/crud/unified/replaceOne-sort.json new file mode 100644 index 0000000000..cf2271dda5 --- /dev/null +++ b/test/crud/unified/replaceOne-sort.json @@ -0,0 +1,232 @@ +{ + "description": "replaceOne-sort", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "ReplaceOne with sort option", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "replaceOne", + "object": "collection0", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "_id": -1 + }, + "replacement": { + "x": 1 + } + }, + "expectResult": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedCount": 0 + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "coll0", + "updates": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "u": { + "x": 1 + }, + "sort": { + "_id": -1 + }, + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } + } + ] + } + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1 + }, + "commandName": "update" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 1 + } + ] + } + ] + }, + { + "description": "replaceOne with sort option unsupported (server-side error)", + "runOnRequirements": [ + { + "maxServerVersion": "7.99" + } + ], + "operations": [ + { + "name": "replaceOne", + "object": "collection0", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "_id": -1 + }, + "replacement": { + "x": 1 + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "coll0", + "updates": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "u": { + "x": 1 + }, + "sort": { + "_id": -1 + }, + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + } + ] +} diff --git a/test/crud/unified/updateOne-sort.json b/test/crud/unified/updateOne-sort.json new file mode 100644 index 0000000000..8fe4f50b94 --- /dev/null +++ b/test/crud/unified/updateOne-sort.json @@ -0,0 +1,240 @@ +{ + "description": "updateOne-sort", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "UpdateOne with sort option", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "updateOne", + "object": "collection0", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "_id": -1 + }, + "update": { + "$inc": { + "x": 1 + } + } + }, + "expectResult": { + "matchedCount": 1, + "modifiedCount": 1, + "upsertedCount": 0 + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "coll0", + "updates": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "u": { + "$inc": { + "x": 1 + } + }, + "sort": { + "_id": -1 + }, + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } + } + ] + } + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1 + }, + "commandName": "update" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 34 + } + ] + } + ] + }, + { + "description": "updateOne with sort option unsupported (server-side error)", + "runOnRequirements": [ + { + "maxServerVersion": "7.99" + } + ], + "operations": [ + { + "name": "updateOne", + "object": "collection0", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "_id": -1 + }, + "update": { + "$inc": { + "x": 1 + } + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "coll0", + "updates": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "u": { + "$inc": { + "x": 1 + } + }, + "sort": { + "_id": -1 + }, + "multi": { + "$$unsetOrMatches": false + }, + "upsert": { + "$$unsetOrMatches": false + } + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + } + ] +} diff --git a/test/test_auth.py b/test/test_auth.py index b311d330bc..310006afff 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -32,6 +32,8 @@ ) from test.utils import AllowListEventListener, delay, ignore_deprecations +import pytest + from pymongo import MongoClient, monitoring from pymongo.auth_shared import _build_credentials_tuple from pymongo.errors import OperationFailure @@ -42,6 +44,8 @@ _IS_SYNC = True +pytestmark = pytest.mark.auth + # YOU MUST RUN KINIT BEFORE RUNNING GSSAPI TESTS ON UNIX. GSSAPI_HOST = os.environ.get("GSSAPI_HOST") GSSAPI_PORT = int(os.environ.get("GSSAPI_PORT", "27017")) diff --git a/test/test_bulk.py b/test/test_bulk.py index ad22c1ce9a..6d29ff510a 100644 --- a/test/test_bulk.py +++ b/test/test_bulk.py @@ -959,6 +959,9 @@ def cause_wtimeout(self, requests, ordered): @client_context.require_replica_set @client_context.require_secondaries_count(1) def test_write_concern_failure_ordered(self): + self.skipTest("Skipping until PYTHON-4865 is resolved.") + details = None + # Ensure we don't raise on wnote. coll_ww = self.coll.with_options(write_concern=WriteConcern(w=self.w)) result = coll_ww.bulk_write([DeleteOne({"something": "that does no exist"})]) @@ -1039,6 +1042,9 @@ def test_write_concern_failure_ordered(self): @client_context.require_replica_set @client_context.require_secondaries_count(1) def test_write_concern_failure_unordered(self): + self.skipTest("Skipping until PYTHON-4865 is resolved.") + details = None + # Ensure we don't raise on wnote. coll_ww = self.coll.with_options(write_concern=WriteConcern(w=self.w)) result = coll_ww.bulk_write([DeleteOne({"something": "that does no exist"})], ordered=False) diff --git a/test/test_change_stream.py b/test/test_change_stream.py index 0742384184..4ed21f55cf 100644 --- a/test/test_change_stream.py +++ b/test/test_change_stream.py @@ -39,6 +39,7 @@ from test.utils import ( AllowListEventListener, EventListener, + OvertCommandListener, wait_until, ) @@ -177,7 +178,7 @@ def _wait_until(): @no_type_check def test_try_next_runs_one_getmore(self): - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener]) # Connect to the cluster. client.admin.command("ping") @@ -235,7 +236,7 @@ def _wait_until(): @no_type_check def test_batch_size_is_honored(self): - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener]) # Connect to the cluster. client.admin.command("ping") diff --git a/test/test_collation.py b/test/test_collation.py index 6d4e958a1f..06436f0638 100644 --- a/test/test_collation.py +++ b/test/test_collation.py @@ -18,7 +18,7 @@ import functools import warnings from test import IntegrationTest, client_context, unittest -from test.utils import EventListener +from test.utils import EventListener, OvertCommandListener from typing import Any from pymongo.collation import ( @@ -100,13 +100,12 @@ class TestCollation(IntegrationTest): @client_context.require_connection def setUp(self) -> None: super().setUp() - self.listener = EventListener() + self.listener = OvertCommandListener() self.client = self.rs_or_single_client(event_listeners=[self.listener]) self.db = self.client.pymongo_test self.collation = Collation("en_US") self.warn_context = warnings.catch_warnings() self.warn_context.__enter__() - warnings.simplefilter("ignore", DeprecationWarning) def tearDown(self) -> None: self.warn_context.__exit__() diff --git a/test/test_collection.py b/test/test_collection.py index 9364d34e34..af524bba47 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -36,6 +36,7 @@ from test.utils import ( IMPOSSIBLE_WRITE_CONCERN, EventListener, + OvertCommandListener, get_pool, is_mongos, wait_until, @@ -2079,7 +2080,7 @@ def test_find_one_and(self): self.assertEqual(4, (c.find_one_and_update({}, {"$inc": {"i": 1}}, sort=sort))["j"]) def test_find_one_and_write_concern(self): - listener = EventListener() + listener = OvertCommandListener() db = (self.single_client(event_listeners=[listener]))[self.db.name] # non-default WriteConcern. c_w0 = db.get_collection("test", write_concern=WriteConcern(w=0)) diff --git a/test/test_collection_management.py b/test/test_collection_management.py index 0eacde1302..063c20df8f 100644 --- a/test/test_collection_management.py +++ b/test/test_collection_management.py @@ -16,6 +16,7 @@ from __future__ import annotations import os +import pathlib import sys sys.path[0:0] = [""] @@ -23,11 +24,18 @@ from test import unittest from test.unified_format import generate_test_classes +_IS_SYNC = True + # Location of JSON test specifications. -TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "collection_management") +if _IS_SYNC: + _TEST_PATH = os.path.join(pathlib.Path(__file__).resolve().parent, "collection_management") +else: + _TEST_PATH = os.path.join( + pathlib.Path(__file__).resolve().parent.parent, "collection_management" + ) # Generate unified tests. -globals().update(generate_test_classes(TEST_PATH, module=__name__)) +globals().update(generate_test_classes(_TEST_PATH, module=__name__)) if __name__ == "__main__": unittest.main() diff --git a/test/test_create_entities.py b/test/test_create_entities.py index b7965d4a1d..ad75fe5702 100644 --- a/test/test_create_entities.py +++ b/test/test_create_entities.py @@ -21,6 +21,8 @@ from test import IntegrationTest from test.unified_format import UnifiedSpecTestMixinV1 +_IS_SYNC = True + class TestCreateEntities(IntegrationTest): def test_store_events_as_entities(self): diff --git a/test/test_cursor.py b/test/test_cursor.py index e687abcfbf..bcc7ed75f1 100644 --- a/test/test_cursor.py +++ b/test/test_cursor.py @@ -1590,7 +1590,7 @@ def test_read_concern(self): next(c.find_raw_batches()) def test_monitoring(self): - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener]) c = client.pymongo_test.test c.drop() @@ -1753,7 +1753,7 @@ def test_collation(self): next(self.db.test.aggregate_raw_batches([], collation=Collation("en_US"))) def test_monitoring(self): - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener]) c = client.pymongo_test.test c.drop() diff --git a/test/test_grid_file.py b/test/test_grid_file.py index 0a5b1ad40a..6534bc11bf 100644 --- a/test/test_grid_file.py +++ b/test/test_grid_file.py @@ -33,7 +33,7 @@ sys.path[0:0] = [""] -from test.utils import EventListener +from test.utils import OvertCommandListener from bson.objectid import ObjectId from gridfs.errors import NoFile @@ -809,7 +809,7 @@ def test_survive_cursor_not_found(self): # Use 102 batches to cause a single getMore. chunk_size = 1024 data = b"d" * (102 * chunk_size) - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener]) db = client.pymongo_test with GridIn(db.fs, chunk_size=chunk_size) as infile: diff --git a/test/test_index_management.py b/test/test_index_management.py index ec1e363737..6ca726e2e0 100644 --- a/test/test_index_management.py +++ b/test/test_index_management.py @@ -27,7 +27,7 @@ from test import IntegrationTest, PyMongoTestCase, unittest from test.unified_format import generate_test_classes -from test.utils import AllowListEventListener, EventListener +from test.utils import AllowListEventListener, EventListener, OvertCommandListener from pymongo.errors import OperationFailure from pymongo.operations import SearchIndexModel @@ -88,7 +88,7 @@ def setUpClass(cls) -> None: url = os.environ.get("MONGODB_URI") username = os.environ["DB_USER"] password = os.environ["DB_PASSWORD"] - cls.listener = listener = EventListener() + cls.listener = listener = OvertCommandListener() cls.client = cls.unmanaged_simple_client( url, username=username, password=password, event_listeners=[listener] ) diff --git a/test/test_monitoring.py b/test/test_monitoring.py index 31f546fe54..670558c0a0 100644 --- a/test/test_monitoring.py +++ b/test/test_monitoring.py @@ -31,6 +31,7 @@ ) from test.utils import ( EventListener, + OvertCommandListener, wait_until, ) @@ -52,7 +53,7 @@ class TestCommandMonitoring(IntegrationTest): @classmethod def setUpClass(cls) -> None: - cls.listener = EventListener() + cls.listener = OvertCommandListener() @client_context.require_connection def setUp(self) -> None: @@ -1092,11 +1093,13 @@ def test_first_batch_helper(self): @client_context.require_version_max(6, 1, 99) def test_sensitive_commands(self): - listeners = self.client._event_listeners + listener = EventListener() + client = self.rs_or_single_client(event_listeners=[listener]) + listeners = client._event_listeners - self.listener.reset() + listener.reset() cmd = SON([("getnonce", 1)]) - listeners.publish_command_start(cmd, "pymongo_test", 12345, self.client.address, None) # type: ignore[arg-type] + listeners.publish_command_start(cmd, "pymongo_test", 12345, client.address, None) # type: ignore[arg-type] delta = datetime.timedelta(milliseconds=100) listeners.publish_command_success( delta, @@ -1107,15 +1110,15 @@ def test_sensitive_commands(self): None, database_name="pymongo_test", ) - started = self.listener.started_events[0] - succeeded = self.listener.succeeded_events[0] - self.assertEqual(0, len(self.listener.failed_events)) + started = listener.started_events[0] + succeeded = listener.succeeded_events[0] + self.assertEqual(0, len(listener.failed_events)) self.assertIsInstance(started, monitoring.CommandStartedEvent) self.assertEqual({}, started.command) self.assertEqual("pymongo_test", started.database_name) self.assertEqual("getnonce", started.command_name) self.assertIsInstance(started.request_id, int) - self.assertEqual(self.client.address, started.connection_id) + self.assertEqual(client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertEqual(succeeded.duration_micros, 100000) self.assertEqual(started.command_name, succeeded.command_name) @@ -1130,7 +1133,7 @@ class TestGlobalListener(IntegrationTest): @classmethod def setUpClass(cls) -> None: - cls.listener = EventListener() + cls.listener = OvertCommandListener() # We plan to call register(), which internally modifies _LISTENERS. cls.saved_listeners = copy.deepcopy(monitoring._LISTENERS) monitoring.register(cls.listener) @@ -1138,17 +1141,11 @@ def setUpClass(cls) -> None: @client_context.require_connection def setUp(self): super().setUp() - self.listener = EventListener() - # We plan to call register(), which internally modifies _LISTENERS. - self.saved_listeners = copy.deepcopy(monitoring._LISTENERS) - monitoring.register(self.listener) + self.listener.reset() self.client = self.single_client() # Get one (authenticated) socket in the pool. self.client.pymongo_test.command("ping") - def tearDown(self) -> None: - self.listener.reset() - @classmethod def tearDownClass(cls): monitoring._LISTENERS = cls.saved_listeners diff --git a/test/test_operations.py b/test/test_operations.py new file mode 100644 index 0000000000..3ee6677735 --- /dev/null +++ b/test/test_operations.py @@ -0,0 +1,80 @@ +# Copyright 2024-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test the operations module.""" +from __future__ import annotations + +from test import UnitTest, unittest + +from pymongo import ASCENDING, DESCENDING +from pymongo.collation import Collation +from pymongo.errors import OperationFailure +from pymongo.operations import IndexModel, SearchIndexModel + + +class TestOperationsBase(UnitTest): + """Base class for testing operations module.""" + + def assertRepr(self, obj): + new_obj = eval(repr(obj)) + self.assertEqual(type(new_obj), type(obj)) + self.assertEqual(repr(new_obj), repr(obj)) + + +class TestIndexModel(TestOperationsBase): + """Test IndexModel features.""" + + def test_repr(self): + # Based on examples in test_collection.py + self.assertRepr(IndexModel("hello")) + self.assertRepr(IndexModel([("hello", DESCENDING), ("world", ASCENDING)])) + self.assertRepr( + IndexModel([("hello", DESCENDING), ("world", ASCENDING)], name="hello_world") + ) + # Test all the kwargs + self.assertRepr(IndexModel("name", name="name")) + self.assertRepr(IndexModel("unique", unique=False)) + self.assertRepr(IndexModel("background", background=True)) + self.assertRepr(IndexModel("sparse", sparse=True)) + self.assertRepr(IndexModel("bucketSize", bucketSize=1)) + self.assertRepr(IndexModel("min", min=1)) + self.assertRepr(IndexModel("max", max=1)) + self.assertRepr(IndexModel("expireAfterSeconds", expireAfterSeconds=1)) + self.assertRepr( + IndexModel("partialFilterExpression", partialFilterExpression={"hello": "world"}) + ) + self.assertRepr(IndexModel("collation", collation=Collation(locale="en_US"))) + self.assertRepr(IndexModel("wildcardProjection", wildcardProjection={"$**": 1})) + self.assertRepr(IndexModel("hidden", hidden=False)) + # Test string literal + self.assertEqual(repr(IndexModel("hello")), "IndexModel({'hello': 1}, name='hello_1')") + self.assertEqual( + repr(IndexModel({"hello": 1, "world": -1})), + "IndexModel({'hello': 1, 'world': -1}, name='hello_1_world_-1')", + ) + + +class TestSearchIndexModel(TestOperationsBase): + """Test SearchIndexModel features.""" + + def test_repr(self): + self.assertRepr(SearchIndexModel({"hello": "hello"}, key=1)) + self.assertEqual( + repr(SearchIndexModel({"hello": "hello"}, key=1)), + "SearchIndexModel(definition={'hello': 'hello'}, key=1)", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_read_write_concern_spec.py b/test/test_read_write_concern_spec.py index 67943d495d..db53b67ae4 100644 --- a/test/test_read_write_concern_spec.py +++ b/test/test_read_write_concern_spec.py @@ -24,7 +24,7 @@ from test import IntegrationTest, client_context, unittest from test.unified_format import generate_test_classes -from test.utils import EventListener +from test.utils import OvertCommandListener from pymongo import DESCENDING from pymongo.errors import ( @@ -44,7 +44,7 @@ class TestReadWriteConcernSpec(IntegrationTest): def test_omit_default_read_write_concern(self): - listener = EventListener() + listener = OvertCommandListener() # Client with default readConcern and writeConcern client = self.rs_or_single_client(event_listeners=[listener]) self.addCleanup(client.close) @@ -205,7 +205,7 @@ def test_error_includes_errInfo(self): @client_context.require_version_min(4, 9) def test_write_error_details_exposes_errinfo(self): - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener]) self.addCleanup(client.close) db = client.errinfotest diff --git a/test/test_server_selection.py b/test/test_server_selection.py index 67e9716bf4..984b967f50 100644 --- a/test/test_server_selection.py +++ b/test/test_server_selection.py @@ -33,6 +33,7 @@ from test.utils import ( EventListener, FunctionCallRecorder, + OvertCommandListener, wait_until, ) from test.utils_selection_tests import ( @@ -74,7 +75,7 @@ def custom_selector(servers): return [servers[idx]] # Initialize client with appropriate listeners. - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client( server_selector=custom_selector, event_listeners=[listener] ) diff --git a/test/test_session.py b/test/test_session.py index 980d9df688..634efa11c0 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -36,6 +36,7 @@ from test.utils import ( EventListener, ExceptionCatchingThread, + OvertCommandListener, wait_until, ) @@ -191,7 +192,7 @@ def test_implicit_sessions_checkout(self): lsid_set = set() failures = 0 for _ in range(5): - listener = EventListener() + listener = OvertCommandListener() client = self.rs_or_single_client(event_listeners=[listener], maxPoolSize=1) cursor = client.db.test.find({}) ops: List[Tuple[Callable, List[Any]]] = [ diff --git a/test/test_ssl.py b/test/test_ssl.py index 36d7ba12b6..04db9b61a4 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -33,6 +33,7 @@ ) from test.utils import ( EventListener, + OvertCommandListener, cat_files, ignore_deprecations, ) diff --git a/test/unified_format.py b/test/unified_format.py index 7d5c4e4e03..a88c51e6d5 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -769,7 +769,7 @@ def _databaseOperation_listCollections(self, target, *args, **kwargs): if "batch_size" in kwargs: kwargs["cursor"] = {"batchSize": kwargs.pop("batch_size")} cursor = target.list_collections(*args, **kwargs) - return list(cursor) + return cursor.to_list() def _databaseOperation_createCollection(self, target, *args, **kwargs): # PYTHON-1936 Ignore the listCollections event from create_collection. diff --git a/test/utils.py b/test/utils.py index 24673b698e..493d6f9422 100644 --- a/test/utils.py +++ b/test/utils.py @@ -967,10 +967,6 @@ def parse_spec_options(opts): def prepare_spec_arguments(spec, arguments, opname, entity_map, with_txn_callback): for arg_name in list(arguments): c2s = camel_to_snake(arg_name) - # PyMongo accepts sort as list of tuples. - if arg_name == "sort": - sort_dict = arguments[arg_name] - arguments[arg_name] = list(sort_dict.items()) # Named "key" instead not fieldName. if arg_name == "fieldName": arguments["key"] = arguments.pop(arg_name) diff --git a/tools/synchro.py b/tools/synchro.py index 17841d3025..47617365f4 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -197,6 +197,7 @@ def async_only_test(f: str) -> bool: "test_client_context.py", "test_collation.py", "test_collection.py", + "test_collection_management.py", "test_command_logging.py", "test_command_logging.py", "test_command_monitoring.py", @@ -204,6 +205,7 @@ def async_only_test(f: str) -> bool: "test_common.py", "test_connection_logging.py", "test_connections_survive_primary_stepdown_spec.py", + "test_create_entities.py", "test_crud_unified.py", "test_cursor.py", "test_database.py",