Merge branch 'master' of github.com:mongodb/mongo-python-driver

Author: blink1073

.evergreen/config.yml

@@ -8,7 +8,17 @@ if [ -z "$PYTHON_BINARY" ]; then
     PYTHON_BINARY=$(find_python3)
 fi
 
-if $PYTHON_BINARY -m hatch --version; then
+# Check if we should skip hatch and run the tests directly.
+if [ -n "$SKIP_HATCH" ]; then
+    ENV_NAME=testenv-$RANDOM
+    createvirtualenv "$PYTHON_BINARY" $ENV_NAME
+    # shellcheck disable=SC2064
+    trap "deactivate; rm -rf $ENV_NAME" EXIT HUP
+    python -m pip install -e ".[test]"
+    run_hatch() {
+      bash ./.evergreen/run-tests.sh
+    }
+elif $PYTHON_BINARY -m hatch --version; then
     run_hatch() {
       $PYTHON_BINARY -m hatch run "$@"
     }

.evergreen/hatch.sh

@@ -0,0 +1,53 @@
+#!/bin/bash -ex
+
+# Get the current unique version of this checkout
+# shellcheck disable=SC2154
+if [ "$is_patch" = "true" ]; then
+    # shellcheck disable=SC2154
+    CURRENT_VERSION="$(git describe)-patch-$version_id"
+else
+    CURRENT_VERSION=latest
+fi
+
+PROJECT_DIRECTORY="$(pwd)"
+DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
+
+# Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
+if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
+    DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
+    PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
+fi
+
+SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
+
+if [ -f "$SCRIPT_DIR/env.sh" ]; then
+  echo "Reading $SCRIPT_DIR/env.sh file"
+  . "$SCRIPT_DIR/env.sh"
+  exit 0
+fi
+
+export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
+export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
+
+cat <<EOT > $SCRIPT_DIR/env.sh
+set -o errexit
+export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
+export CURRENT_VERSION="$CURRENT_VERSION"
+export SKIP_LEGACY_SHELL=1
+export DRIVERS_TOOLS="$DRIVERS_TOOLS"
+export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
+export MONGODB_BINARIES="$MONGODB_BINARIES"
+export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
+
+export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
+export PATH="$MONGODB_BINARIES:$PATH"
+# shellcheck disable=SC2154
+export PROJECT="$project"
+export PIP_QUIET=1
+EOT
+
+# Add these expansions to make it easier to call out tests scripts from the EVG yaml
+cat <<EOT > expansion.yml
+DRIVERS_TOOLS: "$DRIVERS_TOOLS"
+PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
+EOT

.evergreen/scripts/configure-env.sh

@@ -383,10 +383,11 @@ static int millis_from_datetime_ms(PyObject* dt, long long* out){
 static PyObject* decode_datetime(PyObject* self, long long millis, const codec_options_t* options){
     PyObject* naive = NULL;
     PyObject* replace = NULL;
-    PyObject* args = NULL;
-    PyObject* kwargs = NULL;
     PyObject* value = NULL;
     struct module_state *state = GETSTATE(self);
+    if (!state) {
+        goto invalid;
+    }
     if (options->datetime_conversion == DATETIME_MS){
         return datetime_ms_from_millis(self, millis);
     }
@@ -414,8 +415,8 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o
                     Py_DECREF(utcoffset);
                     return 0;
                 }
-                min_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * 86400 +
-                                     PyDateTime_DELTA_GET_SECONDS(utcoffset)) * 1000 +
+                min_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * (int64_t)86400 +
+                                     PyDateTime_DELTA_GET_SECONDS(utcoffset)) * (int64_t)1000 +
                                      (PyDateTime_DELTA_GET_MICROSECONDS(utcoffset) / 1000);
             }
             Py_DECREF(utcoffset);
@@ -433,8 +434,8 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o
                     Py_DECREF(utcoffset);
                     return 0;
                 }
-                max_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * 86400 +
-                                     PyDateTime_DELTA_GET_SECONDS(utcoffset)) * 1000 +
+                max_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * (int64_t)86400 +
+                                     PyDateTime_DELTA_GET_SECONDS(utcoffset)) * (int64_t)1000 +
                                      (PyDateTime_DELTA_GET_MICROSECONDS(utcoffset) / 1000);
             }
             Py_DECREF(utcoffset);
@@ -487,8 +488,6 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o
 invalid:
     Py_XDECREF(naive);
     Py_XDECREF(replace);
-    Py_XDECREF(args);
-    Py_XDECREF(kwargs);
     return value;
 }
 

bson/_cbsonmodule.c

@@ -4,13 +4,18 @@ Changelog
 Changes in Version 4.9.0
 -------------------------
 
+.. warning:: Driver support for MongoDB 3.6 reached end of life in April 2024.
+   PyMongo 4.9 will be the last release to support MongoDB 3.6.
+
 PyMongo 4.9 brings a number of improvements including:
 
 - Added support for MongoDB 8.0.
 - A new asynchronous API with full asyncio support.
 - Added support for In-Use Encryption range queries with MongoDB 8.0.
   Added :attr:`~pymongo.encryption.Algorithm.RANGE`.
   ``sparsity`` and ``trim_factor`` are now optional in :class:`~pymongo.encryption_options.RangeOpts`.
+- Added support for the "delegated" option for the KMIP ``master_key`` in
+  :meth:`~pymongo.encryption.ClientEncryption.create_data_key`.
 - pymongocrypt>=1.10 is now required for :ref:`In-Use Encryption` support.
 - Added :meth:`~pymongo.cursor.Cursor.to_list` to :class:`~pymongo.cursor.Cursor`,
   :class:`~pymongo.command_cursor.CommandCursor`,

doc/changelog.rst

@@ -281,6 +281,7 @@ async def write_command(
                 )
             if bwc.publish:
                 bwc._succeed(request_id, reply, duration)  # type: ignore[arg-type]
+            await client._process_response(reply, bwc.session)  # type: ignore[arg-type]
         except Exception as exc:
             duration = datetime.datetime.now() - bwc.start_time
             if isinstance(exc, (NotPrimaryError, OperationFailure)):
@@ -308,6 +309,9 @@ async def write_command(
 
             if bwc.publish:
                 bwc._fail(request_id, failure, duration)
+            # Process the response from the server.
+            if isinstance(exc, (NotPrimaryError, OperationFailure)):
+                await client._process_response(exc.details, bwc.session)  # type: ignore[arg-type]
             raise
         finally:
             bwc.start_time = datetime.datetime.now()
@@ -449,7 +453,6 @@ async def _execute_batch(
         else:
             request_id, msg, to_send = bwc.batch_command(cmd, ops)
             result = await self.write_command(bwc, cmd, request_id, msg, to_send, client)  # type: ignore[arg-type]
-        await client._process_response(result, bwc.session)  # type: ignore[arg-type]
 
         return result, to_send  # type: ignore[return-value]
 

pymongo/asynchronous/bulk.py

@@ -283,6 +283,8 @@ async def write_command(
                 )
             if bwc.publish:
                 bwc._succeed(request_id, reply, duration)  # type: ignore[arg-type]
+            # Process the response from the server.
+            await self.client._process_response(reply, bwc.session)  # type: ignore[arg-type]
         except Exception as exc:
             duration = datetime.datetime.now() - bwc.start_time
             if isinstance(exc, (NotPrimaryError, OperationFailure)):
@@ -312,6 +314,11 @@ async def write_command(
                 bwc._fail(request_id, failure, duration)
             # Top-level error will be embedded in ClientBulkWriteException.
             reply = {"error": exc}
+            # Process the response from the server.
+            if isinstance(exc, OperationFailure):
+                await self.client._process_response(exc.details, bwc.session)  # type: ignore[arg-type]
+            else:
+                await self.client._process_response({}, bwc.session)  # type: ignore[arg-type]
         finally:
             bwc.start_time = datetime.datetime.now()
         return reply  # type: ignore[return-value]
@@ -431,7 +438,6 @@ async def _execute_batch(
         result = await self.write_command(
             bwc, cmd, request_id, msg, to_send_ops, to_send_ns, self.client
         )  # type: ignore[arg-type]
-        await self.client._process_response(result, bwc.session)  # type: ignore[arg-type]
         return result, to_send_ops, to_send_ns  # type: ignore[return-value]
 
     async def _process_results_cursor(

pymongo/asynchronous/client_bulk.py

@@ -764,6 +764,9 @@ async def create_data_key(
                 Secret Data managed object.
               - `endpoint` (string): Optional. Host with optional
                  port, e.g. "example.vault.azure.net:".
+              - `delegated` (bool): Optional. If True (recommended), the
+                KMIP server will perform encryption and decryption. If
+                delegated is not provided, defaults to false.
 
         :param key_alt_names: An optional list of string alternate
             names used to reference a key. If a key is created with alternate

pymongo/asynchronous/encryption.py

@@ -281,6 +281,7 @@ def write_command(
                 )
             if bwc.publish:
                 bwc._succeed(request_id, reply, duration)  # type: ignore[arg-type]
+            client._process_response(reply, bwc.session)  # type: ignore[arg-type]
         except Exception as exc:
             duration = datetime.datetime.now() - bwc.start_time
             if isinstance(exc, (NotPrimaryError, OperationFailure)):
@@ -308,6 +309,9 @@ def write_command(
 
             if bwc.publish:
                 bwc._fail(request_id, failure, duration)
+            # Process the response from the server.
+            if isinstance(exc, (NotPrimaryError, OperationFailure)):
+                client._process_response(exc.details, bwc.session)  # type: ignore[arg-type]
             raise
         finally:
             bwc.start_time = datetime.datetime.now()
@@ -449,7 +453,6 @@ def _execute_batch(
         else:
             request_id, msg, to_send = bwc.batch_command(cmd, ops)
             result = self.write_command(bwc, cmd, request_id, msg, to_send, client)  # type: ignore[arg-type]
-        client._process_response(result, bwc.session)  # type: ignore[arg-type]
 
         return result, to_send  # type: ignore[return-value]
 

pymongo/synchronous/bulk.py

@@ -283,6 +283,8 @@ def write_command(
                 )
             if bwc.publish:
                 bwc._succeed(request_id, reply, duration)  # type: ignore[arg-type]
+            # Process the response from the server.
+            self.client._process_response(reply, bwc.session)  # type: ignore[arg-type]
         except Exception as exc:
             duration = datetime.datetime.now() - bwc.start_time
             if isinstance(exc, (NotPrimaryError, OperationFailure)):
@@ -312,6 +314,11 @@ def write_command(
                 bwc._fail(request_id, failure, duration)
             # Top-level error will be embedded in ClientBulkWriteException.
             reply = {"error": exc}
+            # Process the response from the server.
+            if isinstance(exc, OperationFailure):
+                self.client._process_response(exc.details, bwc.session)  # type: ignore[arg-type]
+            else:
+                self.client._process_response({}, bwc.session)  # type: ignore[arg-type]
         finally:
             bwc.start_time = datetime.datetime.now()
         return reply  # type: ignore[return-value]
@@ -429,7 +436,6 @@ def _execute_batch(
         """Executes a batch of bulkWrite server commands (ack)."""
         request_id, msg, to_send_ops, to_send_ns = bwc.batch_command(cmd, ops, namespaces)
         result = self.write_command(bwc, cmd, request_id, msg, to_send_ops, to_send_ns, self.client)  # type: ignore[arg-type]
-        self.client._process_response(result, bwc.session)  # type: ignore[arg-type]
         return result, to_send_ops, to_send_ns  # type: ignore[return-value]
 
     def _process_results_cursor(