From 4728868fe4432ad5efc3eddc0a473c078533c7c0 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Tue, 19 Aug 2025 13:04:46 -0700 Subject: [PATCH 01/13] PYTHON-5505 Prototype system overload retry loop for all operations --- pymongo/asynchronous/collection.py | 5 +- pymongo/asynchronous/database.py | 5 ++ pymongo/asynchronous/helpers.py | 38 +++++++++ pymongo/asynchronous/mongo_client.py | 31 ++++++-- pymongo/synchronous/collection.py | 5 +- pymongo/synchronous/database.py | 5 ++ pymongo/synchronous/helpers.py | 38 +++++++++ pymongo/synchronous/mongo_client.py | 31 ++++++-- test/asynchronous/test_backpressure.py | 103 +++++++++++++++++++++++++ test/test_backpressure.py | 103 +++++++++++++++++++++++++ tools/synchro.py | 1 + 11 files changed, 353 insertions(+), 12 deletions(-) create mode 100644 test/asynchronous/test_backpressure.py create mode 100644 test/test_backpressure.py diff --git a/pymongo/asynchronous/collection.py b/pymongo/asynchronous/collection.py index 741c11e551..dead0ed4dc 100644 --- a/pymongo/asynchronous/collection.py +++ b/pymongo/asynchronous/collection.py @@ -58,6 +58,7 @@ AsyncCursor, AsyncRawBatchCursor, ) +from pymongo.asynchronous.helpers import _retry_overload from pymongo.collation import validate_collation_or_none from pymongo.common import _ecoc_coll_name, _esc_coll_name from pymongo.errors import ( @@ -2227,6 +2228,7 @@ async def create_indexes( return await self._create_indexes(indexes, session, **kwargs) @_csot.apply + @_retry_overload async def _create_indexes( self, indexes: Sequence[IndexModel], session: Optional[AsyncClientSession], **kwargs: Any ) -> list[str]: @@ -2422,7 +2424,6 @@ async def drop_indexes( kwargs["comment"] = comment await self._drop_index("*", session=session, **kwargs) - @_csot.apply async def drop_index( self, index_or_name: _IndexKeyHint, @@ -2472,6 +2473,7 @@ async def drop_index( await self._drop_index(index_or_name, session, comment, **kwargs) @_csot.apply + @_retry_overload async def _drop_index( self, index_or_name: _IndexKeyHint, @@ -3079,6 +3081,7 @@ async def aggregate_raw_batches( ) @_csot.apply + @_retry_overload async def rename( self, new_name: str, diff --git a/pymongo/asynchronous/database.py b/pymongo/asynchronous/database.py index f70c2b403f..f3b35a0dcb 100644 --- a/pymongo/asynchronous/database.py +++ b/pymongo/asynchronous/database.py @@ -38,6 +38,7 @@ from pymongo.asynchronous.change_stream import AsyncDatabaseChangeStream from pymongo.asynchronous.collection import AsyncCollection from pymongo.asynchronous.command_cursor import AsyncCommandCursor +from pymongo.asynchronous.helpers import _retry_overload from pymongo.common import _ecoc_coll_name, _esc_coll_name from pymongo.database_shared import _check_name, _CodecDocumentType from pymongo.errors import CollectionInvalid, InvalidOperation @@ -477,6 +478,7 @@ async def watch( return change_stream @_csot.apply + @_retry_overload async def create_collection( self, name: str, @@ -816,6 +818,7 @@ async def command( ... @_csot.apply + @_retry_overload async def command( self, command: Union[str, MutableMapping[str, Any]], @@ -947,6 +950,7 @@ async def command( ) @_csot.apply + @_retry_overload async def cursor_command( self, command: Union[str, MutableMapping[str, Any]], @@ -1264,6 +1268,7 @@ async def _drop_helper( ) @_csot.apply + @_retry_overload async def drop_collection( self, name_or_collection: Union[str, AsyncCollection[_DocumentTypeArg]], diff --git a/pymongo/asynchronous/helpers.py b/pymongo/asynchronous/helpers.py index 54fd64f74a..863e473203 100644 --- a/pymongo/asynchronous/helpers.py +++ b/pymongo/asynchronous/helpers.py @@ -17,8 +17,10 @@ import asyncio import builtins +import random import socket import sys +import time from typing import ( Any, Callable, @@ -28,6 +30,7 @@ from pymongo.errors import ( OperationFailure, + PyMongoError, ) from pymongo.helpers_shared import _REAUTHENTICATION_REQUIRED_CODE @@ -70,6 +73,41 @@ async def inner(*args: Any, **kwargs: Any) -> Any: return cast(F, inner) +_MAX_RETRIES = 3 +_BACKOFF_INITIAL = 0.05 +_BACKOFF_MAX = 10 +_TIME = time + + +async def _backoff( + attempt: int, initial_delay: float = _BACKOFF_INITIAL, max_delay: float = _BACKOFF_MAX +) -> None: + jitter = random.random() # noqa: S311 + backoff = jitter * min(initial_delay * (2**attempt), max_delay) + await asyncio.sleep(backoff) + + +def _retry_overload(func: F) -> F: + async def inner(*args: Any, **kwargs: Any) -> Any: + no_retry = kwargs.pop("no_retry", False) + attempt = 0 + while True: + try: + return await func(*args, **kwargs) + except PyMongoError as exc: + if no_retry or not exc.has_error_label("Retryable"): + raise + attempt += 1 + if attempt > _MAX_RETRIES: + raise + + # Implement exponential backoff on retry. + await _backoff(attempt) + continue + + return cast(F, inner) + + async def _getaddrinfo( host: Any, port: Any, **kwargs: Any ) -> list[ diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index b616647791..65a56524b1 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -67,6 +67,7 @@ from pymongo.asynchronous.client_bulk import _AsyncClientBulk from pymongo.asynchronous.client_session import _EmptyServerSession from pymongo.asynchronous.command_cursor import AsyncCommandCursor +from pymongo.asynchronous.helpers import _MAX_RETRIES, _backoff, _retry_overload from pymongo.asynchronous.settings import TopologySettings from pymongo.asynchronous.topology import Topology, _ErrorContext from pymongo.client_options import ClientOptions @@ -2398,6 +2399,7 @@ async def list_database_names( return [doc["name"] async for doc in res] @_csot.apply + @_retry_overload async def drop_database( self, name_or_database: Union[str, database.AsyncDatabase[_DocumentTypeArg]], @@ -2783,14 +2785,19 @@ async def run(self) -> T: # most likely be a waste of time. raise except PyMongoError as exc: + overload = False # Execute specialized catch on read if self._is_read: if isinstance(exc, (ConnectionFailure, OperationFailure)): # ConnectionFailures do not supply a code property exc_code = getattr(exc, "code", None) - if self._is_not_eligible_for_retry() or ( - isinstance(exc, OperationFailure) - and exc_code not in helpers_shared._RETRYABLE_ERROR_CODES + overload = exc.has_error_label("Retryable") + if not overload and ( + self._is_not_eligible_for_retry() + or ( + isinstance(exc, OperationFailure) + and exc_code not in helpers_shared._RETRYABLE_ERROR_CODES + ) ): raise self._retrying = True @@ -2807,12 +2814,18 @@ async def run(self) -> T: retryable_write_error_exc = isinstance( exc.error, PyMongoError ) and exc.error.has_error_label("RetryableWriteError") + overload = isinstance( + exc.error, PyMongoError + ) and exc.error.has_error_label("Retryable") else: retryable_write_error_exc = exc.has_error_label("RetryableWriteError") - if retryable_write_error_exc: + overload = exc.has_error_label("Retryable") + if retryable_write_error_exc or overload: assert self._session await self._session._unpin() - if not retryable_write_error_exc or self._is_not_eligible_for_retry(): + if not overload and ( + not retryable_write_error_exc or not self._is_not_eligible_for_retry() + ): if exc.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc else: @@ -2830,6 +2843,14 @@ async def run(self) -> T: if self._client.topology_description.topology_type == TOPOLOGY_TYPE.Sharded: self._deprioritized_servers.append(self._server) + if overload: + if self._attempt_number > _MAX_RETRIES: + if exc.has_error_label("NoWritesPerformed") and self._last_error: + raise self._last_error from exc + else: + raise + await _backoff(self._attempt_number) + def _is_not_eligible_for_retry(self) -> bool: """Checks if the exchange is not eligible for retry""" return not self._retryable or (self._is_retrying() and not self._multiple_retries) diff --git a/pymongo/synchronous/collection.py b/pymongo/synchronous/collection.py index 9f32deb765..3df867f7bc 100644 --- a/pymongo/synchronous/collection.py +++ b/pymongo/synchronous/collection.py @@ -89,6 +89,7 @@ Cursor, RawBatchCursor, ) +from pymongo.synchronous.helpers import _retry_overload from pymongo.typings import _CollationIn, _DocumentType, _DocumentTypeArg, _Pipeline from pymongo.write_concern import DEFAULT_WRITE_CONCERN, WriteConcern, validate_boolean @@ -2224,6 +2225,7 @@ def create_indexes( return self._create_indexes(indexes, session, **kwargs) @_csot.apply + @_retry_overload def _create_indexes( self, indexes: Sequence[IndexModel], session: Optional[ClientSession], **kwargs: Any ) -> list[str]: @@ -2419,7 +2421,6 @@ def drop_indexes( kwargs["comment"] = comment self._drop_index("*", session=session, **kwargs) - @_csot.apply def drop_index( self, index_or_name: _IndexKeyHint, @@ -2469,6 +2470,7 @@ def drop_index( self._drop_index(index_or_name, session, comment, **kwargs) @_csot.apply + @_retry_overload def _drop_index( self, index_or_name: _IndexKeyHint, @@ -3072,6 +3074,7 @@ def aggregate_raw_batches( ) @_csot.apply + @_retry_overload def rename( self, new_name: str, diff --git a/pymongo/synchronous/database.py b/pymongo/synchronous/database.py index e30f97817c..d8b9ae6a10 100644 --- a/pymongo/synchronous/database.py +++ b/pymongo/synchronous/database.py @@ -43,6 +43,7 @@ from pymongo.synchronous.change_stream import DatabaseChangeStream from pymongo.synchronous.collection import Collection from pymongo.synchronous.command_cursor import CommandCursor +from pymongo.synchronous.helpers import _retry_overload from pymongo.typings import _CollationIn, _DocumentType, _DocumentTypeArg, _Pipeline if TYPE_CHECKING: @@ -477,6 +478,7 @@ def watch( return change_stream @_csot.apply + @_retry_overload def create_collection( self, name: str, @@ -816,6 +818,7 @@ def command( ... @_csot.apply + @_retry_overload def command( self, command: Union[str, MutableMapping[str, Any]], @@ -945,6 +948,7 @@ def command( ) @_csot.apply + @_retry_overload def cursor_command( self, command: Union[str, MutableMapping[str, Any]], @@ -1257,6 +1261,7 @@ def _drop_helper( ) @_csot.apply + @_retry_overload def drop_collection( self, name_or_collection: Union[str, Collection[_DocumentTypeArg]], diff --git a/pymongo/synchronous/helpers.py b/pymongo/synchronous/helpers.py index bc69a49e80..28cc023ae3 100644 --- a/pymongo/synchronous/helpers.py +++ b/pymongo/synchronous/helpers.py @@ -17,8 +17,10 @@ import asyncio import builtins +import random import socket import sys +import time from typing import ( Any, Callable, @@ -28,6 +30,7 @@ from pymongo.errors import ( OperationFailure, + PyMongoError, ) from pymongo.helpers_shared import _REAUTHENTICATION_REQUIRED_CODE @@ -70,6 +73,41 @@ def inner(*args: Any, **kwargs: Any) -> Any: return cast(F, inner) +_MAX_RETRIES = 3 +_BACKOFF_INITIAL = 0.05 +_BACKOFF_MAX = 10 +_TIME = time + + +def _backoff( + attempt: int, initial_delay: float = _BACKOFF_INITIAL, max_delay: float = _BACKOFF_MAX +) -> None: + jitter = random.random() # noqa: S311 + backoff = jitter * min(initial_delay * (2**attempt), max_delay) + time.sleep(backoff) + + +def _retry_overload(func: F) -> F: + def inner(*args: Any, **kwargs: Any) -> Any: + no_retry = kwargs.pop("no_retry", False) + attempt = 0 + while True: + try: + return func(*args, **kwargs) + except PyMongoError as exc: + if no_retry or not exc.has_error_label("Retryable"): + raise + attempt += 1 + if attempt > _MAX_RETRIES: + raise + + # Implement exponential backoff on retry. + _backoff(attempt) + continue + + return cast(F, inner) + + def _getaddrinfo( host: Any, port: Any, **kwargs: Any ) -> list[ diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index ef0663584c..03d1140f77 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -110,6 +110,7 @@ from pymongo.synchronous.client_bulk import _ClientBulk from pymongo.synchronous.client_session import _EmptyServerSession from pymongo.synchronous.command_cursor import CommandCursor +from pymongo.synchronous.helpers import _MAX_RETRIES, _backoff, _retry_overload from pymongo.synchronous.settings import TopologySettings from pymongo.synchronous.topology import Topology, _ErrorContext from pymongo.topology_description import TOPOLOGY_TYPE, TopologyDescription @@ -2388,6 +2389,7 @@ def list_database_names( return [doc["name"] for doc in res] @_csot.apply + @_retry_overload def drop_database( self, name_or_database: Union[str, database.Database[_DocumentTypeArg]], @@ -2773,14 +2775,19 @@ def run(self) -> T: # most likely be a waste of time. raise except PyMongoError as exc: + overload = False # Execute specialized catch on read if self._is_read: if isinstance(exc, (ConnectionFailure, OperationFailure)): # ConnectionFailures do not supply a code property exc_code = getattr(exc, "code", None) - if self._is_not_eligible_for_retry() or ( - isinstance(exc, OperationFailure) - and exc_code not in helpers_shared._RETRYABLE_ERROR_CODES + overload = exc.has_error_label("Retryable") + if not overload and ( + self._is_not_eligible_for_retry() + or ( + isinstance(exc, OperationFailure) + and exc_code not in helpers_shared._RETRYABLE_ERROR_CODES + ) ): raise self._retrying = True @@ -2797,12 +2804,18 @@ def run(self) -> T: retryable_write_error_exc = isinstance( exc.error, PyMongoError ) and exc.error.has_error_label("RetryableWriteError") + overload = isinstance( + exc.error, PyMongoError + ) and exc.error.has_error_label("Retryable") else: retryable_write_error_exc = exc.has_error_label("RetryableWriteError") - if retryable_write_error_exc: + overload = exc.has_error_label("Retryable") + if retryable_write_error_exc or overload: assert self._session self._session._unpin() - if not retryable_write_error_exc or self._is_not_eligible_for_retry(): + if not overload and ( + not retryable_write_error_exc or not self._is_not_eligible_for_retry() + ): if exc.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc else: @@ -2820,6 +2833,14 @@ def run(self) -> T: if self._client.topology_description.topology_type == TOPOLOGY_TYPE.Sharded: self._deprioritized_servers.append(self._server) + if overload: + if self._attempt_number > _MAX_RETRIES: + if exc.has_error_label("NoWritesPerformed") and self._last_error: + raise self._last_error from exc + else: + raise + _backoff(self._attempt_number) + def _is_not_eligible_for_retry(self) -> bool: """Checks if the exchange is not eligible for retry""" return not self._retryable or (self._is_retrying() and not self._multiple_retries) diff --git a/test/asynchronous/test_backpressure.py b/test/asynchronous/test_backpressure.py new file mode 100644 index 0000000000..f542b898e6 --- /dev/null +++ b/test/asynchronous/test_backpressure.py @@ -0,0 +1,103 @@ +# Copyright 2025-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test the CSOT unified spec tests.""" +from __future__ import annotations + +import sys + +sys.path[0:0] = [""] + +from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest + +from pymongo.asynchronous.helpers import _MAX_RETRIES +from pymongo.errors import PyMongoError + +_IS_SYNC = False + +# Mock an system overload error. +mock_overload_error = { + "configureFailPoint": "failCommand", + "mode": {"times": 1}, + "data": { + "failCommands": ["find", "insert"], + "errorCode": 462, # IngressRequestRateLimitExceeded + "errorLabels": ["Retryable"], + }, +} + + +class TestCSOT(AsyncIntegrationTest): + RUN_ON_LOAD_BALANCER = True + + @async_client_context.require_failCommand_appName + async def test_retry_overload_error_command(self): + await self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_once = mock_overload_error.copy() + fail_once["mode"] = {"times": _MAX_RETRIES} + async with self.fail_point(fail_once): + await self.db.command("find", "t") + + # Ensure command stops retrying after _MAX_RETRIES. + fail_many_times = mock_overload_error.copy() + fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} + async with self.fail_point(fail_many_times): + with self.assertRaises(PyMongoError) as error: + await self.db.command("find", "t") + + self.assertIn("Retryable", str(error.exception)) + + @async_client_context.require_failCommand_appName + async def test_retry_overload_error_find(self): + await self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_once = mock_overload_error.copy() + fail_once["mode"] = {"times": _MAX_RETRIES} + async with self.fail_point(fail_once): + await self.db.t.find_one() + + # Ensure command stops retrying after _MAX_RETRIES. + fail_many_times = mock_overload_error.copy() + fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} + async with self.fail_point(fail_many_times): + with self.assertRaises(PyMongoError) as error: + await self.db.t.find_one() + + self.assertIn("Retryable", str(error.exception)) + + @async_client_context.require_failCommand_appName + async def test_retry_overload_error_insert_one(self): + await self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_once = mock_overload_error.copy() + fail_once["mode"] = {"times": _MAX_RETRIES} + async with self.fail_point(fail_once): + await self.db.t.find_one() + + # Ensure command stops retrying after _MAX_RETRIES. + fail_many_times = mock_overload_error.copy() + fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} + async with self.fail_point(fail_many_times): + with self.assertRaises(PyMongoError) as error: + await self.db.t.find_one() + + self.assertIn("Retryable", str(error.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_backpressure.py b/test/test_backpressure.py new file mode 100644 index 0000000000..c0f12be528 --- /dev/null +++ b/test/test_backpressure.py @@ -0,0 +1,103 @@ +# Copyright 2025-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test the CSOT unified spec tests.""" +from __future__ import annotations + +import sys + +sys.path[0:0] = [""] + +from test import IntegrationTest, client_context, unittest + +from pymongo.errors import PyMongoError +from pymongo.synchronous.helpers import _MAX_RETRIES + +_IS_SYNC = True + +# Mock an system overload error. +mock_overload_error = { + "configureFailPoint": "failCommand", + "mode": {"times": 1}, + "data": { + "failCommands": ["find", "insert"], + "errorCode": 462, # IngressRequestRateLimitExceeded + "errorLabels": ["Retryable"], + }, +} + + +class TestCSOT(IntegrationTest): + RUN_ON_LOAD_BALANCER = True + + @client_context.require_failCommand_appName + def test_retry_overload_error_command(self): + self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_once = mock_overload_error.copy() + fail_once["mode"] = {"times": _MAX_RETRIES} + with self.fail_point(fail_once): + self.db.command("find", "t") + + # Ensure command stops retrying after _MAX_RETRIES. + fail_many_times = mock_overload_error.copy() + fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} + with self.fail_point(fail_many_times): + with self.assertRaises(PyMongoError) as error: + self.db.command("find", "t") + + self.assertIn("Retryable", str(error.exception)) + + @client_context.require_failCommand_appName + def test_retry_overload_error_find(self): + self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_once = mock_overload_error.copy() + fail_once["mode"] = {"times": _MAX_RETRIES} + with self.fail_point(fail_once): + self.db.t.find_one() + + # Ensure command stops retrying after _MAX_RETRIES. + fail_many_times = mock_overload_error.copy() + fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} + with self.fail_point(fail_many_times): + with self.assertRaises(PyMongoError) as error: + self.db.t.find_one() + + self.assertIn("Retryable", str(error.exception)) + + @client_context.require_failCommand_appName + def test_retry_overload_error_insert_one(self): + self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_once = mock_overload_error.copy() + fail_once["mode"] = {"times": _MAX_RETRIES} + with self.fail_point(fail_once): + self.db.t.find_one() + + # Ensure command stops retrying after _MAX_RETRIES. + fail_many_times = mock_overload_error.copy() + fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} + with self.fail_point(fail_many_times): + with self.assertRaises(PyMongoError) as error: + self.db.t.find_one() + + self.assertIn("Retryable", str(error.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/synchro.py b/tools/synchro.py index 9a760c0ad7..44698134cd 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -208,6 +208,7 @@ def async_only_test(f: str) -> bool: "test_auth_oidc.py", "test_auth_spec.py", "test_bulk.py", + "test_backpressure.py", "test_change_stream.py", "test_client.py", "test_client_bulk_write.py", From a83dea8411d6a326f27f7cc5548bc6356a9b8cde Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Tue, 19 Aug 2025 15:47:22 -0700 Subject: [PATCH 02/13] PYTHON-5505 Make cursor getMore retryable --- pymongo/asynchronous/mongo_client.py | 8 ++++--- pymongo/synchronous/mongo_client.py | 8 ++++--- test/asynchronous/test_backpressure.py | 31 ++++++++++++++++++++++++++ test/test_backpressure.py | 31 ++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 6 deletions(-) diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 65a56524b1..ffeb172270 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -2737,6 +2737,7 @@ def __init__( ): self._last_error: Optional[Exception] = None self._retrying = False + self._overload = False self._multiple_retries = _csot.get_timeout() is not None self._client = mongo_client @@ -2808,8 +2809,6 @@ async def run(self) -> T: # Specialized catch on write operation if not self._is_read: - if not self._retryable: - raise if isinstance(exc, ClientBulkWriteException) and exc.error: retryable_write_error_exc = isinstance( exc.error, PyMongoError @@ -2820,6 +2819,8 @@ async def run(self) -> T: else: retryable_write_error_exc = exc.has_error_label("RetryableWriteError") overload = exc.has_error_label("Retryable") + if not self._retryable and not overload: + raise if retryable_write_error_exc or overload: assert self._session await self._session._unpin() @@ -2843,6 +2844,7 @@ async def run(self) -> T: if self._client.topology_description.topology_type == TOPOLOGY_TYPE.Sharded: self._deprioritized_servers.append(self._server) + self._overload = overload if overload: if self._attempt_number > _MAX_RETRIES: if exc.has_error_label("NoWritesPerformed") and self._last_error: @@ -2944,7 +2946,7 @@ async def _read(self) -> T: conn, read_pref, ): - if self._retrying and not self._retryable: + if self._retrying and not self._retryable and not self._overload: self._check_last_error() if self._retrying: _debug_log( diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 03d1140f77..e537902359 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -2727,6 +2727,7 @@ def __init__( ): self._last_error: Optional[Exception] = None self._retrying = False + self._overload = False self._multiple_retries = _csot.get_timeout() is not None self._client = mongo_client @@ -2798,8 +2799,6 @@ def run(self) -> T: # Specialized catch on write operation if not self._is_read: - if not self._retryable: - raise if isinstance(exc, ClientBulkWriteException) and exc.error: retryable_write_error_exc = isinstance( exc.error, PyMongoError @@ -2810,6 +2809,8 @@ def run(self) -> T: else: retryable_write_error_exc = exc.has_error_label("RetryableWriteError") overload = exc.has_error_label("Retryable") + if not self._retryable and not overload: + raise if retryable_write_error_exc or overload: assert self._session self._session._unpin() @@ -2833,6 +2834,7 @@ def run(self) -> T: if self._client.topology_description.topology_type == TOPOLOGY_TYPE.Sharded: self._deprioritized_servers.append(self._server) + self._overload = overload if overload: if self._attempt_number > _MAX_RETRIES: if exc.has_error_label("NoWritesPerformed") and self._last_error: @@ -2934,7 +2936,7 @@ def _read(self) -> T: conn, read_pref, ): - if self._retrying and not self._retryable: + if self._retrying and not self._retryable and not self._overload: self._check_last_error() if self._retrying: _debug_log( diff --git a/test/asynchronous/test_backpressure.py b/test/asynchronous/test_backpressure.py index f542b898e6..2755f24314 100644 --- a/test/asynchronous/test_backpressure.py +++ b/test/asynchronous/test_backpressure.py @@ -98,6 +98,37 @@ async def test_retry_overload_error_insert_one(self): self.assertIn("Retryable", str(error.exception)) + @async_client_context.require_failCommand_appName + async def test_retry_overload_error_getMore(self): + coll = self.db.t + await coll.insert_many([{"x": 1} for _ in range(10)]) + + # Ensure command is retried on overload error. + fail_once = { + "configureFailPoint": "failCommand", + "mode": {"times": _MAX_RETRIES}, + "data": { + "failCommands": ["getMore"], + "errorCode": 462, # IngressRequestRateLimitExceeded + "errorLabels": ["Retryable"], + }, + } + cursor = coll.find(batch_size=2) + await cursor.next() + async with self.fail_point(fail_once): + await cursor.to_list() + + # Ensure command stops retrying after _MAX_RETRIES. + fail_many_times = fail_once.copy() + fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} + cursor = coll.find(batch_size=2) + await cursor.next() + async with self.fail_point(fail_many_times): + with self.assertRaises(PyMongoError) as error: + await cursor.to_list() + + self.assertIn("Retryable", str(error.exception)) + if __name__ == "__main__": unittest.main() diff --git a/test/test_backpressure.py b/test/test_backpressure.py index c0f12be528..c475a7fc8e 100644 --- a/test/test_backpressure.py +++ b/test/test_backpressure.py @@ -98,6 +98,37 @@ def test_retry_overload_error_insert_one(self): self.assertIn("Retryable", str(error.exception)) + @client_context.require_failCommand_appName + def test_retry_overload_error_getMore(self): + coll = self.db.t + coll.insert_many([{"x": 1} for _ in range(10)]) + + # Ensure command is retried on overload error. + fail_once = { + "configureFailPoint": "failCommand", + "mode": {"times": _MAX_RETRIES}, + "data": { + "failCommands": ["getMore"], + "errorCode": 462, # IngressRequestRateLimitExceeded + "errorLabels": ["Retryable"], + }, + } + cursor = coll.find(batch_size=2) + cursor.next() + with self.fail_point(fail_once): + cursor.to_list() + + # Ensure command stops retrying after _MAX_RETRIES. + fail_many_times = fail_once.copy() + fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} + cursor = coll.find(batch_size=2) + cursor.next() + with self.fail_point(fail_many_times): + with self.assertRaises(PyMongoError) as error: + cursor.to_list() + + self.assertIn("Retryable", str(error.exception)) + if __name__ == "__main__": unittest.main() From c2ff971b35b4b0123bc2dd50586772a274043ba1 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Tue, 19 Aug 2025 15:59:57 -0700 Subject: [PATCH 03/13] PYTHON-5505 Use functools.wraps to fix test_comment --- pymongo/asynchronous/helpers.py | 3 +++ pymongo/synchronous/helpers.py | 3 +++ test/asynchronous/test_backpressure.py | 4 ++-- test/test_backpressure.py | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pymongo/asynchronous/helpers.py b/pymongo/asynchronous/helpers.py index 863e473203..10e578f3f4 100644 --- a/pymongo/asynchronous/helpers.py +++ b/pymongo/asynchronous/helpers.py @@ -17,6 +17,7 @@ import asyncio import builtins +import functools import random import socket import sys @@ -41,6 +42,7 @@ def _handle_reauth(func: F) -> F: + @functools.wraps(func) async def inner(*args: Any, **kwargs: Any) -> Any: no_reauth = kwargs.pop("no_reauth", False) from pymongo.asynchronous.pool import AsyncConnection @@ -88,6 +90,7 @@ async def _backoff( def _retry_overload(func: F) -> F: + @functools.wraps(func) async def inner(*args: Any, **kwargs: Any) -> Any: no_retry = kwargs.pop("no_retry", False) attempt = 0 diff --git a/pymongo/synchronous/helpers.py b/pymongo/synchronous/helpers.py index 28cc023ae3..91f73ffbce 100644 --- a/pymongo/synchronous/helpers.py +++ b/pymongo/synchronous/helpers.py @@ -17,6 +17,7 @@ import asyncio import builtins +import functools import random import socket import sys @@ -41,6 +42,7 @@ def _handle_reauth(func: F) -> F: + @functools.wraps(func) def inner(*args: Any, **kwargs: Any) -> Any: no_reauth = kwargs.pop("no_reauth", False) from pymongo.message import _BulkWriteContext @@ -88,6 +90,7 @@ def _backoff( def _retry_overload(func: F) -> F: + @functools.wraps(func) def inner(*args: Any, **kwargs: Any) -> Any: no_retry = kwargs.pop("no_retry", False) attempt = 0 diff --git a/test/asynchronous/test_backpressure.py b/test/asynchronous/test_backpressure.py index 2755f24314..d3100cdb18 100644 --- a/test/asynchronous/test_backpressure.py +++ b/test/asynchronous/test_backpressure.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Test the CSOT unified spec tests.""" +"""Test Client Backpressure spec.""" from __future__ import annotations import sys @@ -38,7 +38,7 @@ } -class TestCSOT(AsyncIntegrationTest): +class TestBackpressure(AsyncIntegrationTest): RUN_ON_LOAD_BALANCER = True @async_client_context.require_failCommand_appName diff --git a/test/test_backpressure.py b/test/test_backpressure.py index c475a7fc8e..b10ae94996 100644 --- a/test/test_backpressure.py +++ b/test/test_backpressure.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Test the CSOT unified spec tests.""" +"""Test Client Backpressure spec.""" from __future__ import annotations import sys @@ -38,7 +38,7 @@ } -class TestCSOT(IntegrationTest): +class TestBackpressure(IntegrationTest): RUN_ON_LOAD_BALANCER = True @client_context.require_failCommand_appName From b6dd34dd99bf1425f05f20c7379e26263a0f263f Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Tue, 19 Aug 2025 16:08:46 -0700 Subject: [PATCH 04/13] PYTHON-5505 Add test that even non-retryable write ops will be retried on overload The server only adds the "Retryable" error label when it knows it is safe to retry. --- test/asynchronous/test_backpressure.py | 23 ++++++++++++++++++++++- test/test_backpressure.py | 23 ++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/test/asynchronous/test_backpressure.py b/test/asynchronous/test_backpressure.py index d3100cdb18..3bf9e767a4 100644 --- a/test/asynchronous/test_backpressure.py +++ b/test/asynchronous/test_backpressure.py @@ -31,7 +31,7 @@ "configureFailPoint": "failCommand", "mode": {"times": 1}, "data": { - "failCommands": ["find", "insert"], + "failCommands": ["find", "insert", "update"], "errorCode": 462, # IngressRequestRateLimitExceeded "errorLabels": ["Retryable"], }, @@ -98,6 +98,27 @@ async def test_retry_overload_error_insert_one(self): self.assertIn("Retryable", str(error.exception)) + @async_client_context.require_failCommand_appName + async def test_retry_overload_error_update_many(self): + # Even though update_many is not a retryable write operation, it will + # still be retried via the "Retryable" error label. + await self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_once = mock_overload_error.copy() + fail_once["mode"] = {"times": _MAX_RETRIES} + async with self.fail_point(fail_once): + await self.db.t.update_many({}, {"$set": {"x": 2}}) + + # Ensure command stops retrying after _MAX_RETRIES. + fail_many_times = mock_overload_error.copy() + fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} + async with self.fail_point(fail_many_times): + with self.assertRaises(PyMongoError) as error: + await self.db.t.update_many({}, {"$set": {"x": 2}}) + + self.assertIn("Retryable", str(error.exception)) + @async_client_context.require_failCommand_appName async def test_retry_overload_error_getMore(self): coll = self.db.t diff --git a/test/test_backpressure.py b/test/test_backpressure.py index b10ae94996..6c12b48f6b 100644 --- a/test/test_backpressure.py +++ b/test/test_backpressure.py @@ -31,7 +31,7 @@ "configureFailPoint": "failCommand", "mode": {"times": 1}, "data": { - "failCommands": ["find", "insert"], + "failCommands": ["find", "insert", "update"], "errorCode": 462, # IngressRequestRateLimitExceeded "errorLabels": ["Retryable"], }, @@ -98,6 +98,27 @@ def test_retry_overload_error_insert_one(self): self.assertIn("Retryable", str(error.exception)) + @client_context.require_failCommand_appName + def test_retry_overload_error_update_many(self): + # Even though update_many is not a retryable write operation, it will + # still be retried via the "Retryable" error label. + self.db.t.insert_one({"x": 1}) + + # Ensure command is retried on overload error. + fail_once = mock_overload_error.copy() + fail_once["mode"] = {"times": _MAX_RETRIES} + with self.fail_point(fail_once): + self.db.t.update_many({}, {"$set": {"x": 2}}) + + # Ensure command stops retrying after _MAX_RETRIES. + fail_many_times = mock_overload_error.copy() + fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} + with self.fail_point(fail_many_times): + with self.assertRaises(PyMongoError) as error: + self.db.t.update_many({}, {"$set": {"x": 2}}) + + self.assertIn("Retryable", str(error.exception)) + @client_context.require_failCommand_appName def test_retry_overload_error_getMore(self): coll = self.db.t From eb113f063c0f8c1b5ef029046c769d8f379c3749 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 20 Aug 2025 10:49:47 -0700 Subject: [PATCH 05/13] PYTHON-5505 Rename overloaded flag --- pymongo/asynchronous/mongo_client.py | 24 ++++++++++++------------ pymongo/synchronous/mongo_client.py | 24 ++++++++++++------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index ffeb172270..dfaad87940 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -2737,7 +2737,7 @@ def __init__( ): self._last_error: Optional[Exception] = None self._retrying = False - self._overload = False + self._overloaded = False self._multiple_retries = _csot.get_timeout() is not None self._client = mongo_client @@ -2786,14 +2786,14 @@ async def run(self) -> T: # most likely be a waste of time. raise except PyMongoError as exc: - overload = False + overloaded = False # Execute specialized catch on read if self._is_read: if isinstance(exc, (ConnectionFailure, OperationFailure)): # ConnectionFailures do not supply a code property exc_code = getattr(exc, "code", None) - overload = exc.has_error_label("Retryable") - if not overload and ( + overloaded = exc.has_error_label("Retryable") + if not overloaded and ( self._is_not_eligible_for_retry() or ( isinstance(exc, OperationFailure) @@ -2813,18 +2813,18 @@ async def run(self) -> T: retryable_write_error_exc = isinstance( exc.error, PyMongoError ) and exc.error.has_error_label("RetryableWriteError") - overload = isinstance( + overloaded = isinstance( exc.error, PyMongoError ) and exc.error.has_error_label("Retryable") else: retryable_write_error_exc = exc.has_error_label("RetryableWriteError") - overload = exc.has_error_label("Retryable") - if not self._retryable and not overload: + overloaded = exc.has_error_label("Retryable") + if not self._retryable and not overloaded: raise - if retryable_write_error_exc or overload: + if retryable_write_error_exc or overloaded: assert self._session await self._session._unpin() - if not overload and ( + if not overloaded and ( not retryable_write_error_exc or not self._is_not_eligible_for_retry() ): if exc.has_error_label("NoWritesPerformed") and self._last_error: @@ -2844,8 +2844,8 @@ async def run(self) -> T: if self._client.topology_description.topology_type == TOPOLOGY_TYPE.Sharded: self._deprioritized_servers.append(self._server) - self._overload = overload - if overload: + self._overloaded = overloaded + if overloaded: if self._attempt_number > _MAX_RETRIES: if exc.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc @@ -2946,7 +2946,7 @@ async def _read(self) -> T: conn, read_pref, ): - if self._retrying and not self._retryable and not self._overload: + if self._retrying and not self._retryable and not self._overloaded: self._check_last_error() if self._retrying: _debug_log( diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index e537902359..431a15551d 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -2727,7 +2727,7 @@ def __init__( ): self._last_error: Optional[Exception] = None self._retrying = False - self._overload = False + self._overloaded = False self._multiple_retries = _csot.get_timeout() is not None self._client = mongo_client @@ -2776,14 +2776,14 @@ def run(self) -> T: # most likely be a waste of time. raise except PyMongoError as exc: - overload = False + overloaded = False # Execute specialized catch on read if self._is_read: if isinstance(exc, (ConnectionFailure, OperationFailure)): # ConnectionFailures do not supply a code property exc_code = getattr(exc, "code", None) - overload = exc.has_error_label("Retryable") - if not overload and ( + overloaded = exc.has_error_label("Retryable") + if not overloaded and ( self._is_not_eligible_for_retry() or ( isinstance(exc, OperationFailure) @@ -2803,18 +2803,18 @@ def run(self) -> T: retryable_write_error_exc = isinstance( exc.error, PyMongoError ) and exc.error.has_error_label("RetryableWriteError") - overload = isinstance( + overloaded = isinstance( exc.error, PyMongoError ) and exc.error.has_error_label("Retryable") else: retryable_write_error_exc = exc.has_error_label("RetryableWriteError") - overload = exc.has_error_label("Retryable") - if not self._retryable and not overload: + overloaded = exc.has_error_label("Retryable") + if not self._retryable and not overloaded: raise - if retryable_write_error_exc or overload: + if retryable_write_error_exc or overloaded: assert self._session self._session._unpin() - if not overload and ( + if not overloaded and ( not retryable_write_error_exc or not self._is_not_eligible_for_retry() ): if exc.has_error_label("NoWritesPerformed") and self._last_error: @@ -2834,8 +2834,8 @@ def run(self) -> T: if self._client.topology_description.topology_type == TOPOLOGY_TYPE.Sharded: self._deprioritized_servers.append(self._server) - self._overload = overload - if overload: + self._overloaded = overloaded + if overloaded: if self._attempt_number > _MAX_RETRIES: if exc.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc @@ -2936,7 +2936,7 @@ def _read(self) -> T: conn, read_pref, ): - if self._retrying and not self._retryable and not self._overload: + if self._retrying and not self._retryable and not self._overloaded: self._check_last_error() if self._retrying: _debug_log( From b4366474d0c95781b28bd1f0568aa3cbd499bf09 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 20 Aug 2025 11:51:06 -0700 Subject: [PATCH 06/13] PYTHON-5505 cleanup tests --- test/asynchronous/test_backpressure.py | 58 +++++++++++++------------- test/test_backpressure.py | 58 +++++++++++++------------- 2 files changed, 58 insertions(+), 58 deletions(-) diff --git a/test/asynchronous/test_backpressure.py b/test/asynchronous/test_backpressure.py index 3bf9e767a4..a9a6fb56f5 100644 --- a/test/asynchronous/test_backpressure.py +++ b/test/asynchronous/test_backpressure.py @@ -46,15 +46,15 @@ async def test_retry_overload_error_command(self): await self.db.t.insert_one({"x": 1}) # Ensure command is retried on overload error. - fail_once = mock_overload_error.copy() - fail_once["mode"] = {"times": _MAX_RETRIES} - async with self.fail_point(fail_once): + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + async with self.fail_point(fail_many): await self.db.command("find", "t") # Ensure command stops retrying after _MAX_RETRIES. - fail_many_times = mock_overload_error.copy() - fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} - async with self.fail_point(fail_many_times): + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + async with self.fail_point(fail_too_many): with self.assertRaises(PyMongoError) as error: await self.db.command("find", "t") @@ -65,15 +65,15 @@ async def test_retry_overload_error_find(self): await self.db.t.insert_one({"x": 1}) # Ensure command is retried on overload error. - fail_once = mock_overload_error.copy() - fail_once["mode"] = {"times": _MAX_RETRIES} - async with self.fail_point(fail_once): + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + async with self.fail_point(fail_many): await self.db.t.find_one() # Ensure command stops retrying after _MAX_RETRIES. - fail_many_times = mock_overload_error.copy() - fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} - async with self.fail_point(fail_many_times): + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + async with self.fail_point(fail_too_many): with self.assertRaises(PyMongoError) as error: await self.db.t.find_one() @@ -84,15 +84,15 @@ async def test_retry_overload_error_insert_one(self): await self.db.t.insert_one({"x": 1}) # Ensure command is retried on overload error. - fail_once = mock_overload_error.copy() - fail_once["mode"] = {"times": _MAX_RETRIES} - async with self.fail_point(fail_once): + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + async with self.fail_point(fail_many): await self.db.t.find_one() # Ensure command stops retrying after _MAX_RETRIES. - fail_many_times = mock_overload_error.copy() - fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} - async with self.fail_point(fail_many_times): + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + async with self.fail_point(fail_too_many): with self.assertRaises(PyMongoError) as error: await self.db.t.find_one() @@ -105,15 +105,15 @@ async def test_retry_overload_error_update_many(self): await self.db.t.insert_one({"x": 1}) # Ensure command is retried on overload error. - fail_once = mock_overload_error.copy() - fail_once["mode"] = {"times": _MAX_RETRIES} - async with self.fail_point(fail_once): + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + async with self.fail_point(fail_many): await self.db.t.update_many({}, {"$set": {"x": 2}}) # Ensure command stops retrying after _MAX_RETRIES. - fail_many_times = mock_overload_error.copy() - fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} - async with self.fail_point(fail_many_times): + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + async with self.fail_point(fail_too_many): with self.assertRaises(PyMongoError) as error: await self.db.t.update_many({}, {"$set": {"x": 2}}) @@ -125,7 +125,7 @@ async def test_retry_overload_error_getMore(self): await coll.insert_many([{"x": 1} for _ in range(10)]) # Ensure command is retried on overload error. - fail_once = { + fail_many = { "configureFailPoint": "failCommand", "mode": {"times": _MAX_RETRIES}, "data": { @@ -136,15 +136,15 @@ async def test_retry_overload_error_getMore(self): } cursor = coll.find(batch_size=2) await cursor.next() - async with self.fail_point(fail_once): + async with self.fail_point(fail_many): await cursor.to_list() # Ensure command stops retrying after _MAX_RETRIES. - fail_many_times = fail_once.copy() - fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} + fail_too_many = fail_many.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} cursor = coll.find(batch_size=2) await cursor.next() - async with self.fail_point(fail_many_times): + async with self.fail_point(fail_too_many): with self.assertRaises(PyMongoError) as error: await cursor.to_list() diff --git a/test/test_backpressure.py b/test/test_backpressure.py index 6c12b48f6b..324dd6f15a 100644 --- a/test/test_backpressure.py +++ b/test/test_backpressure.py @@ -46,15 +46,15 @@ def test_retry_overload_error_command(self): self.db.t.insert_one({"x": 1}) # Ensure command is retried on overload error. - fail_once = mock_overload_error.copy() - fail_once["mode"] = {"times": _MAX_RETRIES} - with self.fail_point(fail_once): + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + with self.fail_point(fail_many): self.db.command("find", "t") # Ensure command stops retrying after _MAX_RETRIES. - fail_many_times = mock_overload_error.copy() - fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} - with self.fail_point(fail_many_times): + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + with self.fail_point(fail_too_many): with self.assertRaises(PyMongoError) as error: self.db.command("find", "t") @@ -65,15 +65,15 @@ def test_retry_overload_error_find(self): self.db.t.insert_one({"x": 1}) # Ensure command is retried on overload error. - fail_once = mock_overload_error.copy() - fail_once["mode"] = {"times": _MAX_RETRIES} - with self.fail_point(fail_once): + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + with self.fail_point(fail_many): self.db.t.find_one() # Ensure command stops retrying after _MAX_RETRIES. - fail_many_times = mock_overload_error.copy() - fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} - with self.fail_point(fail_many_times): + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + with self.fail_point(fail_too_many): with self.assertRaises(PyMongoError) as error: self.db.t.find_one() @@ -84,15 +84,15 @@ def test_retry_overload_error_insert_one(self): self.db.t.insert_one({"x": 1}) # Ensure command is retried on overload error. - fail_once = mock_overload_error.copy() - fail_once["mode"] = {"times": _MAX_RETRIES} - with self.fail_point(fail_once): + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + with self.fail_point(fail_many): self.db.t.find_one() # Ensure command stops retrying after _MAX_RETRIES. - fail_many_times = mock_overload_error.copy() - fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} - with self.fail_point(fail_many_times): + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + with self.fail_point(fail_too_many): with self.assertRaises(PyMongoError) as error: self.db.t.find_one() @@ -105,15 +105,15 @@ def test_retry_overload_error_update_many(self): self.db.t.insert_one({"x": 1}) # Ensure command is retried on overload error. - fail_once = mock_overload_error.copy() - fail_once["mode"] = {"times": _MAX_RETRIES} - with self.fail_point(fail_once): + fail_many = mock_overload_error.copy() + fail_many["mode"] = {"times": _MAX_RETRIES} + with self.fail_point(fail_many): self.db.t.update_many({}, {"$set": {"x": 2}}) # Ensure command stops retrying after _MAX_RETRIES. - fail_many_times = mock_overload_error.copy() - fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} - with self.fail_point(fail_many_times): + fail_too_many = mock_overload_error.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} + with self.fail_point(fail_too_many): with self.assertRaises(PyMongoError) as error: self.db.t.update_many({}, {"$set": {"x": 2}}) @@ -125,7 +125,7 @@ def test_retry_overload_error_getMore(self): coll.insert_many([{"x": 1} for _ in range(10)]) # Ensure command is retried on overload error. - fail_once = { + fail_many = { "configureFailPoint": "failCommand", "mode": {"times": _MAX_RETRIES}, "data": { @@ -136,15 +136,15 @@ def test_retry_overload_error_getMore(self): } cursor = coll.find(batch_size=2) cursor.next() - with self.fail_point(fail_once): + with self.fail_point(fail_many): cursor.to_list() # Ensure command stops retrying after _MAX_RETRIES. - fail_many_times = fail_once.copy() - fail_many_times["mode"] = {"times": _MAX_RETRIES + 1} + fail_too_many = fail_many.copy() + fail_too_many["mode"] = {"times": _MAX_RETRIES + 1} cursor = coll.find(batch_size=2) cursor.next() - with self.fail_point(fail_many_times): + with self.fail_point(fail_too_many): with self.assertRaises(PyMongoError) as error: cursor.to_list() From f3be7f796baa7937a6f6e0198bb0bab5575e8a79 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 20 Aug 2025 12:36:04 -0700 Subject: [PATCH 07/13] PYTHON-5505 Only apply backoff when SystemOverloaded label is present --- pymongo/asynchronous/mongo_client.py | 38 +++++++++++++++------------- pymongo/synchronous/mongo_client.py | 38 +++++++++++++++------------- 2 files changed, 42 insertions(+), 34 deletions(-) diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index dfaad87940..55a9ace144 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -2737,7 +2737,7 @@ def __init__( ): self._last_error: Optional[Exception] = None self._retrying = False - self._overloaded = False + self._always_retryable = False self._multiple_retries = _csot.get_timeout() is not None self._client = mongo_client @@ -2786,14 +2786,16 @@ async def run(self) -> T: # most likely be a waste of time. raise except PyMongoError as exc: + always_retryable = False overloaded = False # Execute specialized catch on read if self._is_read: if isinstance(exc, (ConnectionFailure, OperationFailure)): # ConnectionFailures do not supply a code property exc_code = getattr(exc, "code", None) - overloaded = exc.has_error_label("Retryable") - if not overloaded and ( + always_retryable = exc.has_error_label("Retryable") + overloaded = exc.has_error_label("SystemOverloaded") + if not always_retryable and ( self._is_not_eligible_for_retry() or ( isinstance(exc, OperationFailure) @@ -2810,21 +2812,22 @@ async def run(self) -> T: # Specialized catch on write operation if not self._is_read: if isinstance(exc, ClientBulkWriteException) and exc.error: - retryable_write_error_exc = isinstance( - exc.error, PyMongoError - ) and exc.error.has_error_label("RetryableWriteError") - overloaded = isinstance( - exc.error, PyMongoError - ) and exc.error.has_error_label("Retryable") + if isinstance(exc.error, PyMongoError): + retryable_write_error_exc = exc.error.has_error_label( + "RetryableWriteError" + ) + always_retryable = exc.error.has_error_label("Retryable") + overloaded = exc.error.has_error_label("SystemOverloaded") else: retryable_write_error_exc = exc.has_error_label("RetryableWriteError") - overloaded = exc.has_error_label("Retryable") - if not self._retryable and not overloaded: + always_retryable = exc.has_error_label("Retryable") + overloaded = exc.has_error_label("SystemOverloaded") + if not self._retryable and not always_retryable: raise - if retryable_write_error_exc or overloaded: + if retryable_write_error_exc or always_retryable: assert self._session await self._session._unpin() - if not overloaded and ( + if not always_retryable and ( not retryable_write_error_exc or not self._is_not_eligible_for_retry() ): if exc.has_error_label("NoWritesPerformed") and self._last_error: @@ -2844,14 +2847,15 @@ async def run(self) -> T: if self._client.topology_description.topology_type == TOPOLOGY_TYPE.Sharded: self._deprioritized_servers.append(self._server) - self._overloaded = overloaded - if overloaded: + self._always_retryable = always_retryable + if always_retryable: if self._attempt_number > _MAX_RETRIES: if exc.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc else: raise - await _backoff(self._attempt_number) + if overloaded: + await _backoff(self._attempt_number) def _is_not_eligible_for_retry(self) -> bool: """Checks if the exchange is not eligible for retry""" @@ -2946,7 +2950,7 @@ async def _read(self) -> T: conn, read_pref, ): - if self._retrying and not self._retryable and not self._overloaded: + if self._retrying and not self._retryable and not self._always_retryable: self._check_last_error() if self._retrying: _debug_log( diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 431a15551d..ec80071af0 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -2727,7 +2727,7 @@ def __init__( ): self._last_error: Optional[Exception] = None self._retrying = False - self._overloaded = False + self._always_retryable = False self._multiple_retries = _csot.get_timeout() is not None self._client = mongo_client @@ -2776,14 +2776,16 @@ def run(self) -> T: # most likely be a waste of time. raise except PyMongoError as exc: + always_retryable = False overloaded = False # Execute specialized catch on read if self._is_read: if isinstance(exc, (ConnectionFailure, OperationFailure)): # ConnectionFailures do not supply a code property exc_code = getattr(exc, "code", None) - overloaded = exc.has_error_label("Retryable") - if not overloaded and ( + always_retryable = exc.has_error_label("Retryable") + overloaded = exc.has_error_label("SystemOverloaded") + if not always_retryable and ( self._is_not_eligible_for_retry() or ( isinstance(exc, OperationFailure) @@ -2800,21 +2802,22 @@ def run(self) -> T: # Specialized catch on write operation if not self._is_read: if isinstance(exc, ClientBulkWriteException) and exc.error: - retryable_write_error_exc = isinstance( - exc.error, PyMongoError - ) and exc.error.has_error_label("RetryableWriteError") - overloaded = isinstance( - exc.error, PyMongoError - ) and exc.error.has_error_label("Retryable") + if isinstance(exc.error, PyMongoError): + retryable_write_error_exc = exc.error.has_error_label( + "RetryableWriteError" + ) + always_retryable = exc.error.has_error_label("Retryable") + overloaded = exc.error.has_error_label("SystemOverloaded") else: retryable_write_error_exc = exc.has_error_label("RetryableWriteError") - overloaded = exc.has_error_label("Retryable") - if not self._retryable and not overloaded: + always_retryable = exc.has_error_label("Retryable") + overloaded = exc.has_error_label("SystemOverloaded") + if not self._retryable and not always_retryable: raise - if retryable_write_error_exc or overloaded: + if retryable_write_error_exc or always_retryable: assert self._session self._session._unpin() - if not overloaded and ( + if not always_retryable and ( not retryable_write_error_exc or not self._is_not_eligible_for_retry() ): if exc.has_error_label("NoWritesPerformed") and self._last_error: @@ -2834,14 +2837,15 @@ def run(self) -> T: if self._client.topology_description.topology_type == TOPOLOGY_TYPE.Sharded: self._deprioritized_servers.append(self._server) - self._overloaded = overloaded - if overloaded: + self._always_retryable = always_retryable + if always_retryable: if self._attempt_number > _MAX_RETRIES: if exc.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc else: raise - _backoff(self._attempt_number) + if overloaded: + _backoff(self._attempt_number) def _is_not_eligible_for_retry(self) -> bool: """Checks if the exchange is not eligible for retry""" @@ -2936,7 +2940,7 @@ def _read(self) -> T: conn, read_pref, ): - if self._retrying and not self._retryable and not self._overloaded: + if self._retrying and not self._retryable and not self._always_retryable: self._check_last_error() if self._retrying: _debug_log( From 65fbd7eff41a03029d25e79374aef889134cd02a Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 20 Aug 2025 12:38:23 -0700 Subject: [PATCH 08/13] PYTHON-5505 Fix _retry_overload handling of backoff too --- pymongo/asynchronous/helpers.py | 3 ++- pymongo/synchronous/helpers.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/helpers.py b/pymongo/asynchronous/helpers.py index 10e578f3f4..8aa576a207 100644 --- a/pymongo/asynchronous/helpers.py +++ b/pymongo/asynchronous/helpers.py @@ -105,7 +105,8 @@ async def inner(*args: Any, **kwargs: Any) -> Any: raise # Implement exponential backoff on retry. - await _backoff(attempt) + if exc.has_error_label("SystemOverloaded"): + await _backoff(attempt) continue return cast(F, inner) diff --git a/pymongo/synchronous/helpers.py b/pymongo/synchronous/helpers.py index 91f73ffbce..fa1a79b218 100644 --- a/pymongo/synchronous/helpers.py +++ b/pymongo/synchronous/helpers.py @@ -105,7 +105,8 @@ def inner(*args: Any, **kwargs: Any) -> Any: raise # Implement exponential backoff on retry. - _backoff(attempt) + if exc.has_error_label("SystemOverloaded"): + _backoff(attempt) continue return cast(F, inner) From 194863ff2e1f886351f5daaf77ddca9d4463cb64 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 20 Aug 2025 12:40:26 -0700 Subject: [PATCH 09/13] PYTHON-5505 Remove unused "no_retry" flag --- pymongo/asynchronous/helpers.py | 3 +-- pymongo/synchronous/helpers.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pymongo/asynchronous/helpers.py b/pymongo/asynchronous/helpers.py index 8aa576a207..49d5ec604e 100644 --- a/pymongo/asynchronous/helpers.py +++ b/pymongo/asynchronous/helpers.py @@ -92,13 +92,12 @@ async def _backoff( def _retry_overload(func: F) -> F: @functools.wraps(func) async def inner(*args: Any, **kwargs: Any) -> Any: - no_retry = kwargs.pop("no_retry", False) attempt = 0 while True: try: return await func(*args, **kwargs) except PyMongoError as exc: - if no_retry or not exc.has_error_label("Retryable"): + if not exc.has_error_label("Retryable"): raise attempt += 1 if attempt > _MAX_RETRIES: diff --git a/pymongo/synchronous/helpers.py b/pymongo/synchronous/helpers.py index fa1a79b218..889382b19c 100644 --- a/pymongo/synchronous/helpers.py +++ b/pymongo/synchronous/helpers.py @@ -92,13 +92,12 @@ def _backoff( def _retry_overload(func: F) -> F: @functools.wraps(func) def inner(*args: Any, **kwargs: Any) -> Any: - no_retry = kwargs.pop("no_retry", False) attempt = 0 while True: try: return func(*args, **kwargs) except PyMongoError as exc: - if no_retry or not exc.has_error_label("Retryable"): + if not exc.has_error_label("Retryable"): raise attempt += 1 if attempt > _MAX_RETRIES: From 989fdfe2f3c34ad7c28d37cc11146b8dab014b41 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 20 Aug 2025 13:36:34 -0700 Subject: [PATCH 10/13] PYTHON-5505 Fix RetryableWriteError label check --- pymongo/asynchronous/mongo_client.py | 11 +++++------ pymongo/synchronous/mongo_client.py | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 55a9ace144..fa52288521 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -2811,24 +2811,23 @@ async def run(self) -> T: # Specialized catch on write operation if not self._is_read: + retryable_write_label = False if isinstance(exc, ClientBulkWriteException) and exc.error: if isinstance(exc.error, PyMongoError): - retryable_write_error_exc = exc.error.has_error_label( - "RetryableWriteError" - ) + retryable_write_label = exc.error.has_error_label("RetryableWriteError") always_retryable = exc.error.has_error_label("Retryable") overloaded = exc.error.has_error_label("SystemOverloaded") else: - retryable_write_error_exc = exc.has_error_label("RetryableWriteError") + retryable_write_label = exc.has_error_label("RetryableWriteError") always_retryable = exc.has_error_label("Retryable") overloaded = exc.has_error_label("SystemOverloaded") if not self._retryable and not always_retryable: raise - if retryable_write_error_exc or always_retryable: + if retryable_write_label or always_retryable: assert self._session await self._session._unpin() if not always_retryable and ( - not retryable_write_error_exc or not self._is_not_eligible_for_retry() + not retryable_write_label or not self._is_not_eligible_for_retry() ): if exc.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index ec80071af0..d5a87903c0 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -2801,24 +2801,23 @@ def run(self) -> T: # Specialized catch on write operation if not self._is_read: + retryable_write_label = False if isinstance(exc, ClientBulkWriteException) and exc.error: if isinstance(exc.error, PyMongoError): - retryable_write_error_exc = exc.error.has_error_label( - "RetryableWriteError" - ) + retryable_write_label = exc.error.has_error_label("RetryableWriteError") always_retryable = exc.error.has_error_label("Retryable") overloaded = exc.error.has_error_label("SystemOverloaded") else: - retryable_write_error_exc = exc.has_error_label("RetryableWriteError") + retryable_write_label = exc.has_error_label("RetryableWriteError") always_retryable = exc.has_error_label("Retryable") overloaded = exc.has_error_label("SystemOverloaded") if not self._retryable and not always_retryable: raise - if retryable_write_error_exc or always_retryable: + if retryable_write_label or always_retryable: assert self._session self._session._unpin() if not always_retryable and ( - not retryable_write_error_exc or not self._is_not_eligible_for_retry() + not retryable_write_label or not self._is_not_eligible_for_retry() ): if exc.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc From 1f40e117aef2fe5a44a858eff843177f760d2c21 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 20 Aug 2025 14:41:43 -0700 Subject: [PATCH 11/13] PYTHON-5505 Fix retry eligibility bug --- pymongo/asynchronous/mongo_client.py | 2 +- pymongo/synchronous/mongo_client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index fa52288521..5328a3957e 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -2827,7 +2827,7 @@ async def run(self) -> T: assert self._session await self._session._unpin() if not always_retryable and ( - not retryable_write_label or not self._is_not_eligible_for_retry() + not retryable_write_label or self._is_not_eligible_for_retry() ): if exc.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index d5a87903c0..7e907f93db 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -2817,7 +2817,7 @@ def run(self) -> T: assert self._session self._session._unpin() if not always_retryable and ( - not retryable_write_label or not self._is_not_eligible_for_retry() + not retryable_write_label or self._is_not_eligible_for_retry() ): if exc.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc From 6114788f7590c16f5fa71035bc0cbb0220d9ec10 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 20 Aug 2025 15:12:50 -0700 Subject: [PATCH 12/13] PYTHON-5505 Fix handling of ClientBulkWriteException --- pymongo/asynchronous/mongo_client.py | 24 +++++++++++------------- pymongo/synchronous/mongo_client.py | 24 +++++++++++------------- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 5328a3957e..1192c784af 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -2788,6 +2788,7 @@ async def run(self) -> T: except PyMongoError as exc: always_retryable = False overloaded = False + exc_to_check = exc # Execute specialized catch on read if self._is_read: if isinstance(exc, (ConnectionFailure, OperationFailure)): @@ -2811,16 +2812,13 @@ async def run(self) -> T: # Specialized catch on write operation if not self._is_read: - retryable_write_label = False - if isinstance(exc, ClientBulkWriteException) and exc.error: - if isinstance(exc.error, PyMongoError): - retryable_write_label = exc.error.has_error_label("RetryableWriteError") - always_retryable = exc.error.has_error_label("Retryable") - overloaded = exc.error.has_error_label("SystemOverloaded") - else: - retryable_write_label = exc.has_error_label("RetryableWriteError") - always_retryable = exc.has_error_label("Retryable") - overloaded = exc.has_error_label("SystemOverloaded") + if isinstance(exc, ClientBulkWriteException) and isinstance( + exc.error, PyMongoError + ): + exc_to_check = exc.error + retryable_write_label = exc_to_check.has_error_label("RetryableWriteError") + always_retryable = exc_to_check.has_error_label("Retryable") + overloaded = exc_to_check.has_error_label("SystemOverloaded") if not self._retryable and not always_retryable: raise if retryable_write_label or always_retryable: @@ -2829,7 +2827,7 @@ async def run(self) -> T: if not always_retryable and ( not retryable_write_label or self._is_not_eligible_for_retry() ): - if exc.has_error_label("NoWritesPerformed") and self._last_error: + if exc_to_check.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc else: raise @@ -2838,7 +2836,7 @@ async def run(self) -> T: self._bulk.retrying = True else: self._retrying = True - if not exc.has_error_label("NoWritesPerformed"): + if not exc_to_check.has_error_label("NoWritesPerformed"): self._last_error = exc if self._last_error is None: self._last_error = exc @@ -2849,7 +2847,7 @@ async def run(self) -> T: self._always_retryable = always_retryable if always_retryable: if self._attempt_number > _MAX_RETRIES: - if exc.has_error_label("NoWritesPerformed") and self._last_error: + if exc_to_check.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc else: raise diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 7e907f93db..45fb5bf849 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -2778,6 +2778,7 @@ def run(self) -> T: except PyMongoError as exc: always_retryable = False overloaded = False + exc_to_check = exc # Execute specialized catch on read if self._is_read: if isinstance(exc, (ConnectionFailure, OperationFailure)): @@ -2801,16 +2802,13 @@ def run(self) -> T: # Specialized catch on write operation if not self._is_read: - retryable_write_label = False - if isinstance(exc, ClientBulkWriteException) and exc.error: - if isinstance(exc.error, PyMongoError): - retryable_write_label = exc.error.has_error_label("RetryableWriteError") - always_retryable = exc.error.has_error_label("Retryable") - overloaded = exc.error.has_error_label("SystemOverloaded") - else: - retryable_write_label = exc.has_error_label("RetryableWriteError") - always_retryable = exc.has_error_label("Retryable") - overloaded = exc.has_error_label("SystemOverloaded") + if isinstance(exc, ClientBulkWriteException) and isinstance( + exc.error, PyMongoError + ): + exc_to_check = exc.error + retryable_write_label = exc_to_check.has_error_label("RetryableWriteError") + always_retryable = exc_to_check.has_error_label("Retryable") + overloaded = exc_to_check.has_error_label("SystemOverloaded") if not self._retryable and not always_retryable: raise if retryable_write_label or always_retryable: @@ -2819,7 +2817,7 @@ def run(self) -> T: if not always_retryable and ( not retryable_write_label or self._is_not_eligible_for_retry() ): - if exc.has_error_label("NoWritesPerformed") and self._last_error: + if exc_to_check.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc else: raise @@ -2828,7 +2826,7 @@ def run(self) -> T: self._bulk.retrying = True else: self._retrying = True - if not exc.has_error_label("NoWritesPerformed"): + if not exc_to_check.has_error_label("NoWritesPerformed"): self._last_error = exc if self._last_error is None: self._last_error = exc @@ -2839,7 +2837,7 @@ def run(self) -> T: self._always_retryable = always_retryable if always_retryable: if self._attempt_number > _MAX_RETRIES: - if exc.has_error_label("NoWritesPerformed") and self._last_error: + if exc_to_check.has_error_label("NoWritesPerformed") and self._last_error: raise self._last_error from exc else: raise From d2e3354eaa0f6077259dd9e7ffc314cb1f439233 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 20 Aug 2025 15:41:16 -0700 Subject: [PATCH 13/13] PYTHON-5505 Allow retry write after overload error even if sessions are not supported --- pymongo/asynchronous/mongo_client.py | 2 +- pymongo/synchronous/mongo_client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index 1192c784af..ae6e819334 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -2915,7 +2915,7 @@ async def _write(self) -> T: and conn.supports_sessions ) is_mongos = conn.is_mongos - if not sessions_supported: + if not self._always_retryable and not sessions_supported: # A retry is not possible because this server does # not support sessions raise the last error. self._check_last_error() diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 45fb5bf849..dcd8c50cca 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -2905,7 +2905,7 @@ def _write(self) -> T: and conn.supports_sessions ) is_mongos = conn.is_mongos - if not sessions_supported: + if not self._always_retryable and not sessions_supported: # A retry is not possible because this server does # not support sessions raise the last error. self._check_last_error()