Skip to content

PYTHON-4667 Handle $clusterTime from error responses in client Bulk Write #1822

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Sep 5, 2024
Merged

PYTHON-4667 Handle $clusterTime from error responses in client Bulk Write #1822

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5010f8b
PYTHON-4667 Handle from error responses in client Bulk Write
blink1073 Aug 30, 2024
000fdcb
address review
blink1073 Aug 31, 2024
00fda16
fix typing
blink1073 Aug 31, 2024
879f8d0
fix var handling
blink1073 Aug 31, 2024
b57b32b
address review
blink1073 Sep 4, 2024
48b059f
address review
blink1073 Sep 4, 2024
e8bbb16
simplify
blink1073 Sep 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion pymongo/asynchronous/bulk.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,7 @@ async def write_command(
)
if bwc.publish:
bwc._succeed(request_id, reply, duration) # type: ignore[arg-type]
await client._process_response(reply, bwc.session) # type: ignore[arg-type]
except Exception as exc:
duration = datetime.datetime.now() - bwc.start_time
if isinstance(exc, (NotPrimaryError, OperationFailure)):
Expand Down Expand Up @@ -308,6 +309,9 @@ async def write_command(

if bwc.publish:
bwc._fail(request_id, failure, duration)
# Process the response from the server.
if isinstance(exc, (NotPrimaryError, OperationFailure)):
await client._process_response(exc.details, bwc.session) # type: ignore[arg-type]
raise
finally:
bwc.start_time = datetime.datetime.now()
Expand Down Expand Up @@ -449,7 +453,6 @@ async def _execute_batch(
else:
request_id, msg, to_send = bwc.batch_command(cmd, ops)
result = await self.write_command(bwc, cmd, request_id, msg, to_send, client) # type: ignore[arg-type]
await client._process_response(result, bwc.session) # type: ignore[arg-type]

return result, to_send # type: ignore[return-value]

Expand Down
8 changes: 7 additions & 1 deletion pymongo/asynchronous/client_bulk.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,8 @@ async def write_command(
)
if bwc.publish:
bwc._succeed(request_id, reply, duration) # type: ignore[arg-type]
# Process the response from the server.
await self.client._process_response(reply, bwc.session) # type: ignore[arg-type]
except Exception as exc:
duration = datetime.datetime.now() - bwc.start_time
if isinstance(exc, (NotPrimaryError, OperationFailure)):
Expand Down Expand Up @@ -312,6 +314,11 @@ async def write_command(
bwc._fail(request_id, failure, duration)
# Top-level error will be embedded in ClientBulkWriteException.
reply = {"error": exc}
# Process the response from the server.
if isinstance(exc, OperationFailure):
await self.client._process_response(exc.details, bwc.session) # type: ignore[arg-type]
else:
await self.client._process_response({}, bwc.session) # type: ignore[arg-type]
finally:
bwc.start_time = datetime.datetime.now()
return reply # type: ignore[return-value]
Expand Down Expand Up @@ -431,7 +438,6 @@ async def _execute_batch(
result = await self.write_command(
bwc, cmd, request_id, msg, to_send_ops, to_send_ns, self.client
) # type: ignore[arg-type]
await self.client._process_response(result, bwc.session) # type: ignore[arg-type]
return result, to_send_ops, to_send_ns # type: ignore[return-value]

async def _process_results_cursor(
Expand Down
5 changes: 4 additions & 1 deletion pymongo/synchronous/bulk.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,7 @@ def write_command(
)
if bwc.publish:
bwc._succeed(request_id, reply, duration) # type: ignore[arg-type]
client._process_response(reply, bwc.session) # type: ignore[arg-type]
except Exception as exc:
duration = datetime.datetime.now() - bwc.start_time
if isinstance(exc, (NotPrimaryError, OperationFailure)):
Expand Down Expand Up @@ -308,6 +309,9 @@ def write_command(

if bwc.publish:
bwc._fail(request_id, failure, duration)
# Process the response from the server.
if isinstance(exc, (NotPrimaryError, OperationFailure)):
client._process_response(exc.details, bwc.session) # type: ignore[arg-type]
raise
finally:
bwc.start_time = datetime.datetime.now()
Expand Down Expand Up @@ -449,7 +453,6 @@ def _execute_batch(
else:
request_id, msg, to_send = bwc.batch_command(cmd, ops)
result = self.write_command(bwc, cmd, request_id, msg, to_send, client) # type: ignore[arg-type]
client._process_response(result, bwc.session) # type: ignore[arg-type]

return result, to_send # type: ignore[return-value]

Expand Down
8 changes: 7 additions & 1 deletion pymongo/synchronous/client_bulk.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,8 @@ def write_command(
)
if bwc.publish:
bwc._succeed(request_id, reply, duration) # type: ignore[arg-type]
# Process the response from the server.
self.client._process_response(reply, bwc.session) # type: ignore[arg-type]
except Exception as exc:
duration = datetime.datetime.now() - bwc.start_time
if isinstance(exc, (NotPrimaryError, OperationFailure)):
Expand Down Expand Up @@ -312,6 +314,11 @@ def write_command(
bwc._fail(request_id, failure, duration)
# Top-level error will be embedded in ClientBulkWriteException.
reply = {"error": exc}
# Process the response from the server.
if isinstance(exc, OperationFailure):
self.client._process_response(exc.details, bwc.session) # type: ignore[arg-type]
else:
self.client._process_response({}, bwc.session) # type: ignore[arg-type]
finally:
bwc.start_time = datetime.datetime.now()
return reply # type: ignore[return-value]
Expand Down Expand Up @@ -429,7 +436,6 @@ def _execute_batch(
"""Executes a batch of bulkWrite server commands (ack)."""
request_id, msg, to_send_ops, to_send_ns = bwc.batch_command(cmd, ops, namespaces)
result = self.write_command(bwc, cmd, request_id, msg, to_send_ops, to_send_ns, self.client) # type: ignore[arg-type]
self.client._process_response(result, bwc.session) # type: ignore[arg-type]
return result, to_send_ops, to_send_ns # type: ignore[return-value]

def _process_results_cursor(
Expand Down
12 changes: 12 additions & 0 deletions test/mockupdb/test_cluster_time.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

from bson import Timestamp
from pymongo import DeleteMany, InsertOne, MongoClient, UpdateOne
from pymongo.errors import OperationFailure

pytestmark = pytest.mark.mockupdb

Expand Down Expand Up @@ -166,6 +167,17 @@ def test_monitor(self):
request.reply(reply)
client.close()

def test_bulk_error(self):
def callback(client: MongoClient[dict]) -> None:
client.db.collection.bulk_write([InsertOne({}), InsertOne({})])

with self.assertRaises(OperationFailure) as context:
self.cluster_time_conversation(
callback,
[{"ok": 0, "errmsg": "mock error"}],
)
self.assertIn("$clusterTime", str(context.exception))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test is skipping the clusterTime asserts in cluster_time_conversation. We need to do:

        def callback(client: MongoClient[dict]) -> None:
            with self.assertRaises(OperationFailure):
                client.db.collection.bulk_write([InsertOne({}), InsertOne({})])

        self.cluster_time_conversation(
            callback,
            [{"ok": 0, "errmsg": "mock error"}],
        )

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you also add a similar test for client.bulk_write?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done and done



if __name__ == "__main__":
unittest.main()
Loading