MOTOR-1353 Update Synchro Tests to support PyMongo 4.9 (#298)

Author: blink1073

doc/changelog.rst

@@ -3,6 +3,11 @@ Changelog
 
 .. currentmodule:: motor.motor_tornado
 
+Motor 3.6.0
+-----------
+- Add support for MongoDB 8.0 and PyMongo 4.9.
+- The length parameter in :meth:`MotorCursor.to_list` is now optional.
+
 Motor 3.5.1
 -----------
 - Fix runtime behavior of Motor generic class typing, e.g. ``client: AsyncIOMotorClient[Dict[str, Any]]``.

motor/core.py

@@ -127,6 +127,7 @@ class AgnosticClient(AgnosticBaseProperties):
     server_info = AsyncRead()
     topology_description = ReadOnlyProperty()
     start_session = AsyncCommand(doc=docstrings.start_session_doc).wrap(ClientSession)
+    _connect = AsyncRead()
 
     def __init__(self, *args, **kwargs):
         """Create a new connection to a single MongoDB instance at *host:port*.
@@ -1619,7 +1620,7 @@ def _each_got_more(self, callback, future):
             self._framework.call_soon(self.get_io_loop(), functools.partial(callback, None, None))
 
     @coroutine_annotation
-    def to_list(self, length):
+    def to_list(self, length=None):
         """Get a list of documents.
 
         .. testsetup:: to_list

motor/core.pyi

@@ -697,7 +697,7 @@ class AgnosticBaseCursor(AgnosticBase, Generic[_DocumentType]):
     def next_object(self) -> Any: ...
     def each(self, callback: Callable) -> None: ...
     def _each_got_more(self, callback: Callable, future: Any) -> None: ...
-    def to_list(self, length: Union[int, None]) -> Future[list]: ...
+    def to_list(self, length: Optional[int] = ...) -> Future[list]: ...
     def _to_list(
         self, length: Union[int, None], the_list: list, future: Any, get_more_result: Any
     ) -> None: ...

motor/docstrings.py

@@ -1149,7 +1149,7 @@ async def add_3_to_x():
 
   >>> async def f():
   ...     cursor = collection.find().sort("_id", pymongo.DESCENDING)
-  ...     docs = await cursor.to_list(None)
+  ...     docs = await cursor.to_list()
   ...     print([d["_id"] for d in docs])
   ...
   >>> IOLoop.current().run_sync(f)
@@ -1163,7 +1163,7 @@ async def add_3_to_x():
   ...     cursor = collection.find().sort(
   ...         [("field1", pymongo.ASCENDING), ("field2", pymongo.DESCENDING)]
   ...     )
-  ...     docs = await cursor.to_list(None)
+  ...     docs = await cursor.to_list()
   ...     print([(d["field1"], d["field2"]) for d in docs])
   ...
   >>> IOLoop.current().run_sync(f)

requirements/test.txt

@@ -4,3 +4,4 @@ tornado>=5
 aiohttp>=3.8.7
 motor[encryption]
 cffi>=1.17.0rc1;python_version=="3.13"
+pytest_asyncio

synchro/__init__.py

@@ -28,9 +28,8 @@
 # Make e.g. "from pymongo.errors import AutoReconnect" work. Note that
 # importing * won't pick up underscore-prefixed attrs.
 from gridfs import *
-from gridfs import _disallow_transactions
 from gridfs.errors import *
-from gridfs.grid_file import (
+from gridfs.synchronous.grid_file import (
     _SEEK_CUR,
     _SEEK_END,
     _UPLOAD_BUFFER_CHUNKS,
@@ -59,6 +58,7 @@
     server_type,
     srv_resolver,
     ssl_support,
+    uri_parser,
     write_concern,
 )
 
@@ -67,51 +67,50 @@
     from pymongo import _csot
 except ImportError:
     pass
-from pymongo import client_session
+from pymongo import client_session, synchronous
 from pymongo.auth import *
-from pymongo.auth import _build_credentials_tuple, _password_digest
-from pymongo.client_session import TransactionOptions, _TxnState
+from pymongo.client_session import TransactionOptions
 from pymongo.collation import *
 from pymongo.common import *
 from pymongo.common import _MAX_END_SESSIONS, _UUID_REPRESENTATIONS
 from pymongo.compression_support import _have_snappy, _have_zlib, _have_zstd
 from pymongo.cursor import *
-from pymongo.cursor import _QUERY_OPTIONS
+from pymongo.cursor_shared import _QUERY_OPTIONS
 from pymongo.encryption import *
-from pymongo.encryption import _MONGOCRYPTD_TIMEOUT_MS, _Encrypter
 from pymongo.encryption_options import *
 from pymongo.encryption_options import _HAVE_PYMONGOCRYPT
 from pymongo.errors import *
 from pymongo.event_loggers import *
-from pymongo.helpers import _check_command_response
+from pymongo.helpers_shared import _SENSITIVE_COMMANDS, _check_command_response
 from pymongo.lock import _create_lock
 from pymongo.message import (
     _COMMAND_OVERHEAD,
     _CursorAddress,
     _gen_find_command,
     _maybe_add_read_preference,
 )
-from pymongo.monitor import *
 from pymongo.monitoring import *
-from pymongo.monitoring import _LISTENERS, _SENSITIVE_COMMANDS, _Listeners
+from pymongo.monitoring import _LISTENERS, _Listeners
 from pymongo.ocsp_cache import _OCSPCache
 from pymongo.operations import *
-from pymongo.periodic_executor import *
-from pymongo.periodic_executor import _EXECUTORS
-from pymongo.pool import *
-from pymongo.pool import _METADATA, Connection, Pool, _PoolClosedError
 from pymongo.read_concern import *
 from pymongo.read_preferences import *
 from pymongo.read_preferences import _ServerMode
 from pymongo.results import *
 from pymongo.results import _WriteResult
 from pymongo.saslprep import *
-from pymongo.server import *
 from pymongo.server_selectors import *
-from pymongo.settings import *
 from pymongo.srv_resolver import _resolve, _SrvResolver
 from pymongo.ssl_support import *
-from pymongo.topology import *
+from pymongo.synchronous.client_session import _TxnState
+from pymongo.synchronous.monitor import *
+from pymongo.synchronous.periodic_executor import *
+from pymongo.synchronous.periodic_executor import _EXECUTORS
+from pymongo.synchronous.pool import *
+from pymongo.synchronous.pool import Connection, Pool, _PoolClosedError
+from pymongo.synchronous.server import *
+from pymongo.synchronous.settings import *
+from pymongo.synchronous.topology import *
 from pymongo.topology_description import *
 from pymongo.uri_parser import *
 from pymongo.uri_parser import _have_dnspython
@@ -131,6 +130,7 @@
 
 _MotorRawBatchCursor = create_motor_class(_AgnosticRawBatchCursor)
 _MotorRawBatchCommandCursor = create_motor_class(_AgnosticRawBatchCommandCursor)
+get_hosts_and_min_ttl = _SrvResolver.get_hosts_and_min_ttl
 
 
 def wrap_synchro(fn):
@@ -262,9 +262,6 @@ class SynchroMeta(type):
 
     - Motor methods which return Motor class instances are wrapped to return
       Synchro class instances.
-
-    - Certain internals accessed by PyMongo's unittests, such as _Cursor__data,
-      are delegated from Synchro directly to PyMongo.
     """
 
     def __new__(cls, name, bases, attrs):
@@ -395,9 +392,10 @@ def __getitem__(self, name):
         return Database(self, name, delegate=self.delegate[name])
 
     # For PyMongo tests that access client internals.
-    _MongoClient__all_credentials = SynchroProperty()
-    _MongoClient__kill_cursors_queue = SynchroProperty()
-    _MongoClient__options = SynchroProperty()
+    _connect = Sync()
+    _all_credentials = SynchroProperty()
+    _kill_cursors_queue = SynchroProperty()
+    _options = SynchroProperty()
     _cache_credentials = SynchroProperty()
     _close_cursor_now = SynchroProperty()
     _get_topology = SynchroProperty()
@@ -577,6 +575,16 @@ class Cursor(Synchro):
     rewind = WrapOutgoing()
     clone = WrapOutgoing()
     close = Sync("close")
+    to_list = Sync("to_list")
+
+    _query_flags = SynchroProperty()
+    _data = SynchroProperty()
+    _max_time_ms = SynchroProperty()
+    _max_await_time_ms = SynchroProperty()
+    _retrieved = SynchroProperty()
+    _spec = SynchroProperty()
+    _exhaust = SynchroProperty()
+    _query_spec = SynchroProperty()
 
     _next = Sync("next")
 
@@ -618,22 +626,12 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         # Don't suppress exceptions.
         return False
 
-    # For PyMongo tests that access cursor internals.
-    _Cursor__data = SynchroProperty()
-    _Cursor__exhaust = SynchroProperty()
-    _Cursor__max_await_time_ms = SynchroProperty()
-    _Cursor__max_time_ms = SynchroProperty()
-    _Cursor__query_flags = SynchroProperty()
-    _Cursor__query_spec = SynchroProperty()
-    _Cursor__retrieved = SynchroProperty()
-    _Cursor__spec = SynchroProperty()
-    _read_preference = SynchroProperty()
-
 
 class CommandCursor(Cursor):
     __delegate_class__ = motor.motor_tornado.MotorCommandCursor
 
     try_next = Sync("try_next")
+    to_list = Sync("to_list")
 
 
 class GridOutCursor(Cursor):

synchro/synchrotest.py

@@ -197,6 +197,16 @@
     "TestClient.test_handshake*",
     # This test is not a valid unittest target.
     "TestRangeQueryProse.run_test_cases",
+    # The test logic interferes with the SynchroLoader.
+    "ClientUnitTest.test_detected_environment_logging",
+    "ClientUnitTest.test_detected_environment_warning",
+    # The batch_size portion of this test does not work with synchro.
+    "TestCursor.test_to_list_length",
+    # These tests hang due to internal incompatibilities in to_list.
+    "TestCursor.test_command_cursor_to_list*",
+    # Motor does not allow calling to_list on a tailable cursor.
+    "TestCursor.test_max_await_time_ms",
+    "TestCursor.test_to_list_tailable",
 ]
 
 
@@ -230,11 +240,7 @@ def find_spec(self, fullname, path, target=None):
 class SynchroModuleLoader(importlib.abc.Loader):
     def patch_spec(self, fullname):
         parts = fullname.split(".")
-        if parts[-1] in ("gridfs", "pymongo"):
-            # E.g. "import pymongo"
-            return True
-        elif len(parts) >= 2 and parts[-2] in ("gridfs", "pymongo"):
-            # E.g. "import pymongo.mongo_client"
+        if ("pymongo" in parts or "gridfs" in parts) and "asynchronous" not in parts:
             return True
 
         return False
@@ -328,8 +334,17 @@ def want_method(method, classname):
         "pymongo.mongo_client",
         "pymongo.database",
         "pymongo.srv_resolver",
+        "pymongo.synchronous.collection",
+        "pymongo.synchronous.client_session",
+        "pymongo.synchronous.command_cursor",
+        "pymongo.synchronous.change_stream",
+        "pymongo.synchronous.cursor",
+        "pymongo.synchronous.encryption",
+        "pymongo.synchronous.mongo_client",
+        "pymongo.synchronous.database",
         "gridfs",
         "gridfs.grid_file",
+        "gridfs.synchronous.grid_file",
     ]:
         sys.modules.pop(n)
 
@@ -341,7 +356,9 @@ def want_method(method, classname):
 
     # Run the tests from the pymongo target dir with our custom plugin.
     os.chdir(sys.argv[1])
-    code = pytest.main(sys.argv[2:] + ["-p", "no:warnings"], plugins=[SynchroPytestPlugin()])
+    code = pytest.main(
+        sys.argv[2:] + ["-m", "default"] + ["-p", "no:warnings"], plugins=[SynchroPytestPlugin()]
+    )
 
     if code != 0:
         sys.exit(code)

test/asyncio_tests/test_asyncio_cursor.py

@@ -192,7 +192,6 @@ def expected(start, stop):
         self.assertEqual(expected(100, 101), (await cursor.to_list(1)))
         self.assertEqual(expected(101, 102), (await cursor.to_list(1)))
         self.assertEqual(expected(102, 103), (await cursor.to_list(1)))
-        self.assertEqual([], (await cursor.to_list(0)))
         self.assertEqual(expected(103, 105), (await cursor.to_list(2)))
 
         # Only 95 docs left, make sure length=100 doesn't error or hang

tox.ini

@@ -115,8 +115,7 @@ allowlist_externals =
 setenv =
     PYTHONPATH = {envtmpdir}/mongo-python-driver
 commands =
-    # TODO: Use "master" as part of MOTOR-1330
-    git clone --depth 1 --branch v4.8 https://github.com/mongodb/mongo-python-driver.git {envtmpdir}/mongo-python-driver
+    git clone --depth 1 https://github.com/mongodb/mongo-python-driver.git {envtmpdir}/mongo-python-driver
     python -m pip install -e {envtmpdir}/mongo-python-driver
     python -m synchro.synchrotest {envtmpdir}/mongo-python-driver -v {posargs}
 
@@ -131,8 +130,7 @@ passenv =
 setenv =
     PYTHONPATH = {envtmpdir}/mongo-python-driver
 commands =
-    # TODO: Use "master" as part of MOTOR-1330
-    git clone --depth 1 --branch v4.8 https://github.com/mongodb/mongo-python-driver.git {envtmpdir}/mongo-python-driver
+    git clone --depth 1 https://github.com/mongodb/mongo-python-driver.git {envtmpdir}/mongo-python-driver
     python -m pip install -e {envtmpdir}/mongo-python-driver
     python -m synchro.synchrotest {envtmpdir}/mongo-python-driver -v test/test_auth.py