PYTHON-5154 - Remove PyOpenSSL support from Async PyMongo (mongodb#2246)

Author: NoahStapp

.evergreen/generated_configs/variants.yml

@@ -360,9 +360,9 @@ buildvariants:
     tags: [encryption_tag]
   - name: encryption-pyopenssl-rhel8-python3.9
     tasks:
-      - name: .sharded_cluster .auth .ssl .sync_async
-      - name: .replica_set .noauth .ssl .sync_async
-      - name: .standalone .noauth .nossl .sync_async
+      - name: .sharded_cluster .auth .ssl .sync
+      - name: .replica_set .noauth .ssl .sync
+      - name: .standalone .noauth .nossl .sync
     display_name: Encryption PyOpenSSL RHEL8 Python3.9
     run_on:
       - rhel87-small
@@ -374,9 +374,9 @@ buildvariants:
     tags: [encryption_tag]
   - name: encryption-pyopenssl-rhel8-python3.13
     tasks:
-      - name: .sharded_cluster .auth .ssl .sync_async
-      - name: .replica_set .noauth .ssl .sync_async
-      - name: .standalone .noauth .nossl .sync_async
+      - name: .sharded_cluster .auth .ssl .sync
+      - name: .replica_set .noauth .ssl .sync
+      - name: .standalone .noauth .nossl .sync
     display_name: Encryption PyOpenSSL RHEL8 Python3.13
     run_on:
       - rhel87-small
@@ -388,9 +388,9 @@ buildvariants:
     tags: [encryption_tag]
   - name: encryption-pyopenssl-rhel8-pypy3.10
     tasks:
-      - name: .sharded_cluster .auth .ssl .sync_async
-      - name: .replica_set .noauth .ssl .sync_async
-      - name: .standalone .noauth .nossl .sync_async
+      - name: .sharded_cluster .auth .ssl .sync
+      - name: .replica_set .noauth .ssl .sync
+      - name: .standalone .noauth .nossl .sync
     display_name: Encryption PyOpenSSL RHEL8 PyPy3.10
     run_on:
       - rhel87-small
@@ -419,15 +419,14 @@ buildvariants:
       TEST_NAME: encryption
       TEST_CRYPT_SHARED: "true"
       PYTHON_BINARY: /opt/python/3.11/bin/python3
-  - name: encryption-pyopenssl-rhel8-python3.12
+  - name: encryption-rhel8-python3.12
     tasks:
       - name: .standalone .noauth .nossl .sync_async
-    display_name: Encryption PyOpenSSL RHEL8 Python3.12
+    display_name: Encryption RHEL8 Python3.12
     run_on:
       - rhel87-small
     expansions:
       TEST_NAME: encryption
-      SUB_TEST_NAME: pyopenssl
       PYTHON_BINARY: /opt/python/3.12/bin/python3
   - name: encryption-macos-python3.9
     tasks:
@@ -909,8 +908,8 @@ buildvariants:
   # Pyopenssl tests
   - name: pyopenssl-macos-python3.9
     tasks:
-      - name: .replica_set .noauth .nossl .sync_async
-      - name: .7.0 .noauth .nossl .sync_async
+      - name: .replica_set .noauth .nossl .sync
+      - name: .7.0 .noauth .nossl .sync
     display_name: PyOpenSSL macOS Python3.9
     run_on:
       - macos-14
@@ -920,8 +919,8 @@ buildvariants:
       PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
   - name: pyopenssl-rhel8-python3.10
     tasks:
-      - name: .replica_set .auth .ssl .sync_async
-      - name: .7.0 .auth .ssl .sync_async
+      - name: .replica_set .auth .ssl .sync
+      - name: .7.0 .auth .ssl .sync
     display_name: PyOpenSSL RHEL8 Python3.10
     run_on:
       - rhel87-small
@@ -931,8 +930,8 @@ buildvariants:
       PYTHON_BINARY: /opt/python/3.10/bin/python3
   - name: pyopenssl-rhel8-python3.11
     tasks:
-      - name: .replica_set .auth .ssl .sync_async
-      - name: .7.0 .auth .ssl .sync_async
+      - name: .replica_set .auth .ssl .sync
+      - name: .7.0 .auth .ssl .sync
     display_name: PyOpenSSL RHEL8 Python3.11
     run_on:
       - rhel87-small
@@ -942,8 +941,8 @@ buildvariants:
       PYTHON_BINARY: /opt/python/3.11/bin/python3
   - name: pyopenssl-rhel8-python3.12
     tasks:
-      - name: .replica_set .auth .ssl .sync_async
-      - name: .7.0 .auth .ssl .sync_async
+      - name: .replica_set .auth .ssl .sync
+      - name: .7.0 .auth .ssl .sync
     display_name: PyOpenSSL RHEL8 Python3.12
     run_on:
       - rhel87-small
@@ -953,8 +952,8 @@ buildvariants:
       PYTHON_BINARY: /opt/python/3.12/bin/python3
   - name: pyopenssl-win64-python3.13
     tasks:
-      - name: .replica_set .auth .ssl .sync_async
-      - name: .7.0 .auth .ssl .sync_async
+      - name: .replica_set .auth .ssl .sync
+      - name: .7.0 .auth .ssl .sync
     display_name: PyOpenSSL Win64 Python3.13
     run_on:
       - windows-64-vsMulti-small
@@ -964,8 +963,8 @@ buildvariants:
       PYTHON_BINARY: C:/python/Python313/python.exe
   - name: pyopenssl-rhel8-pypy3.10
     tasks:
-      - name: .replica_set .auth .ssl .sync_async
-      - name: .7.0 .auth .ssl .sync_async
+      - name: .replica_set .auth .ssl .sync
+      - name: .7.0 .auth .ssl .sync
     display_name: PyOpenSSL RHEL8 PyPy3.10
     run_on:
       - rhel87-small

.evergreen/scripts/generate_config.py

@@ -369,7 +369,7 @@ def get_encryption_expansions(encryption):
     host = DEFAULT_HOST
 
     # Test against all server versions for the three main python versions.
-    encryptions = ["Encryption", "Encryption crypt_shared", "Encryption PyOpenSSL"]
+    encryptions = ["Encryption", "Encryption crypt_shared"]
     for encryption, python in product(encryptions, [*MIN_MAX_PYTHON, PYPYS[-1]]):
         expansions = get_encryption_expansions(encryption)
         display_name = get_variant_name(encryption, host, python=python, **expansions)
@@ -384,6 +384,21 @@ def get_encryption_expansions(encryption):
         )
         variants.append(variant)
 
+    # Test PyOpenSSL against on all server versions for all python versions.
+    for encryption, python in product(["Encryption PyOpenSSL"], [*MIN_MAX_PYTHON, PYPYS[-1]]):
+        expansions = get_encryption_expansions(encryption)
+        display_name = get_variant_name(encryption, host, python=python, **expansions)
+        variant = create_variant(
+            [f"{t} .sync" for t in SUB_TASKS],
+            display_name,
+            python=python,
+            host=host,
+            expansions=expansions,
+            batchtime=batchtime,
+            tags=tags,
+        )
+        variants.append(variant)
+
     # Test the rest of the pythons on linux for all server versions.
     for encryption, python, task in zip_cycle(encryptions, CPYTHONS[1:-1] + PYPYS[:-1], SUB_TASKS):
         expansions = get_encryption_expansions(encryption)
@@ -499,7 +514,7 @@ def create_pyopenssl_variants():
 
         display_name = get_variant_name(base_name, host, python=python)
         variant = create_variant(
-            [f".replica_set .{auth} .{ssl} .sync_async", f".7.0 .{auth} .{ssl} .sync_async"],
+            [f".replica_set .{auth} .{ssl} .sync", f".7.0 .{auth} .{ssl} .sync"],
             display_name,
             python=python,
             host=host,

doc/changelog.rst

@@ -23,6 +23,7 @@ PyMongo 4.12 brings a number of changes including:
   :class:`~pymongo.read_preferences.Secondary`,
   :class:`~pymongo.read_preferences.SecondaryPreferred`,
   :class:`~pymongo.read_preferences.Nearest`. Support for ``hedge`` will be removed in PyMongo 5.0.
+- Removed PyOpenSSL support from the asynchronous API due to limitations of the CPython asyncio.Protocol SSL implementation.
 
 Issues Resolved
 ...............

pymongo/pool_shared.py

@@ -280,20 +280,14 @@ async def _async_configured_socket(
         # We have to pass hostname / ip address to wrap_socket
         # to use SSLContext.check_hostname.
         if HAS_SNI:
-            if hasattr(ssl_context, "a_wrap_socket"):
-                ssl_sock = await ssl_context.a_wrap_socket(sock, server_hostname=host)  # type: ignore[assignment, misc, unused-ignore]
-            else:
-                loop = asyncio.get_running_loop()
-                ssl_sock = await loop.run_in_executor(
-                    None,
-                    functools.partial(ssl_context.wrap_socket, sock, server_hostname=host),  # type: ignore[assignment, misc, unused-ignore]
-                )
+            loop = asyncio.get_running_loop()
+            ssl_sock = await loop.run_in_executor(
+                None,
+                functools.partial(ssl_context.wrap_socket, sock, server_hostname=host),  # type: ignore[assignment, misc, unused-ignore]
+            )
         else:
-            if hasattr(ssl_context, "a_wrap_socket"):
-                ssl_sock = await ssl_context.a_wrap_socket(sock)  # type: ignore[assignment, misc, unused-ignore]
-            else:
-                loop = asyncio.get_running_loop()
-                ssl_sock = await loop.run_in_executor(None, ssl_context.wrap_socket, sock)  # type: ignore[assignment, misc, unused-ignore]
+            loop = asyncio.get_running_loop()
+            ssl_sock = await loop.run_in_executor(None, ssl_context.wrap_socket, sock)  # type: ignore[assignment, misc, unused-ignore]
     except _CertificateError:
         sock.close()
         # Raise _CertificateError directly like we do after match_hostname

pymongo/pyopenssl_context.py

@@ -14,10 +14,11 @@
 
 """A CPython compatible SSLContext implementation wrapping PyOpenSSL's
 context.
+
+Due to limitations of the CPython asyncio.Protocol implementation for SSL, the async API does not support PyOpenSSL.
 """
 from __future__ import annotations
 
-import asyncio
 import socket as _socket
 import ssl as _stdlibssl
 import sys as _sys
@@ -109,15 +110,12 @@ def __init__(
         ctx: _SSL.Context,
         sock: Optional[_socket.socket],
         suppress_ragged_eofs: bool,
-        is_async: bool = False,
     ):
         self.socket_checker = _SocketChecker()
         self.suppress_ragged_eofs = suppress_ragged_eofs
         super().__init__(ctx, sock)
-        self._is_async = is_async
 
     def _call(self, call: Callable[..., _T], *args: Any, **kwargs: Any) -> _T:
-        is_async = kwargs.pop("allow_async", True) and self._is_async
         timeout = self.gettimeout()
         if timeout:
             start = _time.monotonic()
@@ -126,7 +124,7 @@ def _call(self, call: Callable[..., _T], *args: Any, **kwargs: Any) -> _T:
                 return call(*args, **kwargs)
             except BLOCKING_IO_ERRORS as exc:
                 # Do not retry if the connection is in non-blocking mode.
-                if is_async or timeout == 0:
+                if timeout == 0:
                     raise exc
                 # Check for closed socket.
                 if self.fileno() == -1:
@@ -148,7 +146,6 @@ def _call(self, call: Callable[..., _T], *args: Any, **kwargs: Any) -> _T:
                 continue
 
     def do_handshake(self, *args: Any, **kwargs: Any) -> None:
-        kwargs["allow_async"] = False
         return self._call(super().do_handshake, *args, **kwargs)
 
     def recv(self, *args: Any, **kwargs: Any) -> bytes:
@@ -379,58 +376,6 @@ def set_default_verify_paths(self) -> None:
         # but not that same as CPython's.
         self._ctx.set_default_verify_paths()
 
-    async def a_wrap_socket(
-        self,
-        sock: _socket.socket,
-        server_side: bool = False,
-        do_handshake_on_connect: bool = True,
-        suppress_ragged_eofs: bool = True,
-        server_hostname: Optional[str] = None,
-        session: Optional[_SSL.Session] = None,
-    ) -> _sslConn:
-        """Wrap an existing Python socket connection and return a TLS socket
-        object.
-        """
-        ssl_conn = _sslConn(self._ctx, sock, suppress_ragged_eofs, True)
-        loop = asyncio.get_running_loop()
-        if session:
-            ssl_conn.set_session(session)
-        if server_side is True:
-            ssl_conn.set_accept_state()
-        else:
-            # SNI
-            if server_hostname and not _is_ip_address(server_hostname):
-                # XXX: Do this in a callback registered with
-                # SSLContext.set_info_callback? See Twisted for an example.
-                ssl_conn.set_tlsext_host_name(server_hostname.encode("idna"))
-            if self.verify_mode != _stdlibssl.CERT_NONE:
-                # Request a stapled OCSP response.
-                await loop.run_in_executor(None, ssl_conn.request_ocsp)
-            ssl_conn.set_connect_state()
-        # If this wasn't true the caller of wrap_socket would call
-        # do_handshake()
-        if do_handshake_on_connect:
-            # XXX: If we do hostname checking in a callback we can get rid
-            # of this call to do_handshake() since the handshake
-            # will happen automatically later.
-            await loop.run_in_executor(None, ssl_conn.do_handshake)
-            # XXX: Do this in a callback registered with
-            # SSLContext.set_info_callback? See Twisted for an example.
-            if self.check_hostname and server_hostname is not None:
-                from service_identity import pyopenssl
-
-                try:
-                    if _is_ip_address(server_hostname):
-                        pyopenssl.verify_ip_address(ssl_conn, server_hostname)
-                    else:
-                        pyopenssl.verify_hostname(ssl_conn, server_hostname)
-                except (  # type:ignore[misc]
-                    service_identity.SICertificateError,
-                    service_identity.SIVerificationError,
-                ) as exc:
-                    raise _CertificateError(str(exc)) from None
-        return ssl_conn
-
     def wrap_socket(
         self,
         sock: _socket.socket,