Skip to content

PYTHON-5021 - Fix usages of getaddrinfo to be non-blocking #2059

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Jan 17, 2025
Merged

PYTHON-5021 - Fix usages of getaddrinfo to be non-blocking #2059

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
f7bb39e
PYTHON-5021 - Fix usages of getaddrinfo to be non-blocking
NoahStapp Jan 14, 2025
882c0db
Merge branch 'master' into PYTHON-5021
NoahStapp Jan 15, 2025
f6c0136
Use get_running_loop instead of get_event_loop
NoahStapp Jan 15, 2025
076a014
Use our own executor
NoahStapp Jan 15, 2025
2821566
Fix import of _PYMONGO_EXECUTOR
NoahStapp Jan 15, 2025
26fe6e1
Merge branch 'master' into PYTHON-5021
NoahStapp Jan 15, 2025
297bf9c
getaddrinfo helper method
NoahStapp Jan 15, 2025
39ade36
cleanup
NoahStapp Jan 15, 2025
5e3bc65
Use run_in_executor for getaddrinfo
NoahStapp Jan 15, 2025
85e59fd
Revert back to using the default executor
NoahStapp Jan 15, 2025
abe6b24
getaddrinfo -> _getaddrinfo
NoahStapp Jan 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 11 additions & 17 deletions pymongo/asynchronous/auth.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
"""Authentication helpers."""
from __future__ import annotations

import asyncio
import functools
import hashlib
import hmac
Expand All @@ -39,6 +38,7 @@
_authenticate_oidc,
_get_authenticator,
)
from pymongo.asynchronous.helpers import getaddrinfo
from pymongo.auth_shared import (
MongoCredential,
_authenticate_scram_start,
Expand Down Expand Up @@ -184,22 +184,16 @@ async def _canonicalize_hostname(hostname: str, option: str | bool) -> str:
if option in [False, "none"]:
return hostname

if not _IS_SYNC:
loop = asyncio.get_running_loop()
af, socktype, proto, canonname, sockaddr = (
await loop.getaddrinfo(
hostname,
None,
family=0,
type=0,
proto=socket.IPPROTO_TCP,
flags=socket.AI_CANONNAME,
)
)[0] # type: ignore[index]
else:
af, socktype, proto, canonname, sockaddr = socket.getaddrinfo(
hostname, None, 0, 0, socket.IPPROTO_TCP, socket.AI_CANONNAME
)[0]
af, socktype, proto, canonname, sockaddr = (
await getaddrinfo(
hostname,
None,
family=0,
type=0,
proto=socket.IPPROTO_TCP,
flags=socket.AI_CANONNAME,
)
)[0] # type: ignore[index]

# For forward just to resolve the cname as dns.lookup() will not return it.
if option == "forward":
Expand Down
12 changes: 12 additions & 0 deletions pymongo/asynchronous/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
"""Miscellaneous pieces that need to be synchronized."""
from __future__ import annotations

import asyncio
import builtins
import socket
import sys
from typing import (
Any,
Expand Down Expand Up @@ -68,6 +70,16 @@ async def inner(*args: Any, **kwargs: Any) -> Any:
return cast(F, inner)


async def getaddrinfo(host, port, **kwargs):
if not _IS_SYNC:
loop = asyncio.get_running_loop()
return await loop.getaddrinfo( # type: ignore[assignment]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we be using run_in_executor here instead as well?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch sorry, juggling too many changes at once 😅

host, port, **kwargs
)
else:
return socket.getaddrinfo(host, port, **kwargs) # type: ignore[assignment]


if sys.version_info >= (3, 10):
anext = builtins.anext
aiter = builtins.aiter
Expand Down
11 changes: 2 additions & 9 deletions pymongo/asynchronous/pool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
from pymongo import _csot, helpers_shared
from pymongo._asyncio_executor import _PYMONGO_EXECUTOR
from pymongo.asynchronous.client_session import _validate_session_write_concern
from pymongo.asynchronous.helpers import _handle_reauth
from pymongo.asynchronous.helpers import _handle_reauth, getaddrinfo
from pymongo.asynchronous.network import command, receive_message
from pymongo.common import (
MAX_BSON_SIZE,
Expand Down Expand Up @@ -815,14 +815,7 @@ async def _create_connection(address: _Address, options: PoolOptions) -> socket.
family = socket.AF_UNSPEC

err = None
if not _IS_SYNC:
loop = asyncio.get_running_loop()
results = await loop.getaddrinfo( # type: ignore[assignment]
host, port, family=family, type=socket.SOCK_STREAM
)
else:
results = socket.getaddrinfo(host, port, family, socket.SOCK_STREAM) # type: ignore[assignment]
for res in results: # type: ignore[attr-defined]
for res in await getaddrinfo(host, port, family=family, type=socket.SOCK_STREAM): # type: ignore[attr-defined]
af, socktype, proto, dummy, sa = res
# SOCK_CLOEXEC was new in CPython 3.2, and only available on a limited
# number of platforms (newer Linux and *BSD). Starting with CPython 3.4
Expand Down
28 changes: 11 additions & 17 deletions pymongo/synchronous/auth.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
"""Authentication helpers."""
from __future__ import annotations

import asyncio
import functools
import hashlib
import hmac
Expand Down Expand Up @@ -46,6 +45,7 @@
_authenticate_oidc,
_get_authenticator,
)
from pymongo.synchronous.helpers import getaddrinfo

if TYPE_CHECKING:
from pymongo.hello import Hello
Expand Down Expand Up @@ -181,22 +181,16 @@ def _canonicalize_hostname(hostname: str, option: str | bool) -> str:
if option in [False, "none"]:
return hostname

if not _IS_SYNC:
loop = asyncio.get_running_loop()
af, socktype, proto, canonname, sockaddr = (
loop.getaddrinfo(
hostname,
None,
family=0,
type=0,
proto=socket.IPPROTO_TCP,
flags=socket.AI_CANONNAME,
)
)[0] # type: ignore[index]
else:
af, socktype, proto, canonname, sockaddr = socket.getaddrinfo(
hostname, None, 0, 0, socket.IPPROTO_TCP, socket.AI_CANONNAME
)[0]
af, socktype, proto, canonname, sockaddr = (
getaddrinfo(
hostname,
None,
family=0,
type=0,
proto=socket.IPPROTO_TCP,
flags=socket.AI_CANONNAME,
)
)[0] # type: ignore[index]

# For forward just to resolve the cname as dns.lookup() will not return it.
if option == "forward":
Expand Down
12 changes: 12 additions & 0 deletions pymongo/synchronous/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
"""Miscellaneous pieces that need to be synchronized."""
from __future__ import annotations

import asyncio
import builtins
import socket
import sys
from typing import (
Any,
Expand Down Expand Up @@ -68,6 +70,16 @@ def inner(*args: Any, **kwargs: Any) -> Any:
return cast(F, inner)


def getaddrinfo(host, port, **kwargs):
if not _IS_SYNC:
loop = asyncio.get_running_loop()
return loop.getaddrinfo( # type: ignore[assignment]
host, port, **kwargs
)
else:
return socket.getaddrinfo(host, port, **kwargs) # type: ignore[assignment]


if sys.version_info >= (3, 10):
next = builtins.next
iter = builtins.iter
Expand Down
11 changes: 2 additions & 9 deletions pymongo/synchronous/pool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@
from pymongo.socket_checker import SocketChecker
from pymongo.ssl_support import HAS_SNI, SSLError
from pymongo.synchronous.client_session import _validate_session_write_concern
from pymongo.synchronous.helpers import _handle_reauth
from pymongo.synchronous.helpers import _handle_reauth, getaddrinfo
from pymongo.synchronous.network import command, receive_message

if TYPE_CHECKING:
Expand Down Expand Up @@ -813,14 +813,7 @@ def _create_connection(address: _Address, options: PoolOptions) -> socket.socket
family = socket.AF_UNSPEC

err = None
if not _IS_SYNC:
loop = asyncio.get_running_loop()
results = loop.getaddrinfo( # type: ignore[assignment]
host, port, family=family, type=socket.SOCK_STREAM
)
else:
results = socket.getaddrinfo(host, port, family, socket.SOCK_STREAM) # type: ignore[assignment]
for res in results: # type: ignore[attr-defined]
for res in getaddrinfo(host, port, family=family, type=socket.SOCK_STREAM): # type: ignore[attr-defined]
af, socktype, proto, dummy, sa = res
# SOCK_CLOEXEC was new in CPython 3.2, and only available on a limited
# number of platforms (newer Linux and *BSD). Starting with CPython 3.4
Expand Down
Loading