From ce817de79efaf80b9092cf059587cb3e31b86537 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 14 May 2025 23:37:19 +0200 Subject: [PATCH] Use os.closerange instead of counting to 1 billion In `billiard.compat.close_open_fds`, after checking whether os.closerange exists, if it exists we did _not_ use it, which makes no sense at all. Instead, the implementation called `os.close`, `os.sysconf("SC_OPENMAX") - len(keep)` times. On most systems, SC_OPENMAX is something like 1024, 4096, or 65536. In the latter case, calling it will already take 30ms on my system, almost noticable for humans. But docker (and some systems) uses a really high value: ``` docker run --rm python python -c 'import os;print(os.sysconf("SC_OPEN_MAX"))' 1073741816 ``` In other words, when running in docker we were counting to 1 billlion, at 2 million per second. The single function call `close_open_fd([0,1,2])` takes almost 10 minutes there! The fix is trivial: When `os.closerange` is available, use the existing implementation. It's right there! An alternative approach would be to just list all open handles with `psutil.Process().open_files()` or so, and then just close those few. --- billiard/compat.py | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/billiard/compat.py b/billiard/compat.py index bea9746..3f411eb 100644 --- a/billiard/compat.py +++ b/billiard/compat.py @@ -130,27 +130,17 @@ def closerange(fd_low, fd_high): # noqa if exc.errno != errno.EBADF: raise - def close_open_fds(keep=None): - # must make sure this is 0-inclusive (Issue #celery/1882) - keep = list(uniq(sorted( - f for f in map(maybe_fileno, keep or []) if f is not None - ))) - maxfd = get_fdmax(default=2048) - kL, kH = iter([-1] + keep), iter(keep + [maxfd]) - for low, high in zip_longest(kL, kH): - if low + 1 != high: - closerange(low + 1, high) -else: - def close_open_fds(keep=None): # noqa - keep = [maybe_fileno(f) - for f in (keep or []) if maybe_fileno(f) is not None] - for fd in reversed(range(get_fdmax(default=2048))): - if fd not in keep: - try: - os.close(fd) - except OSError as exc: - if exc.errno != errno.EBADF: - raise + +def close_open_fds(keep=None): + # must make sure this is 0-inclusive (Issue #celery/1882) + keep = list(uniq(sorted( + f for f in map(maybe_fileno, keep or []) if f is not None + ))) + maxfd = get_fdmax(default=2048) + kL, kH = iter([-1] + keep), iter(keep + [maxfd]) + for low, high in zip_longest(kL, kH): + if low + 1 != high: + closerange(low + 1, high) def get_errno(exc):