From 572d1e5676e814cdf3bc6a0cd1cd5ac9c84144c7 Mon Sep 17 00:00:00 2001 From: Forest Date: Tue, 30 Jul 2024 17:20:41 -0700 Subject: [PATCH 01/78] gh-55454: Add IMAP4 IDLE support to imaplib This extends imaplib with support for the rfc2177 IMAP IDLE command, as requested in #55454. It allows events to be pushed to a client as they occur, rather than having to continually poll for mailbox changes. The interface is a new idle() method, which returns an iterable context manager. Entering the context starts IDLE mode, during which events (untagged responses) can be retrieved using the iteration protocol. Exiting the context sends DONE to the server, ending IDLE mode. An optional time limit for the IDLE session is supported, for use with servers that impose an inactivity timeout. The context manager also offers a burst() method, designed for programs wishing to process events in batch rather than one at a time. Notable differences from other implementations: - It's an extension to imaplib, rather than a replacement. - It doesn't introduce additional threads. - It doesn't impose new requirements on the use of imaplib's existing methods. - It passes the unit tests in CPython's test/test_imaplib.py module (and adds new ones). - It works on Windows, Linux, and other unix-like systems. - It makes IDLE available on all of imaplib's client variants (including IMAP4_stream). - The interface is pythonic and easy to use. Caveats: - Due to a Windows limitation, the special case of IMAP4_stream running on Windows lacks a duration/timeout feature. (This is the stdin/stdout pipe connection variant; timeouts work fine for socket-based connections, even on Windows.) I have documented it where appropriate. - The file-like imaplib instance attributes are changed from buffered to unbuffered mode. This could potentially break any client code that uses those objects directly without expecting partial reads/writes. However, these attributes are undocumented. As such, I think (and PEP 8 confirms) that they are fair game for changes. https://peps.python.org/pep-0008/#public-and-internal-interfaces Usage examples: https://github.com/python/cpython/issues/55454#issuecomment-2227543041 Original discussion: https://discuss.python.org/t/gauging-interest-in-my-imap4-idle-implementation-for-imaplib/59272 Earlier requests and suggestions: https://github.com/python/cpython/issues/55454 https://mail.python.org/archives/list/python-ideas@python.org/thread/C4TVEYL5IBESQQPPS5GBR7WFBXCLQMZ2/ --- Doc/library/imaplib.rst | 101 ++++- Doc/whatsnew/3.14.rst | 5 + Lib/imaplib.py | 368 +++++++++++++++++- Lib/test/test_imaplib.py | 50 +++ Misc/ACKS | 1 + ...4-08-01-01-00-00.gh-issue-55454.wy0vGw.rst | 1 + 6 files changed, 506 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-08-01-01-00-00.gh-issue-55454.wy0vGw.rst diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index a2dad58b00b9fa..55bb66aad1efa7 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -10,6 +10,7 @@ .. changes for IMAP4_SSL by Tino Lange , March 2002 .. changes for IMAP4_stream by Piers Lauder , November 2002 +.. changes for IDLE by Forest August 2024 **Source code:** :source:`Lib/imaplib.py` @@ -187,7 +188,7 @@ However, the *password* argument to the ``LOGIN`` command is always quoted. If you want to avoid having an argument string quoted (eg: the *flags* argument to ``STORE``) then enclose the string in parentheses (eg: ``r'(\Deleted)'``). -Each command returns a tuple: ``(type, [data, ...])`` where *type* is usually +Most commands return a tuple: ``(type, [data, ...])`` where *type* is usually ``'OK'`` or ``'NO'``, and *data* is either the text from the command response, or mandated results from the command. Each *data* is either a ``bytes``, or a tuple. If a tuple, then the first part is the header of the response, and the @@ -307,6 +308,48 @@ An :class:`IMAP4` instance has the following methods: of the IMAP4 QUOTA extension defined in rfc2087. +.. method:: IMAP4.idle([dur]) + + Return an iterable context manager implementing the ``IDLE`` command + as defined in :rfc:`2177`. + + The optional *dur* argument specifies a maximum duration (in seconds) to + keep idling. It defaults to ``None``, meaning no time limit. + To avoid inactivity timeouts on servers that impose them, callers are + advised to keep this <= 29 minutes. See the note below regarding + :class:`IMAP4_stream` on Windows. + + The context manager sends the ``IDLE`` command upon entry, produces + responses via iteration, and sends ``DONE`` upon exit. + It represents responses as ``(type, datum)`` tuples, rather than the + ``(type, [data, ...])`` tuples returned by other methods, because only + one response is represented at a time. + + Example:: + + with M.idle(dur=29*60) as idler: + for response in idler: + typ, datum = response + print(typ, datum) + + It is also possible to process a burst of responses all at once instead + of one at a time. See `IDLE Context Manager`_ for details. + + Responses produced by the iterator will not be returned by + :meth:`IMAP4.response`. + + .. note:: + + Windows :class:`IMAP4_stream` connections have no way to accurately + respect *dur*, since Windows ``select()`` only works on sockets. + However, if the server regularly sends status messages during ``IDLE``, + they will wake our selector and keep iteration from blocking for long. + Dovecot's ``imap_idle_notify_interval`` is two minutes by default. + Assuming that's typical of IMAP servers, subtracting it from the 29 + minutes needed to avoid server inactivity timeouts would make 27 + minutes a sensible value for *dur* in this situation. + + .. method:: IMAP4.list([directory[, pattern]]) List mailbox names in *directory* matching *pattern*. *directory* defaults to @@ -612,6 +655,62 @@ The following attributes are defined on instances of :class:`IMAP4`: .. versionadded:: 3.5 +.. _idle context manager: + +IDLE Context Manager +-------------------- + +The object returned by :meth:`IMAP4.idle` implements the context management +protocol for the :keyword:`with` statement, and the :term:`iterator` protocol +for retrieving untagged responses while the context is active. +It also has the following method: + +.. method:: IdleContextManager.burst([interval]) + + Yield a burst of responses no more than *interval* seconds apart. + + This generator retrieves the next response along with any + immediately available subsequent responses (e.g. a rapid series of + ``EXPUNGE`` responses after a bulk delete) so they can be efficiently + processed as a batch instead of one at a time. + + The optional *interval* argument specifies a time limit (in seconds) + for each response after the first. It defaults to 0.1 seconds. + (The ``IDLE`` context's maximum duration is respected when waiting for the + first response.) + + Represents responses as ``(type, datum)`` tuples, just as when + iterating directly on the context manager. + + Example:: + + with M.idle() as idler: + + # get the next response and any others following by < 0.1 seconds + batch = list(idler.burst()) + + print(f'processing {len(batch)} responses...') + for typ, datum in batch: + print(typ, datum) + + Produces no responses and returns immediately if the ``IDLE`` context's + maximum duration (the *dur* argument to :meth:`IMAP4.idle`) has elapsed. + Callers should plan accordingly if using this method in a loop. + + .. note:: + + Windows :class:`IMAP4_stream` connections will ignore the *interval* + argument, yielding endless responses and blocking indefinitely for each + one, since Windows ``select()`` only works on sockets. It is therefore + advised not to use this method with an :class:`IMAP4_stream` connection + on Windows. + +.. note:: + + The context manager's type name is not part of its public interface, + and is subject to change. + + .. _imap4-example: IMAP4 Example diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2ab4102f32ab0b..57379b6ef5fd3a 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -150,6 +150,11 @@ Added support for converting any objects that have the :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. (Contributed by Serhiy Storchaka in :gh:`82017`.) +imaplib +------- + +* Add :meth:`~imaplib.IMAP4.idle`, implementing the ``IDLE`` command + as defined in :rfc:`2177`. (Contributed by Forest in :gh:`55454`.) json ---- diff --git a/Lib/imaplib.py b/Lib/imaplib.py index e576c29e67dc0a..7bcbe4912191d8 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -19,10 +19,22 @@ # GET/SETQUOTA contributed by Andreas Zeidler June 2002. # PROXYAUTH contributed by Rick Holbert November 2002. # GET/SETANNOTATION contributed by Tomas Lindroos June 2005. - -__version__ = "2.58" - -import binascii, errno, random, re, socket, subprocess, sys, time, calendar +# IDLE contributed by Forest August 2024. + +__version__ = "2.59" + +import binascii +import calendar +import errno +import functools +import platform +import random +import re +import selectors +import socket +import subprocess +import sys +import time from datetime import datetime, timezone, timedelta from io import DEFAULT_BUFFER_SIZE @@ -74,6 +86,7 @@ 'GETANNOTATION':('AUTH', 'SELECTED'), 'GETQUOTA': ('AUTH', 'SELECTED'), 'GETQUOTAROOT': ('AUTH', 'SELECTED'), + 'IDLE': ('AUTH', 'SELECTED'), 'MYRIGHTS': ('AUTH', 'SELECTED'), 'LIST': ('AUTH', 'SELECTED'), 'LOGIN': ('NONAUTH',), @@ -192,10 +205,13 @@ def __init__(self, host='', port=IMAP4_PORT, timeout=None): self.tagged_commands = {} # Tagged commands awaiting response self.untagged_responses = {} # {typ: [data, ...], ...} self.continuation_response = '' # Last continuation response + self._idle_responses = [] # Response queue for idle iteration + self._idle_capture = False # Whether to queue responses for idle self.is_readonly = False # READ-ONLY desired state self.tagnum = 0 self._tls_established = False self._mode_ascii() + self._readbuf = b'' # Open socket to server. @@ -315,14 +331,58 @@ def open(self, host='', port=IMAP4_PORT, timeout=None): def read(self, size): """Read 'size' bytes from remote.""" - return self.file.read(size) + # Read from an unbuffered input, so our select() calls will not be + # defeated by a hidden library buffer. Use our own buffer instead, + # which can be examined before calling select(). + if isinstance(self, IMAP4_stream): + read = self.readfile.read + else: + read = self.sock.recv + + parts = [] + while True: + if len(self._readbuf) >= size: + parts.append(self._readbuf[:size]) + self._readbuf = self._readbuf[size:] + break + parts.append(self._readbuf) + size -= len(self._readbuf) + self._readbuf = read(DEFAULT_BUFFER_SIZE) + if not self._readbuf: + break + return b''.join(parts) def readline(self): """Read line from remote.""" - line = self.file.readline(_MAXLINE + 1) + # Read from an unbuffered input, so our select() calls will not be + # defeated by a hidden library buffer. Use our own buffer instead, + # which can be examined before calling select(). + if isinstance(self, IMAP4_stream): + read = self.readfile.read + else: + read = self.sock.recv + + LF = b'\n' + parts = [] + length = 0 + while length < _MAXLINE: + try: + pos = self._readbuf.index(LF) + 1 + parts.append(self._readbuf[:pos]) + length += len(parts[-1]) + self._readbuf = self._readbuf[pos:] + break + except ValueError: + parts.append(self._readbuf) + length += len(parts[-1]) + self._readbuf = read(DEFAULT_BUFFER_SIZE) + if not self._readbuf: + break + + line = b''.join(parts) if len(line) > _MAXLINE: - raise self.error("got more than %d bytes" % _MAXLINE) + raise self.error(f'got more than {_MAXLINE} bytes') return line @@ -588,6 +648,44 @@ def getquotaroot(self, mailbox): return typ, [quotaroot, quota] + def idle(self, dur=None): + """Return an iterable context manager implementing the IDLE command + + :param dur: Maximum duration (in seconds) to keep idling, + or None for no time limit. + To avoid inactivity timeouts on servers that impose + them, callers are advised to keep this <= 29 minutes. + See the note below regarding IMAP4_stream on Windows. + :type dur: int|float|None + + The context manager sends the IDLE command upon entry, produces + responses via iteration, and sends DONE upon exit. + It represents responses as (type, datum) tuples, rather than the + (type, [data, ...]) tuples returned by other methods, because only one + response is represented at a time. + + Example: + + with imap.idle(dur=29*60) as idler: + for response in idler: + typ, datum = response + print(typ, datum) + + Responses produced by the iterator are not added to the internal + cache for retrieval by response(). + + Note: Windows IMAP4_stream connections have no way to accurately + respect 'dur', since Windows select() only works on sockets. + However, if the server regularly sends status messages during IDLE, + they will wake our selector and keep iteration from blocking for long. + Dovecot's imap_idle_notify_interval is two minutes by default. + Assuming that's typical of IMAP servers, subtracting it from the 29 + minutes needed to avoid server inactivity timeouts would make 27 + minutes a sensible value for 'dur' in this situation. + """ + return _Idler(self, dur) + + def list(self, directory='""', pattern='*'): """List mailbox names in directory matching pattern. @@ -944,6 +1042,14 @@ def xatom(self, name, *args): def _append_untagged(self, typ, dat): if dat is None: dat = b'' + + # During idle, queue untagged responses for delivery via iteration + if self._idle_capture: + self._idle_responses.append((typ, dat)) + if __debug__ and self.debug >= 5: + self._mesg(f'idle: queue untagged {typ} {dat!r}') + return + ur = self.untagged_responses if __debug__: if self.debug >= 5: @@ -1279,6 +1385,236 @@ def print_log(self): n -= 1 +class _Idler: + # Iterable context manager: start IDLE & produce untagged responses + # + # This iterator produces (type, datum) tuples. They slightly differ + # from the tuples returned by IMAP4.response(): The second item in the + # tuple is a single datum, rather than a list of them, because only one + # untagged response is produced at a time. + + def __init__(self, imap, dur=None): + if 'IDLE' not in imap.capabilities: + raise imap.error("Server does not support IDLE") + self._dur = dur + self._imap = imap + self._tag = None + self._sock_timeout = None + self._old_state = None + + def __enter__(self): + imap = self._imap + assert not (imap._idle_responses or imap._idle_capture) + + if __debug__ and imap.debug >= 4: + imap._mesg('idle start' + + ('' if self._dur is None else f' dur={self._dur}')) + + try: + # Start capturing untagged responses before sending IDLE, + # so we can deliver via iteration any that arrive while + # the IDLE command continuation request is still pending. + imap._idle_capture = True + + self._tag = imap._command('IDLE') + # Process responses until the server requests continuation + while resp := imap._get_response(): # Returns None on continuation + if imap.tagged_commands[self._tag]: + raise imap.abort(f'unexpected status response: {resp}') + + if __debug__ and imap.debug >= 4: + prompt = imap.continuation_response + imap._mesg(f'idle continuation prompt: {prompt}') + except: + imap._idle_capture = False + raise + + self._sock_timeout = imap.sock.gettimeout() if imap.sock else None + if self._sock_timeout is not None: + imap.sock.settimeout(None) # Socket timeout would break IDLE + + self._old_state = imap.state + imap.state = 'IDLING' + + return self + + def __iter__(self): + return self + + def _wait(self, timeout=None): + # Block until the next read operation should be attempted, either + # because data becomes availalable within 'timeout' seconds or + # because the OS cannot determine whether data is available. + # Return True when a blocking read() is worth trying + # Return False if the timeout expires while waiting + + imap = self._imap + if timeout is None: + return True + if imap._readbuf: + return True + if timeout <= 0: + return False + + if imap.sock: + fileobj = imap.sock + elif platform.system() == 'Windows': + return True # Cannot select(); allow a possibly-blocking read + else: + fileobj = imap.readfile + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _wait select({timeout})') + + with selectors.DefaultSelector() as sel: + sel.register(fileobj, selectors.EVENT_READ) + readables = sel.select(timeout) + return bool(readables) + + def _pop(self, timeout, default=('', None)): + # Get the next response, or a default value on timeout + # + # :param timeout: Time limit (in seconds) to wait for response + # :type timeout: int|float|None + # :param default: Value to return on timeout + # + # Note: This method ignores 'dur' in favor of the timeout argument. + # + # Note: Windows IMAP4_stream connections will ignore the timeout + # argument and block until the next response arrives, because + # Windows select() only works on sockets. + + imap = self._imap + if imap.state != 'IDLING': + raise imap.error('_pop() only works during IDLE') + + if imap._idle_responses: + resp = imap._idle_responses.pop(0) + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) de-queued {resp[0]}') + return resp + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout})') + + if not self._wait(timeout): + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) done') + return default + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) reading') + imap._get_response() # Reads line, calls _append_untagged() + resp = imap._idle_responses.pop(0) + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) read {resp[0]}') + return resp + + def burst(self, interval=0.1): + """Yield a burst of responses no more than 'interval' seconds apart + + :param interval: Time limit for each response after the first + (The IDLE context's maximum duration is + respected when waiting for the first response.) + :type interval: int|float + + This generator retrieves the next response along with any + immediately available subsequent responses (e.g. a rapid series of + EXPUNGE responses after a bulk delete) so they can be efficiently + processed as a batch instead of one at a time. + + Represents responses as (type, datum) tuples, just as when + iterating directly on the context manager. + + Example: + + with imap.idle() as idler: + batch = list(idler.burst()) + print(f'processing {len(batch)} responses...') + + Produces no responses and returns immediately if the IDLE + context's maximum duration (the 'dur' argument) has elapsed. + Callers should plan accordingly if using this method in a loop. + + Note: Windows IMAP4_stream connections will ignore the interval + argument, yielding endless responses and blocking indefinitely + for each one, because Windows select() only works on sockets. + It is therefore advised not to use this method with an IMAP4_stream + connection on Windows. + """ + try: + yield next(self) + except StopIteration: + return + + start = time.monotonic() + + yield from iter(functools.partial(self._pop, interval, None), None) + + if self._dur is not None: + elapsed = time.monotonic() - start + self._dur = max(self._dur - elapsed, 0) + + def __next__(self): + imap = self._imap + start = time.monotonic() + + typ, datum = self._pop(self._dur) + + if self._dur is not None: + elapsed = time.monotonic() - start + self._dur = max(self._dur - elapsed, 0) + + if not typ: + if __debug__ and imap.debug >= 4: + imap._mesg('idle iterator exhausted') + raise StopIteration + + return typ, datum + + def __exit__(self, exc_type, exc_val, exc_tb): + imap = self._imap + + if __debug__ and imap.debug >= 4: + imap._mesg('idle done') + imap.state = self._old_state + + if self._sock_timeout is not None: + imap.sock.settimeout(self._sock_timeout) + self._sock_timeout = None + + # Stop intercepting untagged responses before sending DONE, + # since we can no longer deliver them via iteration. + imap._idle_capture = False + + # If we captured untagged responses while the IDLE command + # continuation request was still pending, but the user did not + # iterate over them before exiting IDLE, we must put them + # someplace where the user can retrieve them. The only + # sensible place for this is the untagged_responses dict, + # despite its unfortunate inability to preserve the relative + # order of different response types. + if leftovers := len(imap._idle_responses): + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle quit with {leftovers} leftover responses') + while imap._idle_responses: + typ, datum = imap._idle_responses.pop(0) + imap._append_untagged(typ, datum) + + try: + imap.send(b'DONE' + CRLF) + status, [msg] = imap._command_complete('IDLE', self._tag) + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle status: {status} {msg!r}') + + except OSError: + if not exc_type: + raise + + return False # Do not suppress context body exceptions + + if HAVE_SSL: class IMAP4_SSL(IMAP4): @@ -1348,26 +1684,20 @@ def open(self, host=None, port=None, timeout=None): self.sock = None self.file = None self.process = subprocess.Popen(self.command, - bufsize=DEFAULT_BUFFER_SIZE, + bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, close_fds=True) self.writefile = self.process.stdin self.readfile = self.process.stdout - def read(self, size): - """Read 'size' bytes from remote.""" - return self.readfile.read(size) - - - def readline(self): - """Read line from remote.""" - return self.readfile.readline() - def send(self, data): """Send data to remote.""" - self.writefile.write(data) - self.writefile.flush() + # Write with buffered semantics to the unbuffered output, avoiding + # partial writes. + sent = 0 + while sent < len(data): + sent += self.writefile.write(data[sent:]) def shutdown(self): diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 1fd75d0a3f4c7b..374a07f2e59108 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -497,6 +497,56 @@ def test_with_statement_logout(self): # command tests + def test_idle_capability(self): + client, _ = self._setup(SimpleIMAPHandler) + with self.assertRaisesRegex(imaplib.IMAP4.error, + 'does not support IDLE'): + with client.idle(): + pass + + class IdleCmdHandler(SimpleIMAPHandler): + capabilities = 'IDLE' + def cmd_IDLE(self, tag, args): + self._send_textline('+ idling') + self._send_line(b'* 2 EXISTS') + self._send_line(b'* 0 RECENT') + time.sleep(1) + self._send_line(b'* 1 RECENT') + r = yield + if r == b'DONE\r\n': + self._send_tagged(tag, 'OK', 'Idle completed') + else: + self._send_tagged(tag, 'BAD', 'Expected DONE') + + def test_idle_iter(self): + client, _ = self._setup(self.IdleCmdHandler) + client.login('user', 'pass') + with client.idle() as idler: + # iteration should produce responses + typ, datum = next(idler) + self.assertEqual(typ, 'EXISTS') + self.assertEqual(datum, b'2') + typ, datum = next(idler) + self.assertEqual(typ, 'RECENT') + self.assertEqual(datum, b'0') + # iteration should have consumed untagged responses + _, data = client.response('EXISTS') + self.assertEqual(data, [None]) + # responses not iterated should remain after idle + _, data = client.response('RECENT') + self.assertEqual(data, [b'1']) + + def test_idle_burst(self): + client, _ = self._setup(self.IdleCmdHandler) + client.login('user', 'pass') + # burst() should yield immediately available responses + with client.idle() as idler: + batch = list(idler.burst()) + self.assertEqual(len(batch), 2) + # burst() should not have consumed later responses + _, data = client.response('RECENT') + self.assertEqual(data, [b'1']) + def test_login(self): client, _ = self._setup(SimpleIMAPHandler) typ, data = client.login('user', 'pass') diff --git a/Misc/ACKS b/Misc/ACKS index b031eb7c11f73f..c4605c8de2016c 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -572,6 +572,7 @@ Benjamin Fogle Artem Fokin Arnaud Fontaine Michael Foord +Forest Amaury Forgeot d'Arc Doug Fort Daniel Fortunov diff --git a/Misc/NEWS.d/next/Library/2024-08-01-01-00-00.gh-issue-55454.wy0vGw.rst b/Misc/NEWS.d/next/Library/2024-08-01-01-00-00.gh-issue-55454.wy0vGw.rst new file mode 100644 index 00000000000000..58fc85963217c9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-01-01-00-00.gh-issue-55454.wy0vGw.rst @@ -0,0 +1 @@ +Add IMAP4 ``IDLE`` support to the :mod:`imaplib` module. Patch by Forest. From dc71241eda80059238c356ed77bbd50fc1997e60 Mon Sep 17 00:00:00 2001 From: Forest Date: Wed, 11 Sep 2024 15:10:49 -0700 Subject: [PATCH 02/78] gh-55454: Clarify imaplib idle() docs - Add example idle response tuples, to make the minor difference from other imaplib response tuples more obvious. - Merge the idle context manager's burst() method docs with the IMAP object's idle() method docs, for easier understanding. - Upgrade the Windows note regarding lack of pipe timeouts to a warning. - Rephrase various things for clarity. --- Doc/library/imaplib.rst | 143 ++++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 78 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 55bb66aad1efa7..87f4b4fb9062de 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -308,22 +308,28 @@ An :class:`IMAP4` instance has the following methods: of the IMAP4 QUOTA extension defined in rfc2087. -.. method:: IMAP4.idle([dur]) +.. method:: IMAP4.idle(dur=None) Return an iterable context manager implementing the ``IDLE`` command as defined in :rfc:`2177`. - The optional *dur* argument specifies a maximum duration (in seconds) to - keep idling. It defaults to ``None``, meaning no time limit. - To avoid inactivity timeouts on servers that impose them, callers are - advised to keep this <= 29 minutes. See the note below regarding + The context manager sends the ``IDLE`` command when activated by the + :keyword:`with` statement, produces IMAP untagged responses via the + :term:`iterator` protocol, and sends ``DONE`` upon context exit. + + The *dur* argument sets a maximum duration (in seconds) to keep idling, + after which iteration will stop. It defaults to ``None``, meaning no time + limit. Callers wishing to avoid inactivity timeouts on servers that impose + them should keep this <= 29 minutes. + See the :ref:`warning below ` if using :class:`IMAP4_stream` on Windows. - The context manager sends the ``IDLE`` command upon entry, produces - responses via iteration, and sends ``DONE`` upon exit. - It represents responses as ``(type, datum)`` tuples, rather than the - ``(type, [data, ...])`` tuples returned by other methods, because only - one response is represented at a time. + Response tuples produced by the iterator almost exactly match those + returned by other imaplib methods. The difference is that the tuple's + second member is a single response datum, rather than a list of data. + Therefore, in a mailbox where calling ``M.response('EXISTS')`` would + return ``('EXISTS', [b'1'])``, the idle iterator would produce + ``('EXISTS', b'1')``. Example:: @@ -332,22 +338,59 @@ An :class:`IMAP4` instance has the following methods: typ, datum = response print(typ, datum) - It is also possible to process a burst of responses all at once instead - of one at a time. See `IDLE Context Manager`_ for details. + ('EXISTS', b'1') + ('RECENT', b'1') - Responses produced by the iterator will not be returned by - :meth:`IMAP4.response`. + Instead of iterating one response at a time, it is also possible to retrieve + the next response along with any immediately available subsequent responses + (e.g. a rapid series of ``EXPUNGE`` events from a bulk delete). This + batch processing aid is provided by the context's ``burst()`` + :term:`generator`: - .. note:: + .. method:: idler.burst(interval=0.1) + + Yield a burst of responses no more than *interval* seconds apart. + + Example:: + + with M.idle() as idler: + + # get the next response and any others following by < 0.1 seconds + batch = list(idler.burst()) + + print(f'processing {len(batch)} responses...') + print(batch) + + processing 3 responses... + [('EXPUNGE', b'2'), ('EXPUNGE', b'1'), ('RECENT', b'0')] + + The ``IDLE`` context's maximum duration (the *dur* argument to + :meth:`IMAP4.idle`) is respected when waiting for the first response + in a burst. Therefore, an expired idle context will cause this generator + to return immediately without producing anything. Callers should + consider this if using it in a loop. + + + .. _windows-pipe-timeout-warning: + + .. warning:: Windows :class:`IMAP4_stream` connections have no way to accurately - respect *dur*, since Windows ``select()`` only works on sockets. - However, if the server regularly sends status messages during ``IDLE``, - they will wake our selector and keep iteration from blocking for long. - Dovecot's ``imap_idle_notify_interval`` is two minutes by default. - Assuming that's typical of IMAP servers, subtracting it from the 29 - minutes needed to avoid server inactivity timeouts would make 27 - minutes a sensible value for *dur* in this situation. + respect the *dur* or *interval* arguments, since Windows ``select()`` + only works on sockets. + + If the server regularly sends status messages during ``IDLE``, they will + wake our iterator anyway, allowing *dur* to behave roughly as intended, + although usually late. Dovecot's ``imap_idle_notify_interval`` default + setting does this every 2 minutes. Assuming that's typical of IMAP + servers, subtracting it from the 29 minutes needed to avoid server + inactivity timeouts would make 27 minutes a sensible value for *dur* in + this situation. + + There is no such fallback for ``burst()``, which will yield endless + responses and block indefinitely for each one. It is therefore advised + not to use ``burst()`` with an :class:`IMAP4_stream` connection on + Windows. .. method:: IMAP4.list([directory[, pattern]]) @@ -655,62 +698,6 @@ The following attributes are defined on instances of :class:`IMAP4`: .. versionadded:: 3.5 -.. _idle context manager: - -IDLE Context Manager --------------------- - -The object returned by :meth:`IMAP4.idle` implements the context management -protocol for the :keyword:`with` statement, and the :term:`iterator` protocol -for retrieving untagged responses while the context is active. -It also has the following method: - -.. method:: IdleContextManager.burst([interval]) - - Yield a burst of responses no more than *interval* seconds apart. - - This generator retrieves the next response along with any - immediately available subsequent responses (e.g. a rapid series of - ``EXPUNGE`` responses after a bulk delete) so they can be efficiently - processed as a batch instead of one at a time. - - The optional *interval* argument specifies a time limit (in seconds) - for each response after the first. It defaults to 0.1 seconds. - (The ``IDLE`` context's maximum duration is respected when waiting for the - first response.) - - Represents responses as ``(type, datum)`` tuples, just as when - iterating directly on the context manager. - - Example:: - - with M.idle() as idler: - - # get the next response and any others following by < 0.1 seconds - batch = list(idler.burst()) - - print(f'processing {len(batch)} responses...') - for typ, datum in batch: - print(typ, datum) - - Produces no responses and returns immediately if the ``IDLE`` context's - maximum duration (the *dur* argument to :meth:`IMAP4.idle`) has elapsed. - Callers should plan accordingly if using this method in a loop. - - .. note:: - - Windows :class:`IMAP4_stream` connections will ignore the *interval* - argument, yielding endless responses and blocking indefinitely for each - one, since Windows ``select()`` only works on sockets. It is therefore - advised not to use this method with an :class:`IMAP4_stream` connection - on Windows. - -.. note:: - - The context manager's type name is not part of its public interface, - and is subject to change. - - .. _imap4-example: IMAP4 Example From f41f2bbb5892ed6960e70af1a66ea64ac74a6358 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 21 Sep 2024 10:39:53 -0700 Subject: [PATCH 03/78] docs: words instead of <= Co-authored-by: Peter Bierma --- Doc/library/imaplib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 87f4b4fb9062de..936040c47d29d5 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -320,7 +320,7 @@ An :class:`IMAP4` instance has the following methods: The *dur* argument sets a maximum duration (in seconds) to keep idling, after which iteration will stop. It defaults to ``None``, meaning no time limit. Callers wishing to avoid inactivity timeouts on servers that impose - them should keep this <= 29 minutes. + them should keep this at most 29 minutes. See the :ref:`warning below ` if using :class:`IMAP4_stream` on Windows. From 8077f2eab287703b77350f1bfc9db2bd236dd9a7 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 21 Sep 2024 10:40:51 -0700 Subject: [PATCH 04/78] docs: improve style in an example Co-authored-by: Peter Bierma --- Doc/library/imaplib.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 936040c47d29d5..5fdea0b5b33fd7 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -333,9 +333,8 @@ An :class:`IMAP4` instance has the following methods: Example:: - with M.idle(dur=29*60) as idler: - for response in idler: - typ, datum = response + with M.idle(dur=29 * 60) as idler: + for type, datum in idler: print(typ, datum) ('EXISTS', b'1') From 24fcdbb9565da401f49b34bb43ac30aa7405cf57 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 21 Sep 2024 10:42:23 -0700 Subject: [PATCH 05/78] docs: grammatical edit Co-authored-by: Peter Bierma --- Doc/library/imaplib.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 5fdea0b5b33fd7..6c320c2928aedb 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -374,8 +374,8 @@ An :class:`IMAP4` instance has the following methods: .. warning:: - Windows :class:`IMAP4_stream` connections have no way to accurately - respect the *dur* or *interval* arguments, since Windows ``select()`` + Windows' :class:`IMAP4_stream` connections have no way to accurately + respect the *dur* or *interval* arguments, since Windows' ``select()`` only works on sockets. If the server regularly sends status messages during ``IDLE``, they will From 2c76b2fa81e51bfbd09a21437cf9aa3be07efe52 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 21 Sep 2024 10:43:17 -0700 Subject: [PATCH 06/78] docs consistency Co-authored-by: Peter Bierma --- Doc/library/imaplib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 6c320c2928aedb..5c999ab7e93fce 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -365,7 +365,7 @@ An :class:`IMAP4` instance has the following methods: The ``IDLE`` context's maximum duration (the *dur* argument to :meth:`IMAP4.idle`) is respected when waiting for the first response - in a burst. Therefore, an expired idle context will cause this generator + in a burst. Therefore, an expired ``IDLE`` context will cause this generator to return immediately without producing anything. Callers should consider this if using it in a loop. From 5ef5bb2dcb0db52a8bdf4f30708f44cf59695546 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 21 Sep 2024 10:47:50 -0700 Subject: [PATCH 07/78] comment -> docstring Co-authored-by: Peter Bierma --- Lib/imaplib.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 7bcbe4912191d8..0ccb220868c95e 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1472,17 +1472,17 @@ def _wait(self, timeout=None): return bool(readables) def _pop(self, timeout, default=('', None)): - # Get the next response, or a default value on timeout - # - # :param timeout: Time limit (in seconds) to wait for response - # :type timeout: int|float|None - # :param default: Value to return on timeout - # - # Note: This method ignores 'dur' in favor of the timeout argument. - # - # Note: Windows IMAP4_stream connections will ignore the timeout - # argument and block until the next response arrives, because - # Windows select() only works on sockets. + """Get the next response, or a default value on timeout + + :param timeout: Time limit (in seconds) to wait for response + :type timeout: int|float|None + :param default: Value to return on timeout + + Note: This method ignores 'dur' in favor of the timeout argument. + + Note: Windows IMAP4_stream connections will ignore the timeout + argument and block until the next response arrives, because + Windows select() only works on sockets.""" imap = self._imap if imap.state != 'IDLING': From 91266e0a7c58c589996c00427ed825490dd603d2 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 22 Sep 2024 18:45:40 +0000 Subject: [PATCH 08/78] docs: refer to imaplib as "this module" Co-authored-by: Peter Bierma --- Doc/library/imaplib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 5c999ab7e93fce..6731c60228a04b 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -325,7 +325,7 @@ An :class:`IMAP4` instance has the following methods: :class:`IMAP4_stream` on Windows. Response tuples produced by the iterator almost exactly match those - returned by other imaplib methods. The difference is that the tuple's + returned by other methods in this module. The difference is that the tuple's second member is a single response datum, rather than a list of data. Therefore, in a mailbox where calling ``M.response('EXISTS')`` would return ``('EXISTS', [b'1'])``, the idle iterator would produce From 19c98dc2db1334bd3145fd3a924e821c6deb6fd6 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 22 Sep 2024 19:52:14 +0000 Subject: [PATCH 09/78] imaplib: simplify & clarify idle debug message Co-authored-by: Peter Bierma --- Lib/imaplib.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 0ccb220868c95e..25c3e017fe4be2 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1407,8 +1407,7 @@ def __enter__(self): assert not (imap._idle_responses or imap._idle_capture) if __debug__ and imap.debug >= 4: - imap._mesg('idle start' - + ('' if self._dur is None else f' dur={self._dur}')) + imap._mesg(f'idle start duration={self._dur}') try: # Start capturing untagged responses before sending IDLE, From 882bf2c85e027cecd2d5ff6cf1c27f33b0b80e72 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 22 Sep 2024 13:33:14 -0700 Subject: [PATCH 10/78] imaplib: elaborate in idle context manager comment --- Lib/imaplib.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 25c3e017fe4be2..718c53483e3bec 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1416,8 +1416,13 @@ def __enter__(self): imap._idle_capture = True self._tag = imap._command('IDLE') - # Process responses until the server requests continuation - while resp := imap._get_response(): # Returns None on continuation + # As with any command, the server is allowed to send us unrelated, + # untagged responses before acting on IDLE. These lines will be + # returned by _get_response(). When the server is ready, it will + # send an IDLE continuation request, indicated by _get_response() + # returning None. We therefore process responses in a loop until + # this occurs. + while resp := imap._get_response(): if imap.tagged_commands[self._tag]: raise imap.abort(f'unexpected status response: {resp}') From f648fec1608d18bba9a140929864ea37c34837f4 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 22 Sep 2024 21:37:47 +0000 Subject: [PATCH 11/78] imaplib: re-raise BaseException instead of bare except Co-authored-by: Peter Bierma --- Lib/imaplib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 718c53483e3bec..3300d34534bcc2 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1429,7 +1429,7 @@ def __enter__(self): if __debug__ and imap.debug >= 4: prompt = imap.continuation_response imap._mesg(f'idle continuation prompt: {prompt}') - except: + except BaseException: imap._idle_capture = False raise From 48f6f760736a8346fa6649aea46acd4c4acc2378 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 22 Sep 2024 15:11:49 -0700 Subject: [PATCH 12/78] imaplib: convert private doc string to comment --- Lib/imaplib.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 3300d34534bcc2..fec352e9b8884f 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1476,17 +1476,17 @@ def _wait(self, timeout=None): return bool(readables) def _pop(self, timeout, default=('', None)): - """Get the next response, or a default value on timeout - - :param timeout: Time limit (in seconds) to wait for response - :type timeout: int|float|None - :param default: Value to return on timeout - - Note: This method ignores 'dur' in favor of the timeout argument. - - Note: Windows IMAP4_stream connections will ignore the timeout - argument and block until the next response arrives, because - Windows select() only works on sockets.""" + # Get the next response, or a default value on timeout + # + # :param timeout: Time limit (in seconds) to wait for response + # :type timeout: int|float|None + # :param default: Value to return on timeout + # + # Note: This method ignores 'dur' in favor of the timeout argument. + # + # Note: Windows IMAP4_stream connections will ignore the timeout + # argument and block until the next response arrives, because + # Windows select() only works on sockets. imap = self._imap if imap.state != 'IDLING': From 013bbf18fc421956041416ce7c6ff98d9d137b5c Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 22 Sep 2024 16:47:22 -0700 Subject: [PATCH 13/78] docs: correct mistake in imaplib example This is a correction to 8077f2eab287703b77350f1bfc9db2bd236dd9a7, which changed a variable name in only one place and broke the subsequent reference to it, departed from the naming convention used in the rest of the module, and shadowed the type() builtin along the way. --- Doc/library/imaplib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 6731c60228a04b..757cd715704a04 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -334,7 +334,7 @@ An :class:`IMAP4` instance has the following methods: Example:: with M.idle(dur=29 * 60) as idler: - for type, datum in idler: + for typ, datum in idler: print(typ, datum) ('EXISTS', b'1') From acbc4a1fc8eec139de5240cc7efea165b4622c86 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 23 Sep 2024 14:53:59 -0700 Subject: [PATCH 14/78] imaplib: simplify example code in doc string This is for consistency with the documentation change in 8077f2eab287 and subsequent correction in 013bbf18fc42. --- Lib/imaplib.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index fec352e9b8884f..bbb8443121533a 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -667,8 +667,7 @@ def idle(self, dur=None): Example: with imap.idle(dur=29*60) as idler: - for response in idler: - typ, datum = response + for typ, datum in idler: print(typ, datum) Responses produced by the iterator are not added to the internal From 94c02e88a9d3c827624789abb4ad2e0c1fdbfa0a Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 23 Sep 2024 14:59:26 -0700 Subject: [PATCH 15/78] imaplib: rename _Idler to Idler, update its docs --- Doc/library/imaplib.rst | 13 ++++++++++--- Lib/imaplib.py | 34 +++++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 757cd715704a04..6f765bbed1ccf4 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -310,8 +310,8 @@ An :class:`IMAP4` instance has the following methods: .. method:: IMAP4.idle(dur=None) - Return an iterable context manager implementing the ``IDLE`` command - as defined in :rfc:`2177`. + Return an ``Idler``: an iterable context manager implementing the ``IDLE`` + command as defined in :rfc:`2177`. The context manager sends the ``IDLE`` command when activated by the :keyword:`with` statement, produces IMAP untagged responses via the @@ -346,7 +346,7 @@ An :class:`IMAP4` instance has the following methods: batch processing aid is provided by the context's ``burst()`` :term:`generator`: - .. method:: idler.burst(interval=0.1) + .. method:: Idler.burst(interval=0.1) Yield a burst of responses no more than *interval* seconds apart. @@ -391,6 +391,13 @@ An :class:`IMAP4` instance has the following methods: not to use ``burst()`` with an :class:`IMAP4_stream` connection on Windows. + .. note:: + + Note: The ``Idler`` class name and structure are internal interfaces, + subject to change. Calling code can rely on its context management, + iteration, and public method to remain stable, but should not + subclass, instantiate, or otherwise directly reference the class. + .. method:: IMAP4.list([directory[, pattern]]) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index bbb8443121533a..596b3d3f34da8f 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -649,7 +649,7 @@ def getquotaroot(self, mailbox): def idle(self, dur=None): - """Return an iterable context manager implementing the IDLE command + """Return an Idler: an iterable context manager for the IDLE command :param dur: Maximum duration (in seconds) to keep idling, or None for no time limit. @@ -681,8 +681,13 @@ def idle(self, dur=None): Assuming that's typical of IMAP servers, subtracting it from the 29 minutes needed to avoid server inactivity timeouts would make 27 minutes a sensible value for 'dur' in this situation. + + Note: The Idler class name and structure are internal interfaces, + subject to change. Calling code can rely on its context management, + iteration, and public method to remain stable, but should not + subclass, instantiate, or otherwise directly reference the class. """ - return _Idler(self, dur) + return Idler(self, dur) def list(self, directory='""', pattern='*'): @@ -1384,13 +1389,24 @@ def print_log(self): n -= 1 -class _Idler: - # Iterable context manager: start IDLE & produce untagged responses - # - # This iterator produces (type, datum) tuples. They slightly differ - # from the tuples returned by IMAP4.response(): The second item in the - # tuple is a single datum, rather than a list of them, because only one - # untagged response is produced at a time. +class Idler: + """Iterable context manager: start IDLE & produce untagged responses + + An object of this type is returned by the IMAP4.idle() method. + It sends the IDLE command when activated by the 'with' statement, produces + IMAP untagged responses via the iterator protocol, and sends DONE upon + context exit. + + Iteration produces (type, datum) tuples. They slightly differ + from the tuples returned by IMAP4.response(): The second item in the + tuple is a single datum, rather than a list of them, because only one + untagged response is produced at a time. + + Note: The name and structure of this class are internal interfaces, + subject to change. Calling code can rely on its context management, + iteration, and public method to remain stable, but should not + subclass, instantiate, or otherwise directly reference the class. + """ def __init__(self, imap, dur=None): if 'IDLE' not in imap.capabilities: From 0c6c9a494fa09a8e5d4fd87d9c9d3a399768767e Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 23 Sep 2024 22:25:28 +0000 Subject: [PATCH 16/78] imaplib: add comment in Idler._pop() Co-authored-by: Peter Bierma --- Lib/imaplib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 596b3d3f34da8f..1ae2dee224c833 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1508,6 +1508,7 @@ def _pop(self, timeout, default=('', None)): raise imap.error('_pop() only works during IDLE') if imap._idle_responses: + # Response is ready to return to the user resp = imap._idle_responses.pop(0) if __debug__ and imap.debug >= 4: imap._mesg(f'idle _pop({timeout}) de-queued {resp[0]}') From 223c2faa782793ac39e939aab7a1af42f7e9e390 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 23 Sep 2024 22:27:42 +0000 Subject: [PATCH 17/78] imaplib: remove unnecessary blank line Co-authored-by: Peter Bierma --- Lib/imaplib.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 1ae2dee224c833..3f5f3afdda8f5e 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1627,7 +1627,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): status, [msg] = imap._command_complete('IDLE', self._tag) if __debug__ and imap.debug >= 4: imap._mesg(f'idle status: {status} {msg!r}') - except OSError: if not exc_type: raise From e64546cbaec3020fc7c6db5320e852e40f28513b Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 23 Sep 2024 16:04:38 -0700 Subject: [PATCH 18/78] imaplib: comment on use of unbuffered pipes --- Lib/imaplib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 3f5f3afdda8f5e..b006a74a819040 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1703,7 +1703,7 @@ def open(self, host=None, port=None, timeout=None): self.sock = None self.file = None self.process = subprocess.Popen(self.command, - bufsize=0, + bufsize=0, # Unbuffered stdin/stdout, for select() compatibility stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, close_fds=True) self.writefile = self.process.stdin From f385e441df15d962d1f22e9bab2f15a39e5363d5 Mon Sep 17 00:00:00 2001 From: Forest Date: Tue, 24 Sep 2024 18:30:15 +0000 Subject: [PATCH 19/78] docs: imaplib: use the reStructuredText :class: role Co-authored-by: Peter Bierma --- Doc/library/imaplib.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 6f765bbed1ccf4..a8de32ad07a873 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -310,7 +310,7 @@ An :class:`IMAP4` instance has the following methods: .. method:: IMAP4.idle(dur=None) - Return an ``Idler``: an iterable context manager implementing the ``IDLE`` + Return an :class:`Idler`: an iterable context manager implementing the ``IDLE`` command as defined in :rfc:`2177`. The context manager sends the ``IDLE`` command when activated by the @@ -393,7 +393,7 @@ An :class:`IMAP4` instance has the following methods: .. note:: - Note: The ``Idler`` class name and structure are internal interfaces, + Note: The :class:`Idler` class name and structure are internal interfaces, subject to change. Calling code can rely on its context management, iteration, and public method to remain stable, but should not subclass, instantiate, or otherwise directly reference the class. From 3aceaecdf9e4290e4fa1c5dfa137605eaaf93682 Mon Sep 17 00:00:00 2001 From: Forest Date: Tue, 24 Sep 2024 12:23:29 -0700 Subject: [PATCH 20/78] Revert "docs: imaplib: use the reStructuredText :class: role" This reverts commit f385e441df15d962d1f22e9bab2f15a39e5363d5, because it triggers CI failures in the docs by referencing a class that is (deliberately) undocumented. --- Doc/library/imaplib.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index a8de32ad07a873..6f765bbed1ccf4 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -310,7 +310,7 @@ An :class:`IMAP4` instance has the following methods: .. method:: IMAP4.idle(dur=None) - Return an :class:`Idler`: an iterable context manager implementing the ``IDLE`` + Return an ``Idler``: an iterable context manager implementing the ``IDLE`` command as defined in :rfc:`2177`. The context manager sends the ``IDLE`` command when activated by the @@ -393,7 +393,7 @@ An :class:`IMAP4` instance has the following methods: .. note:: - Note: The :class:`Idler` class name and structure are internal interfaces, + Note: The ``Idler`` class name and structure are internal interfaces, subject to change. Calling code can rely on its context management, iteration, and public method to remain stable, but should not subclass, instantiate, or otherwise directly reference the class. From c8d4d6d994e0bbb5406100eabbfea9199037182d Mon Sep 17 00:00:00 2001 From: Forest Date: Thu, 26 Sep 2024 12:02:44 -0700 Subject: [PATCH 21/78] docs: imaplib: use the reST :class: role, escaped This is a different approach to f385e441df15, which was reverted for creating dangling link references. By prefixing the reStructuredText role target with a ! we disable conversion to a link, thereby passing continuous integration checks even though the referenced class is deliberately absent from the documentation. --- Doc/library/imaplib.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 6f765bbed1ccf4..ce47db49c15047 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -310,7 +310,7 @@ An :class:`IMAP4` instance has the following methods: .. method:: IMAP4.idle(dur=None) - Return an ``Idler``: an iterable context manager implementing the ``IDLE`` + Return an :class:`!Idler`: an iterable context manager implementing the ``IDLE`` command as defined in :rfc:`2177`. The context manager sends the ``IDLE`` command when activated by the @@ -393,7 +393,7 @@ An :class:`IMAP4` instance has the following methods: .. note:: - Note: The ``Idler`` class name and structure are internal interfaces, + Note: The :class:`!Idler` class name and structure are internal interfaces, subject to change. Calling code can rely on its context management, iteration, and public method to remain stable, but should not subclass, instantiate, or otherwise directly reference the class. From c7ed3c5cf57a1c7cbb6bb0c6df0b21a7dc5fd5a3 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 1 Dec 2024 20:03:53 +0000 Subject: [PATCH 22/78] docs: refer to IMAP4 IDLE instead of just IDLE This clarifies that we are referring to the email protocol, not the editor with the same name. Co-authored-by: Guido van Rossum --- Doc/library/imaplib.rst | 4 ++-- Doc/whatsnew/3.14.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index ce47db49c15047..329b3ce6825b76 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -10,7 +10,7 @@ .. changes for IMAP4_SSL by Tino Lange , March 2002 .. changes for IMAP4_stream by Piers Lauder , November 2002 -.. changes for IDLE by Forest August 2024 +.. changes for IMAP4 IDLE by Forest August 2024 **Source code:** :source:`Lib/imaplib.py` @@ -310,7 +310,7 @@ An :class:`IMAP4` instance has the following methods: .. method:: IMAP4.idle(dur=None) - Return an :class:`!Idler`: an iterable context manager implementing the ``IDLE`` + Return an :class:`!Idler`: an iterable context manager implementing the IMAP4 ``IDLE`` command as defined in :rfc:`2177`. The context manager sends the ``IDLE`` command when activated by the diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 6f92ecef4f3503..302c8407d148ad 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -406,7 +406,7 @@ inspect imaplib ------- -* Add :meth:`~imaplib.IMAP4.idle`, implementing the ``IDLE`` command +* Add :meth:`~imaplib.IMAP4.idle`, implementing the IMAP4 ``IDLE`` command as defined in :rfc:`2177`. (Contributed by Forest in :gh:`55454`.) json From b01de95171d6124f8acc7b907c1842472ea5f5fb Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 1 Dec 2024 21:52:51 +0000 Subject: [PATCH 23/78] imaplib: IDLE -> IMAP4 IDLE in exception message Co-authored-by: Peter Bierma --- Lib/imaplib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index b006a74a819040..c64dd153e54599 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1410,7 +1410,7 @@ class Idler: def __init__(self, imap, dur=None): if 'IDLE' not in imap.capabilities: - raise imap.error("Server does not support IDLE") + raise imap.error("Server does not support IMAP4 IDLE") self._dur = dur self._imap = imap self._tag = None From a3f21cd75b4d7c97c0d7e46b0a0cc0875e29f6cc Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 1 Dec 2024 13:56:48 -0800 Subject: [PATCH 24/78] docs: imaplib idle() phrasing and linking tweaks --- Doc/library/imaplib.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 329b3ce6825b76..c37ea9f60848db 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -313,14 +313,14 @@ An :class:`IMAP4` instance has the following methods: Return an :class:`!Idler`: an iterable context manager implementing the IMAP4 ``IDLE`` command as defined in :rfc:`2177`. - The context manager sends the ``IDLE`` command when activated by the + The returned object sends the ``IDLE`` command when activated by the :keyword:`with` statement, produces IMAP untagged responses via the :term:`iterator` protocol, and sends ``DONE`` upon context exit. The *dur* argument sets a maximum duration (in seconds) to keep idling, - after which iteration will stop. It defaults to ``None``, meaning no time - limit. Callers wishing to avoid inactivity timeouts on servers that impose - them should keep this at most 29 minutes. + after which any ongoing iteration will stop. It defaults to ``None``, + meaning no time limit. Callers wishing to avoid inactivity timeouts on + servers that impose them should keep this at most 29 minutes. See the :ref:`warning below ` if using :class:`IMAP4_stream` on Windows. @@ -343,7 +343,7 @@ An :class:`IMAP4` instance has the following methods: Instead of iterating one response at a time, it is also possible to retrieve the next response along with any immediately available subsequent responses (e.g. a rapid series of ``EXPUNGE`` events from a bulk delete). This - batch processing aid is provided by the context's ``burst()`` + batch processing aid is provided by the context's :meth:`Idler.burst` :term:`generator`: .. method:: Idler.burst(interval=0.1) @@ -386,14 +386,14 @@ An :class:`IMAP4` instance has the following methods: inactivity timeouts would make 27 minutes a sensible value for *dur* in this situation. - There is no such fallback for ``burst()``, which will yield endless - responses and block indefinitely for each one. It is therefore advised - not to use ``burst()`` with an :class:`IMAP4_stream` connection on - Windows. + There is no such fallback for :meth:`Idler.burst`, which will yield + endless responses and block indefinitely for each one. It is therefore + advised not to use :meth:`Idler.burst` with an :class:`IMAP4_stream` + connection on Windows. .. note:: - Note: The :class:`!Idler` class name and structure are internal interfaces, + The :class:`!Idler` class name and structure are internal interfaces, subject to change. Calling code can rely on its context management, iteration, and public method to remain stable, but should not subclass, instantiate, or otherwise directly reference the class. From 247e6b52cf1d39b4d3e9e4ce0678499e7a33dece Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 1 Dec 2024 16:16:59 -0800 Subject: [PATCH 25/78] docs: imaplib: avoid linking to an invalid target This reverts and rephrases part of a3f21cd75b4d7c97c0d7e46b0a0cc0875e29f6cc which created links to a method on a deliberately undocumented class. The links didn't work consistently, and caused sphinx warnings that broke cpython's continuous integration tests. --- Doc/library/imaplib.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index c37ea9f60848db..e6478155e7a2af 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -343,7 +343,7 @@ An :class:`IMAP4` instance has the following methods: Instead of iterating one response at a time, it is also possible to retrieve the next response along with any immediately available subsequent responses (e.g. a rapid series of ``EXPUNGE`` events from a bulk delete). This - batch processing aid is provided by the context's :meth:`Idler.burst` + batch processing aid is provided by the context object's ``burst()`` :term:`generator`: .. method:: Idler.burst(interval=0.1) @@ -386,9 +386,9 @@ An :class:`IMAP4` instance has the following methods: inactivity timeouts would make 27 minutes a sensible value for *dur* in this situation. - There is no such fallback for :meth:`Idler.burst`, which will yield + There is no such fallback for ``Idler.burst()``, which will yield endless responses and block indefinitely for each one. It is therefore - advised not to use :meth:`Idler.burst` with an :class:`IMAP4_stream` + advised not to use ``Idler.burst()`` with an :class:`IMAP4_stream` connection on Windows. .. note:: From 79e3d839b306100f6080769dd88bd38ccd08b0d0 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 1 Dec 2024 17:58:35 -0800 Subject: [PATCH 26/78] imaplib: update test after recent exception change This fixes a test that was broken by changing an exception in b01de95171d6124f8acc7b907c1842472ea5f5fb --- Lib/test/test_imaplib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 35da78e1f93232..b57886e3d5cc46 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -500,7 +500,7 @@ def test_with_statement_logout(self): def test_idle_capability(self): client, _ = self._setup(SimpleIMAPHandler) with self.assertRaisesRegex(imaplib.IMAP4.error, - 'does not support IDLE'): + 'does not support IMAP4 IDLE'): with client.idle(): pass From 14dfd2166527f11373de1cec752c4d12942e1751 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 1 Dec 2024 18:24:14 -0800 Subject: [PATCH 27/78] imaplib: rename idle() dur argument to duration --- Doc/library/imaplib.rst | 28 +++++++++++++-------------- Lib/imaplib.py | 42 ++++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index e6478155e7a2af..9483e7cf7329e3 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -308,7 +308,7 @@ An :class:`IMAP4` instance has the following methods: of the IMAP4 QUOTA extension defined in rfc2087. -.. method:: IMAP4.idle(dur=None) +.. method:: IMAP4.idle(duration=None) Return an :class:`!Idler`: an iterable context manager implementing the IMAP4 ``IDLE`` command as defined in :rfc:`2177`. @@ -317,7 +317,7 @@ An :class:`IMAP4` instance has the following methods: :keyword:`with` statement, produces IMAP untagged responses via the :term:`iterator` protocol, and sends ``DONE`` upon context exit. - The *dur* argument sets a maximum duration (in seconds) to keep idling, + The *duration* argument sets a maximum duration (in seconds) to keep idling, after which any ongoing iteration will stop. It defaults to ``None``, meaning no time limit. Callers wishing to avoid inactivity timeouts on servers that impose them should keep this at most 29 minutes. @@ -333,7 +333,7 @@ An :class:`IMAP4` instance has the following methods: Example:: - with M.idle(dur=29 * 60) as idler: + with M.idle(duration=29 * 60) as idler: for typ, datum in idler: print(typ, datum) @@ -363,9 +363,9 @@ An :class:`IMAP4` instance has the following methods: processing 3 responses... [('EXPUNGE', b'2'), ('EXPUNGE', b'1'), ('RECENT', b'0')] - The ``IDLE`` context's maximum duration (the *dur* argument to - :meth:`IMAP4.idle`) is respected when waiting for the first response - in a burst. Therefore, an expired ``IDLE`` context will cause this generator + The ``IDLE`` context's maximum duration (the argument to + :meth:`IMAP4.idle`) is respected when waiting for the first response in a + burst. Therefore, an expired ``IDLE`` context will cause this generator to return immediately without producing anything. Callers should consider this if using it in a loop. @@ -375,16 +375,16 @@ An :class:`IMAP4` instance has the following methods: .. warning:: Windows' :class:`IMAP4_stream` connections have no way to accurately - respect the *dur* or *interval* arguments, since Windows' ``select()`` - only works on sockets. + respect the *duration* or *interval* arguments, since Windows' + ``select()`` only works on sockets. If the server regularly sends status messages during ``IDLE``, they will - wake our iterator anyway, allowing *dur* to behave roughly as intended, - although usually late. Dovecot's ``imap_idle_notify_interval`` default - setting does this every 2 minutes. Assuming that's typical of IMAP - servers, subtracting it from the 29 minutes needed to avoid server - inactivity timeouts would make 27 minutes a sensible value for *dur* in - this situation. + wake our iterator anyway, allowing *duration* to behave roughly as + intended, although usually late. Dovecot's ``imap_idle_notify_interval`` + default setting does this every 2 minutes. Assuming that's typical of + IMAP servers, subtracting it from the 29 minutes needed to avoid server + inactivity timeouts would make 27 minutes a sensible value for *duration* + in this situation. There is no such fallback for ``Idler.burst()``, which will yield endless responses and block indefinitely for each one. It is therefore diff --git a/Lib/imaplib.py b/Lib/imaplib.py index c64dd153e54599..d041cdf8dd14ee 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -648,15 +648,15 @@ def getquotaroot(self, mailbox): return typ, [quotaroot, quota] - def idle(self, dur=None): + def idle(self, duration=None): """Return an Idler: an iterable context manager for the IDLE command - :param dur: Maximum duration (in seconds) to keep idling, - or None for no time limit. - To avoid inactivity timeouts on servers that impose - them, callers are advised to keep this <= 29 minutes. - See the note below regarding IMAP4_stream on Windows. - :type dur: int|float|None + :param duration: Maximum duration (in seconds) to keep idling, + or None for no time limit. + To avoid inactivity timeouts on servers that impose + them, callers are advised to keep this <= 29 minutes. + See the note below regarding IMAP4_stream on Windows. + :type duration: int|float|None The context manager sends the IDLE command upon entry, produces responses via iteration, and sends DONE upon exit. @@ -666,7 +666,7 @@ def idle(self, dur=None): Example: - with imap.idle(dur=29*60) as idler: + with imap.idle(duration=29 * 60) as idler: for typ, datum in idler: print(typ, datum) @@ -674,20 +674,20 @@ def idle(self, dur=None): cache for retrieval by response(). Note: Windows IMAP4_stream connections have no way to accurately - respect 'dur', since Windows select() only works on sockets. + respect 'duration', since Windows select() only works on sockets. However, if the server regularly sends status messages during IDLE, they will wake our selector and keep iteration from blocking for long. Dovecot's imap_idle_notify_interval is two minutes by default. Assuming that's typical of IMAP servers, subtracting it from the 29 minutes needed to avoid server inactivity timeouts would make 27 - minutes a sensible value for 'dur' in this situation. + minutes a sensible value for 'duration' in this situation. Note: The Idler class name and structure are internal interfaces, subject to change. Calling code can rely on its context management, iteration, and public method to remain stable, but should not subclass, instantiate, or otherwise directly reference the class. """ - return Idler(self, dur) + return Idler(self, duration) def list(self, directory='""', pattern='*'): @@ -1408,10 +1408,10 @@ class Idler: subclass, instantiate, or otherwise directly reference the class. """ - def __init__(self, imap, dur=None): + def __init__(self, imap, duration=None): if 'IDLE' not in imap.capabilities: raise imap.error("Server does not support IMAP4 IDLE") - self._dur = dur + self._duration = duration self._imap = imap self._tag = None self._sock_timeout = None @@ -1422,7 +1422,7 @@ def __enter__(self): assert not (imap._idle_responses or imap._idle_capture) if __debug__ and imap.debug >= 4: - imap._mesg(f'idle start duration={self._dur}') + imap._mesg(f'idle start duration={self._duration}') try: # Start capturing untagged responses before sending IDLE, @@ -1497,7 +1497,7 @@ def _pop(self, timeout, default=('', None)): # :type timeout: int|float|None # :param default: Value to return on timeout # - # Note: This method ignores 'dur' in favor of the timeout argument. + # Note: This method ignores 'duration' in favor of the timeout argument. # # Note: Windows IMAP4_stream connections will ignore the timeout # argument and block until the next response arrives, because @@ -1554,7 +1554,7 @@ def burst(self, interval=0.1): print(f'processing {len(batch)} responses...') Produces no responses and returns immediately if the IDLE - context's maximum duration (the 'dur' argument) has elapsed. + context's maximum duration (the 'duration' argument) has elapsed. Callers should plan accordingly if using this method in a loop. Note: Windows IMAP4_stream connections will ignore the interval @@ -1572,19 +1572,19 @@ def burst(self, interval=0.1): yield from iter(functools.partial(self._pop, interval, None), None) - if self._dur is not None: + if self._duration is not None: elapsed = time.monotonic() - start - self._dur = max(self._dur - elapsed, 0) + self._duration = max(self._duration - elapsed, 0) def __next__(self): imap = self._imap start = time.monotonic() - typ, datum = self._pop(self._dur) + typ, datum = self._pop(self._duration) - if self._dur is not None: + if self._duration is not None: elapsed = time.monotonic() - start - self._dur = max(self._dur - elapsed, 0) + self._duration = max(self._duration - elapsed, 0) if not typ: if __debug__ and imap.debug >= 4: From 19253d106ed734ca71671488f90cb5f55f521728 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 2 Dec 2024 13:32:43 -0800 Subject: [PATCH 28/78] imaplib: bytes.index() -> bytes.find() This makes it more obvious which statement triggers the branch. --- Lib/imaplib.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index d041cdf8dd14ee..f035e645ea0441 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -367,18 +367,18 @@ def readline(self): parts = [] length = 0 while length < _MAXLINE: - try: - pos = self._readbuf.index(LF) + 1 + pos = self._readbuf.find(LF) + if pos != -1: + pos += 1 parts.append(self._readbuf[:pos]) length += len(parts[-1]) self._readbuf = self._readbuf[pos:] break - except ValueError: - parts.append(self._readbuf) - length += len(parts[-1]) - self._readbuf = read(DEFAULT_BUFFER_SIZE) - if not self._readbuf: - break + parts.append(self._readbuf) + length += len(parts[-1]) + self._readbuf = read(DEFAULT_BUFFER_SIZE) + if not self._readbuf: + break line = b''.join(parts) if len(line) > _MAXLINE: From b65074e0ac40c940f415f6d68c5b9464ca909af0 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 2 Dec 2024 23:39:18 +0000 Subject: [PATCH 29/78] imaplib: remove no-longer-necessary statement Co-authored-by: Martin Panter --- Lib/imaplib.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index f035e645ea0441..7e0a48fe9c257a 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -371,7 +371,6 @@ def readline(self): if pos != -1: pos += 1 parts.append(self._readbuf[:pos]) - length += len(parts[-1]) self._readbuf = self._readbuf[pos:] break parts.append(self._readbuf) From 5d8a40be0cb3514b9fa492dce7c34c3a9e6f78b1 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 2 Dec 2024 16:05:19 -0800 Subject: [PATCH 30/78] docs: imaplib: concise & valid method links The burst() method is a little tricky to link in restructuredText, due to quirks of its parent class. This syntax allows sphinx to generate working links without generating warnings (which break continuous integration) and without burdening the reader with unimportant namespace qualifications. It makes the reST source ugly, but few people read the reST source, so it's a tolerable tradeoff. --- Doc/library/imaplib.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 9483e7cf7329e3..67e4e00675160c 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -343,8 +343,8 @@ An :class:`IMAP4` instance has the following methods: Instead of iterating one response at a time, it is also possible to retrieve the next response along with any immediately available subsequent responses (e.g. a rapid series of ``EXPUNGE`` events from a bulk delete). This - batch processing aid is provided by the context object's ``burst()`` - :term:`generator`: + batch processing aid is provided by the + :meth:`Idler.burst() ` :term:`generator`. .. method:: Idler.burst(interval=0.1) @@ -386,10 +386,11 @@ An :class:`IMAP4` instance has the following methods: inactivity timeouts would make 27 minutes a sensible value for *duration* in this situation. - There is no such fallback for ``Idler.burst()``, which will yield - endless responses and block indefinitely for each one. It is therefore - advised not to use ``Idler.burst()`` with an :class:`IMAP4_stream` - connection on Windows. + There is no such fallback for + :meth:`Idler.burst() `, which will yield endless + responses and block indefinitely for each one. It is therefore advised + not to use :meth:`Idler.burst() ` with an + :class:`IMAP4_stream` connection on Windows. .. note:: From fc13f75aa2535643b027373a55a1ab8c04ef1c21 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 2 Dec 2024 17:15:05 -0800 Subject: [PATCH 31/78] imaplib: note data types present in IDLE responses --- Doc/library/imaplib.rst | 3 ++- Lib/imaplib.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 67e4e00675160c..ade59024ce1782 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -329,7 +329,8 @@ An :class:`IMAP4` instance has the following methods: second member is a single response datum, rather than a list of data. Therefore, in a mailbox where calling ``M.response('EXISTS')`` would return ``('EXISTS', [b'1'])``, the idle iterator would produce - ``('EXISTS', b'1')``. + ``('EXISTS', b'1')``. A datum can be bytes or a tuple, as described in + :ref:`IMAP4 Objects `. Example:: diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 7e0a48fe9c257a..052c9736c629ad 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -661,7 +661,8 @@ def idle(self, duration=None): responses via iteration, and sends DONE upon exit. It represents responses as (type, datum) tuples, rather than the (type, [data, ...]) tuples returned by other methods, because only one - response is represented at a time. + response is represented at a time. A datum can be bytes or a tuple, + as described in the IMAP4 class documentation. Example: From 3221dbc0e302d0c6440db2e4c3eb3d8fee7fe02e Mon Sep 17 00:00:00 2001 From: Forest Date: Wed, 4 Dec 2024 01:01:11 +0000 Subject: [PATCH 32/78] docs: imaplib: add comma to reST changes header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/imaplib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index ade59024ce1782..755b06d143b327 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -10,7 +10,7 @@ .. changes for IMAP4_SSL by Tino Lange , March 2002 .. changes for IMAP4_stream by Piers Lauder , November 2002 -.. changes for IMAP4 IDLE by Forest August 2024 +.. changes for IMAP4 IDLE by Forest , August 2024 **Source code:** :source:`Lib/imaplib.py` From 564c722abac35331bb6fe414be93101e4eac6dbb Mon Sep 17 00:00:00 2001 From: Forest Date: Sat, 7 Dec 2024 11:06:21 -0800 Subject: [PATCH 33/78] imaplib: sync doc strings with reST docs --- Lib/imaplib.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 052c9736c629ad..348936d3edfe8e 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -651,9 +651,9 @@ def idle(self, duration=None): """Return an Idler: an iterable context manager for the IDLE command :param duration: Maximum duration (in seconds) to keep idling, - or None for no time limit. - To avoid inactivity timeouts on servers that impose - them, callers are advised to keep this <= 29 minutes. + or None for no time limit. To avoid inactivity + timeouts on servers that impose them, callers are + advised to keep this at most 29 minutes. See the note below regarding IMAP4_stream on Windows. :type duration: int|float|None @@ -673,7 +673,7 @@ def idle(self, duration=None): Responses produced by the iterator are not added to the internal cache for retrieval by response(). - Note: Windows IMAP4_stream connections have no way to accurately + Warning: Windows IMAP4_stream connections have no way to accurately respect 'duration', since Windows select() only works on sockets. However, if the server regularly sends status messages during IDLE, they will wake our selector and keep iteration from blocking for long. @@ -682,7 +682,7 @@ def idle(self, duration=None): minutes needed to avoid server inactivity timeouts would make 27 minutes a sensible value for 'duration' in this situation. - Note: The Idler class name and structure are internal interfaces, + Note: The Idler class name and structure are internal interfaces, subject to change. Calling code can rely on its context management, iteration, and public method to remain stable, but should not subclass, instantiate, or otherwise directly reference the class. @@ -1402,7 +1402,7 @@ class Idler: tuple is a single datum, rather than a list of them, because only one untagged response is produced at a time. - Note: The name and structure of this class are internal interfaces, + Note: The name and structure of this class are internal interfaces, subject to change. Calling code can rely on its context management, iteration, and public method to remain stable, but should not subclass, instantiate, or otherwise directly reference the class. @@ -1553,15 +1553,17 @@ def burst(self, interval=0.1): batch = list(idler.burst()) print(f'processing {len(batch)} responses...') - Produces no responses and returns immediately if the IDLE - context's maximum duration (the 'duration' argument) has elapsed. - Callers should plan accordingly if using this method in a loop. - - Note: Windows IMAP4_stream connections will ignore the interval - argument, yielding endless responses and blocking indefinitely - for each one, because Windows select() only works on sockets. - It is therefore advised not to use this method with an IMAP4_stream - connection on Windows. + The IDLE context's maximum duration, as passed to IMAP4.idle(), + is respected when waiting for the first response in a burst. + Therefore, an expired IDLE context will cause this generator + to return immediately without producing anything. Callers should + consider this if using it in a loop. + + Warning: Windows IMAP4_stream connections have no way to accurately + respect the 'interval' argument, since Windows select() only works + on sockets. This will cause the generator to yield endless responses + and block indefinitely for each one. It is therefore advised not to + use burst() with an IMAP4_stream connection on Windows. """ try: yield next(self) From c9e8034f9757fa09ddaae52e938dfb8904fcb983 Mon Sep 17 00:00:00 2001 From: Forest Date: Sat, 7 Dec 2024 12:39:44 -0800 Subject: [PATCH 34/78] docs: imaplib: minor Idler clarifications --- Doc/library/imaplib.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 755b06d143b327..279c52e381a3ca 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -364,9 +364,9 @@ An :class:`IMAP4` instance has the following methods: processing 3 responses... [('EXPUNGE', b'2'), ('EXPUNGE', b'1'), ('RECENT', b'0')] - The ``IDLE`` context's maximum duration (the argument to - :meth:`IMAP4.idle`) is respected when waiting for the first response in a - burst. Therefore, an expired ``IDLE`` context will cause this generator + The ``IDLE`` context's maximum duration, as passed to :meth:`IMAP4.idle`, + is respected when waiting for the first response in a burst. + Therefore, an expired :class:`!Idler` will cause this generator to return immediately without producing anything. Callers should consider this if using it in a loop. @@ -397,8 +397,8 @@ An :class:`IMAP4` instance has the following methods: The :class:`!Idler` class name and structure are internal interfaces, subject to change. Calling code can rely on its context management, - iteration, and public method to remain stable, but should not - subclass, instantiate, or otherwise directly reference the class. + iteration, and public method to remain stable, but should not subclass, + instantiate, compare, or otherwise directly reference the class. .. method:: IMAP4.list([directory[, pattern]]) From 9c4af2c4e7e0d8b293c5432ae92afde69005ffb3 Mon Sep 17 00:00:00 2001 From: Forest Date: Sat, 7 Dec 2024 13:01:12 -0800 Subject: [PATCH 35/78] imaplib: idle: emit (type, [data, ...]) tuples This allows our iterator to emit untagged responses that contain literal strings in the same way that imaplib's existing methods do, while still emitting exactly one whole response per iteration. --- Doc/library/imaplib.rst | 39 +++++++++----------- Lib/imaplib.py | 77 ++++++++++++++++++++++++---------------- Lib/test/test_imaplib.py | 30 +++++++++++----- 3 files changed, 84 insertions(+), 62 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 279c52e381a3ca..462feefd02a016 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -317,6 +317,9 @@ An :class:`IMAP4` instance has the following methods: :keyword:`with` statement, produces IMAP untagged responses via the :term:`iterator` protocol, and sends ``DONE`` upon context exit. + Responses are represented as ``(type, [data, ...])`` tuples, as described + in :ref:`IMAP4 Objects `. + The *duration* argument sets a maximum duration (in seconds) to keep idling, after which any ongoing iteration will stop. It defaults to ``None``, meaning no time limit. Callers wishing to avoid inactivity timeouts on @@ -324,22 +327,14 @@ An :class:`IMAP4` instance has the following methods: See the :ref:`warning below ` if using :class:`IMAP4_stream` on Windows. - Response tuples produced by the iterator almost exactly match those - returned by other methods in this module. The difference is that the tuple's - second member is a single response datum, rather than a list of data. - Therefore, in a mailbox where calling ``M.response('EXISTS')`` would - return ``('EXISTS', [b'1'])``, the idle iterator would produce - ``('EXISTS', b'1')``. A datum can be bytes or a tuple, as described in - :ref:`IMAP4 Objects `. - Example:: - with M.idle(duration=29 * 60) as idler: - for typ, datum in idler: - print(typ, datum) - - ('EXISTS', b'1') - ('RECENT', b'1') + >>> with M.idle(duration=29 * 60) as idler: + ... for typ, data in idler: + ... print(typ, data) + ... + EXISTS [b'1'] + RECENT [b'1'] Instead of iterating one response at a time, it is also possible to retrieve the next response along with any immediately available subsequent responses @@ -353,16 +348,14 @@ An :class:`IMAP4` instance has the following methods: Example:: - with M.idle() as idler: - - # get the next response and any others following by < 0.1 seconds - batch = list(idler.burst()) - - print(f'processing {len(batch)} responses...') - print(batch) - + >>> with M.idle() as idler: + ... # get next response and any others following by < 0.1 seconds + ... batch = list(idler.burst()) + ... print(f'processing {len(batch)} responses...') + ... print(batch) + ... processing 3 responses... - [('EXPUNGE', b'2'), ('EXPUNGE', b'1'), ('RECENT', b'0')] + [('EXPUNGE', [b'2']), ('EXPUNGE', [b'1']), ('RECENT', [b'0'])] The ``IDLE`` context's maximum duration, as passed to :meth:`IMAP4.idle`, is respected when waiting for the first response in a burst. diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 348936d3edfe8e..de2c920cd977a4 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -657,21 +657,21 @@ def idle(self, duration=None): See the note below regarding IMAP4_stream on Windows. :type duration: int|float|None - The context manager sends the IDLE command upon entry, produces - responses via iteration, and sends DONE upon exit. - It represents responses as (type, datum) tuples, rather than the - (type, [data, ...]) tuples returned by other methods, because only one - response is represented at a time. A datum can be bytes or a tuple, - as described in the IMAP4 class documentation. + The returned object sends the IDLE command when activated by the + 'with' statement, produces IMAP untagged responses via the iterator + protocol, and sends DONE upon context exit. - Example: + Responses are represented as (type, [data, ...]) tuples, as described + in the IMAP4 class documentation. - with imap.idle(duration=29 * 60) as idler: - for typ, datum in idler: - print(typ, datum) + Example: - Responses produced by the iterator are not added to the internal - cache for retrieval by response(). + >>> with M.idle(duration=29 * 60) as idler: + ... for typ, data in idler: + ... print(typ, data) + ... + EXISTS [b'1'] + RECENT [b'1'] Warning: Windows IMAP4_stream connections have no way to accurately respect 'duration', since Windows select() only works on sockets. @@ -684,8 +684,8 @@ def idle(self, duration=None): Note: The Idler class name and structure are internal interfaces, subject to change. Calling code can rely on its context management, - iteration, and public method to remain stable, but should not - subclass, instantiate, or otherwise directly reference the class. + iteration, and public method to remain stable, but should not subclass, + instantiate, compare, or otherwise directly reference the class. """ return Idler(self, duration) @@ -1049,7 +1049,17 @@ def _append_untagged(self, typ, dat): # During idle, queue untagged responses for delivery via iteration if self._idle_capture: - self._idle_responses.append((typ, dat)) + # Responses containing literal strings are passed to us one data + # fragment at a time, while others arrive in a single call. + if (not self._idle_responses or + isinstance(self._idle_responses[-1][1][-1], bytes)): + # We are not continuing a fragmented response; start a new one + self._idle_responses.append((typ, [dat])) + else: + # We are continuing a fragmented response; append the fragment + response = self._idle_responses[-1] + assert response[0] == typ + response[1].append(dat) if __debug__ and self.debug >= 5: self._mesg(f'idle: queue untagged {typ} {dat!r}') return @@ -1397,10 +1407,8 @@ class Idler: IMAP untagged responses via the iterator protocol, and sends DONE upon context exit. - Iteration produces (type, datum) tuples. They slightly differ - from the tuples returned by IMAP4.response(): The second item in the - tuple is a single datum, rather than a list of them, because only one - untagged response is produced at a time. + Iteration produces (type, [data, ...]) tuples, as described in the IMAP4 + class documentation. Note: The name and structure of this class are internal interfaces, subject to change. Calling code can rely on its context management, @@ -1544,20 +1552,25 @@ def burst(self, interval=0.1): EXPUNGE responses after a bulk delete) so they can be efficiently processed as a batch instead of one at a time. - Represents responses as (type, datum) tuples, just as when - iterating directly on the context manager. + Responses are represented as (type, [data, ...]) tuples, as described + in the IMAP4 class documentation. Example: - with imap.idle() as idler: - batch = list(idler.burst()) - print(f'processing {len(batch)} responses...') + >>> with M.idle() as idler: + ... # get next response and any others following by < 0.1 seconds + ... batch = list(idler.burst()) + ... print(f'processing {len(batch)} responses...') + ... print(batch) + ... + processing 3 responses... + [('EXPUNGE', [b'2']), ('EXPUNGE', [b'1']), ('RECENT', [b'0'])] The IDLE context's maximum duration, as passed to IMAP4.idle(), is respected when waiting for the first response in a burst. - Therefore, an expired IDLE context will cause this generator - to return immediately without producing anything. Callers should - consider this if using it in a loop. + Therefore, an expired Idler will cause this generator to return + immediately without producing anything. Callers should consider + this if using it in a loop. Warning: Windows IMAP4_stream connections have no way to accurately respect the 'interval' argument, since Windows select() only works @@ -1582,7 +1595,7 @@ def __next__(self): imap = self._imap start = time.monotonic() - typ, datum = self._pop(self._duration) + typ, data = self._pop(self._duration) if self._duration is not None: elapsed = time.monotonic() - start @@ -1593,7 +1606,7 @@ def __next__(self): imap._mesg('idle iterator exhausted') raise StopIteration - return typ, datum + return typ, data def __exit__(self, exc_type, exc_val, exc_tb): imap = self._imap @@ -1621,8 +1634,10 @@ def __exit__(self, exc_type, exc_val, exc_tb): if __debug__ and imap.debug >= 4: imap._mesg(f'idle quit with {leftovers} leftover responses') while imap._idle_responses: - typ, datum = imap._idle_responses.pop(0) - imap._append_untagged(typ, datum) + typ, data = imap._idle_responses.pop(0) + # Append one fragment at a time, just as _get_response() does + for datum in data: + imap._append_untagged(typ, datum) try: imap.send(b'DONE' + CRLF) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index b57886e3d5cc46..b7d9a14e0f9c72 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -508,8 +508,15 @@ class IdleCmdHandler(SimpleIMAPHandler): capabilities = 'IDLE' def cmd_IDLE(self, tag, args): self._send_textline('+ idling') + # simple response self._send_line(b'* 2 EXISTS') - self._send_line(b'* 0 RECENT') + # complex response: fragmented data due to literal string + self._send_line(b'* 1 FETCH (BODY[HEADER.FIELDS (DATE)] {41}') + self._send(b'Date: Fri, 06 Dec 2024 06:00:00 +0000\r\n\r\n') + self._send_line(b')') + # simple response following a fragmented one + self._send_line(b'* 3 EXISTS') + # response arriving later time.sleep(1) self._send_line(b'* 1 RECENT') r = yield @@ -523,12 +530,19 @@ def test_idle_iter(self): client.login('user', 'pass') with client.idle() as idler: # iteration should produce responses - typ, datum = next(idler) - self.assertEqual(typ, 'EXISTS') - self.assertEqual(datum, b'2') - typ, datum = next(idler) - self.assertEqual(typ, 'RECENT') - self.assertEqual(datum, b'0') + response = next(idler) + self.assertEqual(response, ('EXISTS', [b'2'])) + # fragmented response (with literal string) should arrive whole + expected_fetch_data = [ + (b'1 (BODY[HEADER.FIELDS (DATE)] {41}', + b'Date: Fri, 06 Dec 2024 06:00:00 +0000\r\n\r\n'), + b')'] + typ, data = next(idler) + self.assertEqual(typ, 'FETCH') + self.assertEqual(data, expected_fetch_data) + # response after a fragmented one should arrive separately + response = next(idler) + self.assertEqual(response, ('EXISTS', [b'3'])) # iteration should have consumed untagged responses _, data = client.response('EXISTS') self.assertEqual(data, [None]) @@ -542,7 +556,7 @@ def test_idle_burst(self): # burst() should yield immediately available responses with client.idle() as idler: batch = list(idler.burst()) - self.assertEqual(len(batch), 2) + self.assertEqual(len(batch), 3) # burst() should not have consumed later responses _, data = client.response('RECENT') self.assertEqual(data, [b'1']) From 59e0c6a5e12ad99d728755c237cf24b84d6b8f3a Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 8 Dec 2024 15:14:12 -0800 Subject: [PATCH 36/78] imaplib: while/yield instead of yield from iter() --- Lib/imaplib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index de2c920cd977a4..e6a7ff8820c068 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -26,7 +26,6 @@ import binascii import calendar import errno -import functools import platform import random import re @@ -1585,7 +1584,8 @@ def burst(self, interval=0.1): start = time.monotonic() - yield from iter(functools.partial(self._pop, interval, None), None) + while response := self._pop(interval, None): + yield response if self._duration is not None: elapsed = time.monotonic() - start From 2e3e956c486b07335f5bc3ed28927a2cfafa4381 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 8 Dec 2024 19:05:32 -0800 Subject: [PATCH 37/78] imaplib: idle: use deadline idiom when iterating This simplifies the code, and avoids idle duration drift from time spent processing each iteration. --- Lib/imaplib.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index e6a7ff8820c068..b5e946c12efaa7 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1419,6 +1419,7 @@ def __init__(self, imap, duration=None): if 'IDLE' not in imap.capabilities: raise imap.error("Server does not support IMAP4 IDLE") self._duration = duration + self._deadline = None self._imap = imap self._tag = None self._sock_timeout = None @@ -1459,6 +1460,9 @@ def __enter__(self): if self._sock_timeout is not None: imap.sock.settimeout(None) # Socket timeout would break IDLE + if self._duration is not None: + self._deadline = time.monotonic() + self._duration + self._old_state = imap.state imap.state = 'IDLING' @@ -1582,24 +1586,17 @@ def burst(self, interval=0.1): except StopIteration: return - start = time.monotonic() - while response := self._pop(interval, None): yield response - if self._duration is not None: - elapsed = time.monotonic() - start - self._duration = max(self._duration - elapsed, 0) - def __next__(self): imap = self._imap - start = time.monotonic() - - typ, data = self._pop(self._duration) - if self._duration is not None: - elapsed = time.monotonic() - start - self._duration = max(self._duration - elapsed, 0) + if self._duration is None: + timeout = None + else: + timeout = self._deadline - time.monotonic() + typ, data = self._pop(timeout) if not typ: if __debug__ and imap.debug >= 4: From bdde9438449432e0b81df28d0398371c5c3cfbe0 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 8 Dec 2024 20:29:53 -0800 Subject: [PATCH 38/78] docs: imaplib: state duration/interval arg types --- Doc/library/imaplib.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 462feefd02a016..b94e4005aa1579 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -322,8 +322,9 @@ An :class:`IMAP4` instance has the following methods: The *duration* argument sets a maximum duration (in seconds) to keep idling, after which any ongoing iteration will stop. It defaults to ``None``, - meaning no time limit. Callers wishing to avoid inactivity timeouts on - servers that impose them should keep this at most 29 minutes. + meaning no time limit, but can also be an :class:`int` or :class:`float`. + Callers wishing to avoid inactivity timeouts on servers that impose them + should keep this at most 29 minutes. See the :ref:`warning below ` if using :class:`IMAP4_stream` on Windows. @@ -344,7 +345,8 @@ An :class:`IMAP4` instance has the following methods: .. method:: Idler.burst(interval=0.1) - Yield a burst of responses no more than *interval* seconds apart. + Yield a burst of responses no more than *interval* seconds apart + (expressed as an :class:`int` or :class:`float`). Example:: From b64c7a5ead36cf71281c89e8dd63f8418f21ee26 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 9 Dec 2024 11:01:04 -0800 Subject: [PATCH 39/78] docs: imaplib: minor rephrasing of a sentence --- Doc/library/imaplib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index b94e4005aa1579..e2b66c991dc1b9 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -375,7 +375,7 @@ An :class:`IMAP4` instance has the following methods: ``select()`` only works on sockets. If the server regularly sends status messages during ``IDLE``, they will - wake our iterator anyway, allowing *duration* to behave roughly as + wake our iterator anyway and allow *duration* to behave roughly as intended, although usually late. Dovecot's ``imap_idle_notify_interval`` default setting does this every 2 minutes. Assuming that's typical of IMAP servers, subtracting it from the 29 minutes needed to avoid server From b73a3656c4b0e2e27906ab1a5b82206c9a18505e Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 9 Dec 2024 13:06:49 -0800 Subject: [PATCH 40/78] docs: imaplib: reposition a paragraph This might improve readability, especially when encountering Idler.burst() for the first time. --- Doc/library/imaplib.rst | 11 ++++++----- Lib/imaplib.py | 9 +++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index e2b66c991dc1b9..15ff57c528c353 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -337,17 +337,18 @@ An :class:`IMAP4` instance has the following methods: EXISTS [b'1'] RECENT [b'1'] - Instead of iterating one response at a time, it is also possible to retrieve - the next response along with any immediately available subsequent responses - (e.g. a rapid series of ``EXPUNGE`` events from a bulk delete). This - batch processing aid is provided by the - :meth:`Idler.burst() ` :term:`generator`. .. method:: Idler.burst(interval=0.1) Yield a burst of responses no more than *interval* seconds apart (expressed as an :class:`int` or :class:`float`). + This :term:`generator` is an alternative to iterating one response at a + time, intended to aid in efficient batch processing. It retrieves the + next response along with any immediately available subsequent responses. + (For example, a rapid series of ``EXPUNGE`` responses after a bulk + delete.) + Example:: >>> with M.idle() as idler: diff --git a/Lib/imaplib.py b/Lib/imaplib.py index b5e946c12efaa7..8f7cc7f826e9c7 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1550,10 +1550,11 @@ def burst(self, interval=0.1): respected when waiting for the first response.) :type interval: int|float - This generator retrieves the next response along with any - immediately available subsequent responses (e.g. a rapid series of - EXPUNGE responses after a bulk delete) so they can be efficiently - processed as a batch instead of one at a time. + This generator is an alternative to iterating one response at a + time, intended to aid in efficient batch processing. It retrieves + the next response along with any immediately available subsequent + responses. (For example, a rapid series of EXPUNGE responses after + a bulk delete.) Responses are represented as (type, [data, ...]) tuples, as described in the IMAP4 class documentation. From ae7449945150e0421423acfd9c6e1f47c687659e Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 9 Dec 2024 13:19:53 -0800 Subject: [PATCH 41/78] docs: imaplib: wrap long lines in idle() section --- Doc/library/imaplib.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 15ff57c528c353..7c47d0b85f5e8d 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -310,8 +310,8 @@ An :class:`IMAP4` instance has the following methods: .. method:: IMAP4.idle(duration=None) - Return an :class:`!Idler`: an iterable context manager implementing the IMAP4 ``IDLE`` - command as defined in :rfc:`2177`. + Return an :class:`!Idler`: an iterable context manager implementing the + IMAP4 ``IDLE`` command as defined in :rfc:`2177`. The returned object sends the ``IDLE`` command when activated by the :keyword:`with` statement, produces IMAP untagged responses via the @@ -384,10 +384,10 @@ An :class:`IMAP4` instance has the following methods: in this situation. There is no such fallback for - :meth:`Idler.burst() `, which will yield endless - responses and block indefinitely for each one. It is therefore advised - not to use :meth:`Idler.burst() ` with an - :class:`IMAP4_stream` connection on Windows. + :meth:`Idler.burst() `, which will yield + endless responses and block indefinitely for each one. It is therefore + advised not to use :meth:`Idler.burst() ` + with an :class:`IMAP4_stream` connection on Windows. .. note:: From 92d7ce7ca4ed25c91df9ff9b205939e14f7380a3 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 9 Dec 2024 13:35:48 -0800 Subject: [PATCH 42/78] docs: imaplib: note: Idler objects require 'with' --- Doc/library/imaplib.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 7c47d0b85f5e8d..80a3ec907cccd7 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -389,6 +389,12 @@ An :class:`IMAP4` instance has the following methods: advised not to use :meth:`Idler.burst() ` with an :class:`IMAP4_stream` connection on Windows. + .. note:: + + The :class:`!Idler` object returned by :meth:`IMAP4.idle` is usable only + within a :keyword:`with` statement. To retrieve unsolicited IMAP + responses outside that context, see :meth:`IMAP4.response`. + .. note:: The :class:`!Idler` class name and structure are internal interfaces, From 171ebf1b65bb51d15c8094bf994e35228d42dab9 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 9 Dec 2024 15:15:48 -0800 Subject: [PATCH 43/78] docs: imaplib: say that 29 minutes is 1740 seconds --- Doc/library/imaplib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 80a3ec907cccd7..f7a1a0d12b1fbd 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -324,7 +324,7 @@ An :class:`IMAP4` instance has the following methods: after which any ongoing iteration will stop. It defaults to ``None``, meaning no time limit, but can also be an :class:`int` or :class:`float`. Callers wishing to avoid inactivity timeouts on servers that impose them - should keep this at most 29 minutes. + should keep this at most 29 minutes (1740 seconds). See the :ref:`warning below ` if using :class:`IMAP4_stream` on Windows. From 5682ef4b6c2b211f61b68c84ff2d935d6b880aeb Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 9 Dec 2024 16:12:12 -0800 Subject: [PATCH 44/78] docs: imaplib: mark a paragraph as a 'tip' --- Doc/library/imaplib.rst | 12 +++++++----- Lib/imaplib.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index f7a1a0d12b1fbd..4e206cf5132318 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -360,11 +360,13 @@ An :class:`IMAP4` instance has the following methods: processing 3 responses... [('EXPUNGE', [b'2']), ('EXPUNGE', [b'1']), ('RECENT', [b'0'])] - The ``IDLE`` context's maximum duration, as passed to :meth:`IMAP4.idle`, - is respected when waiting for the first response in a burst. - Therefore, an expired :class:`!Idler` will cause this generator - to return immediately without producing anything. Callers should - consider this if using it in a loop. + .. tip:: + + The ``IDLE`` context's maximum duration, as passed to + :meth:`IMAP4.idle`, is respected when waiting for the first response + in a burst. Therefore, an expired :class:`!Idler` will cause this + generator to return immediately without producing anything. Callers + should consider this if using it in a loop. .. _windows-pipe-timeout-warning: diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 8f7cc7f826e9c7..8f9d7eb9747dd4 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1570,7 +1570,7 @@ def burst(self, interval=0.1): processing 3 responses... [('EXPUNGE', [b'2']), ('EXPUNGE', [b'1']), ('RECENT', [b'0'])] - The IDLE context's maximum duration, as passed to IMAP4.idle(), + Tip: The IDLE context's maximum duration, as passed to IMAP4.idle(), is respected when waiting for the first response in a burst. Therefore, an expired Idler will cause this generator to return immediately without producing anything. Callers should consider From 656e9f5dcd257199b35410991283a07272801236 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 9 Dec 2024 16:21:21 -0800 Subject: [PATCH 45/78] docs: imaplib: rephrase reference to MS Windows --- Doc/library/imaplib.rst | 2 +- Lib/imaplib.py | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 4e206cf5132318..b45e8e93683839 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -373,7 +373,7 @@ An :class:`IMAP4` instance has the following methods: .. warning:: - Windows' :class:`IMAP4_stream` connections have no way to accurately + On Windows, :class:`IMAP4_stream` connections have no way to accurately respect the *duration* or *interval* arguments, since Windows' ``select()`` only works on sockets. diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 8f9d7eb9747dd4..befb55fd47fc14 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -672,11 +672,11 @@ def idle(self, duration=None): EXISTS [b'1'] RECENT [b'1'] - Warning: Windows IMAP4_stream connections have no way to accurately - respect 'duration', since Windows select() only works on sockets. - However, if the server regularly sends status messages during IDLE, - they will wake our selector and keep iteration from blocking for long. - Dovecot's imap_idle_notify_interval is two minutes by default. + Warning: On Windows, IMAP4_stream connections have no way to + accurately respect 'duration', since Windows select() only works on + sockets. However, if the server regularly sends status messages during + IDLE, they will wake our selector and keep iteration from blocking for + long. Dovecot's imap_idle_notify_interval is two minutes by default. Assuming that's typical of IMAP servers, subtracting it from the 29 minutes needed to avoid server inactivity timeouts would make 27 minutes a sensible value for 'duration' in this situation. @@ -1576,11 +1576,12 @@ def burst(self, interval=0.1): immediately without producing anything. Callers should consider this if using it in a loop. - Warning: Windows IMAP4_stream connections have no way to accurately - respect the 'interval' argument, since Windows select() only works - on sockets. This will cause the generator to yield endless responses - and block indefinitely for each one. It is therefore advised not to - use burst() with an IMAP4_stream connection on Windows. + Warning: On Windows, IMAP4_stream connections have no way to + accurately respect the 'interval' argument, since Windows select() + only works on sockets. This will cause the generator to yield + endless responses and block indefinitely for each one. It is + therefore advised not to use burst() with an IMAP4_stream + connection on Windows. """ try: yield next(self) From 3b7053444e910c576a88161d61252303c0f82d81 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 9 Dec 2024 16:28:49 -0800 Subject: [PATCH 46/78] imaplib: end doc string titles with a period --- Lib/imaplib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index befb55fd47fc14..6afe8f99ec8a81 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -647,7 +647,7 @@ def getquotaroot(self, mailbox): def idle(self, duration=None): - """Return an Idler: an iterable context manager for the IDLE command + """Return an Idler: an iterable context manager for the IDLE command. :param duration: Maximum duration (in seconds) to keep idling, or None for no time limit. To avoid inactivity @@ -1399,7 +1399,7 @@ def print_log(self): class Idler: - """Iterable context manager: start IDLE & produce untagged responses + """Iterable context manager: start IDLE & produce untagged responses. An object of this type is returned by the IMAP4.idle() method. It sends the IDLE command when activated by the 'with' statement, produces From 60e2b6f6bb45f230be5b11fd50174906e2d5ff4f Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 15 Dec 2024 14:38:36 -0800 Subject: [PATCH 47/78] imaplib: idle: socket timeouts instead of select() IDLE timeouts were originally implemented using select() after checking for the presence of already-buffered data. That allowed timeouts on pipe connetions like IMAP4_stream. However, it seemed possible that SSL data arriving without any IMAP data afterward could cause select() to indicate available application data when there was none, leading to a read() call that would block with no timeout. It was unclear under what conditions this would happen in practice. This change switches to socket timeouts instead of select(), just to be safe. This also reverts IMAP4_stream changes that were made to support IDLE timeouts, since our new implementation only supports socket connections. --- Doc/library/imaplib.rst | 33 ++------- Lib/imaplib.py | 144 ++++++++++++++++------------------------ 2 files changed, 65 insertions(+), 112 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index b45e8e93683839..8d32cd3b46e1cd 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -321,12 +321,12 @@ An :class:`IMAP4` instance has the following methods: in :ref:`IMAP4 Objects `. The *duration* argument sets a maximum duration (in seconds) to keep idling, - after which any ongoing iteration will stop. It defaults to ``None``, - meaning no time limit, but can also be an :class:`int` or :class:`float`. + after which any ongoing iteration will stop. It can be an :class:`int` or + :class:`float`, or ``None`` for no time limit. Callers wishing to avoid inactivity timeouts on servers that impose them should keep this at most 29 minutes (1740 seconds). - See the :ref:`warning below ` if using - :class:`IMAP4_stream` on Windows. + Requires a socket connection; *duration* must be ``None`` on + :class:`IMAP4_stream` connections. Example:: @@ -349,6 +349,9 @@ An :class:`IMAP4` instance has the following methods: (For example, a rapid series of ``EXPUNGE`` responses after a bulk delete.) + Requires a socket connection; does not work on :class:`IMAP4_stream` + connections. + Example:: >>> with M.idle() as idler: @@ -369,28 +372,6 @@ An :class:`IMAP4` instance has the following methods: should consider this if using it in a loop. - .. _windows-pipe-timeout-warning: - - .. warning:: - - On Windows, :class:`IMAP4_stream` connections have no way to accurately - respect the *duration* or *interval* arguments, since Windows' - ``select()`` only works on sockets. - - If the server regularly sends status messages during ``IDLE``, they will - wake our iterator anyway and allow *duration* to behave roughly as - intended, although usually late. Dovecot's ``imap_idle_notify_interval`` - default setting does this every 2 minutes. Assuming that's typical of - IMAP servers, subtracting it from the 29 minutes needed to avoid server - inactivity timeouts would make 27 minutes a sensible value for *duration* - in this situation. - - There is no such fallback for - :meth:`Idler.burst() `, which will yield - endless responses and block indefinitely for each one. It is therefore - advised not to use :meth:`Idler.burst() ` - with an :class:`IMAP4_stream` connection on Windows. - .. note:: The :class:`!Idler` object returned by :meth:`IMAP4.idle` is usable only diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 6afe8f99ec8a81..38444cd2873a0f 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -26,10 +26,8 @@ import binascii import calendar import errno -import platform import random import re -import selectors import socket import subprocess import sys @@ -330,13 +328,12 @@ def open(self, host='', port=IMAP4_PORT, timeout=None): def read(self, size): """Read 'size' bytes from remote.""" - # Read from an unbuffered input, so our select() calls will not be - # defeated by a hidden library buffer. Use our own buffer instead, - # which can be examined before calling select(). - if isinstance(self, IMAP4_stream): - read = self.readfile.read - else: - read = self.sock.recv + # We need buffered read() to continue working after socket timeouts, + # since we use them during IDLE. Unfortunately, the standard library's + # SocketIO implementation makes this impossible, by setting a permanent + # error condition instead of letting the caller decide how to handle a + # timeout. We therefore implement our own buffered read(). + # https://github.com/python/cpython/issues/51571 parts = [] while True: @@ -346,7 +343,7 @@ def read(self, size): break parts.append(self._readbuf) size -= len(self._readbuf) - self._readbuf = read(DEFAULT_BUFFER_SIZE) + self._readbuf = self.sock.recv(DEFAULT_BUFFER_SIZE) if not self._readbuf: break return b''.join(parts) @@ -354,13 +351,7 @@ def read(self, size): def readline(self): """Read line from remote.""" - # Read from an unbuffered input, so our select() calls will not be - # defeated by a hidden library buffer. Use our own buffer instead, - # which can be examined before calling select(). - if isinstance(self, IMAP4_stream): - read = self.readfile.read - else: - read = self.sock.recv + # The comment in read() explains why we implement our own readline(). LF = b'\n' parts = [] @@ -374,7 +365,7 @@ def readline(self): break parts.append(self._readbuf) length += len(parts[-1]) - self._readbuf = read(DEFAULT_BUFFER_SIZE) + self._readbuf = self.sock.recv(DEFAULT_BUFFER_SIZE) if not self._readbuf: break @@ -653,7 +644,7 @@ def idle(self, duration=None): or None for no time limit. To avoid inactivity timeouts on servers that impose them, callers are advised to keep this at most 29 minutes. - See the note below regarding IMAP4_stream on Windows. + Requires a socket connection (not IMAP4_stream). :type duration: int|float|None The returned object sends the IDLE command when activated by the @@ -672,15 +663,6 @@ def idle(self, duration=None): EXISTS [b'1'] RECENT [b'1'] - Warning: On Windows, IMAP4_stream connections have no way to - accurately respect 'duration', since Windows select() only works on - sockets. However, if the server regularly sends status messages during - IDLE, they will wake our selector and keep iteration from blocking for - long. Dovecot's imap_idle_notify_interval is two minutes by default. - Assuming that's typical of IMAP servers, subtracting it from the 29 - minutes needed to avoid server inactivity timeouts would make 27 - minutes a sensible value for 'duration' in this situation. - Note: The Idler class name and structure are internal interfaces, subject to change. Calling code can rely on its context management, iteration, and public method to remain stable, but should not subclass, @@ -1418,6 +1400,10 @@ class documentation. def __init__(self, imap, duration=None): if 'IDLE' not in imap.capabilities: raise imap.error("Server does not support IMAP4 IDLE") + if (duration is not None and + not isinstance(imap.sock, socket.socket)): + # IMAP4_stream pipes don't support timeouts + raise imap.error('duration requires a socket connection') self._duration = duration self._deadline = None self._imap = imap @@ -1471,48 +1457,23 @@ def __enter__(self): def __iter__(self): return self - def _wait(self, timeout=None): - # Block until the next read operation should be attempted, either - # because data becomes availalable within 'timeout' seconds or - # because the OS cannot determine whether data is available. - # Return True when a blocking read() is worth trying - # Return False if the timeout expires while waiting - - imap = self._imap - if timeout is None: - return True - if imap._readbuf: - return True - if timeout <= 0: - return False - - if imap.sock: - fileobj = imap.sock - elif platform.system() == 'Windows': - return True # Cannot select(); allow a possibly-blocking read - else: - fileobj = imap.readfile - - if __debug__ and imap.debug >= 4: - imap._mesg(f'idle _wait select({timeout})') - - with selectors.DefaultSelector() as sel: - sel.register(fileobj, selectors.EVENT_READ) - readables = sel.select(timeout) - return bool(readables) - def _pop(self, timeout, default=('', None)): - # Get the next response, or a default value on timeout - # - # :param timeout: Time limit (in seconds) to wait for response - # :type timeout: int|float|None - # :param default: Value to return on timeout - # - # Note: This method ignores 'duration' in favor of the timeout argument. - # - # Note: Windows IMAP4_stream connections will ignore the timeout - # argument and block until the next response arrives, because - # Windows select() only works on sockets. + # Get the next response, or a default value on timeout. + # The timeout arg can be an int or float, or None for no timeout. + # Timeouts require a socket connection (not IMAP4_stream). + # This method ignores self._duration. + + # Historical Note: + # The timeout was originally implemented using select() after + # checking for the presence of already-buffered data. + # That allowed timeouts on pipe connetions like IMAP4_stream. + # However, it seemed possible that SSL data arriving without any + # IMAP data afterward could cause select() to indicate available + # application data when there was none, leading to a read() call + # that would block with no timeout. It was unclear under what + # conditions this would happen in practice. Our implementation was + # changed to use socket timeouts instead of select(), just to be + # safe. imap = self._imap if imap.state != 'IDLING': @@ -1526,16 +1487,23 @@ def _pop(self, timeout, default=('', None)): return resp if __debug__ and imap.debug >= 4: - imap._mesg(f'idle _pop({timeout})') + imap._mesg(f'idle _pop({timeout}) reading') - if not self._wait(timeout): + if timeout is not None: + assert isinstance(imap.sock, socket.socket) + if timeout <= 0: + return default + imap.sock.settimeout(float(timeout)) + try: + imap._get_response() # Reads line, calls _append_untagged() + except TimeoutError: if __debug__ and imap.debug >= 4: imap._mesg(f'idle _pop({timeout}) done') return default + finally: + if timeout is not None: + imap.sock.settimeout(None) - if __debug__ and imap.debug >= 4: - imap._mesg(f'idle _pop({timeout}) reading') - imap._get_response() # Reads line, calls _append_untagged() resp = imap._idle_responses.pop(0) if __debug__ and imap.debug >= 4: @@ -1576,13 +1544,11 @@ def burst(self, interval=0.1): immediately without producing anything. Callers should consider this if using it in a loop. - Warning: On Windows, IMAP4_stream connections have no way to - accurately respect the 'interval' argument, since Windows select() - only works on sockets. This will cause the generator to yield - endless responses and block indefinitely for each one. It is - therefore advised not to use burst() with an IMAP4_stream - connection on Windows. + Note: This generator does not work on IMAP4_stream connections. """ + if not isinstance(self._imap.sock, socket.socket): + raise self._imap.error('burst() requires a socket connection') + try: yield next(self) except StopIteration: @@ -1719,20 +1685,26 @@ def open(self, host=None, port=None, timeout=None): self.sock = None self.file = None self.process = subprocess.Popen(self.command, - bufsize=0, # Unbuffered stdin/stdout, for select() compatibility + bufsize=DEFAULT_BUFFER_SIZE, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, close_fds=True) self.writefile = self.process.stdin self.readfile = self.process.stdout + def read(self, size): + """Read 'size' bytes from remote.""" + return self.readfile.read(size) + + + def readline(self): + """Read line from remote.""" + return self.readfile.readline() + def send(self, data): """Send data to remote.""" - # Write with buffered semantics to the unbuffered output, avoiding - # partial writes. - sent = 0 - while sent < len(data): - sent += self.writefile.write(data[sent:]) + self.writefile.write(data) + self.writefile.flush() def shutdown(self): From def6ab571544ee31ef51d286e9cf3804e329b187 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 15 Dec 2024 14:58:06 -0800 Subject: [PATCH 48/78] imaplib: Idler: rename private state attributes --- Lib/imaplib.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 38444cd2873a0f..b099fd096919fe 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1408,8 +1408,8 @@ def __init__(self, imap, duration=None): self._deadline = None self._imap = imap self._tag = None - self._sock_timeout = None - self._old_state = None + self._saved_timeout = None + self._saved_state = None def __enter__(self): imap = self._imap @@ -1442,14 +1442,14 @@ def __enter__(self): imap._idle_capture = False raise - self._sock_timeout = imap.sock.gettimeout() if imap.sock else None - if self._sock_timeout is not None: + self._saved_timeout = imap.sock.gettimeout() if imap.sock else None + if self._saved_timeout is not None: imap.sock.settimeout(None) # Socket timeout would break IDLE if self._duration is not None: self._deadline = time.monotonic() + self._duration - self._old_state = imap.state + self._saved_state = imap.state imap.state = 'IDLING' return self @@ -1578,11 +1578,11 @@ def __exit__(self, exc_type, exc_val, exc_tb): if __debug__ and imap.debug >= 4: imap._mesg('idle done') - imap.state = self._old_state + imap.state = self._saved_state - if self._sock_timeout is not None: - imap.sock.settimeout(self._sock_timeout) - self._sock_timeout = None + if self._saved_timeout is not None: + imap.sock.settimeout(self._saved_timeout) + self._saved_timeout = None # Stop intercepting untagged responses before sending DONE, # since we can no longer deliver them via iteration. From 8e0b6b004785f6bbff59d772d2c4de84bd7258fb Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 15 Dec 2024 15:00:25 -0800 Subject: [PATCH 49/78] imaplib: rephrase a comment in example code --- Doc/library/imaplib.rst | 2 +- Lib/imaplib.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 8d32cd3b46e1cd..cc01fa5e92d149 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -355,7 +355,7 @@ An :class:`IMAP4` instance has the following methods: Example:: >>> with M.idle() as idler: - ... # get next response and any others following by < 0.1 seconds + ... # get a response and any others following by < 0.1 seconds ... batch = list(idler.burst()) ... print(f'processing {len(batch)} responses...') ... print(batch) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index b099fd096919fe..0aef890fadba5e 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1530,7 +1530,7 @@ def burst(self, interval=0.1): Example: >>> with M.idle() as idler: - ... # get next response and any others following by < 0.1 seconds + ... # get a response and any others following by < 0.1 seconds ... batch = list(idler.burst()) ... print(f'processing {len(batch)} responses...') ... print(batch) From 08a45368310ef0590a6924c508d8d5abdf5cf027 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 15 Dec 2024 15:11:33 -0800 Subject: [PATCH 50/78] docs: imaplib: idle: use Sphinx code-block:: pycon --- Doc/library/imaplib.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index cc01fa5e92d149..619b70b70ef7dc 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -328,7 +328,7 @@ An :class:`IMAP4` instance has the following methods: Requires a socket connection; *duration* must be ``None`` on :class:`IMAP4_stream` connections. - Example:: + .. code-block:: pycon >>> with M.idle(duration=29 * 60) as idler: ... for typ, data in idler: @@ -352,7 +352,7 @@ An :class:`IMAP4` instance has the following methods: Requires a socket connection; does not work on :class:`IMAP4_stream` connections. - Example:: + .. code-block:: pycon >>> with M.idle() as idler: ... # get a response and any others following by < 0.1 seconds From c881c8be0950d9b7857da154c35d2092208b7a6f Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 15 Dec 2024 15:17:53 -0800 Subject: [PATCH 51/78] docs: whatsnew: imaplib: reformat IMAP4.idle entry --- Doc/whatsnew/3.14.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 302c8407d148ad..5dd0a9bc0f5e26 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -406,8 +406,9 @@ inspect imaplib ------- -* Add :meth:`~imaplib.IMAP4.idle`, implementing the IMAP4 ``IDLE`` command - as defined in :rfc:`2177`. (Contributed by Forest in :gh:`55454`.) +* Add :meth:`IMAP4.idle() `, implementing the IMAP4 + ``IDLE`` command as defined in :rfc:`2177`. + (Contributed by Forest in :gh:`55454`.) json ---- From 266a292003d518c13e6b3cc830302ded12fc50ec Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 15 Dec 2024 16:07:13 -0800 Subject: [PATCH 52/78] imaplib: idle: make doc strings brief Since we generally rely on the reST/html documentation for details, we can keep these doc strings short. This matches the module's existing doc string style and avoids having to sync small changes between two files. --- Lib/imaplib.py | 93 +++++++++++--------------------------------------- 1 file changed, 19 insertions(+), 74 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 0aef890fadba5e..0c41d3225d111e 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -638,35 +638,14 @@ def getquotaroot(self, mailbox): def idle(self, duration=None): - """Return an Idler: an iterable context manager for the IDLE command. - - :param duration: Maximum duration (in seconds) to keep idling, - or None for no time limit. To avoid inactivity - timeouts on servers that impose them, callers are - advised to keep this at most 29 minutes. - Requires a socket connection (not IMAP4_stream). - :type duration: int|float|None - - The returned object sends the IDLE command when activated by the - 'with' statement, produces IMAP untagged responses via the iterator - protocol, and sends DONE upon context exit. - - Responses are represented as (type, [data, ...]) tuples, as described - in the IMAP4 class documentation. - - Example: - - >>> with M.idle(duration=29 * 60) as idler: - ... for typ, data in idler: - ... print(typ, data) - ... - EXISTS [b'1'] - RECENT [b'1'] - - Note: The Idler class name and structure are internal interfaces, - subject to change. Calling code can rely on its context management, - iteration, and public method to remain stable, but should not subclass, - instantiate, compare, or otherwise directly reference the class. + """Return an iterable IDLE context manager producing untagged responses. + If the argument is not None, limit iteration to 'duration' seconds. + + with M.idle(duration=29 * 60) as idler: + for typ, data in idler: + print(typ, data) + + Note: 'duration' requires a socket connection (not IMAP4_stream). """ return Idler(self, duration) @@ -1381,20 +1360,11 @@ def print_log(self): class Idler: - """Iterable context manager: start IDLE & produce untagged responses. + """Iterable IDLE context manager: start IDLE & produce untagged responses. An object of this type is returned by the IMAP4.idle() method. - It sends the IDLE command when activated by the 'with' statement, produces - IMAP untagged responses via the iterator protocol, and sends DONE upon - context exit. - - Iteration produces (type, [data, ...]) tuples, as described in the IMAP4 - class documentation. - Note: The name and structure of this class are internal interfaces, - subject to change. Calling code can rely on its context management, - iteration, and public method to remain stable, but should not - subclass, instantiate, or otherwise directly reference the class. + Note: The name and structure of this class are subject to change. """ def __init__(self, imap, duration=None): @@ -1511,40 +1481,15 @@ def _pop(self, timeout, default=('', None)): return resp def burst(self, interval=0.1): - """Yield a burst of responses no more than 'interval' seconds apart - - :param interval: Time limit for each response after the first - (The IDLE context's maximum duration is - respected when waiting for the first response.) - :type interval: int|float - - This generator is an alternative to iterating one response at a - time, intended to aid in efficient batch processing. It retrieves - the next response along with any immediately available subsequent - responses. (For example, a rapid series of EXPUNGE responses after - a bulk delete.) - - Responses are represented as (type, [data, ...]) tuples, as described - in the IMAP4 class documentation. - - Example: - - >>> with M.idle() as idler: - ... # get a response and any others following by < 0.1 seconds - ... batch = list(idler.burst()) - ... print(f'processing {len(batch)} responses...') - ... print(batch) - ... - processing 3 responses... - [('EXPUNGE', [b'2']), ('EXPUNGE', [b'1']), ('RECENT', [b'0'])] - - Tip: The IDLE context's maximum duration, as passed to IMAP4.idle(), - is respected when waiting for the first response in a burst. - Therefore, an expired Idler will cause this generator to return - immediately without producing anything. Callers should consider - this if using it in a loop. - - Note: This generator does not work on IMAP4_stream connections. + """Yield a burst of responses no more than 'interval' seconds apart. + + with M.idle() as idler: + # get a response and any others following by < 0.1 seconds + batch = list(idler.burst()) + print(f'processing {len(batch)} responses...') + print(batch) + + Note: This generator requires a socket connection (not IMAP4_stream). """ if not isinstance(self._imap.sock, socket.socket): raise self._imap.error('burst() requires a socket connection') From 055a9bd7076e2203bab5decf9f96cb5ed15b0fec Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 15 Dec 2024 16:13:38 -0800 Subject: [PATCH 53/78] imaplib: Idler: split assert into two statements --- Lib/imaplib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 0c41d3225d111e..2f2467dc5be4c1 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1383,7 +1383,8 @@ def __init__(self, imap, duration=None): def __enter__(self): imap = self._imap - assert not (imap._idle_responses or imap._idle_capture) + assert not imap._idle_responses + assert not imap._idle_capture if __debug__ and imap.debug >= 4: imap._mesg(f'idle start duration={self._duration}') From 4d3f02091974db4a0c4db9c6ac4b6e96778cce32 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 15 Dec 2024 16:23:21 -0800 Subject: [PATCH 54/78] imaplib: Idler: move assignment out of try: block --- Lib/imaplib.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 2f2467dc5be4c1..f0757c04accb68 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1389,12 +1389,12 @@ def __enter__(self): if __debug__ and imap.debug >= 4: imap._mesg(f'idle start duration={self._duration}') - try: - # Start capturing untagged responses before sending IDLE, - # so we can deliver via iteration any that arrive while - # the IDLE command continuation request is still pending. - imap._idle_capture = True + # Start capturing untagged responses before sending IDLE, + # so we can deliver via iteration any that arrive while + # the IDLE command continuation request is still pending. + imap._idle_capture = True + try: self._tag = imap._command('IDLE') # As with any command, the server is allowed to send us unrelated, # untagged responses before acting on IDLE. These lines will be From 80aaf8df65dfd21ef2766a80eb98f5e2989bf021 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 15 Dec 2024 16:42:03 -0800 Subject: [PATCH 55/78] imaplib: Idler: move __exit__() for readability --- Lib/imaplib.py | 84 +++++++++++++++++++++++++------------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index f0757c04accb68..30397644d722b3 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1425,6 +1425,48 @@ def __enter__(self): return self + def __exit__(self, exc_type, exc_val, exc_tb): + imap = self._imap + + if __debug__ and imap.debug >= 4: + imap._mesg('idle done') + imap.state = self._saved_state + + if self._saved_timeout is not None: + imap.sock.settimeout(self._saved_timeout) + self._saved_timeout = None + + # Stop intercepting untagged responses before sending DONE, + # since we can no longer deliver them via iteration. + imap._idle_capture = False + + # If we captured untagged responses while the IDLE command + # continuation request was still pending, but the user did not + # iterate over them before exiting IDLE, we must put them + # someplace where the user can retrieve them. The only + # sensible place for this is the untagged_responses dict, + # despite its unfortunate inability to preserve the relative + # order of different response types. + if leftovers := len(imap._idle_responses): + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle quit with {leftovers} leftover responses') + while imap._idle_responses: + typ, data = imap._idle_responses.pop(0) + # Append one fragment at a time, just as _get_response() does + for datum in data: + imap._append_untagged(typ, datum) + + try: + imap.send(b'DONE' + CRLF) + status, [msg] = imap._command_complete('IDLE', self._tag) + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle status: {status} {msg!r}') + except OSError: + if not exc_type: + raise + + return False # Do not suppress context body exceptions + def __iter__(self): return self @@ -1519,48 +1561,6 @@ def __next__(self): return typ, data - def __exit__(self, exc_type, exc_val, exc_tb): - imap = self._imap - - if __debug__ and imap.debug >= 4: - imap._mesg('idle done') - imap.state = self._saved_state - - if self._saved_timeout is not None: - imap.sock.settimeout(self._saved_timeout) - self._saved_timeout = None - - # Stop intercepting untagged responses before sending DONE, - # since we can no longer deliver them via iteration. - imap._idle_capture = False - - # If we captured untagged responses while the IDLE command - # continuation request was still pending, but the user did not - # iterate over them before exiting IDLE, we must put them - # someplace where the user can retrieve them. The only - # sensible place for this is the untagged_responses dict, - # despite its unfortunate inability to preserve the relative - # order of different response types. - if leftovers := len(imap._idle_responses): - if __debug__ and imap.debug >= 4: - imap._mesg(f'idle quit with {leftovers} leftover responses') - while imap._idle_responses: - typ, data = imap._idle_responses.pop(0) - # Append one fragment at a time, just as _get_response() does - for datum in data: - imap._append_untagged(typ, datum) - - try: - imap.send(b'DONE' + CRLF) - status, [msg] = imap._command_complete('IDLE', self._tag) - if __debug__ and imap.debug >= 4: - imap._mesg(f'idle status: {status} {msg!r}') - except OSError: - if not exc_type: - raise - - return False # Do not suppress context body exceptions - if HAVE_SSL: From 03b5205a50d944c9a2022eafa25afa17e66e7619 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 15 Dec 2024 16:44:29 -0800 Subject: [PATCH 56/78] imaplib: Idler: move __next__() for readability --- Lib/imaplib.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 30397644d722b3..630e67a8c49a40 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1523,6 +1523,22 @@ def _pop(self, timeout, default=('', None)): imap._mesg(f'idle _pop({timeout}) read {resp[0]}') return resp + def __next__(self): + imap = self._imap + + if self._duration is None: + timeout = None + else: + timeout = self._deadline - time.monotonic() + typ, data = self._pop(timeout) + + if not typ: + if __debug__ and imap.debug >= 4: + imap._mesg('idle iterator exhausted') + raise StopIteration + + return typ, data + def burst(self, interval=0.1): """Yield a burst of responses no more than 'interval' seconds apart. @@ -1545,22 +1561,6 @@ def burst(self, interval=0.1): while response := self._pop(interval, None): yield response - def __next__(self): - imap = self._imap - - if self._duration is None: - timeout = None - else: - timeout = self._deadline - time.monotonic() - typ, data = self._pop(timeout) - - if not typ: - if __debug__ and imap.debug >= 4: - imap._mesg('idle iterator exhausted') - raise StopIteration - - return typ, data - if HAVE_SSL: From 83c89465eefe7663d31f780c29f33e914551a99a Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 15 Dec 2024 17:08:21 -0800 Subject: [PATCH 57/78] imaplib: test: make IdleCmdHandler a global class --- Lib/test/test_imaplib.py | 47 ++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index b7d9a14e0f9c72..a17345fca448eb 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -208,6 +208,28 @@ def cmd_UNSELECT(self, tag, args): self._send_tagged(tag, 'BAD', 'No mailbox selected') +class IdleCmdHandler(SimpleIMAPHandler): + capabilities = 'IDLE' + def cmd_IDLE(self, tag, args): + self._send_textline('+ idling') + # simple response + self._send_line(b'* 2 EXISTS') + # complex response: fragmented data due to literal string + self._send_line(b'* 1 FETCH (BODY[HEADER.FIELDS (DATE)] {41}') + self._send(b'Date: Fri, 06 Dec 2024 06:00:00 +0000\r\n\r\n') + self._send_line(b')') + # simple response following a fragmented one + self._send_line(b'* 3 EXISTS') + # response arriving later + time.sleep(1) + self._send_line(b'* 1 RECENT') + r = yield + if r == b'DONE\r\n': + self._send_tagged(tag, 'OK', 'Idle completed') + else: + self._send_tagged(tag, 'BAD', 'Expected DONE') + + class NewIMAPTestsMixin(): client = None @@ -504,29 +526,8 @@ def test_idle_capability(self): with client.idle(): pass - class IdleCmdHandler(SimpleIMAPHandler): - capabilities = 'IDLE' - def cmd_IDLE(self, tag, args): - self._send_textline('+ idling') - # simple response - self._send_line(b'* 2 EXISTS') - # complex response: fragmented data due to literal string - self._send_line(b'* 1 FETCH (BODY[HEADER.FIELDS (DATE)] {41}') - self._send(b'Date: Fri, 06 Dec 2024 06:00:00 +0000\r\n\r\n') - self._send_line(b')') - # simple response following a fragmented one - self._send_line(b'* 3 EXISTS') - # response arriving later - time.sleep(1) - self._send_line(b'* 1 RECENT') - r = yield - if r == b'DONE\r\n': - self._send_tagged(tag, 'OK', 'Idle completed') - else: - self._send_tagged(tag, 'BAD', 'Expected DONE') - def test_idle_iter(self): - client, _ = self._setup(self.IdleCmdHandler) + client, _ = self._setup(IdleCmdHandler) client.login('user', 'pass') with client.idle() as idler: # iteration should produce responses @@ -551,7 +552,7 @@ def test_idle_iter(self): self.assertEqual(data, [b'1']) def test_idle_burst(self): - client, _ = self._setup(self.IdleCmdHandler) + client, _ = self._setup(IdleCmdHandler) client.login('user', 'pass') # burst() should yield immediately available responses with client.idle() as idler: From be2d2b03bd142063508831c116a9958414fe2f41 Mon Sep 17 00:00:00 2001 From: Forest Date: Sun, 15 Dec 2024 17:14:54 -0800 Subject: [PATCH 58/78] docs: imaplib: idle: collapse double-spaces --- Doc/library/imaplib.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 619b70b70ef7dc..cf893090f03300 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -344,7 +344,7 @@ An :class:`IMAP4` instance has the following methods: (expressed as an :class:`int` or :class:`float`). This :term:`generator` is an alternative to iterating one response at a - time, intended to aid in efficient batch processing. It retrieves the + time, intended to aid in efficient batch processing. It retrieves the next response along with any immediately available subsequent responses. (For example, a rapid series of ``EXPUNGE`` responses after a bulk delete.) @@ -368,20 +368,20 @@ An :class:`IMAP4` instance has the following methods: The ``IDLE`` context's maximum duration, as passed to :meth:`IMAP4.idle`, is respected when waiting for the first response in a burst. Therefore, an expired :class:`!Idler` will cause this - generator to return immediately without producing anything. Callers + generator to return immediately without producing anything. Callers should consider this if using it in a loop. .. note:: The :class:`!Idler` object returned by :meth:`IMAP4.idle` is usable only - within a :keyword:`with` statement. To retrieve unsolicited IMAP + within a :keyword:`with` statement. To retrieve unsolicited IMAP responses outside that context, see :meth:`IMAP4.response`. .. note:: The :class:`!Idler` class name and structure are internal interfaces, - subject to change. Calling code can rely on its context management, + subject to change. Calling code can rely on its context management, iteration, and public method to remain stable, but should not subclass, instantiate, compare, or otherwise directly reference the class. From de62a3ed2242c195684ed5c7be68a11991ff1a25 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 16 Dec 2024 16:22:03 -0800 Subject: [PATCH 59/78] imaplib: warn on use of undocumented 'file' attr --- Lib/imaplib.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 630e67a8c49a40..8bb4d5f856acf8 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -34,6 +34,7 @@ import time from datetime import datetime, timezone, timedelta from io import DEFAULT_BUFFER_SIZE +import warnings try: import ssl @@ -323,7 +324,22 @@ def open(self, host='', port=IMAP4_PORT, timeout=None): self.host = host self.port = port self.sock = self._create_socket(timeout) - self.file = self.sock.makefile('rb') + self._file = self.sock.makefile('rb') + + + @property + def file(self): + # The old 'file' attribute is no longer used now that we do our own + # read() and readline() buffering, with which it conflicts. + # As an undocumented interface, it should never have been accessed by + # external code, and therefore does not warrant deprecation. + # Nevertheless, we provide this property for now, to avoid suddenly + # breaking any code in the wild that might have been using it in a + # harmless way. + warnings.warn( + 'IMAP4.file is unsupported, can cause errors, and may be removed.', + RuntimeWarning) + return self._file def read(self, size): @@ -383,7 +399,7 @@ def send(self, data): def shutdown(self): """Close I/O established in "open".""" - self.file.close() + self._file.close() try: self.sock.shutdown(socket.SHUT_RDWR) except OSError as exc: @@ -883,7 +899,7 @@ def starttls(self, ssl_context=None): if typ == 'OK': self.sock = ssl_context.wrap_socket(self.sock, server_hostname=self.host) - self.file = self.sock.makefile('rb') + self._file = self.sock.makefile('rb') self._tls_established = True self._get_capabilities() else: @@ -1629,7 +1645,7 @@ def open(self, host=None, port=None, timeout=None): self.host = None # For compatibility with parent class self.port = None self.sock = None - self.file = None + self._file = None self.process = subprocess.Popen(self.command, bufsize=DEFAULT_BUFFER_SIZE, stdin=subprocess.PIPE, stdout=subprocess.PIPE, From 7fc8a24d95d855928f13de8328be48ba101af114 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 16 Dec 2024 17:13:58 -0800 Subject: [PATCH 60/78] imaplib: revert import reformatting Since we no longer import platform or selectors, the original import statement style can be restored, reducing the footprint of PR #122542. --- Lib/imaplib.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 8bb4d5f856acf8..6e7e18e9794d8d 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -23,15 +23,7 @@ __version__ = "2.59" -import binascii -import calendar -import errno -import random -import re -import socket -import subprocess -import sys -import time +import binascii, errno, random, re, socket, subprocess, sys, time, calendar from datetime import datetime, timezone, timedelta from io import DEFAULT_BUFFER_SIZE import warnings From 33a1fed515a95e1a880ea4659eb0cffff0203461 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 16 Dec 2024 17:17:24 -0800 Subject: [PATCH 61/78] imaplib: restore original exception msg formatting This reduces the footprint of PR #122542. --- Lib/imaplib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 6e7e18e9794d8d..3f60295c249536 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -379,7 +379,7 @@ def readline(self): line = b''.join(parts) if len(line) > _MAXLINE: - raise self.error(f'got more than {_MAXLINE} bytes') + raise self.error("got more than %d bytes" % _MAXLINE) return line From 40e607a209de9a33d00985bad429d8d0f316b956 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 16 Dec 2024 17:51:07 -0800 Subject: [PATCH 62/78] docs: imaplib: idle: versionadded:: next --- Doc/library/imaplib.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index cf893090f03300..f1be942fe6d27f 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -385,6 +385,8 @@ An :class:`IMAP4` instance has the following methods: iteration, and public method to remain stable, but should not subclass, instantiate, compare, or otherwise directly reference the class. + .. versionadded:: next + .. method:: IMAP4.list([directory[, pattern]]) From 9a07f3bc1cd927ba41a28c57818adaaa610567d0 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 16 Dec 2024 18:35:06 -0800 Subject: [PATCH 63/78] imaplib: move import statement to where it's used This import is only needed if external code tries to use an attribute that it shouldn't be using. Making it a local import reduces module loading time in supported cases. --- Lib/imaplib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 3f60295c249536..d6aa121c8df8e4 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -26,7 +26,6 @@ import binascii, errno, random, re, socket, subprocess, sys, time, calendar from datetime import datetime, timezone, timedelta from io import DEFAULT_BUFFER_SIZE -import warnings try: import ssl @@ -328,6 +327,7 @@ def file(self): # Nevertheless, we provide this property for now, to avoid suddenly # breaking any code in the wild that might have been using it in a # harmless way. + import warnings warnings.warn( 'IMAP4.file is unsupported, can cause errors, and may be removed.', RuntimeWarning) From f47de537985256cbb8a1c4ca357bb80b4deca5fb Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 16 Dec 2024 19:51:47 -0800 Subject: [PATCH 64/78] imaplib test: RuntimeWarning on IMAP4.file access --- Lib/test/test_imaplib.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index a17345fca448eb..91b610af7064f8 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -21,6 +21,7 @@ import ssl except ImportError: ssl = None +import warnings support.requires_working_socket(module=True) @@ -602,6 +603,17 @@ def test_unselect(self): self.assertEqual(data[0], b'Returned to authenticated state. (Success)') self.assertEqual(client.state, 'AUTH') + # property tests + + def test_file_property(self): + client, _ = self._setup(SimpleIMAPHandler) + # 'file' attribute access should trigger a warning + with warnings.catch_warnings(record=True) as warned: + warnings.simplefilter('always') + client.file + self.assertEqual(len(warned), 1) + self.assertTrue(issubclass(warned[-1].category, RuntimeWarning)) + class NewIMAPTests(NewIMAPTestsMixin, unittest.TestCase): imap_class = imaplib.IMAP4 From 66d32a00213e476c6a954ca7bbe99b6e2e5e4427 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 16 Dec 2024 20:18:57 -0800 Subject: [PATCH 65/78] imaplib: use stacklevel=2 in warnings.warn() --- Lib/imaplib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index d6aa121c8df8e4..3792b85b436056 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -330,7 +330,8 @@ def file(self): import warnings warnings.warn( 'IMAP4.file is unsupported, can cause errors, and may be removed.', - RuntimeWarning) + RuntimeWarning, + stacklevel=2) return self._file From d619580eb91f2215a90c118d45f6e2f20ff077a3 Mon Sep 17 00:00:00 2001 From: Forest Date: Tue, 17 Dec 2024 01:44:22 -0800 Subject: [PATCH 66/78] imaplib test: simplify IMAP4.file warning test --- Lib/test/test_imaplib.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 91b610af7064f8..88262cb8bcacdd 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -21,7 +21,6 @@ import ssl except ImportError: ssl = None -import warnings support.requires_working_socket(module=True) @@ -605,14 +604,11 @@ def test_unselect(self): # property tests - def test_file_property(self): + def test_file_property_should_not_be_accessed(self): client, _ = self._setup(SimpleIMAPHandler) - # 'file' attribute access should trigger a warning - with warnings.catch_warnings(record=True) as warned: - warnings.simplefilter('always') + # the 'file' property replaced a private attribute that is now unsafe + with self.assertWarns(RuntimeWarning): client.file - self.assertEqual(len(warned), 1) - self.assertTrue(issubclass(warned[-1].category, RuntimeWarning)) class NewIMAPTests(NewIMAPTestsMixin, unittest.TestCase): From fc8b6f47e78c9482b940b1dba70cd7a6470200e7 Mon Sep 17 00:00:00 2001 From: Forest Date: Sat, 21 Dec 2024 15:26:15 -0800 Subject: [PATCH 67/78] imaplib test: pre-idle-continuation response --- Lib/test/test_imaplib.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 88262cb8bcacdd..651b11a181114c 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -211,6 +211,8 @@ def cmd_UNSELECT(self, tag, args): class IdleCmdHandler(SimpleIMAPHandler): capabilities = 'IDLE' def cmd_IDLE(self, tag, args): + # pre-idle-continuation response + self._send_line(b'* 0 EXISTS') self._send_textline('+ idling') # simple response self._send_line(b'* 2 EXISTS') @@ -530,6 +532,9 @@ def test_idle_iter(self): client, _ = self._setup(IdleCmdHandler) client.login('user', 'pass') with client.idle() as idler: + # iteration should include response between 'IDLE' & '+ idling' + response = next(idler) + self.assertEqual(response, ('EXISTS', [b'0'])) # iteration should produce responses response = next(idler) self.assertEqual(response, ('EXISTS', [b'2'])) @@ -557,7 +562,7 @@ def test_idle_burst(self): # burst() should yield immediately available responses with client.idle() as idler: batch = list(idler.burst()) - self.assertEqual(len(batch), 3) + self.assertEqual(len(batch), 4) # burst() should not have consumed later responses _, data = client.response('RECENT') self.assertEqual(data, [b'1']) From b767ab6ae05914a93e9299e505783928b660e430 Mon Sep 17 00:00:00 2001 From: Forest Date: Sat, 21 Dec 2024 15:56:13 -0800 Subject: [PATCH 68/78] imaplib test: post-done untagged response --- Lib/test/test_imaplib.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 651b11a181114c..33247cfd51c84c 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -227,6 +227,7 @@ def cmd_IDLE(self, tag, args): self._send_line(b'* 1 RECENT') r = yield if r == b'DONE\r\n': + self._send_line(b'* 9 RECENT') self._send_tagged(tag, 'OK', 'Idle completed') else: self._send_tagged(tag, 'BAD', 'Expected DONE') @@ -552,9 +553,11 @@ def test_idle_iter(self): # iteration should have consumed untagged responses _, data = client.response('EXISTS') self.assertEqual(data, [None]) - # responses not iterated should remain after idle + # responses not iterated should be available after idle _, data = client.response('RECENT') - self.assertEqual(data, [b'1']) + self.assertEqual(data[0], b'1') + # responses received after 'DONE' should be available after idle + self.assertEqual(data[1], b'9') def test_idle_burst(self): client, _ = self._setup(IdleCmdHandler) @@ -565,7 +568,7 @@ def test_idle_burst(self): self.assertEqual(len(batch), 4) # burst() should not have consumed later responses _, data = client.response('RECENT') - self.assertEqual(data, [b'1']) + self.assertEqual(data, [b'1', b'9']) def test_login(self): client, _ = self._setup(SimpleIMAPHandler) From dcd0161310eb6cfae7b6789e3211c7f9cef1eace Mon Sep 17 00:00:00 2001 From: Forest Date: Sat, 21 Dec 2024 17:42:36 -0800 Subject: [PATCH 69/78] imaplib: downgrade idle-denied exception to error This makes it easier for client code to distinguish a temporary rejection of the IDLE command from a server responding incorrectly to IDLE. --- Lib/imaplib.py | 3 +++ Lib/test/test_imaplib.py | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 3792b85b436056..27ced075f5d95e 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1413,6 +1413,9 @@ def __enter__(self): # this occurs. while resp := imap._get_response(): if imap.tagged_commands[self._tag]: + typ, data = imap.tagged_commands.pop(self._tag) + if typ == 'NO': + raise imap.error(f'idle denied: {data}') raise imap.abort(f'unexpected status response: {resp}') if __debug__ and imap.debug >= 4: diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 33247cfd51c84c..a6fb5d45a42781 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -208,6 +208,12 @@ def cmd_UNSELECT(self, tag, args): self._send_tagged(tag, 'BAD', 'No mailbox selected') +class IdleCmdDenyHandler(SimpleIMAPHandler): + capabilities = 'IDLE' + def cmd_IDLE(self, tag, args): + self._send_tagged(tag, 'NO', 'IDLE is not allowed at this time') + + class IdleCmdHandler(SimpleIMAPHandler): capabilities = 'IDLE' def cmd_IDLE(self, tag, args): @@ -529,6 +535,13 @@ def test_idle_capability(self): with client.idle(): pass + def test_idle_denied(self): + client, _ = self._setup(IdleCmdDenyHandler) + client.login('user', 'pass') + with self.assertRaises(imaplib.IMAP4.error): + with client.idle() as idler: + pass + def test_idle_iter(self): client, _ = self._setup(IdleCmdHandler) client.login('user', 'pass') From 53c7a19509f2e899b693a1d7fd41d82455759dcd Mon Sep 17 00:00:00 2001 From: Forest Date: Wed, 8 Jan 2025 17:47:03 -0800 Subject: [PATCH 70/78] imaplib: simplify check for socket object --- Lib/imaplib.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 27ced075f5d95e..86c4060eaf50d3 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1379,8 +1379,7 @@ class Idler: def __init__(self, imap, duration=None): if 'IDLE' not in imap.capabilities: raise imap.error("Server does not support IMAP4 IDLE") - if (duration is not None and - not isinstance(imap.sock, socket.socket)): + if duration is not None and not imap.sock: # IMAP4_stream pipes don't support timeouts raise imap.error('duration requires a socket connection') self._duration = duration @@ -1562,7 +1561,7 @@ def burst(self, interval=0.1): Note: This generator requires a socket connection (not IMAP4_stream). """ - if not isinstance(self._imap.sock, socket.socket): + if not self._imap.sock: raise self._imap.error('burst() requires a socket connection') try: From fcaf355eb0b628e5bcb226ba7d5c49976dc72919 Mon Sep 17 00:00:00 2001 From: Forest Date: Thu, 9 Jan 2025 11:24:04 -0800 Subject: [PATCH 71/78] imaplib: narrow the scope of IDLE socket timeouts If an IDLE duration or burst() was in use, and an unsolicited response contained a literal string, and crossed a packet boundary, and the subsequent packet was delayed beyond the IDLE feature's time limit, the timeout would leave the incoming protocol stream in a bad state (with the tail of that response appearing where the start of a response is expected). This change moves the IDLE socket timeout to cover only the start of a response, so it can no longer cause that problem. --- Lib/imaplib.py | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 86c4060eaf50d3..09b2db9ee14274 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -186,6 +186,7 @@ class IMAP4: class error(Exception): pass # Logical errors - debug required class abort(error): pass # Service errors - close and retry class readonly(abort): pass # Mailbox status changed to READ-ONLY + class _responsetimeout(TimeoutError): pass # No response during IDLE def __init__(self, host='', port=IMAP4_PORT, timeout=None): self.debug = Debug @@ -1154,14 +1155,28 @@ def _get_capabilities(self): self.capabilities = tuple(dat.split()) - def _get_response(self): + def _get_response(self, start_timeout=False): # Read response and store. # # Returns None for continuation responses, # otherwise first response line received. - - resp = self._get_line() + # + # If start_timeout is given, temporarily uses it as a socket + # timeout while waiting for the start of a response, raising + # _responsetimeout if one doesn't arrive. (Used by Idler.) + + if start_timeout is not False and self.sock: + assert start_timeout is None or start_timeout > 0 + saved_timeout = self.sock.gettimeout() + self.sock.settimeout(start_timeout) + try: + resp = self._get_line() + except TimeoutError as err: + raise self._responsetimeout from err + finally: + if start_timeout is not False and self.sock: + self.sock.settimeout(saved_timeout) # Command completion response? @@ -1386,7 +1401,6 @@ def __init__(self, imap, duration=None): self._deadline = None self._imap = imap self._tag = None - self._saved_timeout = None self._saved_state = None def __enter__(self): @@ -1424,10 +1438,6 @@ def __enter__(self): imap._idle_capture = False raise - self._saved_timeout = imap.sock.gettimeout() if imap.sock else None - if self._saved_timeout is not None: - imap.sock.settimeout(None) # Socket timeout would break IDLE - if self._duration is not None: self._deadline = time.monotonic() + self._duration @@ -1443,10 +1453,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): imap._mesg('idle done') imap.state = self._saved_state - if self._saved_timeout is not None: - imap.sock.settimeout(self._saved_timeout) - self._saved_timeout = None - # Stop intercepting untagged responses before sending DONE, # since we can no longer deliver them via iteration. imap._idle_capture = False @@ -1514,19 +1520,16 @@ def _pop(self, timeout, default=('', None)): imap._mesg(f'idle _pop({timeout}) reading') if timeout is not None: - assert isinstance(imap.sock, socket.socket) if timeout <= 0: return default - imap.sock.settimeout(float(timeout)) + timeout = float(timeout) # Required by socket.settimeout() + try: - imap._get_response() # Reads line, calls _append_untagged() - except TimeoutError: + imap._get_response(timeout) # Reads line, calls _append_untagged() + except IMAP4._responsetimeout: if __debug__ and imap.debug >= 4: imap._mesg(f'idle _pop({timeout}) done') return default - finally: - if timeout is not None: - imap.sock.settimeout(None) resp = imap._idle_responses.pop(0) From a47bcb4a809b0b0e3f7b0d4cca252e69c77e29e4 Mon Sep 17 00:00:00 2001 From: Forest Date: Fri, 10 Jan 2025 02:20:32 -0800 Subject: [PATCH 72/78] imaplib: preserve partial reads on exception This ensures that short IDLE durations / burst() intervals won't risk corrupting response lines that span multiple packets. --- Lib/imaplib.py | 51 ++++++++++++++++++++++++++-------------- Lib/test/test_imaplib.py | 29 +++++++++++++++++++++++ 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 09b2db9ee14274..4bebb9e4ec4459 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -201,7 +201,7 @@ def __init__(self, host='', port=IMAP4_PORT, timeout=None): self.tagnum = 0 self._tls_established = False self._mode_ascii() - self._readbuf = b'' + self._readbuf = [] # Open socket to server. @@ -346,16 +346,24 @@ def read(self, size): # https://github.com/python/cpython/issues/51571 parts = [] - while True: - if len(self._readbuf) >= size: - parts.append(self._readbuf[:size]) - self._readbuf = self._readbuf[size:] - break - parts.append(self._readbuf) - size -= len(self._readbuf) - self._readbuf = self.sock.recv(DEFAULT_BUFFER_SIZE) - if not self._readbuf: + + while size > 0: + + if len(parts) < len(self._readbuf): + buf = self._readbuf[len(parts)] + else: + buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + if not buf: + break + self._readbuf.append(buf) + + if len(buf) >= size: + parts.append(buf[:size]) + self._readbuf = [buf[size:]] break + parts.append(buf) + size -= len(buf) + return b''.join(parts) @@ -366,18 +374,25 @@ def readline(self): LF = b'\n' parts = [] length = 0 + while length < _MAXLINE: - pos = self._readbuf.find(LF) + + if len(parts) < len(self._readbuf): + buf = self._readbuf[len(parts)] + else: + buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + if not buf: + break + self._readbuf.append(buf) + + pos = buf.find(LF) if pos != -1: pos += 1 - parts.append(self._readbuf[:pos]) - self._readbuf = self._readbuf[pos:] - break - parts.append(self._readbuf) - length += len(parts[-1]) - self._readbuf = self.sock.recv(DEFAULT_BUFFER_SIZE) - if not self._readbuf: + parts.append(buf[:pos]) + self._readbuf = [buf[pos:]] break + parts.append(buf) + length += len(buf) line = b''.join(parts) if len(line) > _MAXLINE: diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index a6fb5d45a42781..2467125f15c9f9 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -239,6 +239,23 @@ def cmd_IDLE(self, tag, args): self._send_tagged(tag, 'BAD', 'Expected DONE') +class IdleCmdDelayedPacketHandler(SimpleIMAPHandler): + capabilities = 'IDLE' + def cmd_IDLE(self, tag, args): + self._send_textline('+ idling') + # response line spanning multiple packets, the last one delayed + self._send(b'* 1 EX') + time.sleep(0.2) + self._send(b'IS') + time.sleep(1) + self._send(b'TS\r\n') + r = yield + if r == b'DONE\r\n': + self._send_tagged(tag, 'OK', 'Idle completed') + else: + self._send_tagged(tag, 'BAD', 'Expected DONE') + + class NewIMAPTestsMixin(): client = None @@ -583,6 +600,18 @@ def test_idle_burst(self): _, data = client.response('RECENT') self.assertEqual(data, [b'1', b'9']) + def test_idle_delayed_packet(self): + client, _ = self._setup(IdleCmdDelayedPacketHandler) + client.login('user', 'pass') + # If our readline() implementation fails to preserve line fragments + # when idle timeouts trigger, a response spanning delayed packets + # can be corrupted, leaving the protocol stream in a bad state. + try: + with client.idle(0.5) as idler: + self.assertRaises(StopIteration, next, idler) + except client.abort as err: + self.fail('multi-packet response was corrupted by idle timeout') + def test_login(self): client, _ = self._setup(SimpleIMAPHandler) typ, data = client.login('user', 'pass') From 7fc4b78dcf23302a73b2313312579a6b43e19441 Mon Sep 17 00:00:00 2001 From: Forest Date: Fri, 10 Jan 2025 12:01:01 -0800 Subject: [PATCH 73/78] imaplib: read/readline: save multipart buffer tail For resilience if read() or readline() ever complete with more than one bytes object remaining in the buffer. This is not expected to happen, but it seems wise to be prepared for a future change making it possible. --- Lib/imaplib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 4bebb9e4ec4459..8f03a42f5e177c 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -359,7 +359,7 @@ def read(self, size): if len(buf) >= size: parts.append(buf[:size]) - self._readbuf = [buf[size:]] + self._readbuf = [buf[size:]] + self._readbuf[len(parts):] break parts.append(buf) size -= len(buf) @@ -389,7 +389,7 @@ def readline(self): if pos != -1: pos += 1 parts.append(buf[:pos]) - self._readbuf = [buf[pos:]] + self._readbuf = [buf[pos:]] + self._readbuf[len(parts):] break parts.append(buf) length += len(buf) From be34141bc4c9cdc76a1f851268575040b6f4acc4 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 20 Jan 2025 11:48:10 -0800 Subject: [PATCH 74/78] imaplib: use TimeoutError subclass only if needed --- Lib/imaplib.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 8f03a42f5e177c..0203feb133bc11 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1185,13 +1185,14 @@ def _get_response(self, start_timeout=False): assert start_timeout is None or start_timeout > 0 saved_timeout = self.sock.gettimeout() self.sock.settimeout(start_timeout) - try: - resp = self._get_line() - except TimeoutError as err: - raise self._responsetimeout from err - finally: - if start_timeout is not False and self.sock: + try: + resp = self._get_line() + except TimeoutError as err: + raise self._responsetimeout from err + finally: self.sock.settimeout(saved_timeout) + else: + resp = self._get_line() # Command completion response? From e8a85094b2a12ebfc2995b27d8816141f1ca4f72 Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 20 Jan 2025 12:15:17 -0800 Subject: [PATCH 75/78] doc: imaplib: elaborate on IDLE response delivery --- Doc/library/imaplib.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index f1be942fe6d27f..1e50d514ba1aeb 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -317,6 +317,12 @@ An :class:`IMAP4` instance has the following methods: :keyword:`with` statement, produces IMAP untagged responses via the :term:`iterator` protocol, and sends ``DONE`` upon context exit. + All untagged responses that arrive after sending the ``IDLE`` command + (including any that arrive before the server acknowledges the command) will + be available via iteration. Any leftover responses (those not iterated in + the :keyword:`with` context) can be retrieved in the usual way after + ``IDLE`` ends, using :meth:`IMAP4.response`. + Responses are represented as ``(type, [data, ...])`` tuples, as described in :ref:`IMAP4 Objects `. From 8d7801054b75ab1a694200892230fd45d8bd5f6a Mon Sep 17 00:00:00 2001 From: Forest Date: Mon, 20 Jan 2025 12:52:37 -0800 Subject: [PATCH 76/78] doc: imaplib: elaborate in note re: IMAP4.response --- Doc/library/imaplib.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 1e50d514ba1aeb..2c5a1f1fbc1213 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -380,9 +380,10 @@ An :class:`IMAP4` instance has the following methods: .. note:: - The :class:`!Idler` object returned by :meth:`IMAP4.idle` is usable only - within a :keyword:`with` statement. To retrieve unsolicited IMAP - responses outside that context, see :meth:`IMAP4.response`. + The iterator returned by :meth:`IMAP4.idle` is usable only within a + :keyword:`with` statement. Before or after that context, unsolicited + responses are collected internally whenever a command finishes, and can + be retrieved with :meth:`IMAP4.response`. .. note:: From f650dfa622df2a3589aced764358d8eafe23957c Mon Sep 17 00:00:00 2001 From: Forest Date: Tue, 4 Feb 2025 14:11:32 -0800 Subject: [PATCH 77/78] imaplib: comment on benefit of reading in chunks Our read() implementation designed to support IDLE replaces the one from PR #119514, fixing the same problem it was addressing. The tests that it added are preserved. --- Lib/imaplib.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 0203feb133bc11..7dc076244a34ed 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -344,6 +344,12 @@ def read(self, size): # error condition instead of letting the caller decide how to handle a # timeout. We therefore implement our own buffered read(). # https://github.com/python/cpython/issues/51571 + # + # Reading in chunks instead of delegating to a single + # BufferedReader.read() call also means we avoid its preallocation + # of an unreasonably large memory block if a malicious server claims + # it will send a huge literal without actually sending one. + # https://github.com/python/cpython/issues/119511 parts = [] From 3512858add800a02b2b8c1a57af37c6dbb598383 Mon Sep 17 00:00:00 2001 From: Forest Date: Tue, 4 Feb 2025 16:14:29 -0800 Subject: [PATCH 78/78] imaplib: readline(): treat ConnectionError as EOF --- Lib/imaplib.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 7dc076244a34ed..2c3925958d011b 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -358,7 +358,10 @@ def read(self, size): if len(parts) < len(self._readbuf): buf = self._readbuf[len(parts)] else: - buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + try: + buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + except ConnectionError: + break if not buf: break self._readbuf.append(buf) @@ -386,7 +389,10 @@ def readline(self): if len(parts) < len(self._readbuf): buf = self._readbuf[len(parts)] else: - buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + try: + buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + except ConnectionError: + break if not buf: break self._readbuf.append(buf)