diff --git a/Doc/Makefile b/Doc/Makefile index c8a749a02a89ec..84578c5c57f478 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -170,6 +170,7 @@ venv: echo "venv already exists."; \ echo "To recreate it, remove it first with \`make clean-venv'."; \ else \ + set -e; \ echo "Creating venv in $(VENVDIR)"; \ if $(UV) --version >/dev/null 2>&1; then \ $(UV) venv --python=$(PYTHON) $(VENVDIR); \ diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 7ccb0e6bdf9406..2ef5c4b35a25cc 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -60,7 +60,7 @@ The :mod:`gc` module provides the following functions: The effect of calling ``gc.collect()`` while the interpreter is already performing a collection is undefined. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 ``generation=1`` performs an increment of collection. @@ -83,13 +83,13 @@ The :mod:`gc` module provides the following functions: returned. If *generation* is not ``None``, return only the objects as follows: * 0: All objects in the young generation - * 1: No objects, as there is no generation 1 (as of Python 3.13) + * 1: No objects, as there is no generation 1 (as of Python 3.14) * 2: All objects in the old generation .. versionchanged:: 3.8 New *generation* parameter. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 Generation 1 is removed .. audit-event:: gc.get_objects generation gc.get_objects @@ -142,7 +142,7 @@ The :mod:`gc` module provides the following functions: See `Garbage collector design `_ for more information. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 *threshold2* is ignored diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 2835c8d0eb711e..07f5ebf57c9b54 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -34,7 +34,7 @@ The module provides the following classes: .. class:: HTTPConnection(host, port=None[, timeout], source_address=None, \ - blocksize=8192) + blocksize=8192, max_response_headers=None) An :class:`HTTPConnection` instance represents one transaction with an HTTP server. It should be instantiated by passing it a host and optional port @@ -46,7 +46,9 @@ The module provides the following classes: The optional *source_address* parameter may be a tuple of a (host, port) to use as the source address the HTTP connection is made from. The optional *blocksize* parameter sets the buffer size in bytes for - sending a file-like message body. + sending a file-like message body. The optional *max_response_headers* + parameter sets the maximum number of allowed response headers to help + prevent denial-of-service attacks, otherwise the default value (100) is used. For example, the following calls all create instances that connect to the server at the same host and port:: @@ -66,10 +68,13 @@ The module provides the following classes: .. versionchanged:: 3.7 *blocksize* parameter was added. + .. versionchanged:: next + *max_response_headers* parameter was added. + .. class:: HTTPSConnection(host, port=None, *[, timeout], \ source_address=None, context=None, \ - blocksize=8192) + blocksize=8192, max_response_headers=None) A subclass of :class:`HTTPConnection` that uses SSL for communication with secure servers. Default port is ``443``. If *context* is specified, it @@ -109,6 +114,9 @@ The module provides the following classes: The deprecated *key_file*, *cert_file* and *check_hostname* parameters have been removed. + .. versionchanged:: next + *max_response_headers* parameter was added. + .. class:: HTTPResponse(sock, debuglevel=0, method=None, url=None) @@ -416,6 +424,14 @@ HTTPConnection Objects .. versionadded:: 3.7 +.. attribute:: HTTPConnection.max_response_headers + + The maximum number of allowed response headers to help prevent denial-of-service + attacks. By default, the maximum number of allowed headers is set to 100. + + .. versionadded:: next + + As an alternative to using the :meth:`~HTTPConnection.request` method described above, you can also send your request step by step, by using the four functions below. diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 99e8ef7b886035..c6c0033837562a 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -1353,6 +1353,9 @@ Command-line options Examples -------- +Reading examples +~~~~~~~~~~~~~~~~~~~ + How to extract an entire tar archive to the current working directory:: import tarfile @@ -1375,6 +1378,23 @@ a generator function instead of a list:: tar.extractall(members=py_files(tar)) tar.close() +How to read a gzip compressed tar archive and display some member information:: + + import tarfile + tar = tarfile.open("sample.tar.gz", "r:gz") + for tarinfo in tar: + print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") + if tarinfo.isreg(): + print("a regular file.") + elif tarinfo.isdir(): + print("a directory.") + else: + print("something else.") + tar.close() + +Writing examples +~~~~~~~~~~~~~~~~ + How to create an uncompressed tar archive from a list of filenames:: import tarfile @@ -1390,19 +1410,15 @@ The same example using the :keyword:`with` statement:: for name in ["foo", "bar", "quux"]: tar.add(name) -How to read a gzip compressed tar archive and display some member information:: +How to create and write an archive to stdout using +:data:`sys.stdout.buffer ` in the *fileobj* parameter +in :meth:`TarFile.add`:: - import tarfile - tar = tarfile.open("sample.tar.gz", "r:gz") - for tarinfo in tar: - print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") - if tarinfo.isreg(): - print("a regular file.") - elif tarinfo.isdir(): - print("a directory.") - else: - print("something else.") - tar.close() + import sys + import tarfile + with tarfile.open("sample.tar.gz", "w|gz", fileobj=sys.stdout.buffer) as tar: + for name in ["foo", "bar", "quux"]: + tar.add(name) How to create an archive and reset the user information using the *filter* parameter in :meth:`TarFile.add`:: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index c108a94692dca7..6ddc77d8f95038 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1058,6 +1058,30 @@ free-threaded build and false for the GIL-enabled build. (Contributed by Neil Schemenauer and Kumar Aditya in :gh:`130010`.) +.. _whatsnew314-incremental-gc: + +Incremental garbage collection +------------------------------ + +The cycle garbage collector is now incremental. +This means that maximum pause times are reduced +by an order of magnitude or more for larger heaps. + +There are now only two generations: young and old. +When :func:`gc.collect` is not called directly, the +GC is invoked a little less frequently. When invoked, it +collects the young generation and an increment of the +old generation, instead of collecting one or more generations. + +The behavior of :func:`!gc.collect` changes slightly: + +* ``gc.collect(1)``: Performs an increment of garbage collection, + rather than collecting generation 1. +* Other calls to :func:`!gc.collect` are unchanged. + +(Contributed by Mark Shannon in :gh:`108362`.) + + Other language changes ====================== @@ -1486,6 +1510,36 @@ functools (Contributed by Sayandip Dutta in :gh:`125916`.) +gc +-- + +The cyclic garbage collector is now incremental, +which changes the meaning of the results of +:meth:`~gc.get_threshold` and :meth:`~gc.set_threshold` +as well as :meth:`~gc.get_count` and :meth:`~gc.get_stats`. + +* For backwards compatibility, :meth:`~gc.get_threshold` continues to return + a three-item tuple. + The first value is the threshold for young collections, as before; + the second value determines the rate at which the old collection is scanned + (the default is 10, and higher values mean that the old collection + is scanned more slowly). + The third value is meaningless and is always zero. + +* :meth:`~gc.set_threshold` ignores any items after the second. + +* :meth:`~gc.get_count` and :meth:`~gc.get_stats` continue to return + the same format of results. + The only difference is that instead of the results referring to + the young, aging and old generations, + the results refer to the young generation + and the aging and collecting spaces of the old generation. + +In summary, code that attempted to manipulate the behavior of the cycle GC +may not work exactly as intended, but it is very unlikely to be harmful. +All other code will work just fine. + + getopt ------ @@ -2233,6 +2287,7 @@ asyncio (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa in :gh:`91048`.) + base64 ------ @@ -2241,6 +2296,15 @@ base64 (Contributed by Bénédikt Tran, Chris Markiewicz, and Adam Turner in :gh:`118761`.) +gc +-- + +* The new :ref:`incremental garbage collector ` + means that maximum pause times are reduced + by an order of magnitude or more for larger heaps. + (Contributed by Mark Shannon in :gh:`108362`.) + + io --- * :mod:`io` which provides the built-in :func:`open` makes less system calls @@ -2707,6 +2771,13 @@ Changes in the Python API Wrap it in :func:`staticmethod` if you want to preserve the old behavior. (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) +* The :ref:`garbage collector is now incremental `, + which means that the behavior of :func:`gc.collect` changes slightly: + + * ``gc.collect(1)``: Performs an increment of garbage collection, + rather than collecting generation 1. + * Other calls to :func:`!gc.collect` are unchanged. + * The :func:`locale.nl_langinfo` function now sets temporarily the ``LC_CTYPE`` locale in some cases. This temporary change affects other threads. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 0f65317633ba70..7e47fa263d9a5e 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -230,6 +230,16 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) +http.client +----------- + +* A new *max_response_headers* keyword-only parameter has been added to + :class:`~http.client.HTTPConnection` and :class:`~http.client.HTTPSConnection` + constructors. This parameter overrides the default maximum number of allowed + response headers. + (Contributed by Alexander Enrique Urieles Nieto in :gh:`131724`.) + + math ---- diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 02470ba0fdd559..a7db778b716537 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -136,12 +136,22 @@ def __get_openssl_constructor(name): # Prefer our builtin blake2 implementation. return __get_builtin_constructor(name) try: - # MD5, SHA1, and SHA2 are in all supported OpenSSL versions - # SHA3/shake are available in OpenSSL 1.1.1+ + # Fetch the OpenSSL hash function if it exists, + # independently of the context security policy. f = getattr(_hashlib, 'openssl_' + name) - # Allow the C module to raise ValueError. The function will be - # defined but the hash not actually available. Don't fall back to - # builtin if the current security policy blocks a digest, bpo#40695. + # Check if the context security policy blocks the digest or not + # by allowing the C module to raise a ValueError. The function + # will be defined but the hash will not be available at runtime. + # + # We use "usedforsecurity=False" to prevent falling back to the + # built-in function in case the security policy does not allow it. + # + # Note that this only affects the explicit named constructors, + # and not the algorithms exposed through hashlib.new() which + # can still be resolved to a built-in function even if the + # current security policy does not allow it. + # + # See https://github.com/python/cpython/issues/84872. f(usedforsecurity=False) # Use the C function directly (very fast) return f diff --git a/Lib/http/client.py b/Lib/http/client.py index e7a1c7bc3b2ae1..0cce49cadc09fa 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -209,22 +209,24 @@ def getallmatchingheaders(self, name): lst.append(line) return lst -def _read_headers(fp): +def _read_headers(fp, max_headers): """Reads potential header lines into a list from a file pointer. Length of line is limited by _MAXLINE, and number of - headers is limited by _MAXHEADERS. + headers is limited by max_headers. """ headers = [] + if max_headers is None: + max_headers = _MAXHEADERS while True: line = fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: raise LineTooLong("header line") - headers.append(line) - if len(headers) > _MAXHEADERS: - raise HTTPException("got more than %d headers" % _MAXHEADERS) if line in (b'\r\n', b'\n', b''): break + headers.append(line) + if len(headers) > max_headers: + raise HTTPException(f"got more than {max_headers} headers") return headers def _parse_header_lines(header_lines, _class=HTTPMessage): @@ -241,10 +243,10 @@ def _parse_header_lines(header_lines, _class=HTTPMessage): hstring = b''.join(header_lines).decode('iso-8859-1') return email.parser.Parser(_class=_class).parsestr(hstring) -def parse_headers(fp, _class=HTTPMessage): +def parse_headers(fp, _class=HTTPMessage, *, _max_headers=None): """Parses only RFC2822 headers from a file pointer.""" - headers = _read_headers(fp) + headers = _read_headers(fp, _max_headers) return _parse_header_lines(headers, _class) @@ -320,7 +322,7 @@ def _read_status(self): raise BadStatusLine(line) return version, status, reason - def begin(self): + def begin(self, *, _max_headers=None): if self.headers is not None: # we've already started reading the response return @@ -331,7 +333,7 @@ def begin(self): if status != CONTINUE: break # skip the header from the 100 response - skipped_headers = _read_headers(self.fp) + skipped_headers = _read_headers(self.fp, _max_headers) if self.debuglevel > 0: print("headers:", skipped_headers) del skipped_headers @@ -346,7 +348,9 @@ def begin(self): else: raise UnknownProtocol(version) - self.headers = self.msg = parse_headers(self.fp) + self.headers = self.msg = parse_headers( + self.fp, _max_headers=_max_headers + ) if self.debuglevel > 0: for hdr, val in self.headers.items(): @@ -864,7 +868,7 @@ def _get_content_length(body, method): return None def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, blocksize=8192): + source_address=None, blocksize=8192, *, max_response_headers=None): self.timeout = timeout self.source_address = source_address self.blocksize = blocksize @@ -877,6 +881,7 @@ def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, self._tunnel_port = None self._tunnel_headers = {} self._raw_proxy_headers = None + self.max_response_headers = max_response_headers (self.host, self.port) = self._get_hostport(host, port) @@ -969,7 +974,7 @@ def _tunnel(self): try: (version, code, message) = response._read_status() - self._raw_proxy_headers = _read_headers(response.fp) + self._raw_proxy_headers = _read_headers(response.fp, self.max_response_headers) if self.debuglevel > 0: for header in self._raw_proxy_headers: @@ -1426,7 +1431,10 @@ def getresponse(self): try: try: - response.begin() + if self.max_response_headers is None: + response.begin() + else: + response.begin(_max_headers=self.max_response_headers) except ConnectionError: self.close() raise @@ -1457,10 +1465,12 @@ class HTTPSConnection(HTTPConnection): def __init__(self, host, port=None, *, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, context=None, blocksize=8192): + source_address=None, context=None, blocksize=8192, + max_response_headers=None): super(HTTPSConnection, self).__init__(host, port, timeout, source_address, - blocksize=blocksize) + blocksize=blocksize, + max_response_headers=max_response_headers) if context is None: context = _create_https_context(self._http_vsn) self._context = context diff --git a/Lib/test/support/hashlib_helper.py b/Lib/test/support/hashlib_helper.py index 337a1e415b0de3..96be74e4105c18 100644 --- a/Lib/test/support/hashlib_helper.py +++ b/Lib/test/support/hashlib_helper.py @@ -1,91 +1,214 @@ import contextlib +import enum import functools -import hashlib import importlib import inspect import unittest import unittest.mock -from collections import namedtuple -from test.support.import_helper import import_module +from test.support import import_helper from types import MappingProxyType -try: - import _hashlib -except ImportError: - _hashlib = None -try: - import _hmac -except ImportError: - _hmac = None +def try_import_module(module_name): + """Try to import a module and return None on failure.""" + try: + return importlib.import_module(module_name) + except ImportError: + return None + + +class HID(enum.StrEnum): + """Enumeration containing the canonical digest names. + + Those names should only be used by hashlib.new() or hmac.new(). + Their support by _hashlib.new() is not necessarily guaranteed. + """ + + md5 = enum.auto() + sha1 = enum.auto() + + sha224 = enum.auto() + sha256 = enum.auto() + sha384 = enum.auto() + sha512 = enum.auto() + sha3_224 = enum.auto() + sha3_256 = enum.auto() + sha3_384 = enum.auto() + sha3_512 = enum.auto() + + shake_128 = enum.auto() + shake_256 = enum.auto() + + blake2s = enum.auto() + blake2b = enum.auto() + + def __repr__(self): + return str(self) + + @property + def is_xof(self): + """Indicate whether the hash is an extendable-output hash function.""" + return self.startswith("shake_") + + @property + def is_keyed(self): + """Indicate whether the hash is a keyed hash function.""" + return self.startswith("blake2") -CANONICAL_DIGEST_NAMES = frozenset(( - 'md5', 'sha1', - 'sha224', 'sha256', 'sha384', 'sha512', - 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', - 'shake_128', 'shake_256', - 'blake2s', 'blake2b', + +CANONICAL_DIGEST_NAMES = frozenset(map(str, HID.__members__)) +NON_HMAC_DIGEST_NAMES = frozenset(( + HID.shake_128, HID.shake_256, + HID.blake2s, HID.blake2b, )) -NON_HMAC_DIGEST_NAMES = frozenset({ - 'shake_128', 'shake_256', - 'blake2s', 'blake2b', -}) +class HashInfo: + """Dataclass storing explicit hash constructor names. + + - *builtin* is the fully-qualified name for the explicit HACL* + hash constructor function, e.g., "_md5.md5". + + - *openssl* is the name of the "_hashlib" module method for the explicit + OpenSSL hash constructor function, e.g., "openssl_md5". + + - *hashlib* is the name of the "hashlib" module method for the explicit + hash constructor function, e.g., "md5". + """ + + def __init__(self, builtin, openssl=None, hashlib=None): + assert isinstance(builtin, str), builtin + assert len(builtin.split(".")) == 2, builtin -class HashAPI(namedtuple("HashAPI", "builtin openssl hashlib")): + self.builtin = builtin + self.builtin_module_name, self.builtin_method_name = ( + self.builtin.split(".", maxsplit=1) + ) + + assert openssl is None or openssl.startswith("openssl_") + self.openssl = self.openssl_method_name = openssl + self.openssl_module_name = "_hashlib" if openssl else None + + assert hashlib is None or isinstance(hashlib, str) + self.hashlib = self.hashlib_method_name = hashlib + self.hashlib_module_name = "hashlib" if hashlib else None + + def module_name(self, implementation): + match implementation: + case "builtin": + return self.builtin_module_name + case "openssl": + return self.openssl_module_name + case "hashlib": + return self.hashlib_module_name + raise AssertionError(f"invalid implementation {implementation}") - def fullname(self, typ): - match typ: + def method_name(self, implementation): + match implementation: case "builtin": - return self.builtin + return self.builtin_method_name case "openssl": - return f"_hashlib.{self.openssl}" if self.openssl else None + return self.openssl_method_name case "hashlib": - return f"hashlib.{self.hashlib}" if self.hashlib else None - case _: - raise AssertionError(f"unknown type: {typ}") + return self.hashlib_method_name + raise AssertionError(f"invalid implementation {implementation}") + + def fullname(self, implementation): + """Get the fully qualified name of a given implementation. + + This returns a string of the form "MODULE_NAME.METHOD_NAME" or None + if the hash function does not have a corresponding implementation. + + *implementation* must be "builtin", "openssl" or "hashlib". + """ + module_name = self.module_name(implementation) + method_name = self.method_name(implementation) + if module_name is None or method_name is None: + return None + return f"{module_name}.{method_name}" # Mapping from a "canonical" name to a pair (HACL*, _hashlib.*, hashlib.*) # constructors. If the constructor name is None, then this means that the # algorithm can only be used by the "agile" new() interfaces. -_EXPLICIT_CONSTRUCTORS = MappingProxyType({ - "md5": HashAPI("_md5.md5", "openssl_md5", "md5"), - "sha1": HashAPI("_sha1.sha1", "openssl_sha1", "sha1"), - "sha224": HashAPI("_sha2.sha224", "openssl_sha224", "sha224"), - "sha256": HashAPI("_sha2.sha256", "openssl_sha256", "sha256"), - "sha384": HashAPI("_sha2.sha384", "openssl_sha384", "sha384"), - "sha512": HashAPI("_sha2.sha512", "openssl_sha512", "sha512"), - "sha3_224": HashAPI("_sha3.sha3_224", "openssl_sha3_224", "sha3_224"), - "sha3_256": HashAPI("_sha3.sha3_256", "openssl_sha3_256", "sha3_256"), - "sha3_384": HashAPI("_sha3.sha3_384", "openssl_sha3_384", "sha3_384"), - "sha3_512": HashAPI("_sha3.sha3_512", "openssl_sha3_512", "sha3_512"), - "shake_128": HashAPI("_sha3.shake_128", "openssl_shake_128", "shake_128"), - "shake_256": HashAPI("_sha3.shake_256", "openssl_shake_256", "shake_256"), - "blake2s": HashAPI("_blake2.blake2s", None, "blake2s"), - "blake2b": HashAPI("_blake2.blake2b", None, "blake2b"), +_EXPLICIT_CONSTRUCTORS = MappingProxyType({ # fmt: skip + HID.md5: HashInfo("_md5.md5", "openssl_md5", "md5"), + HID.sha1: HashInfo("_sha1.sha1", "openssl_sha1", "sha1"), + HID.sha224: HashInfo("_sha2.sha224", "openssl_sha224", "sha224"), + HID.sha256: HashInfo("_sha2.sha256", "openssl_sha256", "sha256"), + HID.sha384: HashInfo("_sha2.sha384", "openssl_sha384", "sha384"), + HID.sha512: HashInfo("_sha2.sha512", "openssl_sha512", "sha512"), + HID.sha3_224: HashInfo( + "_sha3.sha3_224", "openssl_sha3_224", "sha3_224" + ), + HID.sha3_256: HashInfo( + "_sha3.sha3_256", "openssl_sha3_256", "sha3_256" + ), + HID.sha3_384: HashInfo( + "_sha3.sha3_384", "openssl_sha3_384", "sha3_384" + ), + HID.sha3_512: HashInfo( + "_sha3.sha3_512", "openssl_sha3_512", "sha3_512" + ), + HID.shake_128: HashInfo( + "_sha3.shake_128", "openssl_shake_128", "shake_128" + ), + HID.shake_256: HashInfo( + "_sha3.shake_256", "openssl_shake_256", "shake_256" + ), + HID.blake2s: HashInfo("_blake2.blake2s", None, "blake2s"), + HID.blake2b: HashInfo("_blake2.blake2b", None, "blake2b"), }) assert _EXPLICIT_CONSTRUCTORS.keys() == CANONICAL_DIGEST_NAMES +get_hash_info = _EXPLICIT_CONSTRUCTORS.__getitem__ +# Mapping from canonical hash names to their explicit HACL* HMAC constructor. +# There is currently no OpenSSL one-shot named function and there will likely +# be none in the future. _EXPLICIT_HMAC_CONSTRUCTORS = { - name: f'_hmac.compute_{name}' for name in ( - 'md5', 'sha1', - 'sha224', 'sha256', 'sha384', 'sha512', - 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', - ) + HID(name): f"_hmac.compute_{name}" + for name in CANONICAL_DIGEST_NAMES } -_EXPLICIT_HMAC_CONSTRUCTORS['shake_128'] = None -_EXPLICIT_HMAC_CONSTRUCTORS['shake_256'] = None +# Neither HACL* nor OpenSSL supports HMAC over XOFs. +_EXPLICIT_HMAC_CONSTRUCTORS[HID.shake_128] = None +_EXPLICIT_HMAC_CONSTRUCTORS[HID.shake_256] = None # Strictly speaking, HMAC-BLAKE is meaningless as BLAKE2 is already a # keyed hash function. However, as it's exposed by HACL*, we test it. -_EXPLICIT_HMAC_CONSTRUCTORS['blake2s'] = '_hmac.compute_blake2s_32' -_EXPLICIT_HMAC_CONSTRUCTORS['blake2b'] = '_hmac.compute_blake2b_32' +_EXPLICIT_HMAC_CONSTRUCTORS[HID.blake2s] = '_hmac.compute_blake2s_32' +_EXPLICIT_HMAC_CONSTRUCTORS[HID.blake2b] = '_hmac.compute_blake2b_32' _EXPLICIT_HMAC_CONSTRUCTORS = MappingProxyType(_EXPLICIT_HMAC_CONSTRUCTORS) assert _EXPLICIT_HMAC_CONSTRUCTORS.keys() == CANONICAL_DIGEST_NAMES +def _decorate_func_or_class(decorator_func, func_or_class): + if not isinstance(func_or_class, type): + return decorator_func(func_or_class) + + decorated_class = func_or_class + setUpClass = decorated_class.__dict__.get('setUpClass') + if setUpClass is None: + def setUpClass(cls): + super(decorated_class, cls).setUpClass() + setUpClass.__qualname__ = decorated_class.__qualname__ + '.setUpClass' + setUpClass.__module__ = decorated_class.__module__ + else: + setUpClass = setUpClass.__func__ + setUpClass = classmethod(decorator_func(setUpClass)) + decorated_class.setUpClass = setUpClass + return decorated_class + + +def _chain_decorators(decorators): + """Obtain a decorator by chaining multiple decorators. + + The decorators are applied in the order they are given. + """ + def decorator_func(func): + return functools.reduce(lambda w, deco: deco(w), decorators, func) + return functools.partial(_decorate_func_or_class, decorator_func) + + def _ensure_wrapper_signature(wrapper, wrapped): """Ensure that a wrapper has the same signature as the wrapped function. @@ -108,49 +231,129 @@ def _ensure_wrapper_signature(wrapper, wrapped): def requires_hashlib(): + _hashlib = try_import_module("_hashlib") return unittest.skipIf(_hashlib is None, "requires _hashlib") def requires_builtin_hmac(): + _hmac = try_import_module("_hmac") return unittest.skipIf(_hmac is None, "requires _hmac") -def _missing_hash(digestname, implementation=None, *, exc=None): - parts = ["missing", implementation, f"hash algorithm: {digestname!r}"] - msg = " ".join(filter(None, parts)) - raise unittest.SkipTest(msg) from exc +class SkipNoHash(unittest.SkipTest): + """A SkipTest exception raised when a hash is not available.""" + def __init__(self, digestname, implementation=None, interface=None): + parts = ["missing", implementation, f"hash algorithm {digestname!r}"] + if interface is not None: + parts.append(f"for {interface}") + super().__init__(" ".join(filter(None, parts))) -def _openssl_availabillity(digestname, *, usedforsecurity): + +def _hashlib_new(digestname, openssl, /, **kwargs): + """Check availability of [hashlib|_hashlib].new(digestname, **kwargs). + + If *openssl* is True, module is "_hashlib" (C extension module), + otherwise it is "hashlib" (pure Python interface). + + The constructor function is returned (without binding **kwargs), + or SkipTest is raised if none exists. + """ assert isinstance(digestname, str), digestname + # Re-import 'hashlib' in case it was mocked, but propagate + # exceptions as it should be unconditionally available. + hashlib = importlib.import_module("hashlib") + # re-import '_hashlib' in case it was mocked + _hashlib = try_import_module("_hashlib") + module = _hashlib if openssl and _hashlib is not None else hashlib try: - _hashlib.new(digestname, usedforsecurity=usedforsecurity) - except AttributeError: - assert _hashlib is None - _missing_hash(digestname, "OpenSSL") + module.new(digestname, **kwargs) except ValueError as exc: - _missing_hash(digestname, "OpenSSL", exc=exc) + interface = f"{module.__name__}.new" + raise SkipNoHash(digestname, interface=interface) from exc + return functools.partial(module.new, digestname) -def _decorate_func_or_class(func_or_class, decorator_func): - if not isinstance(func_or_class, type): - return decorator_func(func_or_class) +def _builtin_hash(module_name, digestname, /, **kwargs): + """Check availability of .(**kwargs). + + - The *module_name* is the C extension module name based on HACL*. + - The *digestname* is one of its member, e.g., 'md5'. + + The constructor function is returned, or SkipTest is raised if none exists. + """ + assert isinstance(module_name, str), module_name + assert isinstance(digestname, str), digestname + fullname = f'{module_name}.{digestname}' + try: + builtin_module = importlib.import_module(module_name) + except ImportError as exc: + raise SkipNoHash(fullname, "builtin") from exc + try: + constructor = getattr(builtin_module, digestname) + except AttributeError as exc: + raise SkipNoHash(fullname, "builtin") from exc + try: + constructor(**kwargs) + except ValueError as exc: + raise SkipNoHash(fullname, "builtin") from exc + return constructor - decorated_class = func_or_class - setUpClass = decorated_class.__dict__.get('setUpClass') - if setUpClass is None: - def setUpClass(cls): - super(decorated_class, cls).setUpClass() - setUpClass.__qualname__ = decorated_class.__qualname__ + '.setUpClass' - setUpClass.__module__ = decorated_class.__module__ - else: - setUpClass = setUpClass.__func__ - setUpClass = classmethod(decorator_func(setUpClass)) - decorated_class.setUpClass = setUpClass - return decorated_class +def _openssl_new(digestname, /, **kwargs): + """Check availability of _hashlib.new(digestname, **kwargs). -def requires_hashdigest(digestname, openssl=None, usedforsecurity=True): + The constructor function is returned (without binding **kwargs), + or SkipTest is raised if none exists. + """ + assert isinstance(digestname, str), digestname + try: + # re-import '_hashlib' in case it was mocked + _hashlib = importlib.import_module("_hashlib") + except ImportError as exc: + raise SkipNoHash(digestname, "openssl") from exc + try: + _hashlib.new(digestname, **kwargs) + except ValueError as exc: + raise SkipNoHash(digestname, interface="_hashlib.new") from exc + return functools.partial(_hashlib.new, digestname) + + +def _openssl_hash(digestname, /, **kwargs): + """Check availability of _hashlib.openssl_(**kwargs). + + The constructor function is returned (without binding **kwargs), + or SkipTest is raised if none exists. + """ + assert isinstance(digestname, str), digestname + fullname = f"_hashlib.openssl_{digestname}" + try: + # re-import '_hashlib' in case it was mocked + _hashlib = importlib.import_module("_hashlib") + except ImportError as exc: + raise SkipNoHash(fullname, "openssl") from exc + try: + constructor = getattr(_hashlib, f"openssl_{digestname}", None) + except AttributeError as exc: + raise SkipNoHash(fullname, "openssl") from exc + try: + constructor(**kwargs) + except ValueError as exc: + raise SkipNoHash(fullname, "openssl") from exc + return constructor + + +def _make_requires_hashdigest_decorator(test, /, *test_args, **test_kwargs): + def decorator_func(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + test(*test_args, **test_kwargs) + return func(*args, **kwargs) + return wrapper + return functools.partial(_decorate_func_or_class, decorator_func) + + +def requires_hashdigest(digestname, openssl=None, *, usedforsecurity=True): """Decorator raising SkipTest if a hashing algorithm is not available. The hashing algorithm may be missing, blocked by a strict crypto policy, @@ -167,27 +370,9 @@ def requires_hashdigest(digestname, openssl=None, usedforsecurity=True): ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS ValueError: unsupported hash type md4 """ - assert isinstance(digestname, str), digestname - if openssl and _hashlib is not None: - def test_availability(): - _hashlib.new(digestname, usedforsecurity=usedforsecurity) - else: - def test_availability(): - hashlib.new(digestname, usedforsecurity=usedforsecurity) - - def decorator_func(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - test_availability() - except ValueError as exc: - _missing_hash(digestname, exc=exc) - return func(*args, **kwargs) - return wrapper - - def decorator(func_or_class): - return _decorate_func_or_class(func_or_class, decorator_func) - return decorator + return _make_requires_hashdigest_decorator( + _hashlib_new, digestname, openssl, usedforsecurity=usedforsecurity + ) def requires_openssl_hashdigest(digestname, *, usedforsecurity=True): @@ -195,27 +380,9 @@ def requires_openssl_hashdigest(digestname, *, usedforsecurity=True): The hashing algorithm may be missing or blocked by a strict crypto policy. """ - assert isinstance(digestname, str), digestname - def decorator_func(func): - @requires_hashlib() # avoid checking at each call - @functools.wraps(func) - def wrapper(*args, **kwargs): - _openssl_availabillity(digestname, usedforsecurity=usedforsecurity) - return func(*args, **kwargs) - return wrapper - - def decorator(func_or_class): - return _decorate_func_or_class(func_or_class, decorator_func) - return decorator - - -def find_openssl_hashdigest_constructor(digestname, *, usedforsecurity=True): - """Find the OpenSSL hash function constructor by its name.""" - assert isinstance(digestname, str), digestname - _openssl_availabillity(digestname, usedforsecurity=usedforsecurity) - # This returns a function of the form _hashlib.openssl_ and - # not a lambda function as it is rejected by _hashlib.hmac_new(). - return getattr(_hashlib, f"openssl_{digestname}") + return _make_requires_hashdigest_decorator( + _openssl_new, digestname, usedforsecurity=usedforsecurity + ) def requires_builtin_hashdigest( @@ -226,40 +393,22 @@ def requires_builtin_hashdigest( - The *module_name* is the C extension module name based on HACL*. - The *digestname* is one of its member, e.g., 'md5'. """ - assert isinstance(digestname, str), digestname - def decorator_func(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - module = import_module(module_name) - try: - getattr(module, digestname) - except AttributeError: - fullname = f'{module_name}.{digestname}' - _missing_hash(fullname, implementation="HACL") - return func(*args, **kwargs) - return wrapper - - def decorator(func_or_class): - return _decorate_func_or_class(func_or_class, decorator_func) - return decorator + return _make_requires_hashdigest_decorator( + _builtin_hash, module_name, digestname, usedforsecurity=usedforsecurity + ) -def find_builtin_hashdigest_constructor( - module_name, digestname, *, usedforsecurity=True -): - """Find the HACL* hash function constructor. - - - The *module_name* is the C extension module name based on HACL*. - - The *digestname* is one of its member, e.g., 'md5'. - """ - assert isinstance(digestname, str), digestname - module = import_module(module_name) - try: - constructor = getattr(module, digestname) - constructor(b'', usedforsecurity=usedforsecurity) - except (AttributeError, TypeError, ValueError): - _missing_hash(f'{module_name}.{digestname}', implementation="HACL") - return constructor +def requires_builtin_hashes(*ignored, usedforsecurity=True): + """Decorator raising SkipTest if one HACL* hashing algorithm is missing.""" + return _chain_decorators(( + requires_builtin_hashdigest( + api.builtin_module_name, + api.builtin_method_name, + usedforsecurity=usedforsecurity, + ) + for name, api in _EXPLICIT_CONSTRUCTORS.items() + if name not in ignored + )) class HashFunctionsTrait: @@ -281,7 +430,9 @@ class HashFunctionsTrait: 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', ] - # Default 'usedforsecurity' to use when looking up a hash function. + # Default 'usedforsecurity' to use when checking a hash function. + # When the trait properties are callables (e.g., _md5.md5) and + # not strings, they must be called with the same 'usedforsecurity'. usedforsecurity = True @classmethod @@ -357,9 +508,9 @@ class OpenSSLHashFunctionsTrait(HashFunctionsTrait): def _find_constructor(self, digestname): self.is_valid_digest_name(digestname) - return find_openssl_hashdigest_constructor( - digestname, usedforsecurity=self.usedforsecurity - ) + # This returns a function of the form _hashlib.openssl_ and + # not a lambda function as it is rejected by _hashlib.hmac_new(). + return _openssl_hash(digestname, usedforsecurity=self.usedforsecurity) class BuiltinHashFunctionsTrait(HashFunctionsTrait): @@ -370,49 +521,14 @@ class BuiltinHashFunctionsTrait(HashFunctionsTrait): is not since the former is unconditionally built. """ - def _find_constructor_in(self, module, digestname): + def _find_constructor(self, digestname): self.is_valid_digest_name(digestname) - return find_builtin_hashdigest_constructor(module, digestname) - - @property - def md5(self): - return self._find_constructor_in("_md5", "md5") - - @property - def sha1(self): - return self._find_constructor_in("_sha1", "sha1") - - @property - def sha224(self): - return self._find_constructor_in("_sha2", "sha224") - - @property - def sha256(self): - return self._find_constructor_in("_sha2", "sha256") - - @property - def sha384(self): - return self._find_constructor_in("_sha2", "sha384") - - @property - def sha512(self): - return self._find_constructor_in("_sha2", "sha512") - - @property - def sha3_224(self): - return self._find_constructor_in("_sha3", "sha3_224") - - @property - def sha3_256(self): - return self._find_constructor_in("_sha3","sha3_256") - - @property - def sha3_384(self): - return self._find_constructor_in("_sha3","sha3_384") - - @property - def sha3_512(self): - return self._find_constructor_in("_sha3","sha3_512") + info = _EXPLICIT_CONSTRUCTORS[digestname] + return _builtin_hash( + info.builtin_module_name, + info.builtin_method_name, + usedforsecurity=self.usedforsecurity, + ) def find_gil_minsize(modules_names, default=2048): @@ -426,38 +542,45 @@ def find_gil_minsize(modules_names, default=2048): """ sizes = [] for module_name in modules_names: - try: - module = importlib.import_module(module_name) - except ImportError: - continue - sizes.append(getattr(module, '_GIL_MINSIZE', default)) + module = try_import_module(module_name) + if module is not None: + sizes.append(getattr(module, '_GIL_MINSIZE', default)) return max(sizes, default=default) def _block_openssl_hash_new(blocked_name): """Block OpenSSL implementation of _hashlib.new().""" assert isinstance(blocked_name, str), blocked_name - if _hashlib is None: + + # re-import '_hashlib' in case it was mocked + if (_hashlib := try_import_module("_hashlib")) is None: return contextlib.nullcontext() + @functools.wraps(wrapped := _hashlib.new) - def wrapper(name, data=b'', *, usedforsecurity=True, string=None): + def _hashlib_new(name, data=b'', *, usedforsecurity=True, string=None): if name == blocked_name: raise _hashlib.UnsupportedDigestmodError(blocked_name) - return wrapped(*args, **kwargs) - _ensure_wrapper_signature(wrapper, wrapped) - return unittest.mock.patch('_hashlib.new', wrapper) + return wrapped(name, data, + usedforsecurity=usedforsecurity, string=string) + + _ensure_wrapper_signature(_hashlib_new, wrapped) + return unittest.mock.patch('_hashlib.new', _hashlib_new) def _block_openssl_hmac_new(blocked_name): """Block OpenSSL HMAC-HASH implementation.""" assert isinstance(blocked_name, str), blocked_name - if _hashlib is None: + + # re-import '_hashlib' in case it was mocked + if (_hashlib := try_import_module("_hashlib")) is None: return contextlib.nullcontext() + @functools.wraps(wrapped := _hashlib.hmac_new) def wrapper(key, msg=b'', digestmod=None): if digestmod == blocked_name: raise _hashlib.UnsupportedDigestmodError(blocked_name) return wrapped(key, msg, digestmod) + _ensure_wrapper_signature(wrapper, wrapped) return unittest.mock.patch('_hashlib.hmac_new', wrapper) @@ -465,112 +588,132 @@ def wrapper(key, msg=b'', digestmod=None): def _block_openssl_hmac_digest(blocked_name): """Block OpenSSL HMAC-HASH one-shot digest implementation.""" assert isinstance(blocked_name, str), blocked_name - if _hashlib is None: + + # re-import '_hashlib' in case it was mocked + if (_hashlib := try_import_module("_hashlib")) is None: return contextlib.nullcontext() + @functools.wraps(wrapped := _hashlib.hmac_digest) - def wrapper(key, msg, digest): + def _hashlib_hmac_digest(key, msg, digest): if digest == blocked_name: raise _hashlib.UnsupportedDigestmodError(blocked_name) - return wrapped(key, msg, digestmod) - _ensure_wrapper_signature(wrapper, wrapped) - return unittest.mock.patch('_hashlib.hmac_digest', wrapper) + return wrapped(key, msg, digest) + + _ensure_wrapper_signature(_hashlib_hmac_digest, wrapped) + return unittest.mock.patch('_hashlib.hmac_digest', _hashlib_hmac_digest) -@contextlib.contextmanager def _block_builtin_hash_new(name): + """Block a buitin-in hash name from the hashlib.new() interface.""" assert isinstance(name, str), name assert name.lower() == name, f"invalid name: {name}" - - builtin_cache = getattr(hashlib, '__builtin_constructor_cache') - if name in builtin_cache: - f = builtin_cache.pop(name) - F = builtin_cache.pop(name.upper(), None) - else: - f = F = None - try: - yield - finally: - if f is not None: - builtin_cache[name] = f - if F is not None: - builtin_cache[name.upper()] = F + assert name in HID, f"invalid hash: {name}" + + # Re-import 'hashlib' in case it was mocked + hashlib = importlib.import_module('hashlib') + builtin_constructor_cache = getattr(hashlib, '__builtin_constructor_cache') + builtin_constructor_cache_mock = builtin_constructor_cache.copy() + builtin_constructor_cache_mock.pop(name, None) + builtin_constructor_cache_mock.pop(name.upper(), None) + + # __get_builtin_constructor() imports the HACL* modules on demand, + # so we need to block the possibility of importing it, but only + # during the call to __get_builtin_constructor(). + get_builtin_constructor = getattr(hashlib, '__get_builtin_constructor') + builtin_module_name = _EXPLICIT_CONSTRUCTORS[name].builtin_module_name + + @functools.wraps(get_builtin_constructor) + def get_builtin_constructor_mock(name): + with import_helper.isolated_modules(): + sys = importlib.import_module("sys") + sys.modules[builtin_module_name] = None # block module's import + return get_builtin_constructor(name) + + return unittest.mock.patch.multiple( + hashlib, + __get_builtin_constructor=get_builtin_constructor_mock, + __builtin_constructor_cache=builtin_constructor_cache_mock + ) def _block_builtin_hmac_new(blocked_name): assert isinstance(blocked_name, str), blocked_name - if _hmac is None: + + # re-import '_hmac' in case it was mocked + if (_hmac := try_import_module("_hmac")) is None: return contextlib.nullcontext() + @functools.wraps(wrapped := _hmac.new) - def wrapper(key, msg=None, digestmod=None): + def _hmac_new(key, msg=None, digestmod=None): if digestmod == blocked_name: raise _hmac.UnknownHashError(blocked_name) return wrapped(key, msg, digestmod) - _ensure_wrapper_signature(wrapper, wrapped) - return unittest.mock.patch('_hmac.new', wrapper) + + _ensure_wrapper_signature(_hmac_new, wrapped) + return unittest.mock.patch('_hmac.new', _hmac_new) def _block_builtin_hmac_digest(blocked_name): assert isinstance(blocked_name, str), blocked_name - if _hmac is None: + + # re-import '_hmac' in case it was mocked + if (_hmac := try_import_module("_hmac")) is None: return contextlib.nullcontext() + @functools.wraps(wrapped := _hmac.compute_digest) - def wrapper(key, msg, digest): + def _hmac_compute_digest(key, msg, digest): if digest == blocked_name: raise _hmac.UnknownHashError(blocked_name) return wrapped(key, msg, digest) - _ensure_wrapper_signature(wrapper, wrapped) - return unittest.mock.patch('_hmac.compute_digest', wrapper) + _ensure_wrapper_signature(_hmac_compute_digest, wrapped) + return unittest.mock.patch('_hmac.compute_digest', _hmac_compute_digest) -def _make_hash_constructor_blocker(name, dummy, *, interface): - assert isinstance(name, str), name - assert interface in ('builtin', 'openssl', 'hashlib') - assert name in _EXPLICIT_CONSTRUCTORS, f"invalid hash: {name}" - fullname = _EXPLICIT_CONSTRUCTORS[name].fullname(interface) - if fullname is None: + +def _make_hash_constructor_blocker(name, dummy, implementation): + info = _EXPLICIT_CONSTRUCTORS[name] + module_name = info.module_name(implementation) + method_name = info.method_name(implementation) + if module_name is None or method_name is None: # function shouldn't exist for this implementation return contextlib.nullcontext() - assert fullname.count('.') == 1, fullname - module_name, method = fullname.split('.', maxsplit=1) + try: module = importlib.import_module(module_name) except ImportError: # module is already disabled return contextlib.nullcontext() - wrapped = getattr(module, method) + + wrapped = getattr(module, method_name) wrapper = functools.wraps(wrapped)(dummy) _ensure_wrapper_signature(wrapper, wrapped) - return unittest.mock.patch(fullname, wrapper) + return unittest.mock.patch(info.fullname(implementation), wrapper) def _block_hashlib_hash_constructor(name): """Block explicit public constructors.""" - assert isinstance(name, str), name def dummy(data=b'', *, usedforsecurity=True, string=None): - raise ValueError(f"unsupported hash name: {name}") - return _make_hash_constructor_blocker(name, dummy, interface='hashlib') + raise ValueError(f"blocked explicit public hash name: {name}") + + return _make_hash_constructor_blocker(name, dummy, 'hashlib') def _block_openssl_hash_constructor(name): """Block explicit OpenSSL constructors.""" - assert isinstance(name, str), name def dummy(data=b'', *, usedforsecurity=True, string=None): - raise ValueError(f"unsupported hash name: {name}") - return _make_hash_constructor_blocker(name, dummy, interface='openssl') + raise ValueError(f"blocked explicit OpenSSL hash name: {name}") + return _make_hash_constructor_blocker(name, dummy, 'openssl') def _block_builtin_hash_constructor(name): """Block explicit HACL* constructors.""" - assert isinstance(name, str), name def dummy(data=b'', *, usedforsecurity=True, string=b''): - raise ValueError(f"unsupported hash name: {name}") - return _make_hash_constructor_blocker(name, dummy, interface='builtin') + raise ValueError(f"blocked explicit builtin hash name: {name}") + return _make_hash_constructor_blocker(name, dummy, 'builtin') def _block_builtin_hmac_constructor(name): """Block explicit HACL* HMAC constructors.""" - assert isinstance(name, str), name - assert name in _EXPLICIT_HMAC_CONSTRUCTORS, f"invalid hash: {name}" fullname = _EXPLICIT_HMAC_CONSTRUCTORS[name] if fullname is None: # function shouldn't exist for this implementation @@ -585,7 +728,7 @@ def _block_builtin_hmac_constructor(name): return contextlib.nullcontext() @functools.wraps(wrapped := getattr(module, method)) def wrapper(key, obj): - raise ValueError(f"unsupported hash name: {name}") + raise ValueError(f"blocked hash name: {name}") _ensure_wrapper_signature(wrapper, wrapped) return unittest.mock.patch(fullname, wrapper) @@ -600,22 +743,54 @@ def block_algorithm(name, *, allow_openssl=False, allow_builtin=False): """ with contextlib.ExitStack() as stack: if not (allow_openssl or allow_builtin): - # If one of the private interface is allowed, then the - # public interface will fallback to it even though the - # comment in hashlib.py says otherwise. + # Named constructors have a different behavior in the sense + # that they are either built-ins or OpenSSL ones, but not + # "agile" ones (namely once "hashlib" has been imported, + # they are fixed). # - # So we should only block it if the private interfaces - # are blocked as well. + # If OpenSSL is not available, hashes fall back to built-in ones, + # in which case we don't need to block the explicit public hashes + # as they will call a mocked one. + # + # If OpenSSL is available, hashes fall back to "openssl_*" ones, + # except for BLAKE2b and BLAKE2s. + stack.enter_context(_block_hashlib_hash_constructor(name)) + elif ( + # In FIPS mode, hashlib.() functions may raise if they use + # the OpenSSL implementation, except with usedforsecurity=False. + # However, blocking such functions also means blocking them + # so we again need to block them if we want to. + (_hashlib := try_import_module("_hashlib")) + and _hashlib.get_fips_mode() + and not allow_openssl + ) or ( + # Without OpenSSL, hashlib.() functions are aliases + # to built-in functions, so both of them must be blocked + # as the module may have been imported before the HACL ones. + not (_hashlib := try_import_module("_hashlib")) + and not allow_builtin + ): stack.enter_context(_block_hashlib_hash_constructor(name)) + if not allow_openssl: + # _hashlib.new() stack.enter_context(_block_openssl_hash_new(name)) + # _hashlib.openssl_*() + stack.enter_context(_block_openssl_hash_constructor(name)) + # _hashlib.hmac_new() stack.enter_context(_block_openssl_hmac_new(name)) + # _hashlib.hmac_digest() stack.enter_context(_block_openssl_hmac_digest(name)) - stack.enter_context(_block_openssl_hash_constructor(name)) + if not allow_builtin: + # __get_builtin_constructor(name) stack.enter_context(_block_builtin_hash_new(name)) - stack.enter_context(_block_builtin_hmac_new(name)) - stack.enter_context(_block_builtin_hmac_digest(name)) + # .() stack.enter_context(_block_builtin_hash_constructor(name)) + # _hmac.new(..., name) + stack.enter_context(_block_builtin_hmac_new(name)) + # _hmac.compute_() stack.enter_context(_block_builtin_hmac_constructor(name)) + # _hmac.compute_digest(..., name) + stack.enter_context(_block_builtin_hmac_digest(name)) yield diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index 33b7d75e3ff203..9ad37909a8ec4e 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -937,6 +937,13 @@ def test_lazy_import(self): ensure_lazy_imports("gettext", {"re", "warnings", "locale"}) +class TranslationFallbackTestCase(unittest.TestCase): + def test_translation_fallback(self): + with os_helper.temp_cwd() as tempdir: + t = gettext.translation('gettext', localedir=tempdir, fallback=True) + self.assertIsInstance(t, gettext.NullTranslations) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 7123641650263b..b2b64a76a9f0f6 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -545,13 +545,17 @@ def check(self, name, data, hexdigest, shake=False, **kwargs): def check_file_digest(self, name, data, hexdigest): hexdigest = hexdigest.lower() - try: - hashlib.new(name) - except ValueError: - # skip, algorithm is blocked by security policy. - return - digests = [name] - digests.extend(self.constructors_to_test[name]) + digests = [] + for digest in [name, *self.constructors_to_test[name]]: + try: + if callable(digest): + digest(b"") + else: + hashlib.new(digest) + except ValueError: + # skip, algorithm is blocked by security policy. + continue + digests.append(digest) with tempfile.TemporaryFile() as f: f.write(data) diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 38429ad480ff1c..47e3914d1dd62e 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -386,6 +386,52 @@ def test_headers_debuglevel(self): self.assertEqual(lines[2], "header: Second: val1") self.assertEqual(lines[3], "header: Second: val2") + def test_max_response_headers(self): + max_headers = client._MAXHEADERS + 20 + headers = [f"Name{i}: Value{i}".encode() for i in range(max_headers)] + body = b"HTTP/1.1 200 OK\r\n" + b"\r\n".join(headers) + + with self.subTest(max_headers=None): + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + with self.assertRaisesRegex( + client.HTTPException, f"got more than 100 headers" + ): + resp.begin() + + with self.subTest(max_headers=max_headers): + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin(_max_headers=max_headers) + + def test_max_connection_headers(self): + max_headers = client._MAXHEADERS + 20 + headers = ( + f"Name{i}: Value{i}".encode() for i in range(max_headers - 1) + ) + body = ( + b"HTTP/1.1 200 OK\r\n" + + b"\r\n".join(headers) + + b"\r\nContent-Length: 12\r\n\r\nDummy body\r\n" + ) + + with self.subTest(max_headers=None): + conn = client.HTTPConnection("example.com") + conn.sock = FakeSocket(body) + conn.request("GET", "/") + with self.assertRaisesRegex( + client.HTTPException, f"got more than {client._MAXHEADERS} headers" + ): + response = conn.getresponse() + + with self.subTest(max_headers=None): + conn = client.HTTPConnection( + "example.com", max_response_headers=max_headers + ) + conn.sock = FakeSocket(body) + conn.request("GET", "/") + response = conn.getresponse() + response.read() class HttpMethodTests(TestCase): def test_invalid_method_names(self): diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index cb31122fee9642..92b3ef26cd979a 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -2,6 +2,7 @@ import errno import importlib import itertools +import inspect import io import logging import os @@ -820,6 +821,7 @@ def test_linked_to_musl(self): # SuppressCrashReport +@hashlib_helper.requires_builtin_hashes() class TestHashlibSupport(unittest.TestCase): @classmethod @@ -828,11 +830,20 @@ def setUpClass(cls): cls.hashlib = import_helper.import_module("hashlib") cls.hmac = import_helper.import_module("hmac") - # We required the extension modules to be present since blocking - # HACL* implementations while allowing OpenSSL ones would still - # result in failures. + # All C extension modules must be present since blocking + # the built-in implementation while allowing OpenSSL or vice-versa + # may result in failures depending on the exposed built-in hashes. cls._hashlib = import_helper.import_module("_hashlib") cls._hmac = import_helper.import_module("_hmac") + cls._md5 = import_helper.import_module("_md5") + + def skip_if_fips_mode(self): + if self._hashlib.get_fips_mode(): + self.skipTest("disabled in FIPS mode") + + def skip_if_not_fips_mode(self): + if not self._hashlib.get_fips_mode(): + self.skipTest("requires FIPS mode") def check_context(self, disabled=True): if disabled: @@ -853,25 +864,19 @@ def try_import_attribute(self, fullname, default=None): except TypeError: return default - def validate_modules(self): - if hasattr(hashlib_helper, 'hashlib'): - self.assertIs(hashlib_helper.hashlib, self.hashlib) - if hasattr(hashlib_helper, 'hmac'): - self.assertIs(hashlib_helper.hmac, self.hmac) - - def fetch_hash_function(self, name, typ): - entry = hashlib_helper._EXPLICIT_CONSTRUCTORS[name] - match typ: + def fetch_hash_function(self, name, implementation): + info = hashlib_helper.get_hash_info(name) + match implementation: case "hashlib": - assert entry.hashlib is not None, entry - return getattr(self.hashlib, entry.hashlib) + assert info.hashlib is not None, info + return getattr(self.hashlib, info.hashlib) case "openssl": try: - return getattr(self._hashlib, entry.openssl, None) + return getattr(self._hashlib, info.openssl, None) except TypeError: return None - case "builtin": - return self.try_import_attribute(entry.fullname(typ)) + fullname = info.fullname(implementation) + return self.try_import_attribute(fullname) def fetch_hmac_function(self, name): fullname = hashlib_helper._EXPLICIT_HMAC_CONSTRUCTORS[name] @@ -936,16 +941,12 @@ def check_builtin_hmac(self, name, *, disabled=True): ) def test_disable_hash(self, name, allow_openssl, allow_builtin): # In FIPS mode, the function may be available but would still need - # to raise a ValueError. For simplicity, we don't test the helper - # when we're in FIPS mode. - if self._hashlib.get_fips_mode(): - self.skipTest("hash functions may still be blocked in FIPS mode") + # to raise a ValueError, so we will test the helper separately. + self.skip_if_fips_mode() flags = dict(allow_openssl=allow_openssl, allow_builtin=allow_builtin) - is_simple_disabled = not allow_builtin and not allow_openssl + is_fully_disabled = not allow_builtin and not allow_openssl with hashlib_helper.block_algorithm(name, **flags): - self.validate_modules() - # OpenSSL's blake2s and blake2b are unknown names # when only the OpenSSL interface is available. if allow_openssl and not allow_builtin: @@ -954,25 +955,104 @@ def test_disable_hash(self, name, allow_openssl, allow_builtin): else: name_for_hashlib_new = name - with self.check_context(is_simple_disabled): + with self.check_context(is_fully_disabled): _ = self.hashlib.new(name_for_hashlib_new) - with self.check_context(is_simple_disabled): - _ = getattr(self.hashlib, name)(b"") + + # Since _hashlib is present, explicit blake2b/blake2s constructors + # use the built-in implementation, while others (since we are not + # in FIPS mode and since _hashlib exists) use the OpenSSL function. + with self.check_context(is_fully_disabled): + _ = getattr(self.hashlib, name)() self.check_openssl_hash(name, disabled=not allow_openssl) self.check_builtin_hash(name, disabled=not allow_builtin) if name not in hashlib_helper.NON_HMAC_DIGEST_NAMES: - with self.check_context(is_simple_disabled): + with self.check_context(is_fully_disabled): _ = self.hmac.new(b"", b"", name) - with self.check_context(is_simple_disabled): + with self.check_context(is_fully_disabled): _ = self.hmac.HMAC(b"", b"", name) - with self.check_context(is_simple_disabled): + with self.check_context(is_fully_disabled): _ = self.hmac.digest(b"", b"", name) self.check_openssl_hmac(name, disabled=not allow_openssl) self.check_builtin_hmac(name, disabled=not allow_builtin) + @hashlib_helper.block_algorithm("md5") + def test_disable_hash_md5_in_fips_mode(self): + self.skip_if_not_fips_mode() + + self.assertRaises(ValueError, self.hashlib.new, "md5") + self.assertRaises(ValueError, self._hashlib.new, "md5") + self.assertRaises(ValueError, self.hashlib.md5) + self.assertRaises(ValueError, self._hashlib.openssl_md5) + + kwargs = dict(usedforsecurity=True) + self.assertRaises(ValueError, self.hashlib.new, "md5", **kwargs) + self.assertRaises(ValueError, self._hashlib.new, "md5", **kwargs) + self.assertRaises(ValueError, self.hashlib.md5, **kwargs) + self.assertRaises(ValueError, self._hashlib.openssl_md5, **kwargs) + + @hashlib_helper.block_algorithm("md5", allow_openssl=True) + def test_disable_hash_md5_in_fips_mode_allow_openssl(self): + self.skip_if_not_fips_mode() + # Allow the OpenSSL interface to be used but not the HACL* one. + # hashlib.new("md5") is dispatched to hashlib.openssl_md5() + self.assertRaises(ValueError, self.hashlib.new, "md5") + # dispatched to hashlib.openssl_md5() in FIPS mode + h2 = self.hashlib.new("md5", usedforsecurity=False) + self.assertIsInstance(h2, self._hashlib.HASH) + + # block_algorithm() does not mock hashlib.md5 and _hashlib.openssl_md5 + self.assertNotHasAttr(self.hashlib.md5, "__wrapped__") + self.assertNotHasAttr(self._hashlib.openssl_md5, "__wrapped__") + + hashlib_md5 = inspect.unwrap(self.hashlib.md5) + self.assertIs(hashlib_md5, self._hashlib.openssl_md5) + self.assertRaises(ValueError, self.hashlib.md5) + # allow MD5 to be used in FIPS mode if usedforsecurity=False + h3 = self.hashlib.md5(usedforsecurity=False) + self.assertIsInstance(h3, self._hashlib.HASH) + + @hashlib_helper.block_algorithm("md5", allow_builtin=True) + def test_disable_hash_md5_in_fips_mode_allow_builtin(self): + self.skip_if_not_fips_mode() + # Allow the HACL* interface to be used but not the OpenSSL one. + h1 = self.hashlib.new("md5") # dispatched to _md5.md5() + self.assertNotIsInstance(h1, self._hashlib.HASH) + h2 = self.hashlib.new("md5", usedforsecurity=False) + self.assertIsInstance(h2, type(h1)) + + # block_algorithm() mocks hashlib.md5 and _hashlib.openssl_md5 + self.assertHasAttr(self.hashlib.md5, "__wrapped__") + self.assertHasAttr(self._hashlib.openssl_md5, "__wrapped__") + + hashlib_md5 = inspect.unwrap(self.hashlib.md5) + openssl_md5 = inspect.unwrap(self._hashlib.openssl_md5) + self.assertIs(hashlib_md5, openssl_md5) + self.assertRaises(ValueError, self.hashlib.md5) + self.assertRaises(ValueError, self.hashlib.md5, + usedforsecurity=False) + + @hashlib_helper.block_algorithm("md5", + allow_openssl=True, + allow_builtin=True) + def test_disable_hash_md5_in_fips_mode_allow_all(self): + self.skip_if_not_fips_mode() + # hashlib.new() isn't blocked as it falls back to _md5.md5 + self.assertIsInstance(self.hashlib.new("md5"), self._md5.MD5Type) + self.assertRaises(ValueError, self._hashlib.new, "md5") + h = self._hashlib.new("md5", usedforsecurity=False) + self.assertIsInstance(h, self._hashlib.HASH) + + self.assertNotHasAttr(self.hashlib.md5, "__wrapped__") + self.assertNotHasAttr(self._hashlib.openssl_md5, "__wrapped__") + + self.assertIs(self.hashlib.md5, self._hashlib.openssl_md5) + self.assertRaises(ValueError, self.hashlib.md5) + h = self.hashlib.md5(usedforsecurity=False) + self.assertIsInstance(h, self._hashlib.HASH) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 74b979d009664d..11b7f419bddbe4 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -4748,7 +4748,26 @@ class MyList(list): with self.assertRaises(TypeError): _suggestions._generate_suggestions(MyList(), "") + def test_no_site_package_flavour(self): + code = """import boo""" + _, _, stderr = assert_python_failure('-S', '-c', code) + self.assertIn( + (b"Site initialization is disabled, did you forget to " + b"add the site-packages directory to sys.path?"), stderr + ) + + code = """ + import sys + sys.stdlib_module_names = sys.stdlib_module_names + ("boo",) + import boo + """ + _, _, stderr = assert_python_failure('-S', '-c', code) + + self.assertNotIn( + (b"Site initialization is disabled, did you forget to " + b"add the site-packages directory to sys.path?"), stderr + ) class TestColorizedTraceback(unittest.TestCase): diff --git a/Lib/traceback.py b/Lib/traceback.py index 31aa8695735f2b..f0dbb6352f7760 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1106,6 +1106,11 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) if suggestion: self._str += f". Did you mean: '{suggestion}'?" + elif exc_type and issubclass(exc_type, ModuleNotFoundError) and \ + sys.flags.no_site and \ + getattr(exc_value, "name", None) not in sys.stdlib_module_names: + self._str += (". Site initialization is disabled, did you forget to " + + "add the site-packages directory to sys.path?") elif exc_type and issubclass(exc_type, (NameError, AttributeError)) and \ getattr(exc_value, "name", None) is not None: wrong_name = getattr(exc_value, "name", None) diff --git a/Misc/ACKS b/Misc/ACKS index fabd79b9f74210..35826bd713c0f6 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1954,6 +1954,7 @@ Adnan Umer Utkarsh Upadhyay Roger Upole Daniel Urban +Alexander Enrique Urieles Nieto Matthias Urlichs Michael Urman Hector Urtubia diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-17-08-09.gh-issue-127598.Mx8S-y.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-17-08-09.gh-issue-127598.Mx8S-y.rst new file mode 100644 index 00000000000000..aff047bbef0f07 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-17-08-09.gh-issue-127598.Mx8S-y.rst @@ -0,0 +1,2 @@ +Improve :exc:`ModuleNotFoundError` by adding flavour text to the exception when the +:option:`-S` option is passed. Patch by Andrea Mattei. diff --git a/Misc/NEWS.d/next/Library/2025-07-19-15-40-47.gh-issue-131724.LS59nA.rst b/Misc/NEWS.d/next/Library/2025-07-19-15-40-47.gh-issue-131724.LS59nA.rst new file mode 100644 index 00000000000000..71a991aa2c5ae6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-19-15-40-47.gh-issue-131724.LS59nA.rst @@ -0,0 +1,4 @@ +In :mod:`http.client`, a new *max_response_headers* keyword-only parameter has been +added to :class:`~http.client.HTTPConnection` and :class:`~http.client.HTTPSConnection` +constructors. This parameter sets the maximum number of allowed response headers, +helping to prevent denial-of-service attacks. diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 7086a3f6530841..d79e4b360e95c5 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -495,8 +495,8 @@ raise_unsupported_algorithm_impl(PyObject *exc_type, { // Since OpenSSL 3.0, if the algorithm is not supported or fetching fails, // the reason lacks the algorithm name. - int errcode = ERR_peek_last_error(), reason_id; - switch (reason_id = ERR_GET_REASON(errcode)) { + int errcode = ERR_peek_last_error(); + switch (ERR_GET_REASON(errcode)) { case ERR_R_UNSUPPORTED: { PyObject *text = PyUnicode_FromFormat(fallback_format, format_arg); if (text != NULL) { @@ -651,14 +651,11 @@ disable_fips_property(Py_hash_type py_ht) * If 'name' is an OpenSSL indexed name, the return value is cached. */ static PY_EVP_MD * -get_openssl_evp_md_by_utf8name(PyObject *module, const char *name, +get_openssl_evp_md_by_utf8name(_hashlibstate *state, const char *name, Py_hash_type py_ht) { PY_EVP_MD *digest = NULL, *other_digest = NULL; - _hashlibstate *state = get_hashlib_state(module); - py_hashentry_t *entry = (py_hashentry_t *)_Py_hashtable_get( - state->hashtable, (const void*)name - ); + py_hashentry_t *entry = _Py_hashtable_get(state->hashtable, name); if (entry != NULL) { if (!disable_fips_property(py_ht)) { @@ -715,26 +712,25 @@ get_openssl_evp_md_by_utf8name(PyObject *module, const char *name, * py_ht The message digest purpose. */ static PY_EVP_MD * -get_openssl_evp_md(PyObject *module, PyObject *digestmod, Py_hash_type py_ht) +get_openssl_evp_md(_hashlibstate *state, PyObject *digestmod, Py_hash_type py_ht) { const char *name; if (PyUnicode_Check(digestmod)) { name = PyUnicode_AsUTF8(digestmod); } else { - PyObject *dict = get_hashlib_state(module)->constructs; + PyObject *dict = state->constructs; assert(dict != NULL); PyObject *borrowed_ref = PyDict_GetItemWithError(dict, digestmod); name = borrowed_ref == NULL ? NULL : PyUnicode_AsUTF8(borrowed_ref); } if (name == NULL) { if (!PyErr_Occurred()) { - _hashlibstate *state = get_hashlib_state(module); raise_unsupported_algorithm_error(state, digestmod); } return NULL; } - return get_openssl_evp_md_by_utf8name(module, name, py_ht); + return get_openssl_evp_md_by_utf8name(state, name, py_ht); } // --- OpenSSL HASH wrappers -------------------------------------------------- @@ -1191,7 +1187,7 @@ static PyType_Spec HASHXOFobject_type_spec = { #endif static PyObject * -_hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj, +_hashlib_HASH(_hashlibstate *state, const char *digestname, PyObject *data_obj, int usedforsecurity) { Py_buffer view = { 0 }; @@ -1203,19 +1199,13 @@ _hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj, GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); } - digest = get_openssl_evp_md_by_utf8name( - module, digestname, usedforsecurity ? Py_ht_evp : Py_ht_evp_nosecurity - ); + Py_hash_type purpose = usedforsecurity ? Py_ht_evp : Py_ht_evp_nosecurity; + digest = get_openssl_evp_md_by_utf8name(state, digestname, purpose); if (digest == NULL) { goto exit; } - if ((EVP_MD_flags(digest) & EVP_MD_FLAG_XOF) == EVP_MD_FLAG_XOF) { - type = get_hashlib_state(module)->HASHXOF_type; - } else { - type = get_hashlib_state(module)->HASH_type; - } - + type = PY_EVP_MD_xof(digest) ? state->HASHXOF_type : state->HASH_type; self = new_hash_object(type); if (self == NULL) { goto exit; @@ -1267,7 +1257,8 @@ _hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj, if (_Py_hashlib_data_argument(&data_obj, DATA, STRING) < 0) { \ return NULL; \ } \ - return _hashlib_HASH(MODULE, NAME, data_obj, USEDFORSECURITY); \ + _hashlibstate *state = get_hashlib_state(MODULE); \ + return _hashlib_HASH(state, NAME, data_obj, USEDFORSECURITY); \ } while (0) /* The module-level function: new() */ @@ -1569,12 +1560,14 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, PyObject *dklen_obj) /*[clinic end generated code: output=144b76005416599b input=ed3ab0d2d28b5d5c]*/ { + _hashlibstate *state = get_hashlib_state(module); PyObject *key_obj = NULL; char *key; long dklen; int retval; - PY_EVP_MD *digest = get_openssl_evp_md_by_utf8name(module, hash_name, Py_ht_pbkdf2); + PY_EVP_MD *digest = get_openssl_evp_md_by_utf8name(state, hash_name, + Py_ht_pbkdf2); if (digest == NULL) { goto end; } @@ -1773,6 +1766,7 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, Py_buffer *msg, PyObject *digest) /*[clinic end generated code: output=82f19965d12706ac input=0a0790cc3db45c2e]*/ { + _hashlibstate *state = get_hashlib_state(module); unsigned char md[EVP_MAX_MD_SIZE] = {0}; unsigned int md_len = 0; unsigned char *result; @@ -1790,7 +1784,7 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, return NULL; } - evp = get_openssl_evp_md(module, digest, Py_ht_mac); + evp = get_openssl_evp_md(state, digest, Py_ht_mac); if (evp == NULL) { return NULL; } @@ -1808,7 +1802,6 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, if (result == NULL) { if (is_xof) { - _hashlibstate *state = get_hashlib_state(module); /* use a better default error message if an XOF is used */ raise_unsupported_algorithm_error(state, digest); } @@ -1862,6 +1855,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, PyObject *digestmod) /*[clinic end generated code: output=c20d9e4d9ed6d219 input=5f4071dcc7f34362]*/ { + _hashlibstate *state = get_hashlib_state(module); PY_EVP_MD *digest; HMAC_CTX *ctx = NULL; HMACobject *self = NULL; @@ -1879,7 +1873,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, return NULL; } - digest = get_openssl_evp_md(module, digestmod, Py_ht_mac); + digest = get_openssl_evp_md(state, digestmod, Py_ht_mac); if (digest == NULL) { return NULL; } @@ -1895,7 +1889,6 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, PY_EVP_MD_free(digest); if (r == 0) { if (is_xof) { - _hashlibstate *state = get_hashlib_state(module); /* use a better default error message if an XOF is used */ raise_unsupported_algorithm_error(state, digestmod); } @@ -1905,7 +1898,6 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, goto error; } - _hashlibstate *state = get_hashlib_state(module); self = PyObject_New(HMACobject, state->HMAC_type); if (self == NULL) { goto error;