diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 64ef2c91329d81..d77723ef27c2dc 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -48,8 +48,10 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - run: make -j8 + run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt - name: Display build info run: make pythoninfo + - name: Check compiler warnings + run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang - name: Tests run: make test diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 8dd5f559585368..92069fddc31217 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -80,7 +80,7 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo - name: Check compiler warnings - run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu + run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index 434d2021cea8e6..bf9df62c6f1706 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -10,7 +10,8 @@ There are two functions specifically for working with iterators. .. c:function:: int PyIter_Check(PyObject *o) Return non-zero if the object *o* can be safely passed to - :c:func:`PyIter_Next`, and ``0`` otherwise. This function always succeeds. + :c:func:`PyIter_NextItem` and ``0`` otherwise. + This function always succeeds. .. c:function:: int PyAIter_Check(PyObject *o) @@ -19,41 +20,27 @@ There are two functions specifically for working with iterators. .. versionadded:: 3.10 +.. c:function:: int PyIter_NextItem(PyObject *iter, PyObject **item) + + Return ``1`` and set *item* to a :term:`strong reference` of the + next value of the iterator *iter* on success. + Return ``0`` and set *item* to ``NULL`` if there are no remaining values. + Return ``-1``, set *item* to ``NULL`` and set an exception on error. + + .. versionadded:: 3.14 + .. c:function:: PyObject* PyIter_Next(PyObject *o) + This is an older version of :c:func:`!PyIter_NextItem`, + which is retained for backwards compatibility. + Prefer :c:func:`PyIter_NextItem`. + Return the next value from the iterator *o*. The object must be an iterator according to :c:func:`PyIter_Check` (it is up to the caller to check this). If there are no remaining values, returns ``NULL`` with no exception set. If an error occurs while retrieving the item, returns ``NULL`` and passes along the exception. -To write a loop which iterates over an iterator, the C code should look -something like this:: - - PyObject *iterator = PyObject_GetIter(obj); - PyObject *item; - - if (iterator == NULL) { - /* propagate error */ - } - - while ((item = PyIter_Next(iterator))) { - /* do something with item */ - ... - /* release reference when done */ - Py_DECREF(item); - } - - Py_DECREF(iterator); - - if (PyErr_Occurred()) { - /* propagate error */ - } - else { - /* continue doing useful work */ - } - - .. c:type:: PySendResult The enum value used to represent different results of :c:func:`PyIter_Send`. diff --git a/Doc/constraints.txt b/Doc/constraints.txt index ab3b39bf380dad..5f9709ac18ec99 100644 --- a/Doc/constraints.txt +++ b/Doc/constraints.txt @@ -13,11 +13,11 @@ packaging<25 Pygments<3 requests<3 snowballstemmer<3 -sphinxcontrib-applehelp<2.1 -sphinxcontrib-devhelp<2.1 +sphinxcontrib-applehelp>=1.0.6,<2.1 +sphinxcontrib-devhelp>=1.0.6,<2.1 sphinxcontrib-htmlhelp<2.2 sphinxcontrib-jsmath<1.1 -sphinxcontrib-qthelp<2.1 +sphinxcontrib-qthelp>=1.0.6,<2.1 sphinxcontrib-serializinghtml<2.1 # Direct dependencies of Jinja2 (Jinja is a dependency of Sphinx, see above) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index ccef104eeefde5..65d48f8bea7de8 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1132,6 +1132,10 @@ PyAIter_Check:PyObject*:o:0: PyIter_Next:PyObject*::+1: PyIter_Next:PyObject*:o:0: +PyIter_NextItem:int::: +PyIter_NextItem:PyObject*:iter:0: +PyIter_NextItem:PyObject**:item:+1: + PyIter_Send:int::: PyIter_Send:PyObject*:iter:0: PyIter_Send:PyObject*:arg:0: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 90ddb3fd8213ca..592e3465824893 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -335,6 +335,7 @@ func,PyInterpreterState_GetID,3.7,, func,PyInterpreterState_New,3.2,, func,PyIter_Check,3.8,, func,PyIter_Next,3.2,, +func,PyIter_NextItem,3.14,, func,PyIter_Send,3.10,, data,PyListIter_Type,3.2,, data,PyListRevIter_Type,3.2,, diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index 7f10d9a98257f9..6942b9d62cb8f2 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -67,6 +67,9 @@ although there is currently no date scheduled for their removal. * ``EntryPoints`` tuple interface. * Implicit ``None`` on return values. +* :mod:`logging`: the ``warn()`` method has been deprecated + since Python 3.3, use :meth:`~logging.warning()` instead. + * :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use BytesIO and binary mode instead. diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 4ba520c139ebce..204d7e423012d2 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -352,10 +352,6 @@ in a module, ``__name__`` is the module's name in the Python package namespace. .. versionchanged:: 3.8 The *stacklevel* parameter was added. - .. versionchanged:: 3.13 - Remove the undocumented ``warn()`` method which was an alias to the - :meth:`warning` method. - .. method:: Logger.info(msg, *args, **kwargs) @@ -368,6 +364,10 @@ in a module, ``__name__`` is the module's name in the Python package namespace. Logs a message with level :const:`WARNING` on this logger. The arguments are interpreted as for :meth:`debug`. + .. note:: There is an obsolete method ``warn`` which is functionally + identical to ``warning``. As ``warn`` is deprecated, please do not use + it - use ``warning`` instead. + .. method:: Logger.error(msg, *args, **kwargs) Logs a message with level :const:`ERROR` on this logger. The arguments are @@ -1124,11 +1124,6 @@ information into logging calls. For a usage example, see the section on Attribute :attr:`!manager` and method :meth:`!_log` were added, which delegate to the underlying logger and allow adapters to be nested. - .. versionchanged:: 3.13 - - Remove the undocumented :meth:`!warn`` method which was an alias to the - :meth:`!warning` method. - .. versionchanged:: 3.13 The *merge_extra* argument was added. @@ -1224,10 +1219,6 @@ functions. identical to ``warning``. As ``warn`` is deprecated, please do not use it - use ``warning`` instead. - .. versionchanged:: 3.13 - Remove the undocumented ``warn()`` function which was an alias to the - :func:`warning` function. - .. function:: error(msg, *args, **kwargs) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 60c099af928123..3df446a39ea8e9 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1636,7 +1636,7 @@ Copying, renaming and deleting .. method:: Path.unlink(missing_ok=False) Remove this file or symbolic link. If the path points to a directory, - use :func:`Path.rmdir` instead. + use :func:`Path.rmdir` or :func:`Path.delete` instead. If *missing_ok* is false (the default), :exc:`FileNotFoundError` is raised if the path does not exist. @@ -1650,33 +1650,40 @@ Copying, renaming and deleting .. method:: Path.rmdir() - Remove this directory. The directory must be empty. + Remove this directory. The directory must be empty; use + :meth:`Path.delete` to remove a non-empty directory. -.. method:: Path.rmtree(ignore_errors=False, on_error=None) +.. method:: Path.delete(ignore_errors=False, on_error=None) - Recursively delete this entire directory tree. The path must not refer to a symlink. + Delete this file or directory. If this path refers to a non-empty + directory, its files and sub-directories are deleted recursively. - If *ignore_errors* is true, errors resulting from failed removals will be - ignored. If *ignore_errors* is false or omitted, and a function is given to - *on_error*, it will be called each time an exception is raised. If neither - *ignore_errors* nor *on_error* are supplied, exceptions are propagated to - the caller. + If *ignore_errors* is true, errors resulting from failed deletions will be + ignored. If *ignore_errors* is false or omitted, and a callable is given as + the optional *on_error* argument, it will be called with one argument of + type :exc:`OSError` each time an exception is raised. The callable can + handle the error to continue the deletion process or re-raise it to stop. + Note that the filename is available as the :attr:`~OSError.filename` + attribute of the exception object. If neither *ignore_errors* nor + *on_error* are supplied, exceptions are propagated to the caller. .. note:: - On platforms that support the necessary fd-based functions, a symlink - attack-resistant version of :meth:`~Path.rmtree` is used by default. On - other platforms, the :func:`~Path.rmtree` implementation is susceptible - to a symlink attack: given proper timing and circumstances, attackers - can manipulate symlinks on the filesystem to delete files they would not - be able to access otherwise. - - If the optional argument *on_error* is specified, it should be a callable; - it will be called with one argument of type :exc:`OSError`. The - callable can handle the error to continue the deletion process or re-raise - it to stop. Note that the filename is available as the :attr:`~OSError.filename` - attribute of the exception object. + When deleting non-empty directories on platforms that lack the necessary + file descriptor-based functions, the :meth:`~Path.delete` implementation + is susceptible to a symlink attack: given proper timing and + circumstances, attackers can manipulate symlinks on the filesystem to + delete files they would not be able to access otherwise. Applications + can use the :data:`~Path.delete.avoids_symlink_attacks` method attribute + to determine whether the implementation is immune to this attack. + + .. attribute:: delete.avoids_symlink_attacks + + Indicates whether the current platform and implementation provides a + symlink attack resistant version of :meth:`~Path.delete`. Currently + this is only true for platforms supporting fd-based directory access + functions. .. versionadded:: 3.14 diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 1c420419568a90..871cfefc8de310 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -32,7 +32,10 @@ It starts by constructing up to four directories from a head and a tail part. For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads are skipped. For the tail part, it uses the empty string and then :file:`lib/site-packages` (on Windows) or -:file:`lib/python{X.Y}/site-packages` (on Unix and macOS). For each +:file:`lib/python{X.Y[t]}/site-packages` (on Unix and macOS). (The +optional suffix "t" indicates the :term:`free threading` build, and is +appended if ``"t"`` is present in the :attr:`sys.abiflags` constant.) +For each of the distinct head-tail combinations, it sees if it refers to an existing directory, and if so, adds it to ``sys.path`` and also inspects the newly added path for configuration files. @@ -40,6 +43,11 @@ added path for configuration files. .. versionchanged:: 3.5 Support for the "site-python" directory has been removed. +.. versionchanged:: 3.13 + On Unix, :term:`Free threading ` Python installations are + identified by the "t" suffix in the version-specific directory name, such as + :file:`lib/python3.13t/`. + If a file named "pyvenv.cfg" exists one directory above sys.executable, sys.prefix and sys.exec_prefix are set to that directory and it is also checked for site-packages (sys.base_prefix and @@ -188,11 +196,12 @@ Module contents Path to the user site-packages for the running Python. Can be ``None`` if :func:`getusersitepackages` hasn't been called yet. Default value is - :file:`~/.local/lib/python{X.Y}/site-packages` for UNIX and non-framework + :file:`~/.local/lib/python{X.Y}[t]/site-packages` for UNIX and non-framework macOS builds, :file:`~/Library/Python/{X.Y}/lib/python/site-packages` for macOS framework builds, and :file:`{%APPDATA%}\\Python\\Python{XY}\\site-packages` - on Windows. This directory is a site directory, which means that - :file:`.pth` files in it will be processed. + on Windows. The optional "t" indicates the free-threaded build. This + directory is a site directory, which means that :file:`.pth` files in it + will be processed. .. data:: USER_BASE diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index b44ac69606706c..01121feb2b2311 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1245,8 +1245,9 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). | ``s.pop()`` or ``s.pop(i)`` | retrieves the item at *i* and | \(2) | | | also removes it from *s* | | +------------------------------+--------------------------------+---------------------+ -| ``s.remove(x)`` | remove the first item from *s* | \(3) | -| | where ``s[i]`` is equal to *x* | | +| ``s.remove(x)`` | removes the first item from | \(3) | +| | *s* where ``s[i]`` is equal to | | +| | *x* | | +------------------------------+--------------------------------+---------------------+ | ``s.reverse()`` | reverses the items of *s* in | \(4) | | | place | | diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index aa61fbdd3131f6..f099d5553963e0 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -106,12 +106,16 @@ that mutable object is changed. Types affect almost all aspects of object behavior. Even the importance of object identity is affected in some sense: for immutable types, operations that compute new values may actually return a reference to any existing object with -the same type and value, while for mutable objects this is not allowed. E.g., -after ``a = 1; b = 1``, ``a`` and ``b`` may or may not refer to the same object -with the value one, depending on the implementation, but after ``c = []; d = -[]``, ``c`` and ``d`` are guaranteed to refer to two different, unique, newly -created empty lists. (Note that ``c = d = []`` assigns the same object to both -``c`` and ``d``.) +the same type and value, while for mutable objects this is not allowed. +For example, after ``a = 1; b = 1``, *a* and *b* may or may not refer to +the same object with the value one, depending on the implementation. +This is because :class:`int` is an immutable type, so the reference to ``1`` +can be reused. This behaviour depends on the implementation used, so should +not be relied upon, but is something to be aware of when making use of object +identity tests. +However, after ``c = []; d = []``, *c* and *d* are guaranteed to refer to two +different, unique, newly created empty lists. (Note that ``e = f = []`` assigns +the *same* object to both *e* and *f*.) .. _types: diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index c175c4f8b5b1eb..aa1edeb884e0df 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -24,7 +24,7 @@ Command line When invoking Python, you may specify any of these options:: - python [-bBdEhiIOqsSuvVWx?] [-c command | -m module-name | script | - ] [args] + python [-bBdEhiIOPqRsSuvVWx?] [-c command | -m module-name | script | - ] [args] The most common use case is, of course, a simple invocation of a script:: diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 6a4a52bb6e8b12..e00d1ee3e716e7 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -909,19 +909,32 @@ Security Options .. option:: --disable-safety - Disable compiler options that are recommended by `OpenSSF`_ for security reasons with no performance overhead. + Disable compiler options that are `recommended by OpenSSF`_ for security reasons with no performance overhead. If this option is not enabled, CPython will be built based on safety compiler options with no slow down. + When this option is enabled, CPython will not be built with the compiler options listed below. - .. _OpenSSF: https://openssf.org/ + The following compiler options are disabled with :option:`!--disable-safety`: + + * `-fstack-protector-strong`_: Enable run-time checks for stack-based buffer overflows. + * `-Wtrampolines`_: Enable warnings about trampolines that require executable stacks. + + .. _recommended by OpenSSF: https://github.com/ossf/wg-best-practices-os-developers/blob/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.md + .. _-fstack-protector-strong: https://github.com/ossf/wg-best-practices-os-developers/blob/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.md#enable-run-time-checks-for-stack-based-buffer-overflows + .. _-Wtrampolines: https://github.com/ossf/wg-best-practices-os-developers/blob/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.md#enable-warning-about-trampolines-that-require-executable-stacks .. versionadded:: 3.14 .. option:: --enable-slower-safety - Enable compiler options that are recommended by `OpenSSF`_ for security reasons which require overhead. + Enable compiler options that are `recommended by OpenSSF`_ for security reasons which require overhead. If this option is not enabled, CPython will not be built based on safety compiler options which performance impact. + When this option is enabled, CPython will be built with the compiler options listed below. + + The following compiler options are enabled with :option:`!--enable-slower-safety`: + + * `-D_FORTIFY_SOURCE=3`_: Fortify sources with compile- and run-time checks for unsafe libc usage and buffer overflows. - .. _OpenSSF: https://openssf.org/ + .. _-D_FORTIFY_SOURCE=3: https://github.com/ossf/wg-best-practices-os-developers/blob/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.md#fortify-sources-for-unsafe-libc-usage-and-buffer-overflows .. versionadded:: 3.14 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 5c57b5d7ebe2ff..aa06e92afc9a43 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1397,16 +1397,6 @@ locale use ``locale.setlocale(locale.LC_ALL, "")`` instead. (Contributed by Victor Stinner in :gh:`104783`.) -logging -------- - -* :mod:`logging`: Remove undocumented and untested ``Logger.warn()`` and - ``LoggerAdapter.warn()`` methods and ``logging.warn()`` function. Deprecated - since Python 3.3, they were aliases to the :meth:`logging.Logger.warning` - method, :meth:`!logging.LoggerAdapter.warning` method and - :func:`logging.warning` function. - (Contributed by Victor Stinner in :gh:`105376`.) - pathlib ------- diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index aecc7cabd0d1f5..b975f6a4f8a931 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -141,8 +141,7 @@ pathlib :func:`shutil.copyfile`. * :meth:`~pathlib.Path.copytree` copies one directory tree to another, like :func:`shutil.copytree`. - * :meth:`~pathlib.Path.rmtree` recursively removes a directory tree, like - :func:`shutil.rmtree`. + * :meth:`~pathlib.Path.delete` removes a file or directory tree. (Contributed by Barney Gale in :gh:`73991`.) @@ -405,6 +404,10 @@ New Features (Contributed by Victor Stinner in :gh:`119182`.) +* Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, + which has an ambiguous return value. + (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) + Porting to Python 3.14 ---------------------- diff --git a/Include/abstract.h b/Include/abstract.h index f0e49c1afb8164..7cfee1332ccaa4 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -397,13 +397,23 @@ PyAPI_FUNC(int) PyIter_Check(PyObject *); This function always succeeds. */ PyAPI_FUNC(int) PyAIter_Check(PyObject *); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000 +/* Return 1 and set 'item' to the next item of 'iter' on success. + * Return 0 and set 'item' to NULL when there are no remaining values. + * Return -1, set 'item' to NULL and set an exception on error. + */ +PyAPI_FUNC(int) PyIter_NextItem(PyObject *iter, PyObject **item); +#endif + /* Takes an iterator object and calls its tp_iternext slot, returning the next value. If the iterator is exhausted, this returns NULL without setting an exception. - NULL with an exception means an error occurred. */ + NULL with an exception means an error occurred. + + Prefer PyIter_NextItem() instead. */ PyAPI_FUNC(PyObject *) PyIter_Next(PyObject *); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 90cd7b54b34161..e1024ddbdf6062 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -270,6 +270,9 @@ typedef struct _heaptypeobject { PyObject *ht_module; char *_ht_tpname; // Storage for "tp_name"; see PyType_FromModuleAndSpec struct _specialization_cache _spec_cache; // For use by the specializer. +#ifdef Py_GIL_DISABLED + Py_ssize_t unique_id; // ID used for thread-local refcounting +#endif /* here are optional user slots, followed by the members. */ } PyHeapTypeObject; diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index b4bf36e82e376a..5dd5b0c78d42fa 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -381,10 +381,6 @@ extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); -#ifdef Py_GIL_DISABLED -// gh-117783: Immortalize objects that use deferred reference counting -extern void _PyGC_ImmortalizeDeferredObjects(PyInterpreterState *interp); -#endif #ifdef __cplusplus } diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 4a83862ac13e26..a1c1dd0c957230 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -35,6 +35,7 @@ extern "C" { #include "pycore_qsbr.h" // struct _qsbr_state #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state +#include "pycore_typeid.h" // struct _Py_type_id_pool #include "pycore_typeobject.h" // struct types_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_warnings.h" // struct _warnings_runtime_state @@ -220,6 +221,7 @@ struct _is { #if defined(Py_GIL_DISABLED) struct _mimalloc_interp_state mimalloc; struct _brc_state brc; // biased reference counting state + struct _Py_type_id_pool type_ids; PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS]; #endif diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 155810d00bef5b..a5640b7bcb7d60 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -14,10 +14,19 @@ extern "C" { #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_typeid.h" // _PyType_IncrefSlow #define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1) +// This value is added to `ob_ref_shared` for objects that use deferred +// reference counting so that they are not immediately deallocated when the +// non-deferred reference count drops to zero. +// +// The value is half the maximum shared refcount because the low two bits of +// `ob_ref_shared` are used for flags. +#define _Py_REF_DEFERRED (PY_SSIZE_T_MAX / 8) + // gh-121528, gh-118997: Similar to _Py_IsImmortal() but be more loose when // comparing the reference count to stay compatible with C extensions built // with the stable ABI 3.11 or older. Such extensions implement INCREF/DECREF @@ -280,6 +289,67 @@ extern PyStatus _PyObject_InitState(PyInterpreterState *interp); extern void _PyObject_FiniState(PyInterpreterState *interp); extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); +#ifndef Py_GIL_DISABLED +# define _Py_INCREF_TYPE Py_INCREF +# define _Py_DECREF_TYPE Py_DECREF +#else +static inline void +_Py_INCREF_TYPE(PyTypeObject *type) +{ + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + assert(_Py_IsImmortal(type)); + return; + } + + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + PyHeapTypeObject *ht = (PyHeapTypeObject *)type; + + // Unsigned comparison so that `unique_id=-1`, which indicates that + // per-thread refcounting has been disabled on this type, is handled by + // the "else". + if ((size_t)ht->unique_id < (size_t)tstate->types.size) { +# ifdef Py_REF_DEBUG + _Py_INCREF_IncRefTotal(); +# endif + _Py_INCREF_STAT_INC(); + tstate->types.refcounts[ht->unique_id]++; + } + else { + // The slow path resizes the thread-local refcount array if necessary. + // It handles the unique_id=-1 case to keep the inlinable function smaller. + _PyType_IncrefSlow(ht); + } +} + +static inline void +_Py_DECREF_TYPE(PyTypeObject *type) +{ + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + assert(_Py_IsImmortal(type)); + return; + } + + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + PyHeapTypeObject *ht = (PyHeapTypeObject *)type; + + // Unsigned comparison so that `unique_id=-1`, which indicates that + // per-thread refcounting has been disabled on this type, is handled by + // the "else". + if ((size_t)ht->unique_id < (size_t)tstate->types.size) { +# ifdef Py_REF_DEBUG + _Py_DECREF_DecRefTotal(); +# endif + _Py_DECREF_STAT_INC(); + tstate->types.refcounts[ht->unique_id]--; + } + else { + // Directly decref the type if the type id is not assigned or if + // per-thread refcounting has been disabled on this type. + Py_DECREF(type); + } +} +#endif + /* Inline functions trading binary compatibility for speed: _PyObject_Init() is the fast version of PyObject_Init(), and _PyObject_InitVar() is the fast version of PyObject_InitVar(). @@ -291,7 +361,7 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj) assert(op != NULL); Py_SET_TYPE(op, typeobj); assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortalLoose(typeobj)); - Py_INCREF(typeobj); + _Py_INCREF_TYPE(typeobj); _Py_NewReference(op); } diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 18c972bd367599..f681b644c9ad5d 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -31,6 +31,16 @@ typedef struct _PyThreadStateImpl { struct _mimalloc_thread_state mimalloc; struct _Py_freelists freelists; struct _brc_thread_state brc; + struct { + // The thread-local refcounts for heap type objects + Py_ssize_t *refcounts; + + // Size of the refcounts array. + Py_ssize_t size; + + // If set, don't use thread-local refcounts + int is_finalized; + } types; #endif #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) diff --git a/Include/internal/pycore_typeid.h b/Include/internal/pycore_typeid.h new file mode 100644 index 00000000000000..e64d1447f6b51d --- /dev/null +++ b/Include/internal/pycore_typeid.h @@ -0,0 +1,75 @@ +#ifndef Py_INTERNAL_TYPEID_H +#define Py_INTERNAL_TYPEID_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED + +// This contains code for allocating unique ids to heap type objects +// and re-using those ids when the type is deallocated. +// +// The type ids are used to implement per-thread reference counts of +// heap type objects to avoid contention on the reference count fields +// of heap type objects. Static type objects are immortal, so contention +// is not an issue for those types. +// +// Type id of -1 is used to indicate a type doesn't use thread-local +// refcounting. This value is used when a type object is finalized by the GC +// and during interpreter shutdown to allow the type object to be +// deallocated promptly when the object's refcount reaches zero. +// +// Each entry implicitly represents a type id based on it's offset in the +// table. Non-allocated entries form a free-list via the 'next' pointer. +// Allocated entries store the corresponding PyTypeObject. +typedef union _Py_type_id_entry { + // Points to the next free type id, when part of the freelist + union _Py_type_id_entry *next; + + // Stores the type object when the id is assigned + PyHeapTypeObject *type; +} _Py_type_id_entry; + +struct _Py_type_id_pool { + PyMutex mutex; + + // combined table of types with allocated type ids and unallocated + // type ids. + _Py_type_id_entry *table; + + // Next entry to allocate inside 'table' or NULL + _Py_type_id_entry *freelist; + + // size of 'table' + Py_ssize_t size; +}; + +// Assigns the next id from the pool of type ids. +extern void _PyType_AssignId(PyHeapTypeObject *type); + +// Releases the allocated type id back to the pool. +extern void _PyType_ReleaseId(PyHeapTypeObject *type); + +// Merges the thread-local reference counts into the corresponding types. +extern void _PyType_MergeThreadLocalRefcounts(_PyThreadStateImpl *tstate); + +// Like _PyType_MergeThreadLocalRefcounts, but also frees the thread-local +// array of refcounts. +extern void _PyType_FinalizeThreadLocalRefcounts(_PyThreadStateImpl *tstate); + +// Frees the interpreter's pool of type ids. +extern void _PyType_FinalizeIdPool(PyInterpreterState *interp); + +// Increfs the type, resizing the thread-local refcount array if necessary. +PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type); + +#endif /* Py_GIL_DISABLED */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_TYPEID_H */ diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 52a3ab2f0a4abd..ba31e16c3bbeaa 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -555,7 +555,7 @@ Important files * [Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c): Constructs a code object from a sequence of pseudo instructions. - * [Python/instruction_sequence.c.c](https://github.com/python/cpython/blob/main/Python/instruction_sequence.c.c): + * [Python/instruction_sequence.c](https://github.com/python/cpython/blob/main/Python/instruction_sequence.c): A data structure representing a sequence of bytecode-like pseudo-instructions. * [Include/](https://github.com/python/cpython/blob/main/Include/) diff --git a/Lib/argparse.py b/Lib/argparse.py index 83189115bf85f7..100ef9f55cd2f7 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -447,15 +447,24 @@ def _get_actions_usage_parts(self, actions, groups): parts.append(part) # group mutually exclusive actions + inserted_separators_indices = set() for start, end in sorted(inserts, reverse=True): group = inserts[start, end] group_parts = [item for item in parts[start:end] if item is not None] + group_size = len(group_parts) if group.required: - open, close = "()" if len(group_parts) > 1 else ("", "") + open, close = "()" if group_size > 1 else ("", "") else: open, close = "[]" - parts[start] = open + " | ".join(group_parts) + close - for i in range(start + 1, end): + group_parts[0] = open + group_parts[0] + group_parts[-1] = group_parts[-1] + close + for i, part in enumerate(group_parts[:-1], start=start): + # insert a separator if not already done in a nested group + if i not in inserted_separators_indices: + parts[i] = part + ' |' + inserted_separators_indices.add(i) + parts[start + group_size - 1] = group_parts[-1] + for i in range(start + group_size, end): parts[i] = None # return the usage parts diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index e4a39f4d345c79..f614fddcac7026 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -812,7 +812,7 @@ def call_at(self, when, callback, *args, context=None): timer = events.TimerHandle(when, callback, args, self, context) if timer._source_traceback: del timer._source_traceback[-1] - heapq.heappush(self._scheduled, timer) + heapq.heappush(self._scheduled, (when, timer)) timer._scheduled = True return timer @@ -1959,20 +1959,21 @@ def _run_once(self): # Remove delayed calls that were cancelled if their number # is too high new_scheduled = [] - for handle in self._scheduled: + for when_handle in self._scheduled: + handle = when_handle[1] if handle._cancelled: handle._scheduled = False else: - new_scheduled.append(handle) + new_scheduled.append(when_handle) heapq.heapify(new_scheduled) self._scheduled = new_scheduled self._timer_cancelled_count = 0 else: # Remove delayed calls that were cancelled from head of queue. - while self._scheduled and self._scheduled[0]._cancelled: + while self._scheduled and self._scheduled[0][0]._cancelled: self._timer_cancelled_count -= 1 - handle = heapq.heappop(self._scheduled) + _, handle = heapq.heappop(self._scheduled) handle._scheduled = False timeout = None @@ -1980,7 +1981,7 @@ def _run_once(self): timeout = 0 elif self._scheduled: # Compute the desired timeout. - timeout = self._scheduled[0]._when - self.time() + timeout = self._scheduled[0][0]._when - self.time() if timeout > MAXIMUM_SELECT_TIMEOUT: timeout = MAXIMUM_SELECT_TIMEOUT elif timeout < 0: @@ -1993,13 +1994,14 @@ def _run_once(self): # Handle 'later' callbacks that are ready. end_time = self.time() + self._clock_resolution + ready = self._ready while self._scheduled: - handle = self._scheduled[0] - if handle._when >= end_time: + when, handle = self._scheduled[0] + if when >= end_time: break - handle = heapq.heappop(self._scheduled) + heapq.heappop(self._scheduled) handle._scheduled = False - self._ready.append(handle) + ready.append(handle) # This is the only place where callbacks are actually *called*. # All other places just add them to ready. @@ -2007,9 +2009,9 @@ def _run_once(self): # callbacks scheduled by callbacks run this time around -- # they will be run the next time (after another I/O poll). # Use an idiom that is thread-safe without using locks. - ntodo = len(self._ready) + ntodo = len(ready) for i in range(ntodo): - handle = self._ready.popleft() + handle = ready.popleft() if handle._cancelled: continue if self._debug: diff --git a/Lib/code.py b/Lib/code.py index cd3889067d068d..a1fd389b5a1015 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -107,29 +107,21 @@ def showsyntaxerror(self, filename=None, **kwargs): """ colorize = kwargs.pop('colorize', False) - type, value, tb = sys.exc_info() - sys.last_exc = value - sys.last_type = type - sys.last_value = value - sys.last_traceback = tb - if filename and type is SyntaxError: - # Work hard to stuff the correct filename in the exception - try: - msg, (dummy_filename, lineno, offset, line) = value.args - except ValueError: - # Not the format we expect; leave it alone - pass - else: - # Stuff in the right filename - value = SyntaxError(msg, (filename, lineno, offset, line)) - sys.last_exc = sys.last_value = value - if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception_only(type, value, colorize=colorize) - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - self._call_excepthook(type, value, tb) + try: + typ, value, tb = sys.exc_info() + if filename and typ is SyntaxError: + # Work hard to stuff the correct filename in the exception + try: + msg, (dummy_filename, lineno, offset, line) = value.args + except ValueError: + # Not the format we expect; leave it alone + pass + else: + # Stuff in the right filename + value = SyntaxError(msg, (filename, lineno, offset, line)) + self._showtraceback(typ, value, None, colorize) + finally: + typ = value = tb = None def showtraceback(self, **kwargs): """Display the exception that just occurred. @@ -140,32 +132,35 @@ def showtraceback(self, **kwargs): """ colorize = kwargs.pop('colorize', False) - sys.last_type, sys.last_value, last_tb = ei = sys.exc_info() - sys.last_traceback = last_tb - sys.last_exc = ei[1] try: - if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next, colorize=colorize) - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - self._call_excepthook(ei[0], ei[1], last_tb) + typ, value, tb = sys.exc_info() + self._showtraceback(typ, value, tb.tb_next, colorize) finally: - last_tb = ei = None + typ = value = tb = None - def _call_excepthook(self, typ, value, tb): - try: - sys.excepthook(typ, value, tb) - except SystemExit: - raise - except BaseException as e: - e.__context__ = None - print('Error in sys.excepthook:', file=sys.stderr) - sys.__excepthook__(type(e), e, e.__traceback__.tb_next) - print(file=sys.stderr) - print('Original exception was:', file=sys.stderr) - sys.__excepthook__(typ, value, tb) + def _showtraceback(self, typ, value, tb, colorize): + sys.last_type = typ + sys.last_traceback = tb + sys.last_exc = sys.last_value = value = value.with_traceback(tb) + if sys.excepthook is sys.__excepthook__: + lines = traceback.format_exception(typ, value, tb, + colorize=colorize) + self.write(''.join(lines)) + else: + # If someone has set sys.excepthook, we let that take precedence + # over self.write + try: + sys.excepthook(typ, value, tb) + except SystemExit: + raise + except BaseException as e: + e.__context__ = None + e = e.with_traceback(e.__traceback__.tb_next) + print('Error in sys.excepthook:', file=sys.stderr) + sys.__excepthook__(type(e), e, e.__traceback__) + print(file=sys.stderr) + print('Original exception was:', file=sys.stderr) + sys.__excepthook__(typ, value, tb) def write(self, data): """Write a string. diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 23fe7a82eb029d..c5350df270487a 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "24.1.1" +_PIP_VERSION = "24.2" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora diff --git a/Lib/ensurepip/_bundled/pip-24.1.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-24.2-py3-none-any.whl similarity index 76% rename from Lib/ensurepip/_bundled/pip-24.1.1-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-24.2-py3-none-any.whl index e27568eb8b39c9..542cdd1e7284ae 100644 Binary files a/Lib/ensurepip/_bundled/pip-24.1.1-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-24.2-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index c36fc75a24a239..c86d7ddb15dbba 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -207,7 +207,7 @@ def __get__(self, instance, ownerclass=None): # use previous enum.property return self.fget(instance) elif self._attr_type == 'attr': - # look up previous attibute + # look up previous attribute return getattr(self._cls_type, self.name) elif self._attr_type == 'desc': # use previous descriptor diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index eedf97bf74fe6a..4d2adb48570d49 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -111,7 +111,7 @@ def create_widgets(self): load_configs: Load pages except for extensions. activate_config_changes: Tell editors to reload. """ - self.frame = frame = Frame(self, padding="5px") + self.frame = frame = Frame(self, padding=5) self.frame.grid(sticky="nwes") self.note = note = Notebook(frame) self.extpage = ExtPage(note) diff --git a/Lib/idlelib/searchbase.py b/Lib/idlelib/searchbase.py index 64ed50c7364be3..c68a6ca339af04 100644 --- a/Lib/idlelib/searchbase.py +++ b/Lib/idlelib/searchbase.py @@ -86,7 +86,7 @@ def create_widgets(self): top.wm_iconname(self.icon) _setup_dialog(top) self.top = top - self.frame = Frame(top, padding="5px") + self.frame = Frame(top, padding=5) self.frame.grid(sticky="nwes") top.grid_columnconfigure(0, weight=100) top.grid_rowconfigure(0, weight=100) diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 3f4144226b40ec..aa9b79d8cab4bb 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -37,7 +37,7 @@ 'captureWarnings', 'critical', 'debug', 'disable', 'error', 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown', - 'warning', 'getLogRecordFactory', 'setLogRecordFactory', + 'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory', 'lastResort', 'raiseExceptions', 'getLevelNamesMapping', 'getHandlerByName', 'getHandlerNames'] @@ -1530,6 +1530,11 @@ def warning(self, msg, *args, **kwargs): if self.isEnabledFor(WARNING): self._log(WARNING, msg, args, **kwargs) + def warn(self, msg, *args, **kwargs): + warnings.warn("The 'warn' method is deprecated, " + "use 'warning' instead", DeprecationWarning, 2) + self.warning(msg, *args, **kwargs) + def error(self, msg, *args, **kwargs): """ Log 'msg % args' with severity 'ERROR'. @@ -1906,6 +1911,11 @@ def warning(self, msg, *args, **kwargs): """ self.log(WARNING, msg, *args, **kwargs) + def warn(self, msg, *args, **kwargs): + warnings.warn("The 'warn' method is deprecated, " + "use 'warning' instead", DeprecationWarning, 2) + self.warning(msg, *args, **kwargs) + def error(self, msg, *args, **kwargs): """ Delegate an error call to the underlying logger. @@ -2169,6 +2179,11 @@ def warning(msg, *args, **kwargs): basicConfig() root.warning(msg, *args, **kwargs) +def warn(msg, *args, **kwargs): + warnings.warn("The 'warn' function is deprecated, " + "use 'warning' instead", DeprecationWarning, 2) + warning(msg, *args, **kwargs) + def info(msg, *args, **kwargs): """ Log a message with severity 'INFO' on the root logger. If the logger has diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 0fa40f56e998d5..1cba64fd554100 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -196,9 +196,12 @@ def shouldRollover(self, record): if self.stream is None: # delay was set... self.stream = self._open() if self.maxBytes > 0: # are we rolling over? + pos = self.stream.tell() + if not pos: + # gh-116263: Never rollover an empty file + return False msg = "%s\n" % self.format(record) - self.stream.seek(0, 2) #due to non-posix-compliant Windows feature - if self.stream.tell() + len(msg) >= self.maxBytes: + if pos + len(msg) >= self.maxBytes: # See bpo-45401: Never rollover anything other than regular files if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): return False diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index bacee8ba164b48..d7c4e8444f8dec 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -151,7 +151,7 @@ def guess_type(self, url, strict=True): def guess_file_type(self, path, *, strict=True): """Guess the type of a file based on its path. - Similar to guess_type(), but takes file path istead of URL. + Similar to guess_type(), but takes file path instead of URL. """ path = os.fsdecode(path) path = os.path.splitdrive(path)[1] @@ -325,7 +325,7 @@ def guess_type(url, strict=True): def guess_file_type(path, *, strict=True): """Guess the type of a file based on its path. - Similar to guess_type(), but takes file path istead of URL. + Similar to guess_type(), but takes file path instead of URL. """ if _db is None: init() diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index c32e7762cefea3..8c799196e4703a 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -63,7 +63,7 @@ def splitdrive(self, path): def splitext(self, path): """Split the path into a pair (root, ext), where *ext* is empty or - begins with a begins with a period and contains at most one period, + begins with a period and contains at most one period, and *root* is everything before the extension.""" raise UnsupportedOperation(self._unsupported_msg('splitext()')) @@ -919,15 +919,15 @@ def rmdir(self): """ raise UnsupportedOperation(self._unsupported_msg('rmdir()')) - def rmtree(self, ignore_errors=False, on_error=None): + def delete(self, ignore_errors=False, on_error=None): """ - Recursively delete this directory tree. + Delete this file or directory (including all sub-directories). - If *ignore_errors* is true, exceptions raised from scanning the tree - and removing files and directories are ignored. Otherwise, if - *on_error* is set, it will be called to handle the error. If neither - *ignore_errors* nor *on_error* are set, exceptions are propagated to - the caller. + If *ignore_errors* is true, exceptions raised from scanning the + filesystem and removing files and directories are ignored. Otherwise, + if *on_error* is set, it will be called to handle the error. If + neither *ignore_errors* nor *on_error* are set, exceptions are + propagated to the caller. """ if ignore_errors: def on_error(err): @@ -935,14 +935,10 @@ def on_error(err): elif on_error is None: def on_error(err): raise err - try: - if self.is_symlink(): - raise OSError("Cannot call rmtree on a symbolic link") - elif self.is_junction(): - raise OSError("Cannot call rmtree on a junction") + if self.is_dir(follow_symlinks=False): results = self.walk( on_error=on_error, - top_down=False, # Bottom-up so we rmdir() empty directories. + top_down=False, # So we rmdir() empty directories. follow_symlinks=False) for dirpath, dirnames, filenames in results: for name in filenames: @@ -955,10 +951,15 @@ def on_error(err): dirpath.joinpath(name).rmdir() except OSError as err: on_error(err) - self.rmdir() + delete_self = self.rmdir + else: + delete_self = self.unlink + try: + delete_self() except OSError as err: err.filename = str(self) on_error(err) + delete.avoids_symlink_attacks = False def owner(self, *, follow_symlinks=True): """ diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 4fd5279f9fe9ce..6e2f88c93422a4 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -3,6 +3,7 @@ import operator import os import posixpath +import shutil import sys from glob import _StringGlobber from itertools import chain @@ -830,24 +831,34 @@ def rmdir(self): """ os.rmdir(self) - def rmtree(self, ignore_errors=False, on_error=None): + def delete(self, ignore_errors=False, on_error=None): """ - Recursively delete this directory tree. + Delete this file or directory (including all sub-directories). - If *ignore_errors* is true, exceptions raised from scanning the tree - and removing files and directories are ignored. Otherwise, if - *on_error* is set, it will be called to handle the error. If neither - *ignore_errors* nor *on_error* are set, exceptions are propagated to - the caller. + If *ignore_errors* is true, exceptions raised from scanning the + filesystem and removing files and directories are ignored. Otherwise, + if *on_error* is set, it will be called to handle the error. If + neither *ignore_errors* nor *on_error* are set, exceptions are + propagated to the caller. """ - if on_error: - def onexc(func, filename, err): - err.filename = filename - on_error(err) - else: + if self.is_dir(follow_symlinks=False): onexc = None - import shutil - shutil.rmtree(str(self), ignore_errors, onexc=onexc) + if on_error: + def onexc(func, filename, err): + err.filename = filename + on_error(err) + shutil.rmtree(str(self), ignore_errors, onexc=onexc) + else: + try: + self.unlink() + except OSError as err: + if not ignore_errors: + if on_error: + on_error(err) + else: + raise + + delete.avoids_symlink_attacks = shutil.rmtree.avoids_symlink_attacks def rename(self, target): """ diff --git a/Lib/pdb.py b/Lib/pdb.py index 7ff973149b167b..228de489a9cef1 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -614,7 +614,7 @@ def interaction(self, frame, tb_or_exc): # We should print the stack entry if and only if the user input # is expected, and we should print it right before the user input. # We achieve this by appending _pdbcmd_print_frame_status to the - # command queue. If cmdqueue is not exausted, the user input is + # command queue. If cmdqueue is not exhausted, the user input is # not expected and we will not print the stack entry. self.cmdqueue.append('_pdbcmd_print_frame_status') self._cmdloop() diff --git a/Lib/pprint.py b/Lib/pprint.py index 9314701db340c7..dc0953cec67a58 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -35,8 +35,6 @@ """ import collections as _collections -import dataclasses as _dataclasses -import re import sys as _sys import types as _types from io import StringIO as _StringIO @@ -54,6 +52,7 @@ def pprint(object, stream=None, indent=1, width=80, depth=None, *, underscore_numbers=underscore_numbers) printer.pprint(object) + def pformat(object, indent=1, width=80, depth=None, *, compact=False, sort_dicts=True, underscore_numbers=False): """Format a Python object into a pretty-printed representation.""" @@ -61,22 +60,27 @@ def pformat(object, indent=1, width=80, depth=None, *, compact=compact, sort_dicts=sort_dicts, underscore_numbers=underscore_numbers).pformat(object) + def pp(object, *args, sort_dicts=False, **kwargs): """Pretty-print a Python object""" pprint(object, *args, sort_dicts=sort_dicts, **kwargs) + def saferepr(object): """Version of repr() which can handle recursive data structures.""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[0] + def isreadable(object): """Determine if saferepr(object) is readable by eval().""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[1] + def isrecursive(object): """Determine if object requires a recursive representation.""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[2] + class _safe_key: """Helper function for key functions when sorting unorderable objects. @@ -99,10 +103,12 @@ def __lt__(self, other): return ((str(type(self.obj)), id(self.obj)) < \ (str(type(other.obj)), id(other.obj))) + def _safe_tuple(t): "Helper function for comparing 2-tuples" return _safe_key(t[0]), _safe_key(t[1]) + class PrettyPrinter: def __init__(self, indent=1, width=80, depth=None, stream=None, *, compact=False, sort_dicts=True, underscore_numbers=False): @@ -179,12 +185,15 @@ def _format(self, object, stream, indent, allowance, context, level): max_width = self._width - indent - allowance if len(rep) > max_width: p = self._dispatch.get(type(object).__repr__, None) + # Lazy import to improve module import time + from dataclasses import is_dataclass + if p is not None: context[objid] = 1 p(self, object, stream, indent, allowance, context, level + 1) del context[objid] return - elif (_dataclasses.is_dataclass(object) and + elif (is_dataclass(object) and not isinstance(object, type) and object.__dataclass_params__.repr and # Check dataclass has generated repr method. @@ -197,9 +206,12 @@ def _format(self, object, stream, indent, allowance, context, level): stream.write(rep) def _pprint_dataclass(self, object, stream, indent, allowance, context, level): + # Lazy import to improve module import time + from dataclasses import fields as dataclass_fields + cls_name = object.__class__.__name__ indent += len(cls_name) + 1 - items = [(f.name, getattr(object, f.name)) for f in _dataclasses.fields(object) if f.repr] + items = [(f.name, getattr(object, f.name)) for f in dataclass_fields(object) if f.repr] stream.write(cls_name + '(') self._format_namespace_items(items, stream, indent, allowance, context, level) stream.write(')') @@ -291,6 +303,9 @@ def _pprint_str(self, object, stream, indent, allowance, context, level): if len(rep) <= max_width1: chunks.append(rep) else: + # Lazy import to improve module import time + import re + # A list of alternating (non-space, space) strings parts = re.findall(r'\S*\s*', line) assert parts @@ -632,9 +647,11 @@ def _safe_repr(self, object, context, maxlevels, level): rep = repr(object) return rep, (rep and not rep.startswith('<')), False + _builtin_scalars = frozenset({str, bytes, bytearray, float, complex, bool, type(None)}) + def _recursion(object): return ("" % (type(object).__name__, id(object))) diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py index f3c779340fe230..0990255b22c219 100644 --- a/Lib/re/_parser.py +++ b/Lib/re/_parser.py @@ -807,14 +807,6 @@ def _parse(source, state, verbose, nested, first=False): state.grouprefpos[condgroup] = ( source.tell() - len(condname) - 1 ) - if not (condname.isdecimal() and condname.isascii()): - import warnings - warnings.warn( - "bad character in group name %s at position %d" % - (repr(condname) if source.istext else ascii(condname), - source.tell() - len(condname) - 1), - DeprecationWarning, stacklevel=nested + 6 - ) state.checklookbehindgroup(condgroup, source) item_yes = _parse(source, state, verbose, nested + 1) if source.match("|"): @@ -1038,14 +1030,6 @@ def addgroup(index, pos): if index >= MAXGROUPS: raise s.error("invalid group reference %d" % index, len(name) + 1) - if not (name.isdecimal() and name.isascii()): - import warnings - warnings.warn( - "bad character in group name %s at position %d" % - (repr(name) if s.istext else ascii(name), - s.tell() - len(name) - 1), - DeprecationWarning, stacklevel=5 - ) addgroup(index, len(name) + 1) elif c == "0": if s.next in OCTDIGITS: diff --git a/Lib/shutil.py b/Lib/shutil.py index 0235f6bae32f14..72b2d834dc387e 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -149,7 +149,7 @@ def _fastcopy_sendfile(fsrc, fdst): try: sent = os.sendfile(outfd, infd, offset, blocksize) except OSError as err: - # ...in oder to have a more informative exception. + # ...in order to have a more informative exception. err.filename = fsrc.name err.filename2 = fdst.name diff --git a/Lib/ssl.py b/Lib/ssl.py index cc685c2cc405ab..a3ecf5380e4e30 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -513,18 +513,17 @@ def set_alpn_protocols(self, alpn_protocols): self._set_alpn_protocols(protos) def _load_windows_store_certs(self, storename, purpose): - certs = bytearray() try: for cert, encoding, trust in enum_certificates(storename): # CA certs are never PKCS#7 encoded if encoding == "x509_asn": if trust is True or purpose.oid in trust: - certs.extend(cert) + try: + self.load_verify_locations(cadata=cert) + except SSLError as exc: + warnings.warn(f"Bad certificate in Windows certificate store: {exc!s}") except PermissionError: warnings.warn("unable to enumerate Windows certificate store") - if certs: - self.load_verify_locations(cadata=certs) - return certs def load_default_certs(self, purpose=Purpose.SERVER_AUTH): if not isinstance(purpose, _ASN1Object): diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 76214e6cda93b0..2a071f8485a2b8 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -4134,8 +4134,8 @@ test_vararg_and_posonly a: object - *args: object / + *args: object [clinic start generated code]*/ @@ -4177,7 +4177,7 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg static PyObject * test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=79b75dc07decc8d6 input=08dc2bf7afbf1613]*/ +/*[clinic end generated code: output=79b75dc07decc8d6 input=9cfa748bbff09877]*/ /*[clinic input] test_vararg @@ -4920,7 +4920,7 @@ Test_an_metho_arg_named_arg_impl(TestObj *self, int arg) /*[clinic input] Test.__init__ *args: object - / + Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE. [clinic start generated code]*/ @@ -4958,14 +4958,14 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) static int Test___init___impl(TestObj *self, PyObject *args) -/*[clinic end generated code: output=0ed1009fe0dcf98d input=96c3ddc0cd38fc0c]*/ +/*[clinic end generated code: output=0ed1009fe0dcf98d input=2a8bd0033c9ac772]*/ /*[clinic input] @classmethod Test.__new__ *args: object - / + Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE. [clinic start generated code]*/ @@ -5002,7 +5002,7 @@ Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) static PyObject * Test_impl(PyTypeObject *type, PyObject *args) -/*[clinic end generated code: output=8b219f6633e2a2e9 input=26a672e2e9750120]*/ +/*[clinic end generated code: output=8b219f6633e2a2e9 input=70ad829df3dd9b84]*/ /*[clinic input] diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index b756413d7bf375..38de1101072e52 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -3367,8 +3367,8 @@ def test_fromisoformat_fails_datetime(self): '2009-04-19T12:', # Ends with time separator '2009-04-19T12:30:', # Ends with time separator '2009-04-19T12:30:45.', # Ends with time separator - '2009-04-19T12:30:45.123456+', # Ends with timzone separator - '2009-04-19T12:30:45.123456-', # Ends with timzone separator + '2009-04-19T12:30:45.123456+', # Ends with timezone separator + '2009-04-19T12:30:45.123456-', # Ends with timezone separator '2009-04-19T12:30:45.123456-05:00a', # Extra text '2009-04-19T12:30:45.123-05:00a', # Extra text '2009-04-19T12:30:45-05:00a', # Extra text @@ -6884,13 +6884,28 @@ def test_update_type_cache(self): import sys for i in range(5): import _datetime - _datetime.date.max > _datetime.date.min - _datetime.time.max > _datetime.time.min - _datetime.datetime.max > _datetime.datetime.min - _datetime.timedelta.max > _datetime.timedelta.min - isinstance(_datetime.timezone.min, _datetime.tzinfo) - isinstance(_datetime.timezone.utc, _datetime.tzinfo) - isinstance(_datetime.timezone.max, _datetime.tzinfo) + assert _datetime.date.max > _datetime.date.min + assert _datetime.time.max > _datetime.time.min + assert _datetime.datetime.max > _datetime.datetime.min + assert _datetime.timedelta.max > _datetime.timedelta.min + assert _datetime.date.__dict__["min"] is _datetime.date.min + assert _datetime.date.__dict__["max"] is _datetime.date.max + assert _datetime.date.__dict__["resolution"] is _datetime.date.resolution + assert _datetime.time.__dict__["min"] is _datetime.time.min + assert _datetime.time.__dict__["max"] is _datetime.time.max + assert _datetime.time.__dict__["resolution"] is _datetime.time.resolution + assert _datetime.datetime.__dict__["min"] is _datetime.datetime.min + assert _datetime.datetime.__dict__["max"] is _datetime.datetime.max + assert _datetime.datetime.__dict__["resolution"] is _datetime.datetime.resolution + assert _datetime.timedelta.__dict__["min"] is _datetime.timedelta.min + assert _datetime.timedelta.__dict__["max"] is _datetime.timedelta.max + assert _datetime.timedelta.__dict__["resolution"] is _datetime.timedelta.resolution + assert _datetime.timezone.__dict__["min"] is _datetime.timezone.min + assert _datetime.timezone.__dict__["max"] is _datetime.timezone.max + assert _datetime.timezone.__dict__["utc"] is _datetime.timezone.utc + assert isinstance(_datetime.timezone.min, _datetime.tzinfo) + assert isinstance(_datetime.timezone.max, _datetime.tzinfo) + assert isinstance(_datetime.timezone.utc, _datetime.tzinfo) del sys.modules['_datetime'] """) script_helper.assert_python_ok('-c', script) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index beeb280894ea99..f493a92e0ddce8 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -43,7 +43,7 @@ def _run_output(interp, request): def _wait_for_interp_to_run(interp, timeout=None): # bpo-37224: Running this test file in multiprocesses will fail randomly. # The failure reason is that the thread can't acquire the cpu to - # run subinterpreter eariler than the main thread in multiprocess. + # run subinterpreter earlier than the main thread in multiprocess. if timeout is None: timeout = support.SHORT_TIMEOUT for _ in support.sleeping_retry(timeout, error=False): diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 84fe74a1063fb6..fd111be18aed6e 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2959,12 +2959,12 @@ def get_parser(self, required=None): ] usage_when_not_required = '''\ - usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] - [--klmno KLMNO | --pqrst PQRST] + usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] [--klmno KLMNO | + --pqrst PQRST] ''' usage_when_required = '''\ - usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] - (--klmno KLMNO | --pqrst PQRST) + usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] (--klmno KLMNO | + --pqrst PQRST) ''' help = '''\ @@ -4347,6 +4347,24 @@ def test_nested_mutex_groups(self): ''') self.assertEqual(parser.format_usage(), usage) + def test_long_mutex_groups_wrap(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument('--op1', metavar='MET', nargs='?') + g.add_argument('--op2', metavar=('MET1', 'MET2'), nargs='*') + g.add_argument('--op3', nargs='*') + g.add_argument('--op4', metavar=('MET1', 'MET2'), nargs='+') + g.add_argument('--op5', nargs='+') + g.add_argument('--op6', nargs=3) + g.add_argument('--op7', metavar=('MET1', 'MET2', 'MET3'), nargs=3) + + usage = textwrap.dedent('''\ + usage: PROG [-h] [--op1 [MET] | --op2 [MET1 [MET2 ...]] | --op3 [OP3 ...] | + --op4 MET1 [MET2 ...] | --op5 OP5 [OP5 ...] | --op6 OP6 OP6 OP6 | + --op7 MET1 MET2 MET3] + ''') + self.assertEqual(parser.format_usage(), usage) + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 47cbe60bfca4e4..f621f343eb062a 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1492,7 +1492,7 @@ def test_byteswap(self): if a.itemsize==1: self.assertEqual(a, b) else: - # On alphas treating the byte swapped bit patters as + # On alphas treating the byte swapped bit patterns as # floats/doubles results in floating-point exceptions # => compare the 8bit string values instead self.assertNotEqual(a.tobytes(), b.tobytes()) diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index bc39036e90bf8b..3a8c224126a672 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -1007,6 +1007,46 @@ def test_object_generichash(self): for obj in object(), 1, 'string', []: self.assertEqual(generichash(obj), object.__hash__(obj)) + def run_iter_api_test(self, next_func): + for data in (), [], (1, 2, 3), [1 , 2, 3], "123": + with self.subTest(data=data): + items = [] + it = iter(data) + while (item := next_func(it)) is not None: + items.append(item) + self.assertEqual(items, list(data)) + + class Broken: + def __init__(self): + self.count = 0 + + def __next__(self): + if self.count < 3: + self.count += 1 + return self.count + else: + raise TypeError('bad type') + + it = Broken() + self.assertEqual(next_func(it), 1) + self.assertEqual(next_func(it), 2) + self.assertEqual(next_func(it), 3) + with self.assertRaisesRegex(TypeError, 'bad type'): + next_func(it) + + def test_iter_next(self): + from _testcapi import PyIter_Next + self.run_iter_api_test(PyIter_Next) + # CRASHES PyIter_Next(10) + + def test_iter_nextitem(self): + from _testcapi import PyIter_NextItem + self.run_iter_api_test(PyIter_NextItem) + + regex = "expected.*iterator.*got.*'int'" + with self.assertRaisesRegex(TypeError, regex): + PyIter_NextItem(10) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_eval.py b/Lib/test/test_capi/test_eval.py new file mode 100644 index 00000000000000..20ef2695ef3e27 --- /dev/null +++ b/Lib/test/test_capi/test_eval.py @@ -0,0 +1,103 @@ +import sys +import unittest +from test.support import import_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + + +class Tests(unittest.TestCase): + def test_eval_get_func_name(self): + eval_get_func_name = _testlimitedcapi.eval_get_func_name + + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(eval_get_func_name(function_example), + "function_example") + self.assertEqual(eval_get_func_name(A.method_example), + "method_example") + self.assertEqual(eval_get_func_name(A().method_example), + "method_example") + self.assertEqual(eval_get_func_name(sum), "sum") # c function + self.assertEqual(eval_get_func_name(A), "type") + + def test_eval_get_func_desc(self): + eval_get_func_desc = _testlimitedcapi.eval_get_func_desc + + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(eval_get_func_desc(function_example), + "()") + self.assertEqual(eval_get_func_desc(A.method_example), + "()") + self.assertEqual(eval_get_func_desc(A().method_example), + "()") + self.assertEqual(eval_get_func_desc(sum), "()") # c function + self.assertEqual(eval_get_func_desc(A), " object") + + def test_eval_getlocals(self): + # Test PyEval_GetLocals() + x = 1 + self.assertEqual(_testlimitedcapi.eval_getlocals(), + {'self': self, + 'x': 1}) + + y = 2 + self.assertEqual(_testlimitedcapi.eval_getlocals(), + {'self': self, + 'x': 1, + 'y': 2}) + + def test_eval_getglobals(self): + # Test PyEval_GetGlobals() + self.assertEqual(_testlimitedcapi.eval_getglobals(), + globals()) + + def test_eval_getbuiltins(self): + # Test PyEval_GetBuiltins() + self.assertEqual(_testlimitedcapi.eval_getbuiltins(), + globals()['__builtins__']) + + def test_eval_getframe(self): + # Test PyEval_GetFrame() + self.assertEqual(_testlimitedcapi.eval_getframe(), + sys._getframe()) + + def test_eval_getframe_builtins(self): + # Test PyEval_GetFrameBuiltins() + self.assertEqual(_testlimitedcapi.eval_getframe_builtins(), + sys._getframe().f_builtins) + + def test_eval_getframe_globals(self): + # Test PyEval_GetFrameGlobals() + self.assertEqual(_testlimitedcapi.eval_getframe_globals(), + sys._getframe().f_globals) + + def test_eval_getframe_locals(self): + # Test PyEval_GetFrameLocals() + self.assertEqual(_testlimitedcapi.eval_getframe_locals(), + sys._getframe().f_locals) + + def test_eval_get_recursion_limit(self): + # Test Py_GetRecursionLimit() + self.assertEqual(_testlimitedcapi.eval_get_recursion_limit(), + sys.getrecursionlimit()) + + def test_eval_set_recursion_limit(self): + # Test Py_SetRecursionLimit() + old_limit = sys.getrecursionlimit() + try: + limit = old_limit + 123 + _testlimitedcapi.eval_set_recursion_limit(limit) + self.assertEqual(sys.getrecursionlimit(), limit) + finally: + sys.setrecursionlimit(old_limit) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 5c4547da1bdc53..b103bf2450bde0 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -869,36 +869,6 @@ def __init__(self): _testcapi.clear_managed_dict(c) self.assertEqual(c.__dict__, {}) - def test_eval_get_func_name(self): - def function_example(): ... - - class A: - def method_example(self): ... - - self.assertEqual(_testcapi.eval_get_func_name(function_example), - "function_example") - self.assertEqual(_testcapi.eval_get_func_name(A.method_example), - "method_example") - self.assertEqual(_testcapi.eval_get_func_name(A().method_example), - "method_example") - self.assertEqual(_testcapi.eval_get_func_name(sum), "sum") # c function - self.assertEqual(_testcapi.eval_get_func_name(A), "type") - - def test_eval_get_func_desc(self): - def function_example(): ... - - class A: - def method_example(self): ... - - self.assertEqual(_testcapi.eval_get_func_desc(function_example), - "()") - self.assertEqual(_testcapi.eval_get_func_desc(A.method_example), - "()") - self.assertEqual(_testcapi.eval_get_func_desc(A().method_example), - "()") - self.assertEqual(_testcapi.eval_get_func_desc(sum), "()") # c function - self.assertEqual(_testcapi.eval_get_func_desc(A), " object") - def test_function_get_code(self): import types diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f3fd610414cd8a..a7ba7f3d99860e 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -322,7 +322,7 @@ def __init__(self): """ self.expect_failure(block, err, lineno=8) - def test_multiple_star_in_args(self): + def test_star_after_vararg(self): err = "'my_test_func' uses '*' more than once." block = """ /*[clinic input] @@ -336,6 +336,20 @@ def test_multiple_star_in_args(self): """ self.expect_failure(block, err, lineno=6) + def test_vararg_after_star(self): + err = "'my_test_func' uses '*' more than once." + block = """ + /*[clinic input] + my_test_func + + pos_arg: object + * + *args: object + kw_arg: object + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + def test_module_already_got_one(self): err = "Already defined module 'm'!" block = """ @@ -1787,13 +1801,43 @@ def test_parameters_required_after_depr_star2(self): ) self.expect_failure(block, err, lineno=4) + def test_parameters_required_after_depr_star3(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + *args: object + b: int + Docstring. + """ + err = ( + "Function 'bar' specifies '* [from ...]' without " + "following parameters." + ) + self.expect_failure(block, err, lineno=4) + def test_depr_star_must_come_before_star(self): block = """ module foo foo.bar - this: int + a: int * * [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '* [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_before_vararg(self): + block = """ + module foo + foo.bar + a: int + *args: object + * [from 3.14] + b: int Docstring. """ err = "Function 'bar': '* [from ...]' must precede '*'" @@ -1908,7 +1952,7 @@ def test_double_slash(self): err = "Function 'bar' uses '/' more than once." self.expect_failure(block, err) - def test_mix_star_and_slash(self): + def test_slash_after_star(self): block = """ module foo foo.bar @@ -1921,6 +1965,19 @@ def test_mix_star_and_slash(self): err = "Function 'bar': '/' must precede '*'" self.expect_failure(block, err) + def test_slash_after_vararg(self): + block = """ + module foo + foo.bar + x: int + y: int + *args: object + z: int + / + """ + err = "Function 'bar': '/' must precede '*'" + self.expect_failure(block, err) + def test_depr_star_must_come_after_slash(self): block = """ module foo @@ -1960,6 +2017,19 @@ def test_star_must_come_after_depr_slash(self): err = "Function 'bar': '/ [from ...]' must precede '*'" self.expect_failure(block, err, lineno=4) + def test_vararg_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + *args: object + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + def test_depr_slash_must_come_after_slash(self): block = """ module foo @@ -1987,7 +2057,7 @@ def test_parameters_not_permitted_after_slash_for_now(self): self.expect_failure(block, err) def test_parameters_no_more_than_one_vararg(self): - err = "Too many var args" + err = "Function 'bar' uses '*' more than once." block = """ module foo foo.bar @@ -3319,13 +3389,6 @@ def test_posonly_vararg(self): with self.assertRaises(TypeError): ac_tester.posonly_vararg(1, 2, 3, b=4) - def test_vararg_and_posonly(self): - with self.assertRaises(TypeError): - ac_tester.vararg_and_posonly() - with self.assertRaises(TypeError): - ac_tester.vararg_and_posonly(1, b=2) - self.assertEqual(ac_tester.vararg_and_posonly(1, 2, 3, 4), (1, (2, 3, 4))) - def test_vararg(self): with self.assertRaises(TypeError): ac_tester.vararg() @@ -3334,11 +3397,27 @@ def test_vararg(self): self.assertEqual(ac_tester.vararg(1, 2, 3, 4), (1, (2, 3, 4))) def test_vararg_with_default(self): - with self.assertRaises(TypeError): - ac_tester.vararg_with_default() - self.assertEqual(ac_tester.vararg_with_default(1, b=False), (1, (), False)) - self.assertEqual(ac_tester.vararg_with_default(1, 2, 3, 4), (1, (2, 3, 4), False)) - self.assertEqual(ac_tester.vararg_with_default(1, 2, 3, 4, b=True), (1, (2, 3, 4), True)) + fn = ac_tester.vararg_with_default + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1, a=2) + self.assertEqual(fn(1, b=2), (1, (), True)) + self.assertEqual(fn(1, 2, 3, 4), (1, (2, 3, 4), False)) + self.assertEqual(fn(1, 2, 3, 4, b=5), (1, (2, 3, 4), True)) + self.assertEqual(fn(a=1), (1, (), False)) + self.assertEqual(fn(a=1, b=2), (1, (), True)) + + def test_vararg_with_default2(self): + fn = ac_tester.vararg_with_default2 + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1, a=2) + self.assertEqual(fn(1, b=2), (1, (), 2, None)) + self.assertEqual(fn(1, b=2, c=3), (1, (), 2, 3)) + self.assertEqual(fn(1, 2, 3), (1, (2, 3), None, None)) + self.assertEqual(fn(1, 2, 3, b=4), (1, (2, 3), 4, None)) + self.assertEqual(fn(1, 2, 3, b=4, c=5), (1, (2, 3), 4, 5)) + self.assertEqual(fn(a=1), (1, (), None, None)) + self.assertEqual(fn(a=1, b=2), (1, (), 2, None)) + self.assertEqual(fn(a=1, b=2, c=3), (1, (), 2, 3)) def test_vararg_with_only_defaults(self): self.assertEqual(ac_tester.vararg_with_only_defaults(), ((), None)) @@ -3347,6 +3426,17 @@ def test_vararg_with_only_defaults(self): self.assertEqual(ac_tester.vararg_with_only_defaults(1, 2, 3, 4), ((1, 2, 3, 4), None)) self.assertEqual(ac_tester.vararg_with_only_defaults(1, 2, 3, 4, b=5), ((1, 2, 3, 4), 5)) + def test_vararg_kwonly_req_opt(self): + fn = ac_tester.vararg_kwonly_req_opt + self.assertRaises(TypeError, fn) + self.assertEqual(fn(a=1), ((), 1, None, None)) + self.assertEqual(fn(a=1, b=2), ((), 1, 2, None)) + self.assertEqual(fn(a=1, b=2, c=3), ((), 1, 2, 3)) + self.assertRaises(TypeError, fn, 1) + self.assertEqual(fn(1, a=2), ((1,), 2, None, None)) + self.assertEqual(fn(1, a=2, b=3), ((1,), 2, 3, None)) + self.assertEqual(fn(1, a=2, b=3, c=4), ((1,), 2, 3, 4)) + def test_gh_32092_oob(self): ac_tester.gh_32092_oob(1, 2, 3, 4, kw1=5, kw2=6) diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 5dc89108f0ad88..37c7bc772ed8c7 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -1,5 +1,6 @@ "Test InteractiveConsole and InteractiveInterpreter from code module" import sys +import traceback import unittest from textwrap import dedent from contextlib import ExitStack @@ -30,6 +31,7 @@ def mock_sys(self): class TestInteractiveConsole(unittest.TestCase, MockSys): + maxDiff = None def setUp(self): self.console = code.InteractiveConsole() @@ -61,21 +63,118 @@ def test_console_stderr(self): raise AssertionError("no console stdout") def test_syntax_error(self): - self.infunc.side_effect = ["undefined", EOFError('Finished')] + self.infunc.side_effect = ["def f():", + " x = ?", + "", + EOFError('Finished')] self.console.interact() - for call in self.stderr.method_calls: - if 'NameError' in ''.join(call[1]): - break - else: - raise AssertionError("No syntax error from console") + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 2', + ' x = ?', + ' ^', + 'SyntaxError: invalid syntax']) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + def test_indentation_error(self): + self.infunc.side_effect = [" 1", EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 1', + ' 1', + 'IndentationError: unexpected indent']) + self.assertIs(self.sysmod.last_type, IndentationError) + self.assertIs(type(self.sysmod.last_value), IndentationError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + def test_unicode_error(self): + self.infunc.side_effect = ["'\ud800'", EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[output.index('\n') + 1:] + self.assertTrue(output.startswith('UnicodeEncodeError: '), output) + self.assertIs(self.sysmod.last_type, UnicodeEncodeError) + self.assertIs(type(self.sysmod.last_value), UnicodeEncodeError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) def test_sysexcepthook(self): - self.infunc.side_effect = ["raise ValueError('')", + self.infunc.side_effect = ["def f():", + " raise ValueError('BOOM!')", + "", + "f()", EOFError('Finished')] hook = mock.Mock() self.sysmod.excepthook = hook self.console.interact() - self.assertTrue(hook.called) + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, ValueError) + self.assertIs(type(self.sysmod.last_value), ValueError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + 'Traceback (most recent call last):\n', + ' File "", line 1, in \n', + ' File "", line 2, in f\n', + 'ValueError: BOOM!\n']) + + def test_sysexcepthook_syntax_error(self): + self.infunc.side_effect = ["def f():", + " x = ?", + "", + EOFError('Finished')] + hook = mock.Mock() + self.sysmod.excepthook = hook + self.console.interact() + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + ' File "", line 2\n', + ' x = ?\n', + ' ^\n', + 'SyntaxError: invalid syntax\n']) + + def test_sysexcepthook_indentation_error(self): + self.infunc.side_effect = [" 1", EOFError('Finished')] + hook = mock.Mock() + self.sysmod.excepthook = hook + self.console.interact() + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, IndentationError) + self.assertIs(type(self.sysmod.last_value), IndentationError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + ' File "", line 1\n', + ' 1\n', + 'IndentationError: unexpected indent\n']) def test_sysexcepthook_crashing_doesnt_close_repl(self): self.infunc.side_effect = ["1/0", "a = 123", "print(a)", EOFError('Finished')] @@ -167,6 +266,11 @@ def test_cause_tb(self): ValueError """) self.assertIn(expected, output) + self.assertIs(self.sysmod.last_type, ValueError) + self.assertIs(type(self.sysmod.last_value), ValueError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIsNotNone(self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) def test_context_tb(self): self.infunc.side_effect = ["try: ham\nexcept: eggs\n", @@ -185,6 +289,11 @@ def test_context_tb(self): NameError: name 'eggs' is not defined """) self.assertIn(expected, output) + self.assertIs(self.sysmod.last_type, NameError) + self.assertIs(type(self.sysmod.last_value), NameError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIsNotNone(self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) class TestInteractiveConsoleLocalExit(unittest.TestCase, MockSys): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 49523756e115c6..391894956541d7 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -6196,13 +6196,28 @@ def test_emit_after_closing_in_write_mode(self): self.assertEqual(fp.read().strip(), '1') class RotatingFileHandlerTest(BaseFileTest): - @unittest.skipIf(support.is_wasi, "WASI does not have /dev/null.") def test_should_not_rollover(self): - # If maxbytes is zero rollover never occurs + # If file is empty rollover never occurs + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", maxBytes=1) + self.assertFalse(rh.shouldRollover(None)) + rh.close() + + # If maxBytes is zero rollover never occurs rh = logging.handlers.RotatingFileHandler( self.fn, encoding="utf-8", maxBytes=0) self.assertFalse(rh.shouldRollover(None)) rh.close() + + with open(self.fn, 'wb') as f: + f.write(b'\n') + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", maxBytes=0) + self.assertFalse(rh.shouldRollover(None)) + rh.close() + + @unittest.skipIf(support.is_wasi, "WASI does not have /dev/null.") + def test_should_not_rollover_non_file(self): # bpo-45401 - test with special file # We set maxBytes to 1 so that rollover would normally happen, except # for the check for regular files @@ -6212,18 +6227,47 @@ def test_should_not_rollover(self): rh.close() def test_should_rollover(self): - rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8", maxBytes=1) + with open(self.fn, 'wb') as f: + f.write(b'\n') + rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8", maxBytes=2) self.assertTrue(rh.shouldRollover(self.next_rec())) rh.close() def test_file_created(self): # checks that the file is created and assumes it was created # by us + os.unlink(self.fn) rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8") rh.emit(self.next_rec()) self.assertLogFile(self.fn) rh.close() + def test_max_bytes(self, delay=False): + kwargs = {'delay': delay} if delay else {} + os.unlink(self.fn) + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", backupCount=2, maxBytes=100, **kwargs) + self.assertIs(os.path.exists(self.fn), not delay) + small = logging.makeLogRecord({'msg': 'a'}) + large = logging.makeLogRecord({'msg': 'b'*100}) + self.assertFalse(rh.shouldRollover(small)) + self.assertFalse(rh.shouldRollover(large)) + rh.emit(small) + self.assertLogFile(self.fn) + self.assertFalse(os.path.exists(self.fn + ".1")) + self.assertFalse(rh.shouldRollover(small)) + self.assertTrue(rh.shouldRollover(large)) + rh.emit(large) + self.assertTrue(os.path.exists(self.fn)) + self.assertLogFile(self.fn + ".1") + self.assertFalse(os.path.exists(self.fn + ".2")) + self.assertTrue(rh.shouldRollover(small)) + self.assertTrue(rh.shouldRollover(large)) + rh.close() + + def test_max_bytes_delay(self): + self.test_max_bytes(delay=True) + def test_rollover_filenames(self): def namer(name): return name + ".test" @@ -6232,11 +6276,15 @@ def namer(name): rh.namer = namer rh.emit(self.next_rec()) self.assertLogFile(self.fn) + self.assertFalse(os.path.exists(namer(self.fn + ".1"))) rh.emit(self.next_rec()) self.assertLogFile(namer(self.fn + ".1")) + self.assertFalse(os.path.exists(namer(self.fn + ".2"))) rh.emit(self.next_rec()) self.assertLogFile(namer(self.fn + ".2")) self.assertFalse(os.path.exists(namer(self.fn + ".3"))) + rh.emit(self.next_rec()) + self.assertFalse(os.path.exists(namer(self.fn + ".3"))) rh.close() def test_namer_rotator_inheritance(self): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index d7043cd4866a1c..8ef8c374cd6cc5 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1863,6 +1863,21 @@ def f(a=1, b=2): self.assertEqual(call_data[0], (f, 1)) self.assertEqual(call_data[1], (f, sys.monitoring.MISSING)) + def test_instruction_explicit_callback(self): + # gh-122247 + # Calling the instruction event callback explicitly should not + # crash CPython + def callback(code, instruction_offset): + pass + + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.register_callback(0, sys.monitoring.events.INSTRUCTION, callback) + sys.monitoring.set_events(0, sys.monitoring.events.INSTRUCTION) + callback(None, 0) # call the *same* handler while it is registered + sys.monitoring.restart_events() + sys.monitoring.set_events(0, 0) + class TestOptimizer(MonitoringTestBase, unittest.TestCase): diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 7160e764dfb2fa..9e922259cbaa6a 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -32,7 +32,7 @@ if hasattr(os, 'geteuid'): root_in_posix = (os.geteuid() == 0) -rmtree_use_fd_functions = ( +delete_use_fd_functions = ( {os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and os.listdir in os.supports_fd and os.stat in os.supports_follow_symlinks) @@ -862,8 +862,9 @@ def test_group_no_follow_symlinks(self): self.assertEqual(expected_gid, gid_2) self.assertEqual(expected_name, link.group(follow_symlinks=False)) - def test_rmtree_uses_safe_fd_version_if_available(self): - if rmtree_use_fd_functions: + def test_delete_uses_safe_fd_version_if_available(self): + if delete_use_fd_functions: + self.assertTrue(self.cls.delete.avoids_symlink_attacks) d = self.cls(self.base, 'a') d.mkdir() try: @@ -876,16 +877,18 @@ def _raiser(*args, **kwargs): raise Called os.open = _raiser - self.assertRaises(Called, d.rmtree) + self.assertRaises(Called, d.delete) finally: os.open = real_open + else: + self.assertFalse(self.cls.delete.avoids_symlink_attacks) @unittest.skipIf(sys.platform[:6] == 'cygwin', "This test can't be run on Cygwin (issue #1071513).") @os_helper.skip_if_dac_override @os_helper.skip_unless_working_chmod - def test_rmtree_unwritable(self): - tmp = self.cls(self.base, 'rmtree') + def test_delete_unwritable(self): + tmp = self.cls(self.base, 'delete') tmp.mkdir() child_file_path = tmp / 'a' child_dir_path = tmp / 'b' @@ -902,7 +905,7 @@ def test_rmtree_unwritable(self): tmp.chmod(new_mode) errors = [] - tmp.rmtree(on_error=errors.append) + tmp.delete(on_error=errors.append) # Test whether onerror has actually been called. self.assertEqual(len(errors), 3) finally: @@ -911,9 +914,9 @@ def test_rmtree_unwritable(self): child_dir_path.chmod(old_child_dir_mode) @needs_windows - def test_rmtree_inner_junction(self): + def test_delete_inner_junction(self): import _winapi - tmp = self.cls(self.base, 'rmtree') + tmp = self.cls(self.base, 'delete') tmp.mkdir() dir1 = tmp / 'dir1' dir2 = dir1 / 'dir2' @@ -929,15 +932,15 @@ def test_rmtree_inner_junction(self): link3 = dir1 / 'link3' _winapi.CreateJunction(str(file1), str(link3)) # make sure junctions are removed but not followed - dir1.rmtree() + dir1.delete() self.assertFalse(dir1.exists()) self.assertTrue(dir3.exists()) self.assertTrue(file1.exists()) @needs_windows - def test_rmtree_outer_junction(self): + def test_delete_outer_junction(self): import _winapi - tmp = self.cls(self.base, 'rmtree') + tmp = self.cls(self.base, 'delete') tmp.mkdir() try: src = tmp / 'cheese' @@ -946,22 +949,22 @@ def test_rmtree_outer_junction(self): spam = src / 'spam' spam.write_text('') _winapi.CreateJunction(str(src), str(dst)) - self.assertRaises(OSError, dst.rmtree) - dst.rmtree(ignore_errors=True) + self.assertRaises(OSError, dst.delete) + dst.delete(ignore_errors=True) finally: - tmp.rmtree(ignore_errors=True) + tmp.delete(ignore_errors=True) @needs_windows - def test_rmtree_outer_junction_on_error(self): + def test_delete_outer_junction_on_error(self): import _winapi - tmp = self.cls(self.base, 'rmtree') + tmp = self.cls(self.base, 'delete') tmp.mkdir() dir_ = tmp / 'dir' dir_.mkdir() link = tmp / 'link' _winapi.CreateJunction(str(dir_), str(link)) try: - self.assertRaises(OSError, link.rmtree) + self.assertRaises(OSError, link.delete) self.assertTrue(dir_.exists()) self.assertTrue(link.exists(follow_symlinks=False)) errors = [] @@ -969,18 +972,18 @@ def test_rmtree_outer_junction_on_error(self): def on_error(error): errors.append(error) - link.rmtree(on_error=on_error) + link.delete(on_error=on_error) self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0], OSError) self.assertEqual(errors[0].filename, str(link)) finally: os.unlink(str(link)) - @unittest.skipUnless(rmtree_use_fd_functions, "requires safe rmtree") - def test_rmtree_fails_on_close(self): + @unittest.skipUnless(delete_use_fd_functions, "requires safe delete") + def test_delete_fails_on_close(self): # Test that the error handler is called for failed os.close() and that # os.close() is only called once for a file descriptor. - tmp = self.cls(self.base, 'rmtree') + tmp = self.cls(self.base, 'delete') tmp.mkdir() dir1 = tmp / 'dir1' dir1.mkdir() @@ -996,7 +999,7 @@ def close(fd): close_count = 0 with swap_attr(os, 'close', close) as orig_close: with self.assertRaises(OSError): - dir1.rmtree() + dir1.delete() self.assertTrue(dir2.is_dir()) self.assertEqual(close_count, 2) @@ -1004,7 +1007,7 @@ def close(fd): errors = [] with swap_attr(os, 'close', close) as orig_close: - dir1.rmtree(on_error=errors.append) + dir1.delete(on_error=errors.append) self.assertEqual(len(errors), 2) self.assertEqual(errors[0].filename, str(dir2)) self.assertEqual(errors[1].filename, str(dir1)) @@ -1013,27 +1016,23 @@ def close(fd): @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') @unittest.skipIf(sys.platform == "vxworks", "fifo requires special path on VxWorks") - def test_rmtree_on_named_pipe(self): + def test_delete_on_named_pipe(self): p = self.cls(self.base, 'pipe') os.mkfifo(p) - try: - with self.assertRaises(NotADirectoryError): - p.rmtree() - self.assertTrue(p.exists()) - finally: - p.unlink() + p.delete() + self.assertFalse(p.exists()) p = self.cls(self.base, 'dir') p.mkdir() os.mkfifo(p / 'mypipe') - p.rmtree() + p.delete() self.assertFalse(p.exists()) @unittest.skipIf(sys.platform[:6] == 'cygwin', "This test can't be run on Cygwin (issue #1071513).") @os_helper.skip_if_dac_override @os_helper.skip_unless_working_chmod - def test_rmtree_deleted_race_condition(self): + def test_delete_deleted_race_condition(self): # bpo-37260 # # Test that a file or a directory deleted after it is enumerated @@ -1057,7 +1056,7 @@ def on_error(exc): if p != keep: p.unlink() - tmp = self.cls(self.base, 'rmtree') + tmp = self.cls(self.base, 'delete') tmp.mkdir() paths = [tmp] + [tmp / f'child{i}' for i in range(6)] dirs = paths[1::2] @@ -1075,7 +1074,7 @@ def on_error(exc): path.chmod(new_mode) try: - tmp.rmtree(on_error=on_error) + tmp.delete(on_error=on_error) except: # Test failed, so cleanup artifacts. for path, mode in zip(paths, old_modes): @@ -1083,13 +1082,13 @@ def on_error(exc): path.chmod(mode) except OSError: pass - tmp.rmtree() + tmp.delete() raise - def test_rmtree_does_not_choke_on_failing_lstat(self): + def test_delete_does_not_choke_on_failing_lstat(self): try: orig_lstat = os.lstat - tmp = self.cls(self.base, 'rmtree') + tmp = self.cls(self.base, 'delete') def raiser(fn, *args, **kwargs): if fn != str(tmp): @@ -1102,7 +1101,7 @@ def raiser(fn, *args, **kwargs): tmp.mkdir() foo = tmp / 'foo' foo.write_text('') - tmp.rmtree() + tmp.delete() finally: os.lstat = orig_lstat diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 37678c5d799e9a..443a4e989fb54b 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -2641,85 +2641,43 @@ def test_rmdir(self): self.assertFileNotFound(p.stat) self.assertFileNotFound(p.unlink) - def test_rmtree(self): + def test_delete_file(self): + p = self.cls(self.base) / 'fileA' + p.delete() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + + def test_delete_dir(self): base = self.cls(self.base) - base.joinpath('dirA').rmtree() + base.joinpath('dirA').delete() self.assertRaises(FileNotFoundError, base.joinpath('dirA').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirA', 'linkC').lstat) - base.joinpath('dirB').rmtree() + base.joinpath('dirB').delete() self.assertRaises(FileNotFoundError, base.joinpath('dirB').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'fileB').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'linkD').lstat) - base.joinpath('dirC').rmtree() + base.joinpath('dirC').delete() self.assertRaises(FileNotFoundError, base.joinpath('dirC').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'dirD').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'dirD', 'fileD').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'fileC').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'novel.txt').stat) - def test_rmtree_errors(self): - tmp = self.cls(self.base, 'rmtree') - tmp.mkdir() - # filename is guaranteed not to exist - filename = tmp / 'foo' - self.assertRaises(FileNotFoundError, filename.rmtree) - # test that ignore_errors option is honored - filename.rmtree(ignore_errors=True) - - # existing file - filename = tmp / "tstfile" - filename.write_text("") - with self.assertRaises(NotADirectoryError) as cm: - filename.rmtree() - self.assertEqual(cm.exception.filename, str(filename)) - self.assertTrue(filename.exists()) - # test that ignore_errors option is honored - filename.rmtree(ignore_errors=True) - self.assertTrue(filename.exists()) - - def test_rmtree_on_error(self): - tmp = self.cls(self.base, 'rmtree') - tmp.mkdir() - filename = tmp / "tstfile" - filename.write_text("") - errors = [] - - def on_error(error): - errors.append(error) - - filename.rmtree(on_error=on_error) - self.assertEqual(len(errors), 2) - # First from scandir() - self.assertIsInstance(errors[0], NotADirectoryError) - self.assertEqual(errors[0].filename, str(filename)) - # Then from munlink() - self.assertIsInstance(errors[1], NotADirectoryError) - self.assertEqual(errors[1].filename, str(filename)) - @needs_symlinks - def test_rmtree_outer_symlink(self): - tmp = self.cls(self.base, 'rmtree') + def test_delete_symlink(self): + tmp = self.cls(self.base, 'delete') tmp.mkdir() dir_ = tmp / 'dir' dir_.mkdir() link = tmp / 'link' link.symlink_to(dir_) - self.assertRaises(OSError, link.rmtree) + link.delete() self.assertTrue(dir_.exists()) - self.assertTrue(link.exists(follow_symlinks=False)) - errors = [] - - def on_error(error): - errors.append(error) - - link.rmtree(on_error=on_error) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], OSError) - self.assertEqual(errors[0].filename, str(link)) + self.assertFalse(link.exists(follow_symlinks=False)) @needs_symlinks - def test_rmtree_inner_symlink(self): - tmp = self.cls(self.base, 'rmtree') + def test_delete_inner_symlink(self): + tmp = self.cls(self.base, 'delete') tmp.mkdir() dir1 = tmp / 'dir1' dir2 = dir1 / 'dir2' @@ -2735,11 +2693,26 @@ def test_rmtree_inner_symlink(self): link3 = dir1 / 'link3' link3.symlink_to(file1) # make sure symlinks are removed but not followed - dir1.rmtree() + dir1.delete() self.assertFalse(dir1.exists()) self.assertTrue(dir3.exists()) self.assertTrue(file1.exists()) + def test_delete_missing(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + # filename is guaranteed not to exist + filename = tmp / 'foo' + self.assertRaises(FileNotFoundError, filename.delete) + # test that ignore_errors option is honored + filename.delete(ignore_errors=True) + # test on_error + errors = [] + filename.delete(on_error=errors.append) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], FileNotFoundError) + self.assertEqual(errors[0].filename, str(filename)) + def setUpWalk(self): # Build: # TESTFN/ diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index a93c2aef170fc8..c1d4653121a029 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1115,47 +1115,76 @@ def test_not_literal(self): def test_possible_set_operations(self): s = bytes(range(128)).decode() - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: p = re.compile(r'[0-9--1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('-./0123456789')) + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: + self.assertEqual(re.findall(r'[0-9--2]', s), list('-./0123456789')) + self.assertEqual(w.filename, __file__) + self.assertEqual(re.findall(r'[--1]', s), list('-./01')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: p = re.compile(r'[%--1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list("%&'()*+,-1")) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set difference ') as w: p = re.compile(r'[%--]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list("%&'()*+,-")) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: p = re.compile(r'[0-9&&1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('&0123456789')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: + self.assertEqual(re.findall(r'[0-8&&1]', s), list('&012345678')) + self.assertEqual(w.filename, __file__) + + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: p = re.compile(r'[\d&&1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('&0123456789')) + self.assertEqual(re.findall(r'[&&1]', s), list('&1')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set union ') as w: p = re.compile(r'[0-9||a]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789a|')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set union ') as w: p = re.compile(r'[\d||a]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789a|')) + self.assertEqual(re.findall(r'[||1]', s), list('1|')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set symmetric difference ') as w: p = re.compile(r'[0-9~~1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789~')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set symmetric difference ') as w: p = re.compile(r'[\d~~1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789~')) + self.assertEqual(re.findall(r'[~~1]', s), list('1~')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: p = re.compile(r'[[0-9]|]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789[]')) + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: + self.assertEqual(re.findall(r'[[0-8]|]', s), list('012345678[]')) + self.assertEqual(w.filename, __file__) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: p = re.compile(r'[[:digit:]|]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list(':[]dgit')) def test_search_coverage(self): diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index d1d8a967dbe62f..fedad17621cb02 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -371,6 +371,7 @@ def test_windows_feature_macros(self): "PyInterpreterState_New", "PyIter_Check", "PyIter_Next", + "PyIter_NextItem", "PyIter_Send", "PyListIter_Type", "PyListRevIter_Type", diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 709355e293f2fc..42b5a7c94e7700 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1710,6 +1710,7 @@ def delx(self): del self.__x fmt = 'P2nPI13Pl4Pn9Pn12PIPc' s = vsize(fmt) check(int, s) + typeid = 'n' if support.Py_GIL_DISABLED else '' # class s = vsize(fmt + # PyTypeObject '4P' # PyAsyncMethods @@ -1718,7 +1719,8 @@ def delx(self): del self.__x '10P' # PySequenceMethods '2P' # PyBufferProcs '6P' - '1PIP' # Specializer cache + '1PIP' # Specializer cache + + typeid # heap type id (free-threaded only) ) class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 290b3c63a762e9..ef2ad30317bf34 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -579,6 +579,55 @@ def test_constructor(self): self.assertEqual(T.__name__, "T") self.assertEqual(T.__constraints__, ()) self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", bound=type) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, type) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", default=()) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, ()) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", covariant=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, True) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", contravariant=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, True) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", infer_variance=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, True) class TypeParameterDefaultsTests(BaseTestCase): @@ -8828,7 +8877,7 @@ class X(TypedDict): class Y(TypedDict): a: None b: "int" - fwdref = ForwardRef('int', module='test.test_typing') + fwdref = ForwardRef('int', module=__name__) self.assertEqual(Y.__annotations__, {'a': type(None), 'b': fwdref}) self.assertEqual(Y.__annotate__(annotationlib.Format.FORWARDREF), {'a': type(None), 'b': fwdref}) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 2b7d297f011741..b9fcc59d49668c 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -886,6 +886,14 @@ def do_test_with_pip(self, system_site_packages): err = re.sub("^(WARNING: )?The directory .* or its parent directory " "is not owned or is not writable by the current user.*$", "", err, flags=re.MULTILINE) + # Ignore warning about missing optional module: + try: + import ssl + except ImportError: + err = re.sub( + "^WARNING: Disabling truststore since ssl support is missing$", + "", + err, flags=re.MULTILINE) self.assertEqual(err.rstrip(), "") # Being fairly specific regarding the expected behaviour for the # initial bundling phase in Python 3.4. If the output changes in diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 95515a4b694315..8b59630717e790 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -640,6 +640,97 @@ class NonWarningSubclass: self.module.warn('good warning category', MyWarningClass) self.assertIsInstance(cm.warning, Warning) + def check_module_globals(self, module_globals): + with original_warnings.catch_warnings(module=self.module, record=True) as w: + self.module.filterwarnings('default') + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, UserWarning) + self.assertEqual(str(w[0].message), 'eggs') + + def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError): + if self.module is py_warnings: + self.check_module_globals(module_globals) + return + with original_warnings.catch_warnings(module=self.module, record=True) as w: + self.module.filterwarnings('always') + with self.assertRaisesRegex(errtype, re.escape(errmsg)): + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 0) + + def check_module_globals_deprecated(self, module_globals, msg): + if self.module is py_warnings: + self.check_module_globals(module_globals) + return + with original_warnings.catch_warnings(module=self.module, record=True) as w: + self.module.filterwarnings('always') + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 2) + self.assertEqual(w[0].category, DeprecationWarning) + self.assertEqual(str(w[0].message), msg) + self.assertEqual(w[1].category, UserWarning) + self.assertEqual(str(w[1].message), 'eggs') + + def test_gh86298_no_loader_and_no_spec(self): + self.check_module_globals({'__name__': 'bar'}) + + def test_gh86298_loader_is_none_and_no_spec(self): + self.check_module_globals({'__name__': 'bar', '__loader__': None}) + + def test_gh86298_no_loader_and_spec_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_is_none_and_spec_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__loader__': None, '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_is_none_and_spec_loader_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__loader__': None, + '__spec__': types.SimpleNamespace(loader=None)}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_no_spec(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object()}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_spec_is_none(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_no_spec_loader(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), + '__spec__': types.SimpleNamespace()}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_and_spec_loader_disagree(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), + '__spec__': types.SimpleNamespace(loader=object())}, + 'Module globals; __loader__ != __spec__.loader') + + def test_gh86298_no_loader_and_no_spec_loader(self): + self.check_module_globals_error( + {'__name__': 'bar', '__spec__': types.SimpleNamespace()}, + 'Module globals is missing a __spec__.loader', AttributeError) + + def test_gh86298_no_loader_with_spec_loader_okay(self): + self.check_module_globals( + {'__name__': 'bar', + '__spec__': types.SimpleNamespace(loader=object())}) + class CWarnTests(WarnTests, unittest.TestCase): module = c_warnings diff --git a/Lib/typing.py b/Lib/typing.py index 626053d8166160..39a14ae6f83c28 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1054,7 +1054,7 @@ def evaluate_forward_ref( evaluating the forward reference. This parameter must be provided (though it may be an empty tuple) if *owner* is not given and the forward reference does not already have an owner set. *format* specifies the format of the - annotation and is a member of the annoations.Format enum. + annotation and is a member of the annotationlib.Format enum. """ if type_params is _sentinel: diff --git a/Lib/uuid.py b/Lib/uuid.py index c286eac38e1ef4..4d4f06cfc9ebbe 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -374,7 +374,7 @@ def _get_command_stdout(command, *args): # for are actually localized, but in theory some system could do so.) env = dict(os.environ) env['LC_ALL'] = 'C' - # Empty strings will be quoted by popen so we should just ommit it + # Empty strings will be quoted by popen so we should just omit it if args != ('',): command = (executable, *args) else: diff --git a/Makefile.pre.in b/Makefile.pre.in index 5608e593ac9aca..6eb9afefada313 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -43,7 +43,7 @@ SOABI= @SOABI@ ABIFLAGS= @ABIFLAGS@ ABI_THREAD= @ABI_THREAD@ LDVERSION= @LDVERSION@ -MODULE_LDFLAGS=@MODULE_LDFLAGS@ +LIBPYTHON=@LIBPYTHON@ GITVERSION= @GITVERSION@ GITTAG= @GITTAG@ GITBRANCH= @GITBRANCH@ @@ -483,6 +483,7 @@ PYTHON_OBJS= \ Python/thread.o \ Python/traceback.o \ Python/tracemalloc.o \ + Python/typeid.o \ Python/getopt.o \ Python/pystrcmp.o \ Python/pystrtod.o \ @@ -1257,6 +1258,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_tracemalloc.h \ $(srcdir)/Include/internal/pycore_tstate.h \ $(srcdir)/Include/internal/pycore_tuple.h \ + $(srcdir)/Include/internal/pycore_typeid.h \ $(srcdir)/Include/internal/pycore_typeobject.h \ $(srcdir)/Include/internal/pycore_typevarobject.h \ $(srcdir)/Include/internal/pycore_ucnhash.h \ @@ -1665,7 +1667,7 @@ regen-unicodedata: regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ - regen-test-levenshtein regen-global-objects regen-jit + regen-test-levenshtein regen-global-objects @echo @echo "Note: make regen-stdlib-module-names, make regen-limited-abi, " @echo "make regen-configure, make regen-sbom, and make regen-unicodedata should be run manually" diff --git a/Misc/ACKS b/Misc/ACKS index 6008f9e1770d1d..b031eb7c11f73f 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -521,6 +521,7 @@ Michael Ernst Ben Escoto Andy Eskilsson André Espaze +Lucas Esposito Stefan Esser Nicolas Estibals Jonathan Eunice diff --git a/Misc/NEWS.d/next/Build/2024-08-02-12-59-15.gh-issue-118943.vZQtET.rst b/Misc/NEWS.d/next/Build/2024-08-02-12-59-15.gh-issue-118943.vZQtET.rst new file mode 100644 index 00000000000000..42cda69e01cd4c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-08-02-12-59-15.gh-issue-118943.vZQtET.rst @@ -0,0 +1 @@ +Fix an issue where the experimental JIT could be built several times by the ``make regen-all`` target, leading to possible race conditions on heavily parallelized builds. diff --git a/Misc/NEWS.d/next/Build/2024-08-07-00-20-18.gh-issue-116622.U9cxHM.rst b/Misc/NEWS.d/next/Build/2024-08-07-00-20-18.gh-issue-116622.U9cxHM.rst new file mode 100644 index 00000000000000..c9a56d65ad0cd3 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-08-07-00-20-18.gh-issue-116622.U9cxHM.rst @@ -0,0 +1,2 @@ +Rename build variable ``MODULE_LDFLAGS`` back to ``LIBPYTHON``, as it's used by +package build systems (e.g. Meson). diff --git a/Misc/NEWS.d/next/C_API/2024-07-27-00-28-35.gh-issue-105201.0-xUWq.rst b/Misc/NEWS.d/next/C_API/2024-07-27-00-28-35.gh-issue-105201.0-xUWq.rst new file mode 100644 index 00000000000000..bf5300b1c5d5f8 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-07-27-00-28-35.gh-issue-105201.0-xUWq.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, which has an +ambiguous return value. Patch by Irit Katriel and Erlend Aasland. diff --git a/Misc/NEWS.d/next/C_API/2024-08-06-14-23-11.gh-issue-122728.l-fQ-v.rst b/Misc/NEWS.d/next/C_API/2024-08-06-14-23-11.gh-issue-122728.l-fQ-v.rst new file mode 100644 index 00000000000000..a128d6aef34dfc --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-08-06-14-23-11.gh-issue-122728.l-fQ-v.rst @@ -0,0 +1,2 @@ +Fix :c:func:`PyEval_GetLocals` to avoid :exc:`SystemError` ("bad argument to +internal function"). Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-07-29-19-20-25.gh-issue-122417.NVgs0a.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-07-29-19-20-25.gh-issue-122417.NVgs0a.rst new file mode 100644 index 00000000000000..b050c9ce39c054 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-07-29-19-20-25.gh-issue-122417.NVgs0a.rst @@ -0,0 +1,4 @@ +In the free-threaded build, the reference counts for heap type objects are now +partially stored in a distributed manner in per-thread arrays. This reduces +contention on the heap type's reference count fields when creating or +destroying instances of the same type from multiple threads concurrently. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-05-19-28-12.gh-issue-122697.17MvYl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-05-19-28-12.gh-issue-122697.17MvYl.rst new file mode 100644 index 00000000000000..34ee6a916bcf33 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-05-19-28-12.gh-issue-122697.17MvYl.rst @@ -0,0 +1,2 @@ +Fixed memory leaks at interpreter shutdown in the free-threaded build, and +also reporting of leaked memory blocks via :option:`-X showrefcount <-X>`. diff --git a/Misc/NEWS.d/next/IDLE/2024-06-05-14-54-24.gh-issue-120104.j_thj4.rst b/Misc/NEWS.d/next/IDLE/2024-06-05-14-54-24.gh-issue-120104.j_thj4.rst new file mode 100644 index 00000000000000..10f5e345bf3e4f --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2024-06-05-14-54-24.gh-issue-120104.j_thj4.rst @@ -0,0 +1 @@ +Fix padding in config and search dialog windows in IDLE. diff --git a/Misc/NEWS.d/next/Library/2024-05-15-01-21-44.gh-issue-73991.bNDqQN.rst b/Misc/NEWS.d/next/Library/2024-05-15-01-21-44.gh-issue-73991.bNDqQN.rst index 9aa7a7dba666af..5806fed91c7880 100644 --- a/Misc/NEWS.d/next/Library/2024-05-15-01-21-44.gh-issue-73991.bNDqQN.rst +++ b/Misc/NEWS.d/next/Library/2024-05-15-01-21-44.gh-issue-73991.bNDqQN.rst @@ -1 +1 @@ -Add :meth:`pathlib.Path.rmtree`, which recursively removes a directory. +Add :meth:`pathlib.Path.delete`, which recursively removes a file or directory. diff --git a/Misc/NEWS.d/next/Library/2024-06-29-15-23-26.gh-issue-121151.HeLEvq.rst b/Misc/NEWS.d/next/Library/2024-06-29-15-23-26.gh-issue-121151.HeLEvq.rst new file mode 100644 index 00000000000000..f08b6131a702f7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-29-15-23-26.gh-issue-121151.HeLEvq.rst @@ -0,0 +1,2 @@ +Fix wrapping of long usage text of arguments inside a mutually exclusive +group in :mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2024-07-17-12-55-22.gh-issue-121268.41RmjR.rst b/Misc/NEWS.d/next/Library/2024-07-17-12-55-22.gh-issue-121268.41RmjR.rst new file mode 100644 index 00000000000000..f88e363da16124 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-17-12-55-22.gh-issue-121268.41RmjR.rst @@ -0,0 +1 @@ +Remove workarounds for non-IEEE 754 systems in :mod:`cmath`. diff --git a/Misc/NEWS.d/next/Library/2024-07-31-20-43-21.gh-issue-122478.sCU2Le.rst b/Misc/NEWS.d/next/Library/2024-07-31-20-43-21.gh-issue-122478.sCU2Le.rst new file mode 100644 index 00000000000000..6071324593a9ed --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-31-20-43-21.gh-issue-122478.sCU2Le.rst @@ -0,0 +1,3 @@ +Remove internal frames from tracebacks shown in +:class:`code.InteractiveInterpreter` with non-default :func:`sys.excepthook`. +Save correct tracebacks in :attr:`sys.last_traceback` and update ``__traceback__`` attribute of :attr:`sys.last_value` and :attr:`sys.last_exc`. diff --git a/Misc/NEWS.d/next/Library/2024-08-04-14-07-18.gh-issue-118814.uiyks1.rst b/Misc/NEWS.d/next/Library/2024-08-04-14-07-18.gh-issue-118814.uiyks1.rst new file mode 100644 index 00000000000000..14ef6c070603ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-04-14-07-18.gh-issue-118814.uiyks1.rst @@ -0,0 +1 @@ +Fix the :class:`typing.TypeVar` constructor when name is passed by keyword. diff --git a/Misc/NEWS.d/next/Library/2024-08-06-10-36-55.gh-issue-118761.q_x_1A.rst b/Misc/NEWS.d/next/Library/2024-08-06-10-36-55.gh-issue-118761.q_x_1A.rst new file mode 100644 index 00000000000000..3f3e870b0b9565 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-06-10-36-55.gh-issue-118761.q_x_1A.rst @@ -0,0 +1,2 @@ +Improve import time of :mod:`pprint` by around seven times. Patch by Hugo +van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-08-06-18-07-19.gh-issue-122744.kCzNDI.rst b/Misc/NEWS.d/next/Library/2024-08-06-18-07-19.gh-issue-122744.kCzNDI.rst new file mode 100644 index 00000000000000..18ac3dd10d6553 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-06-18-07-19.gh-issue-122744.kCzNDI.rst @@ -0,0 +1 @@ +Bump the version of pip bundled in ensurepip to version 24.2. diff --git a/Misc/NEWS.d/next/Library/2024-08-07-14-12-19.gh-issue-105376.QbGPdE.rst b/Misc/NEWS.d/next/Library/2024-08-07-14-12-19.gh-issue-105376.QbGPdE.rst new file mode 100644 index 00000000000000..9756a14cbcf67e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-07-14-12-19.gh-issue-105376.QbGPdE.rst @@ -0,0 +1,3 @@ +Restore the deprecated :mod:`logging` ``warn()`` method. It was removed in +Python 3.13 alpha 1. Keep the deprecated ``warn()`` method in Python 3.13. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-08-07-17-41-16.gh-issue-116263.EcXir0.rst b/Misc/NEWS.d/next/Library/2024-08-07-17-41-16.gh-issue-116263.EcXir0.rst new file mode 100644 index 00000000000000..167ca943b3527c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-07-17-41-16.gh-issue-116263.EcXir0.rst @@ -0,0 +1,2 @@ +:class:`logging.handlers.RotatingFileHandler` no longer rolls over empty log +files. diff --git a/Misc/NEWS.d/next/Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst b/Misc/NEWS.d/next/Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst new file mode 100644 index 00000000000000..81237e735ebdb7 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-07-24-05-18-25.gh-issue-112301.lfINgZ.rst @@ -0,0 +1,2 @@ +Add macOS warning tracking to warning check tooling. +Patch by Nate Ohlson. diff --git a/Misc/NEWS.d/next/Windows/2022-04-20-18-32-30.gh-issue-79846.Vggv3f.rst b/Misc/NEWS.d/next/Windows/2022-04-20-18-32-30.gh-issue-79846.Vggv3f.rst new file mode 100644 index 00000000000000..82c26701e0e0bc --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2022-04-20-18-32-30.gh-issue-79846.Vggv3f.rst @@ -0,0 +1,2 @@ +Makes :code:`ssl.create_default_context()` ignore invalid certificates in +the Windows certificate store diff --git a/Misc/NEWS.d/next/Windows/2024-07-19-21-50-54.gh-issue-100256.GDrKba.rst b/Misc/NEWS.d/next/Windows/2024-07-19-21-50-54.gh-issue-100256.GDrKba.rst new file mode 100644 index 00000000000000..f0156ddd4772ed --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-07-19-21-50-54.gh-issue-100256.GDrKba.rst @@ -0,0 +1 @@ +:mod:`mimetypes` no longer fails when it encounters an inaccessible registry key. diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 9929f5b2653dca..555b0cb6ba2a48 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -47,7 +47,7 @@ LIBM="@LIBM@" LIBC="@LIBC@" SYSLIBS="$LIBM $LIBC" ABIFLAGS="@ABIFLAGS@" -LIBS="@MODULE_LDFLAGS@ @LIBS@ $SYSLIBS" +LIBS="@LIBPYTHON@ @LIBS@ $SYSLIBS" LIBS_EMBED="-lpython${VERSION}${ABIFLAGS} @LIBS@ $SYSLIBS" BASECFLAGS="@BASECFLAGS@" LDLIBRARY="@LDLIBRARY@" diff --git a/Misc/python.pc.in b/Misc/python.pc.in index c2c740e82b1fde..027dba38585a89 100644 --- a/Misc/python.pc.in +++ b/Misc/python.pc.in @@ -9,5 +9,5 @@ Description: Build a C extension for Python Requires: Version: @VERSION@ Libs.private: @LIBS@ -Libs: -L${libdir} @MODULE_LDFLAGS@ +Libs: -L${libdir} @LIBPYTHON@ Cflags: -I${includedir}/python@VERSION@@ABIFLAGS@ diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 73012193d61485..c38671e389ac5e 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2508,3 +2508,5 @@ [function.Py_TYPE] added = '3.14' +[function.PyIter_NextItem] + added = '3.14' diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index dfc75077650df8..9da4e785804886 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c index b126aee5b9777b..8c2c7137cdce40 100644 --- a/Modules/_testcapi/abstract.c +++ b/Modules/_testcapi/abstract.c @@ -129,6 +129,33 @@ mapping_getoptionalitem(PyObject *self, PyObject *args) } } +static PyObject * +pyiter_next(PyObject *self, PyObject *iter) +{ + PyObject *item = PyIter_Next(iter); + if (item == NULL && !PyErr_Occurred()) { + Py_RETURN_NONE; + } + return item; +} + +static PyObject * +pyiter_nextitem(PyObject *self, PyObject *iter) +{ + PyObject *item; + int rc = PyIter_NextItem(iter, &item); + if (rc < 0) { + assert(PyErr_Occurred()); + assert(item == NULL); + return NULL; + } + assert(!PyErr_Occurred()); + if (item == NULL) { + Py_RETURN_NONE; + } + return item; +} + static PyMethodDef test_methods[] = { {"object_getoptionalattr", object_getoptionalattr, METH_VARARGS}, @@ -138,6 +165,8 @@ static PyMethodDef test_methods[] = { {"mapping_getoptionalitem", mapping_getoptionalitem, METH_VARARGS}, {"mapping_getoptionalitemstring", mapping_getoptionalitemstring, METH_VARARGS}, + {"PyIter_Next", pyiter_next, METH_O}, + {"PyIter_NextItem", pyiter_nextitem, METH_O}, {NULL}, }; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 4a371a5ce33ebe..981efb9629031b 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2655,18 +2655,6 @@ test_frame_getvarstring(PyObject *self, PyObject *args) } -static PyObject * -eval_get_func_name(PyObject *self, PyObject *func) -{ - return PyUnicode_FromString(PyEval_GetFuncName(func)); -} - -static PyObject * -eval_get_func_desc(PyObject *self, PyObject *func) -{ - return PyUnicode_FromString(PyEval_GetFuncDesc(func)); -} - static PyObject * gen_get_code(PyObject *self, PyObject *gen) { @@ -3461,8 +3449,6 @@ static PyMethodDef TestMethods[] = { {"frame_new", frame_new, METH_VARARGS, NULL}, {"frame_getvar", test_frame_getvar, METH_VARARGS, NULL}, {"frame_getvarstring", test_frame_getvarstring, METH_VARARGS, NULL}, - {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, - {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL}, {"gen_get_code", gen_get_code, METH_O, NULL}, {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL}, {"test_code_api", test_code_api, METH_NOARGS, NULL}, diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 4187e13231dc69..2dae8accf01182 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -982,54 +982,56 @@ posonly_vararg_impl(PyObject *module, PyObject *a, PyObject *b, /*[clinic input] -vararg_and_posonly +vararg a: object *args: object - / [clinic start generated code]*/ static PyObject * -vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=42792f799465a14d input=defe017b19ba52e8]*/ +vararg_impl(PyObject *module, PyObject *a, PyObject *args) +/*[clinic end generated code: output=91ab7a0efc52dd5e input=02c0f772d05f591e]*/ { return pack_arguments_newref(2, a, args); } /*[clinic input] -vararg +vararg_with_default a: object *args: object + b: bool = False [clinic start generated code]*/ static PyObject * -vararg_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=91ab7a0efc52dd5e input=02c0f772d05f591e]*/ +vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, + int b) +/*[clinic end generated code: output=182c01035958ce92 input=68cafa6a79f89e36]*/ { - return pack_arguments_newref(2, a, args); + PyObject *obj_b = b ? Py_True : Py_False; + return pack_arguments_newref(3, a, args, obj_b); } /*[clinic input] -vararg_with_default +vararg_with_default2 a: object *args: object - b: bool = False + b: object = None + c: object = None [clinic start generated code]*/ static PyObject * -vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, - int b) -/*[clinic end generated code: output=182c01035958ce92 input=68cafa6a79f89e36]*/ +vararg_with_default2_impl(PyObject *module, PyObject *a, PyObject *args, + PyObject *b, PyObject *c) +/*[clinic end generated code: output=a0fb7c37796e2129 input=59fb22f5f0a8925f]*/ { - PyObject *obj_b = b ? Py_True : Py_False; - return pack_arguments_newref(3, a, args, obj_b); + return pack_arguments_newref(4, a, args, b, c); } @@ -1049,6 +1051,25 @@ vararg_with_only_defaults_impl(PyObject *module, PyObject *args, PyObject *b) } +/*[clinic input] +vararg_kwonly_req_opt + + *args: object + a: object + b: object = None + c: object = None + +[clinic start generated code]*/ + +static PyObject * +vararg_kwonly_req_opt_impl(PyObject *module, PyObject *args, PyObject *a, + PyObject *b, PyObject *c) +/*[clinic end generated code: output=54694a99c3da370a input=b0d8bf09e540d400]*/ +{ + return pack_arguments_newref(4, args, a, b, c); +} + + /*[clinic input] gh_32092_oob @@ -1096,7 +1117,6 @@ gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, gh_99233_refcount *args: object - / Proof-of-concept of GH-99233 refcount error bug. @@ -1104,7 +1124,7 @@ Proof-of-concept of GH-99233 refcount error bug. static PyObject * gh_99233_refcount_impl(PyObject *module, PyObject *args) -/*[clinic end generated code: output=585855abfbca9a7f input=85f5fb47ac91a626]*/ +/*[clinic end generated code: output=585855abfbca9a7f input=eecfdc2092d90dc3]*/ { Py_RETURN_NONE; } @@ -1904,10 +1924,11 @@ static PyMethodDef tester_methods[] = { POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF KEYWORD_ONLY_PARAMETER_METHODDEF POSONLY_VARARG_METHODDEF - VARARG_AND_POSONLY_METHODDEF VARARG_METHODDEF VARARG_WITH_DEFAULT_METHODDEF + VARARG_WITH_DEFAULT2_METHODDEF VARARG_WITH_ONLY_DEFAULTS_METHODDEF + VARARG_KWONLY_REQ_OPT_METHODDEF GH_32092_OOB_METHODDEF GH_32092_KW_PASS_METHODDEF GH_99233_REFCOUNT_METHODDEF diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index fb5cdb6ca9e1d3..2f1a25ae4519b3 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -44,6 +44,9 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_Dict(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Eval(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_Float(mod) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi/eval.c b/Modules/_testlimitedcapi/eval.c new file mode 100644 index 00000000000000..28f5746dfb1783 --- /dev/null +++ b/Modules/_testlimitedcapi/eval.c @@ -0,0 +1,95 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +eval_get_func_name(PyObject *self, PyObject *func) +{ + return PyUnicode_FromString(PyEval_GetFuncName(func)); +} + +static PyObject * +eval_get_func_desc(PyObject *self, PyObject *func) +{ + return PyUnicode_FromString(PyEval_GetFuncDesc(func)); +} + +static PyObject * +eval_getlocals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetLocals()); +} + +static PyObject * +eval_getglobals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetGlobals()); +} + +static PyObject * +eval_getbuiltins(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetBuiltins()); +} + +static PyObject * +eval_getframe(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetFrame()); +} + +static PyObject * +eval_getframe_builtins(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return PyEval_GetFrameBuiltins(); +} + +static PyObject * +eval_getframe_globals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return PyEval_GetFrameGlobals(); +} + +static PyObject * +eval_getframe_locals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return PyEval_GetFrameLocals(); +} + +static PyObject * +eval_get_recursion_limit(PyObject *module, PyObject *Py_UNUSED(args)) +{ + int limit = Py_GetRecursionLimit(); + return PyLong_FromLong(limit); +} + +static PyObject * +eval_set_recursion_limit(PyObject *module, PyObject *args) +{ + int limit; + if (!PyArg_ParseTuple(args, "i", &limit)) { + return NULL; + } + Py_SetRecursionLimit(limit); + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, + {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL}, + {"eval_getlocals", eval_getlocals, METH_NOARGS}, + {"eval_getglobals", eval_getglobals, METH_NOARGS}, + {"eval_getbuiltins", eval_getbuiltins, METH_NOARGS}, + {"eval_getframe", eval_getframe, METH_NOARGS}, + {"eval_getframe_builtins", eval_getframe_builtins, METH_NOARGS}, + {"eval_getframe_globals", eval_getframe_globals, METH_NOARGS}, + {"eval_getframe_locals", eval_getframe_locals, METH_NOARGS}, + {"eval_get_recursion_limit", eval_get_recursion_limit, METH_NOARGS}, + {"eval_set_recursion_limit", eval_set_recursion_limit, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Eval(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index d5e590a8dcd679..c5758605fb71fa 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -27,6 +27,7 @@ int _PyTestLimitedCAPI_Init_ByteArray(PyObject *module); int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); int _PyTestLimitedCAPI_Init_Complex(PyObject *module); int _PyTestLimitedCAPI_Init_Dict(PyObject *module); +int _PyTestLimitedCAPI_Init_Eval(PyObject *module); int _PyTestLimitedCAPI_Init_Float(PyObject *module); int _PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *module); int _PyTestLimitedCAPI_Init_Object(PyObject *module); diff --git a/Modules/_winapi.c b/Modules/_winapi.c index c90d6c5a9ef3ef..a330b3ff68db62 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -2803,7 +2803,7 @@ _winapi__mimetypes_read_windows_registry_impl(PyObject *module, } err = RegOpenKeyExW(hkcr, ext, 0, KEY_READ, &subkey); - if (err == ERROR_FILE_NOT_FOUND) { + if (err == ERROR_FILE_NOT_FOUND || err == ERROR_ACCESS_DENIED) { err = ERROR_SUCCESS; continue; } else if (err != ERROR_SUCCESS) { diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index e02f39d15cce0f..240342b438a893 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -2581,42 +2581,6 @@ posonly_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje return return_value; } -PyDoc_STRVAR(vararg_and_posonly__doc__, -"vararg_and_posonly($module, a, /, *args)\n" -"--\n" -"\n"); - -#define VARARG_AND_POSONLY_METHODDEF \ - {"vararg_and_posonly", _PyCFunction_CAST(vararg_and_posonly), METH_FASTCALL, vararg_and_posonly__doc__}, - -static PyObject * -vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args); - -static PyObject * -vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *a; - PyObject *__clinic_args = NULL; - - if (!_PyArg_CheckPositional("vararg_and_posonly", nargs, 1, PY_SSIZE_T_MAX)) { - goto exit; - } - a = args[0]; - __clinic_args = PyTuple_New(nargs - 1); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 1; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); - } - return_value = vararg_and_posonly_impl(module, a, __clinic_args); - -exit: - Py_XDECREF(__clinic_args); - return return_value; -} - PyDoc_STRVAR(vararg__doc__, "vararg($module, /, a, *args)\n" "--\n" @@ -2742,6 +2706,78 @@ vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P return return_value; } +PyDoc_STRVAR(vararg_with_default2__doc__, +"vararg_with_default2($module, /, a, *args, b=None, c=None)\n" +"--\n" +"\n"); + +#define VARARG_WITH_DEFAULT2_METHODDEF \ + {"vararg_with_default2", _PyCFunction_CAST(vararg_with_default2), METH_FASTCALL|METH_KEYWORDS, vararg_with_default2__doc__}, + +static PyObject * +vararg_with_default2_impl(PyObject *module, PyObject *a, PyObject *args, + PyObject *b, PyObject *c); + +static PyObject * +vararg_with_default2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "vararg_with_default2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *__clinic_args = NULL; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + __clinic_args = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + b = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + c = args[3]; +skip_optional_kwonly: + return_value = vararg_with_default2_impl(module, a, __clinic_args, b, c); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(vararg_with_only_defaults__doc__, "vararg_with_only_defaults($module, /, *args, b=None)\n" "--\n" @@ -2804,6 +2840,78 @@ vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t na return return_value; } +PyDoc_STRVAR(vararg_kwonly_req_opt__doc__, +"vararg_kwonly_req_opt($module, /, *args, a, b=None, c=None)\n" +"--\n" +"\n"); + +#define VARARG_KWONLY_REQ_OPT_METHODDEF \ + {"vararg_kwonly_req_opt", _PyCFunction_CAST(vararg_kwonly_req_opt), METH_FASTCALL|METH_KEYWORDS, vararg_kwonly_req_opt__doc__}, + +static PyObject * +vararg_kwonly_req_opt_impl(PyObject *module, PyObject *args, PyObject *a, + PyObject *b, PyObject *c); + +static PyObject * +vararg_kwonly_req_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "vararg_kwonly_req_opt", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *__clinic_args = NULL; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + a = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + b = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + c = args[3]; +skip_optional_kwonly: + return_value = vararg_kwonly_req_opt_impl(module, __clinic_args, a, b, c); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(gh_32092_oob__doc__, "gh_32092_oob($module, /, pos1, pos2, *varargs, kw1=None, kw2=None)\n" "--\n" @@ -3418,4 +3526,4 @@ _testclinic_TestClass_get_defining_class_arg(PyObject *self, PyTypeObject *cls, exit: return return_value; } -/*[clinic end generated code: output=0d0ceed6c46547bb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cacfbed4b2e50ba6 input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 71686518d84aa7..49ff093871d156 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -185,15 +185,8 @@ cmath_acos_impl(PyObject *module, Py_complex z) if (fabs(z.real) > CM_LARGE_DOUBLE || fabs(z.imag) > CM_LARGE_DOUBLE) { /* avoid unnecessary overflow for large arguments */ r.real = atan2(fabs(z.imag), z.real); - /* split into cases to make sure that the branch cut has the - correct continuity on systems with unsigned zeros */ - if (z.real < 0.) { - r.imag = -copysign(log(hypot(z.real/2., z.imag/2.)) + - M_LN2*2., z.imag); - } else { - r.imag = copysign(log(hypot(z.real/2., z.imag/2.)) + - M_LN2*2., -z.imag); - } + r.imag = -copysign(log(hypot(z.real/2., z.imag/2.)) + + M_LN2*2., z.imag); } else { s1.real = 1.-z.real; s1.imag = -z.imag; @@ -356,11 +349,7 @@ cmath_atanh_impl(PyObject *module, Py_complex z) */ h = hypot(z.real/2., z.imag/2.); /* safe from overflow */ r.real = z.real/4./h/h; - /* the two negations in the next line cancel each other out - except when working with unsigned zeros: they're there to - ensure that the branch cut has the correct continuity on - systems that don't support signed zeros */ - r.imag = -copysign(Py_MATH_PI/2., -z.imag); + r.imag = copysign(Py_MATH_PI/2., z.imag); errno = 0; } else if (z.real == 1. && ay < CM_SQRT_DBL_MIN) { /* C99 standard says: atanh(1+/-0.) should be inf +/- 0i */ diff --git a/Modules/makesetup b/Modules/makesetup index d41b6640bb5186..8bb971b152a522 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -274,7 +274,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | ;; esac rule="$file: $objs" - rule="$rule; \$(BLDSHARED) $objs $libs \$(MODULE_LDFLAGS) -o $file" + rule="$rule; \$(BLDSHARED) $objs $libs \$(LIBPYTHON) -o $file" echo "$rule" >>$rulesf done done diff --git a/Objects/abstract.c b/Objects/abstract.c index afb068718bb010..8626584e9bf56c 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2881,7 +2881,50 @@ PyAIter_Check(PyObject *obj) tp->tp_as_async->am_anext != &_PyObject_NextNotImplemented); } +static int +iternext(PyObject *iter, PyObject **item) +{ + iternextfunc tp_iternext = Py_TYPE(iter)->tp_iternext; + if ((*item = tp_iternext(iter))) { + return 1; + } + + PyThreadState *tstate = _PyThreadState_GET(); + /* When the iterator is exhausted it must return NULL; + * a StopIteration exception may or may not be set. */ + if (!_PyErr_Occurred(tstate)) { + return 0; + } + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + _PyErr_Clear(tstate); + return 0; + } + + /* Error case: an exception (different than StopIteration) is set. */ + return -1; +} + +/* Return 1 and set 'item' to the next item of 'iter' on success. + * Return 0 and set 'item' to NULL when there are no remaining values. + * Return -1, set 'item' to NULL and set an exception on error. + */ +int +PyIter_NextItem(PyObject *iter, PyObject **item) +{ + assert(iter != NULL); + assert(item != NULL); + + if (Py_TYPE(iter)->tp_iternext == NULL) { + *item = NULL; + PyErr_Format(PyExc_TypeError, "expected an iterator, got '%T'", iter); + return -1; + } + + return iternext(iter, item); +} + /* Return next item. + * * If an error occurs, return NULL. PyErr_Occurred() will be true. * If the iteration terminates normally, return NULL and clear the * PyExc_StopIteration exception (if it was set). PyErr_Occurred() @@ -2891,17 +2934,9 @@ PyAIter_Check(PyObject *obj) PyObject * PyIter_Next(PyObject *iter) { - PyObject *result; - result = (*Py_TYPE(iter)->tp_iternext)(iter); - if (result == NULL) { - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyErr_Occurred(tstate) - && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - { - _PyErr_Clear(tstate); - } - } - return result; + PyObject *item; + (void)iternext(iter, &item); + return item; } PySendResult diff --git a/Objects/object.c b/Objects/object.c index db9d3e46795668..c6d46caa0bb62b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2477,15 +2477,7 @@ _PyObject_SetDeferredRefcount(PyObject *op) assert(_Py_IsOwnedByCurrentThread(op)); assert(op->ob_ref_shared == 0); _PyObject_SET_GC_BITS(op, _PyGC_BITS_DEFERRED); - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (_Py_atomic_load_int_relaxed(&interp->gc.immortalize) == 1) { - // gh-117696: immortalize objects instead of using deferred reference - // counting for now. - _Py_SetImmortal(op); - return; - } - op->ob_ref_local += 1; - op->ob_ref_shared = _Py_REF_QUEUED; + op->ob_ref_shared = _Py_REF_SHARED(_Py_REF_DEFERRED, 0); #endif } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index a6a71802ef8e01..dfeccfa4dd7658 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1109,9 +1109,12 @@ free_delayed(uintptr_t ptr) #ifndef Py_GIL_DISABLED free_work_item(ptr); #else - if (_PyRuntime.stoptheworld.world_stopped) { - // Free immediately if the world is stopped, including during - // interpreter shutdown. + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyInterpreterState_GetFinalizing(interp) != NULL || + interp->stoptheworld.world_stopped) + { + // Free immediately during interpreter shutdown or if the world is + // stopped. free_work_item(ptr); return; } @@ -1474,6 +1477,8 @@ _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *interp) { #ifdef WITH_MIMALLOC if (_PyMem_MimallocEnabled()) { + Py_ssize_t leaked = _PyInterpreterState_GetAllocatedBlocks(interp); + interp->runtime->obmalloc.interpreter_leaks += leaked; return; } #endif diff --git a/Objects/setobject.c b/Objects/setobject.c index 9c17e3eef2fe00..c5f96d25585fa4 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1049,14 +1049,13 @@ set_update_internal(PySetObject *so, PyObject *other) set.update so: setobject *others as args: object - / Update the set, adding elements from all others. [clinic start generated code]*/ static PyObject * set_update_impl(PySetObject *so, PyObject *args) -/*[clinic end generated code: output=34f6371704974c8a input=eb47c4fbaeb3286e]*/ +/*[clinic end generated code: output=34f6371704974c8a input=df4fe486e38cd337]*/ { Py_ssize_t i; @@ -1279,14 +1278,13 @@ set_clear_impl(PySetObject *so) set.union so: setobject *others as args: object - / Return a new set with elements from the set and all others. [clinic start generated code]*/ static PyObject * set_union_impl(PySetObject *so, PyObject *args) -/*[clinic end generated code: output=2c83d05a446a1477 input=2e2024fa1e40ac84]*/ +/*[clinic end generated code: output=2c83d05a446a1477 input=ddf088706e9577b2]*/ { PySetObject *result; PyObject *other; @@ -1429,14 +1427,13 @@ set_intersection(PySetObject *so, PyObject *other) set.intersection as set_intersection_multi so: setobject *others as args: object - / Return a new set with elements common to the set and all others. [clinic start generated code]*/ static PyObject * set_intersection_multi_impl(PySetObject *so, PyObject *args) -/*[clinic end generated code: output=2406ef3387adbe2f input=04108ea6d7f0532b]*/ +/*[clinic end generated code: output=2406ef3387adbe2f input=0d9f3805ccbba6a4]*/ { Py_ssize_t i; @@ -1477,14 +1474,13 @@ set_intersection_update(PySetObject *so, PyObject *other) set.intersection_update as set_intersection_update_multi so: setobject *others as args: object - / Update the set, keeping only elements found in it and all others. [clinic start generated code]*/ static PyObject * set_intersection_update_multi_impl(PySetObject *so, PyObject *args) -/*[clinic end generated code: output=251c1f729063609d input=ff8f119f97458d16]*/ +/*[clinic end generated code: output=251c1f729063609d input=223c1e086aa669a9]*/ { PyObject *tmp; @@ -1665,14 +1661,13 @@ set_difference_update_internal(PySetObject *so, PyObject *other) set.difference_update so: setobject *others as args: object - / Update the set, removing elements found in others. [clinic start generated code]*/ static PyObject * set_difference_update_impl(PySetObject *so, PyObject *args) -/*[clinic end generated code: output=28685b2fc63e41c4 input=e7abb43c9f2c5a73]*/ +/*[clinic end generated code: output=28685b2fc63e41c4 input=024e6baa6fbcbb3d]*/ { Py_ssize_t i; @@ -1783,14 +1778,13 @@ set_difference(PySetObject *so, PyObject *other) set.difference as set_difference_multi so: setobject *others as args: object - / Return a new set with elements in the set that are not in the others. [clinic start generated code]*/ static PyObject * set_difference_multi_impl(PySetObject *so, PyObject *args) -/*[clinic end generated code: output=3130c3bb3cac873d input=d8ae9bb6d518ab95]*/ +/*[clinic end generated code: output=3130c3bb3cac873d input=ba78ea5f099e58df]*/ { Py_ssize_t i; PyObject *result, *other; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a2d82e65b6ad9f..00f0dc9849b5c8 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2452,7 +2452,7 @@ subtype_dealloc(PyObject *self) reference counting. Only decref if the base type is not already a heap allocated type. Otherwise, basedealloc should have decref'd it already */ if (type_needs_decref) { - Py_DECREF(type); + _Py_DECREF_TYPE(type); } /* Done */ @@ -2562,7 +2562,7 @@ subtype_dealloc(PyObject *self) reference counting. Only decref if the base type is not already a heap allocated type. Otherwise, basedealloc should have decref'd it already */ if (type_needs_decref) { - Py_DECREF(type); + _Py_DECREF_TYPE(type); } endlabel: @@ -3913,7 +3913,9 @@ type_new_alloc(type_new_ctx *ctx) et->ht_module = NULL; et->_ht_tpname = NULL; - _PyObject_SetDeferredRefcount((PyObject *)et); +#ifdef Py_GIL_DISABLED + _PyType_AssignId(et); +#endif return type; } @@ -4965,6 +4967,11 @@ _PyType_FromMetaclass_impl( type->tp_weaklistoffset = weaklistoffset; type->tp_dictoffset = dictoffset; +#ifdef Py_GIL_DISABLED + // Assign a type id to enable thread-local refcounting + _PyType_AssignId(res); +#endif + /* Ready the type (which includes inheritance). * * After this call we should generally only touch up what's @@ -5914,6 +5921,9 @@ type_dealloc(PyObject *self) } Py_XDECREF(et->ht_module); PyMem_Free(et->_ht_tpname); +#ifdef Py_GIL_DISABLED + _PyType_ReleaseId(et); +#endif Py_TYPE(type)->tp_free((PyObject *)type); } diff --git a/PC/python3dll.c b/PC/python3dll.c index aa3c3965908ff4..78bcef155f51d5 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -326,6 +326,7 @@ EXPORT_FUNC(PyInterpreterState_GetID) EXPORT_FUNC(PyInterpreterState_New) EXPORT_FUNC(PyIter_Check) EXPORT_FUNC(PyIter_Next) +EXPORT_FUNC(PyIter_NextItem) EXPORT_FUNC(PyIter_Send) EXPORT_FUNC(PyList_Append) EXPORT_FUNC(PyList_AsTuple) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index e5e18de60ec349..962d754e4a121d 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -267,6 +267,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 9630f54ae4ea29..86146f73857bd4 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -464,6 +464,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 252039d93103bd..bcb2ce24fcb2bf 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -99,6 +99,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 7efbb0acf8f960..df3324b71b2f60 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -14,6 +14,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 9e3af689f4a288..7991eb93aa2c8a 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -304,6 +304,7 @@ + @@ -643,6 +644,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 31f7971bda845d..075910915fb912 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -831,6 +831,9 @@ Include\internal + + Include\internal + Include\internal @@ -1493,6 +1496,9 @@ Python + + Python + Python diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9a1af0e920188b..b68f9327d898c2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -862,13 +862,14 @@ dummy_func( PyStackRef_CLOSE(list_st); } - inst(STORE_SUBSCR_DICT, (unused/1, value, dict_st, sub_st -- )) { - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + inst(STORE_SUBSCR_DICT, (unused/1, value, dict_st, sub -- )) { PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); PyStackRef_CLOSE(dict_st); ERROR_IF(err, error); } @@ -1182,7 +1183,6 @@ dummy_func( assert(!_PyErr_Occurred(tstate)); } else { - assert(PyLong_Check(lasti)); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); ERROR_NO_POP(); } @@ -1424,7 +1424,7 @@ dummy_func( "no locals found"); ERROR_IF(true, error); } - locals = PyStackRef_FromPyObjectNew(l);; + locals = PyStackRef_FromPyObjectNew(l); } inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { diff --git a/Python/ceval.c b/Python/ceval.c index f1663ee539aeac..c685a95b2ef088 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2499,7 +2499,7 @@ PyEval_GetLocals(void) PyFrameObject *f = _PyFrame_GetFrameObject(current_frame); PyObject *ret = f->f_locals_cache; if (ret == NULL) { - PyObject *ret = PyDict_New(); + ret = PyDict_New(); if (ret == NULL) { Py_DECREF(locals); return NULL; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index afc7786c9e434d..f2741286d60197 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1105,20 +1105,21 @@ } case _STORE_SUBSCR_DICT: { - _PyStackRef sub_st; + _PyStackRef sub; _PyStackRef dict_st; _PyStackRef value; - sub_st = stack_pointer[-1]; + sub = stack_pointer[-1]; dict_st = stack_pointer[-2]; value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); PyStackRef_CLOSE(dict_st); if (err) JUMP_TO_ERROR(); stack_pointer += -3; @@ -1468,10 +1469,10 @@ } STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + stack_pointer[0] = val0; val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - PyStackRef_CLOSE(seq); stack_pointer[-1] = val1; - stack_pointer[0] = val0; + PyStackRef_CLOSE(seq); stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1611,7 +1612,7 @@ "no locals found"); if (true) JUMP_TO_ERROR(); } - locals = PyStackRef_FromPyObjectNew(l);; + locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2421,8 +2422,8 @@ STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); - PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; + PyStackRef_CLOSE(owner); break; } @@ -2443,8 +2444,8 @@ STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); - PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; + PyStackRef_CLOSE(owner); stack_pointer[0] = null; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2480,9 +2481,9 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + stack_pointer[-1] = attr; null = PyStackRef_NULL; PyStackRef_CLOSE(owner); - stack_pointer[-1] = attr; break; } @@ -2496,9 +2497,9 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + stack_pointer[-1] = attr; null = PyStackRef_NULL; PyStackRef_CLOSE(owner); - stack_pointer[-1] = attr; stack_pointer[0] = null; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3436,8 +3437,8 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - self = owner; stack_pointer[-1] = attr; + self = owner; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3457,8 +3458,8 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - self = owner; stack_pointer[-1] = attr; + self = owner; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3522,8 +3523,8 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - self = owner; stack_pointer[-1] = attr; + self = owner; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3545,8 +3546,10 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; maybe_self = PyStackRef_FromPyObjectNew(self); + stack_pointer[-1 - oparg] = maybe_self; PyObject *method = ((PyMethodObject *)callable_o)->im_func; func = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = func; /* Make sure that callable and all args are in memory */ args[-2] = func; args[-1] = maybe_self; @@ -3556,8 +3559,6 @@ func = callable; maybe_self = self_or_null; } - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; break; } @@ -3665,11 +3666,11 @@ assert(PyStackRef_IsNull(null)); assert(Py_TYPE(callable_o) == &PyMethod_Type); self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-1 - oparg] = self; method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = method; assert(PyStackRef_FunctionCheck(method)); PyStackRef_CLOSE(callable); - stack_pointer[-2 - oparg] = method; - stack_pointer[-1 - oparg] = self; break; } @@ -3762,10 +3763,10 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-1 - oparg] = self; func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - PyStackRef_CLOSE(callable); stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = self; + PyStackRef_CLOSE(callable); break; } @@ -5087,8 +5088,8 @@ _PyStackRef null; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); value = PyStackRef_FromPyObjectNew(ptr); - null = PyStackRef_NULL; stack_pointer[0] = value; + null = PyStackRef_NULL; stack_pointer[1] = null; stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 53f04160c38841..543bee24652dc9 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -15,6 +15,7 @@ #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" +#include "pycore_typeid.h" // _PyType_MergeThreadLocalRefcounts #ifdef Py_GIL_DISABLED @@ -54,6 +55,7 @@ struct collection_state { struct visitor_args base; PyInterpreterState *interp; GCState *gcstate; + _PyGC_Reason reason; Py_ssize_t collected; Py_ssize_t uncollectable; Py_ssize_t long_lived_total; @@ -164,7 +166,15 @@ disable_deferred_refcounting(PyObject *op) { if (_PyObject_HasDeferredRefcount(op)) { op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED; - op->ob_ref_shared -= (1 << _Py_REF_SHARED_SHIFT); + op->ob_ref_shared -= _Py_REF_SHARED(_Py_REF_DEFERRED, 0); + + if (PyType_Check(op)) { + // Disable thread-local refcounting for heap types + PyTypeObject *type = (PyTypeObject *)op; + if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + _PyType_ReleaseId((PyHeapTypeObject *)op); + } + } } } @@ -328,16 +338,6 @@ merge_queued_objects(_PyThreadStateImpl *tstate, struct collection_state *state) } } -static void -merge_all_queued_objects(PyInterpreterState *interp, struct collection_state *state) -{ - HEAD_LOCK(&_PyRuntime); - for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { - merge_queued_objects((_PyThreadStateImpl *)p, state); - } - HEAD_UNLOCK(&_PyRuntime); -} - static void process_delayed_frees(PyInterpreterState *interp) { @@ -389,7 +389,9 @@ update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, } Py_ssize_t refcount = Py_REFCNT(op); - refcount -= _PyObject_HasDeferredRefcount(op); + if (_PyObject_HasDeferredRefcount(op)) { + refcount -= _Py_REF_DEFERRED; + } _PyObject_ASSERT(op, refcount >= 0); if (refcount > 0 && !_PyObject_HasDeferredRefcount(op)) { @@ -571,6 +573,16 @@ scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, worklist_push(&state->unreachable, op); } } + else if (state->reason == _Py_GC_REASON_SHUTDOWN && + _PyObject_HasDeferredRefcount(op)) + { + // Disable deferred refcounting for reachable objects as well during + // interpreter shutdown. This ensures that these objects are collected + // immediately when their last reference is removed. + disable_deferred_refcounting(op); + merge_refcount(op, 0); + state->long_lived_total++; + } else { // object is reachable, restore `ob_tid`; we're done with these objects gc_restore_tid(op); @@ -754,10 +766,6 @@ _PyGC_Init(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; - // gh-117783: immortalize objects that would use deferred refcounting - // once the first non-main thread is created (but not in subinterpreters). - gcstate->immortalize = _Py_IsMainInterpreter(interp) ? 0 : -1; - gcstate->garbage = PyList_New(0); if (gcstate->garbage == NULL) { return _PyStatus_NO_MEMORY(); @@ -1105,8 +1113,18 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, state->gcstate->old[i-1].count = 0; } - // merge refcounts for all queued objects - merge_all_queued_objects(interp, state); + HEAD_LOCK(&_PyRuntime); + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p; + + // merge per-thread refcount for types into the type's actual refcount + _PyType_MergeThreadLocalRefcounts(tstate); + + // merge refcounts for all queued objects + merge_queued_objects(tstate, state); + } + HEAD_UNLOCK(&_PyRuntime); + process_delayed_frees(interp); // Find unreachable objects @@ -1221,6 +1239,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) struct collection_state state = { .interp = interp, .gcstate = gcstate, + .reason = reason, }; gc_collect_internal(interp, &state, generation); @@ -1835,32 +1854,6 @@ custom_visitor_wrapper(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } -// gh-117783: Immortalize objects that use deferred reference counting to -// temporarily work around scaling bottlenecks. -static bool -immortalize_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, - void *block, size_t block_size, void *args) -{ - PyObject *op = op_from_block(block, args, false); - if (op != NULL && _PyObject_HasDeferredRefcount(op)) { - _Py_SetImmortal(op); - op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED; - } - return true; -} - -void -_PyGC_ImmortalizeDeferredObjects(PyInterpreterState *interp) -{ - struct visitor_args args; - _PyEval_StopTheWorld(interp); - if (interp->gc.immortalize == 0) { - gc_visit_heaps(interp, &immortalize_visitor, &args); - interp->gc.immortalize = 1; - } - _PyEval_StartTheWorld(interp); -} - void PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f670353cdbde56..d0aa0f9fe83182 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -867,8 +867,10 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; maybe_self = PyStackRef_FromPyObjectNew(self); + stack_pointer[-1 - oparg] = maybe_self; PyObject *method = ((PyMethodObject *)callable_o)->im_func; func = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = func; /* Make sure that callable and all args are in memory */ args[-2] = func; args[-1] = maybe_self; @@ -880,7 +882,6 @@ } } // _DO_CALL - args = &stack_pointer[-oparg]; self_or_null = maybe_self; callable = func; { @@ -1063,12 +1064,12 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-1 - oparg] = self; func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = func; PyStackRef_CLOSE(callable); } // flush - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = self; // _CHECK_FUNCTION_VERSION callable = stack_pointer[-2 - oparg]; { @@ -1172,13 +1173,13 @@ assert(PyStackRef_IsNull(null)); assert(Py_TYPE(callable_o) == &PyMethod_Type); self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + stack_pointer[-1 - oparg] = self; method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = method; assert(PyStackRef_FunctionCheck(method)); PyStackRef_CLOSE(callable); } // flush - stack_pointer[-2 - oparg] = method; - stack_pointer[-1 - oparg] = self; // _PY_FRAME_GENERAL args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -2534,6 +2535,7 @@ int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); if (matches) { value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + stack_pointer[-2] = value; PyStackRef_CLOSE(sub_iter_st); PyStackRef_CLOSE(last_sent_val_st); PyStackRef_CLOSE(exc_value_st); @@ -2545,7 +2547,6 @@ goto exception_unwind; } stack_pointer[-3] = none; - stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3330,8 +3331,8 @@ assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + stack_pointer[0] = next; } - stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3423,8 +3424,8 @@ assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + stack_pointer[0] = next; } - stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3634,8 +3635,10 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; maybe_self = PyStackRef_FromPyObjectNew(self); + stack_pointer[-1 - oparg] = maybe_self; PyObject *method = ((PyMethodObject *)callable_o)->im_func; func = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = func; /* Make sure that callable and all args are in memory */ args[-2] = func; args[-1] = maybe_self; @@ -3647,7 +3650,6 @@ } } // _MONITOR_CALL - args = &stack_pointer[-oparg]; { int is_meth = !PyStackRef_IsNull(maybe_self); PyObject *function = PyStackRef_AsPyObjectBorrow(func); @@ -3668,7 +3670,6 @@ if (err) goto error; } // _DO_CALL - args = &stack_pointer[-oparg]; self_or_null = maybe_self; callable = func; { @@ -4071,6 +4072,7 @@ // _LOAD_CONST { value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + stack_pointer[0] = value; } // _RETURN_VALUE_EVENT val = value; @@ -4453,10 +4455,10 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + stack_pointer[-1] = attr; null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -4577,9 +4579,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); + stack_pointer[-1] = attr; self = owner; } - stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4613,9 +4615,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); + stack_pointer[-1] = attr; self = owner; } - stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4661,9 +4663,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); + stack_pointer[-1] = attr; self = owner; } - stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4739,8 +4741,8 @@ assert(descr != NULL); PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); + stack_pointer[-1] = attr; } - stack_pointer[-1] = attr; DISPATCH(); } @@ -4781,8 +4783,8 @@ assert(descr != NULL); PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); + stack_pointer[-1] = attr; } - stack_pointer[-1] = attr; DISPATCH(); } @@ -4878,10 +4880,10 @@ STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); + stack_pointer[-1] = attr; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -5290,7 +5292,7 @@ "no locals found"); if (true) goto error; } - locals = PyStackRef_FromPyObjectNew(l);; + locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5875,7 +5877,6 @@ assert(!_PyErr_Occurred(tstate)); } else { - assert(PyLong_Check(lasti)); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); goto error; } @@ -5959,6 +5960,7 @@ // _LOAD_CONST { value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + stack_pointer[0] = value; } // _RETURN_VALUE retval = value; @@ -6625,16 +6627,17 @@ static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef dict_st; - _PyStackRef sub_st; + _PyStackRef sub; /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; + sub = stack_pointer[-1]; dict_st = stack_pointer[-2]; value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); PyStackRef_CLOSE(dict_st); if (err) goto pop_3_error; stack_pointer += -3; @@ -7018,10 +7021,10 @@ DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + stack_pointer[0] = val0; val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - PyStackRef_CLOSE(seq); stack_pointer[-1] = val1; - stack_pointer[0] = val0; + PyStackRef_CLOSE(seq); stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); diff --git a/Python/getargs.c b/Python/getargs.c index ec2eeb15c832c3..1b9b39939ae94e 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -2601,7 +2601,7 @@ _PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, * * Otherwise, we leave a place at `buf[vararg]` for vararg tuple * so the index is `i + 1`. */ - if (nargs < vararg && i != vararg) { + if (i < vararg) { buf[i] = current_arg; } else { diff --git a/Python/instrumentation.c b/Python/instrumentation.c index ae790a1441b933..3481b5df14288b 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1344,7 +1344,6 @@ int _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { PyCodeObject *code = _PyFrame_GetCode(frame); - assert(debug_check_sanity(tstate->interp, code)); int offset = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *instrumentation_data = code->_co_monitoring; assert(instrumentation_data->per_instruction_opcodes); @@ -1352,6 +1351,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* if (tstate->tracing) { return next_opcode; } + assert(debug_check_sanity(tstate->interp, code)); PyInterpreterState *interp = tstate->interp; uint8_t tools = instrumentation_data->per_instruction_tools != NULL ? instrumentation_data->per_instruction_tools[offset] : diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 6b641c0775f533..3a41c640fd9599 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -28,6 +28,7 @@ #include "pycore_sliceobject.h" // _PySlice_Fini() #include "pycore_sysmodule.h" // _PySys_ClearAuditHooks() #include "pycore_traceback.h" // _Py_DumpTracebackThreads() +#include "pycore_typeid.h" // _PyType_FinalizeIdPool() #include "pycore_typeobject.h" // _PyTypes_InitTypes() #include "pycore_typevarobject.h" // _Py_clear_generic_types() #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() @@ -1832,6 +1833,9 @@ finalize_interp_types(PyInterpreterState *interp) _PyTypes_FiniTypes(interp); _PyTypes_Fini(interp); +#ifdef Py_GIL_DISABLED + _PyType_FinalizeIdPool(interp); +#endif _PyCode_Fini(interp); diff --git a/Python/pystate.c b/Python/pystate.c index 6fbd17f7eaeaa9..bba88b76088e71 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -20,6 +20,7 @@ #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() +#include "pycore_typeid.h" // _PyType_FinalizeThreadLocalRefcounts() /* -------------------------------------------------------------------------- CAUTION @@ -1584,13 +1585,6 @@ new_threadstate(PyInterpreterState *interp, int whence) PyMem_RawFree(new_tstate); } else { -#ifdef Py_GIL_DISABLED - if (_Py_atomic_load_int(&interp->gc.immortalize) == 0) { - // Immortalize objects marked as using deferred reference counting - // the first time a non-main thread is created. - _PyGC_ImmortalizeDeferredObjects(interp); - } -#endif } #ifdef Py_GIL_DISABLED @@ -1741,6 +1735,10 @@ PyThreadState_Clear(PyThreadState *tstate) struct _Py_freelists *freelists = _Py_freelists_GET(); _PyObject_ClearFreeLists(freelists, 1); + // Merge our thread-local refcounts into the type's own refcount and + // free our local refcount array. + _PyType_FinalizeThreadLocalRefcounts((_PyThreadStateImpl *)tstate); + // Remove ourself from the biased reference counting table of threads. _Py_brc_remove_thread(tstate); #endif @@ -1799,6 +1797,7 @@ tstate_delete_common(PyThreadState *tstate, int release_gil) _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; tstate->interp->object_state.reftotal += tstate_impl->reftotal; tstate_impl->reftotal = 0; + assert(tstate_impl->types.refcounts == NULL); #endif HEAD_UNLOCK(runtime); diff --git a/Python/symtable.c b/Python/symtable.c index 4acf762f8fca39..527bc367e8ea6e 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -286,7 +286,7 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) ( "%U=== Symtable for %U ===\n" "%U%s%s\n" - "%U%s%s%s%s%s%s%s%s%s%s%s%s%s\n" + "%U%s%s%s%s%s%s%s%s%s%s%s\n" "%Ulineno: %d col_offset: %d\n" "%U--- Symbols ---\n" ), diff --git a/Python/typeid.c b/Python/typeid.c new file mode 100644 index 00000000000000..83a68723ded61b --- /dev/null +++ b/Python/typeid.c @@ -0,0 +1,200 @@ +#include "Python.h" + +#include "pycore_lock.h" // PyMutex_LockFlags() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_object.h" // _Py_IncRefTotal +#include "pycore_typeid.h" + +// This contains code for allocating unique ids to heap type objects +// and re-using those ids when the type is deallocated. +// +// See Include/internal/pycore_typeid.h for more details. + +#ifdef Py_GIL_DISABLED + +#define POOL_MIN_SIZE 8 + +#define LOCK_POOL(pool) PyMutex_LockFlags(&pool->mutex, _Py_LOCK_DONT_DETACH) +#define UNLOCK_POOL(pool) PyMutex_Unlock(&pool->mutex) + +static int +resize_interp_type_id_pool(struct _Py_type_id_pool *pool) +{ + if ((size_t)pool->size > PY_SSIZE_T_MAX / (2 * sizeof(*pool->table))) { + return -1; + } + + Py_ssize_t new_size = pool->size * 2; + if (new_size < POOL_MIN_SIZE) { + new_size = POOL_MIN_SIZE; + } + + _Py_type_id_entry *table = PyMem_Realloc(pool->table, + new_size * sizeof(*pool->table)); + if (table == NULL) { + return -1; + } + + Py_ssize_t start = pool->size; + for (Py_ssize_t i = start; i < new_size - 1; i++) { + table[i].next = &table[i + 1]; + } + table[new_size - 1].next = NULL; + + pool->table = table; + pool->freelist = &table[start]; + _Py_atomic_store_ssize(&pool->size, new_size); + return 0; +} + +static int +resize_local_refcounts(_PyThreadStateImpl *tstate) +{ + if (tstate->types.is_finalized) { + return -1; + } + + struct _Py_type_id_pool *pool = &tstate->base.interp->type_ids; + Py_ssize_t size = _Py_atomic_load_ssize(&pool->size); + + Py_ssize_t *refcnts = PyMem_Realloc(tstate->types.refcounts, + size * sizeof(Py_ssize_t)); + if (refcnts == NULL) { + return -1; + } + + Py_ssize_t old_size = tstate->types.size; + if (old_size < size) { + memset(refcnts + old_size, 0, (size - old_size) * sizeof(Py_ssize_t)); + } + + tstate->types.refcounts = refcnts; + tstate->types.size = size; + return 0; +} + +void +_PyType_AssignId(PyHeapTypeObject *type) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _Py_type_id_pool *pool = &interp->type_ids; + + LOCK_POOL(pool); + if (pool->freelist == NULL) { + if (resize_interp_type_id_pool(pool) < 0) { + type->unique_id = -1; + UNLOCK_POOL(pool); + return; + } + } + + _Py_type_id_entry *entry = pool->freelist; + pool->freelist = entry->next; + entry->type = type; + _PyObject_SetDeferredRefcount((PyObject *)type); + type->unique_id = (entry - pool->table); + UNLOCK_POOL(pool); +} + +void +_PyType_ReleaseId(PyHeapTypeObject *type) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _Py_type_id_pool *pool = &interp->type_ids; + + if (type->unique_id < 0) { + // The type doesn't have an id assigned. + return; + } + + LOCK_POOL(pool); + _Py_type_id_entry *entry = &pool->table[type->unique_id]; + assert(entry->type == type); + entry->next = pool->freelist; + pool->freelist = entry; + + type->unique_id = -1; + UNLOCK_POOL(pool); +} + +void +_PyType_IncrefSlow(PyHeapTypeObject *type) +{ + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (type->unique_id < 0 || resize_local_refcounts(tstate) < 0) { + // just incref the type directly. + Py_INCREF(type); + return; + } + + assert(type->unique_id < tstate->types.size); + tstate->types.refcounts[type->unique_id]++; +#ifdef Py_REF_DEBUG + _Py_IncRefTotal((PyThreadState *)tstate); +#endif + _Py_INCREF_STAT_INC(); +} + +void +_PyType_MergeThreadLocalRefcounts(_PyThreadStateImpl *tstate) +{ + if (tstate->types.refcounts == NULL) { + return; + } + + struct _Py_type_id_pool *pool = &tstate->base.interp->type_ids; + + LOCK_POOL(pool); + for (Py_ssize_t i = 0, n = tstate->types.size; i < n; i++) { + Py_ssize_t refcnt = tstate->types.refcounts[i]; + if (refcnt != 0) { + PyObject *type = (PyObject *)pool->table[i].type; + assert(PyType_Check(type)); + + _Py_atomic_add_ssize(&type->ob_ref_shared, + refcnt << _Py_REF_SHARED_SHIFT); + tstate->types.refcounts[i] = 0; + } + } + UNLOCK_POOL(pool); +} + +void +_PyType_FinalizeThreadLocalRefcounts(_PyThreadStateImpl *tstate) +{ + _PyType_MergeThreadLocalRefcounts(tstate); + + PyMem_Free(tstate->types.refcounts); + tstate->types.refcounts = NULL; + tstate->types.size = 0; + tstate->types.is_finalized = 1; +} + +void +_PyType_FinalizeIdPool(PyInterpreterState *interp) +{ + struct _Py_type_id_pool *pool = &interp->type_ids; + + // First, set the free-list to NULL values + while (pool->freelist) { + _Py_type_id_entry *next = pool->freelist->next; + pool->freelist->type = NULL; + pool->freelist = next; + } + + // Now everything non-NULL is a type. Set the type's id to -1 in case it + // outlives the interpreter. + for (Py_ssize_t i = 0; i < pool->size; i++) { + PyHeapTypeObject *ht = pool->table[i].type; + if (ht) { + ht->unique_id = -1; + pool->table[i].type = NULL; + } + } + PyMem_Free(pool->table); + pool->table = NULL; + pool->freelist = NULL; + pool->size = 0; +} + +#endif /* Py_GIL_DISABLED */ diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos new file mode 100644 index 00000000000000..1b504dfc54000f --- /dev/null +++ b/Tools/build/.warningignore_macos @@ -0,0 +1,3 @@ +# Files listed will be ignored by the compiler warning checker +# for the macOS/build and test job. +# Keep lines sorted lexicographically to help avoid merge conflicts. diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index af9f7f169ad943..31258932dbd4ca 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -2,39 +2,87 @@ Parses compiler output with -fdiagnostics-format=json and checks that warnings exist only in files that are expected to have warnings. """ + import argparse +from collections import defaultdict import json import re import sys from pathlib import Path -def extract_warnings_from_compiler_output(compiler_output: str) -> list[dict]: +def extract_warnings_from_compiler_output_clang( + compiler_output: str, +) -> list[dict]: """ - Extracts warnings from the compiler output when using - -fdiagnostics-format=json + Extracts warnings from the compiler output when using clang + """ + # Regex to find warnings in the compiler output + clang_warning_regex = re.compile( + r"(?P.*):(?P\d+):(?P\d+): warning: (?P.*)" + ) + compiler_warnings = [] + for line in compiler_output.splitlines(): + if match := clang_warning_regex.match(line): + compiler_warnings.append( + { + "file": match.group("file"), + "line": match.group("line"), + "column": match.group("column"), + "message": match.group("message"), + } + ) - Compiler output as a whole is not a valid json document, but includes many - json objects and may include other output that is not json. + return compiler_warnings + + +def extract_warnings_from_compiler_output_json( + compiler_output: str, +) -> list[dict]: """ + Extracts warnings from the compiler output when using + -fdiagnostics-format=json. + Compiler output as a whole is not a valid json document, + but includes many json objects and may include other output + that is not json. + """ # Regex to find json arrays at the top level of the file # in the compiler output json_arrays = re.findall( - r"\[(?:[^\[\]]|\[(?:[^\[\]]|\[[^\[\]]*\])*\])*\]", compiler_output + r"\[(?:[^[\]]|\[[^\]]*\])*\]", compiler_output ) compiler_warnings = [] for array in json_arrays: try: json_data = json.loads(array) json_objects_in_array = [entry for entry in json_data] - compiler_warnings.extend( - [ - entry - for entry in json_objects_in_array - if entry.get("kind") == "warning" - ] - ) + warning_list = [ + entry + for entry in json_objects_in_array + if entry.get("kind") == "warning" + ] + for warning in warning_list: + locations = warning["locations"] + for location in locations: + for key in ["caret", "start", "end"]: + if key in location: + compiler_warnings.append( + { + # Remove leading current directory if present + "file": location[key]["file"].lstrip("./"), + "line": location[key]["line"], + "column": location[key]["column"], + "message": warning["message"], + } + ) + # Found a caret, start, or end in location so + # break out completely to address next warning + break + else: + continue + break + except json.JSONDecodeError: continue # Skip malformed JSON @@ -46,27 +94,16 @@ def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: Returns a dictionary where the key is the file and the data is the warnings in that file """ - warnings_by_file = {} + warnings_by_file = defaultdict(list) for warning in warnings: - locations = warning["locations"] - for location in locations: - for key in ["caret", "start", "end"]: - if key in location: - file = location[key]["file"] - file = file.lstrip( - "./" - ) # Remove leading current directory if present - if file not in warnings_by_file: - warnings_by_file[file] = [] - warnings_by_file[file].append(warning) + warnings_by_file[warning["file"]].append(warning) return warnings_by_file def get_unexpected_warnings( - warnings: list[dict], files_with_expected_warnings: set[str], - files_with_warnings: set[str], + files_with_warnings: dict[str, list[dict]], ) -> int: """ Returns failure status if warnings discovered in list of warnings @@ -88,13 +125,12 @@ def get_unexpected_warnings( def get_unexpected_improvements( - warnings: list[dict], files_with_expected_warnings: set[str], - files_with_warnings: set[str], + files_with_warnings: dict[str, list[dict]], ) -> int: """ - Returns failure status if there are no warnings in the list of warnings for - a file that is in the list of files with expected warnings + Returns failure status if there are no warnings in the list of warnings + for a file that is in the list of files with expected warnings """ unexpected_improvements = [] for file in files_with_expected_warnings: @@ -123,7 +159,6 @@ def main(argv: list[str] | None = None) -> int: "-i", "--warning-ignore-file-path", type=str, - required=True, help="Path to the warning ignore file", ) parser.add_argument( @@ -141,6 +176,14 @@ def main(argv: list[str] | None = None) -> int: help="Flag to fail if files that were expected " "to have warnings have no warnings", ) + parser.add_argument( + "-t", + "--compiler-output-type", + type=str, + required=True, + choices=["json", "clang"], + help="Type of compiler output file (json or clang)", + ) args = parser.parse_args(argv) @@ -149,44 +192,56 @@ def main(argv: list[str] | None = None) -> int: # Check that the compiler output file is a valid path if not Path(args.compiler_output_file_path).is_file(): print( - "Compiler output file does not exist: " - f"{args.compiler_output_file_path}" + f"Compiler output file does not exist:" + f" {args.compiler_output_file_path}" ) return 1 - # Check that the warning ignore file is a valid path - if not Path(args.warning_ignore_file_path).is_file(): + # Check that a warning ignore file was specified and if so is a valid path + if not args.warning_ignore_file_path: print( - "Warning ignore file does not exist: " - f"{args.warning_ignore_file_path}" + "Warning ignore file not specified." + " Continuing without it (no warnings ignored)." ) - return 1 + files_with_expected_warnings = set() + else: + if not Path(args.warning_ignore_file_path).is_file(): + print( + f"Warning ignore file does not exist:" + f" {args.warning_ignore_file_path}" + ) + return 1 + with Path(args.warning_ignore_file_path).open( + encoding="UTF-8" + ) as clean_files: + files_with_expected_warnings = { + file.strip() + for file in clean_files + if file.strip() and not file.startswith("#") + } with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f: compiler_output_file_contents = f.read() - with Path(args.warning_ignore_file_path).open( - encoding="UTF-8" - ) as clean_files: - files_with_expected_warnings = { - file.strip() - for file in clean_files - if file.strip() and not file.startswith("#") - } - - warnings = extract_warnings_from_compiler_output( - compiler_output_file_contents - ) + if args.compiler_output_type == "json": + warnings = extract_warnings_from_compiler_output_json( + compiler_output_file_contents + ) + elif args.compiler_output_type == "clang": + warnings = extract_warnings_from_compiler_output_clang( + compiler_output_file_contents + ) + files_with_warnings = get_warnings_by_file(warnings) status = get_unexpected_warnings( - warnings, files_with_expected_warnings, files_with_warnings + files_with_expected_warnings, files_with_warnings ) if args.fail_on_regression: exit_code |= status status = get_unexpected_improvements( - warnings, files_with_expected_warnings, files_with_warnings + files_with_expected_warnings, files_with_warnings ) if args.fail_on_improvement: exit_code |= status diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index f6625a3f7322d5..c91edff24bc665 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -62,7 +62,6 @@ def infallible(self) -> bool: return not self.error_with_pop and not self.error_without_pop - SKIP_PROPERTIES = Properties( escapes=False, error_with_pop=False, @@ -99,7 +98,6 @@ def properties(self) -> Properties: class Flush: - @property def properties(self) -> Properties: return SKIP_PROPERTIES @@ -112,6 +110,7 @@ def name(self) -> str: def size(self) -> int: return 0 + @dataclass class StackItem: name: str @@ -133,6 +132,7 @@ def is_array(self) -> bool: def get_size(self) -> str: return self.size if self.size else "1" + @dataclass class StackEffect: inputs: list[StackItem] @@ -150,6 +150,7 @@ class CacheEntry: def __str__(self) -> str: return f"{self.name}/{self.size}" + @dataclass class Uop: name: str @@ -157,12 +158,13 @@ class Uop: annotations: list[str] stack: StackEffect caches: list[CacheEntry] + deferred_refs: dict[lexer.Token, str | None] body: list[lexer.Token] properties: Properties _size: int = -1 implicitly_created: bool = False replicated = 0 - replicates : "Uop | None" = None + replicates: "Uop | None" = None def dump(self, indent: str) -> None: print( @@ -307,19 +309,26 @@ def override_error( ) -def convert_stack_item(item: parser.StackEffect, replace_op_arg_1: str | None) -> StackItem: +def convert_stack_item( + item: parser.StackEffect, replace_op_arg_1: str | None +) -> StackItem: cond = item.cond if replace_op_arg_1 and OPARG_AND_1.match(item.cond): cond = replace_op_arg_1 - return StackItem( - item.name, item.type, cond, item.size - ) + return StackItem(item.name, item.type, cond, item.size) + -def analyze_stack(op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | None = None) -> StackEffect: +def analyze_stack( + op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | None = None +) -> StackEffect: inputs: list[StackItem] = [ - convert_stack_item(i, replace_op_arg_1) for i in op.inputs if isinstance(i, parser.StackEffect) + convert_stack_item(i, replace_op_arg_1) + for i in op.inputs + if isinstance(i, parser.StackEffect) + ] + outputs: list[StackItem] = [ + convert_stack_item(i, replace_op_arg_1) for i in op.outputs ] - outputs: list[StackItem] = [convert_stack_item(i, replace_op_arg_1) for i in op.outputs] # Mark variables with matching names at the base of the stack as "peek" modified = False for input, output in zip(inputs, outputs): @@ -330,9 +339,11 @@ def analyze_stack(op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | No if isinstance(op, parser.InstDef): output_names = [out.name for out in outputs] for input in inputs: - if (variable_used(op, input.name) or - variable_used(op, "DECREF_INPUTS") or - (not input.peek and input.name in output_names)): + if ( + variable_used(op, input.name) + or variable_used(op, "DECREF_INPUTS") + or (not input.peek and input.name in output_names) + ): input.used = True for output in outputs: if variable_used(op, output.name): @@ -352,18 +363,71 @@ def analyze_caches(inputs: list[parser.InputEffect]) -> list[CacheEntry]: return [CacheEntry(i.name, int(i.size)) for i in caches] +def analyze_deferred_refs(node: parser.InstDef) -> dict[lexer.Token, str | None]: + """Look for PyStackRef_FromPyObjectNew() calls""" + + def find_assignment_target(idx: int) -> list[lexer.Token]: + """Find the tokens that make up the left-hand side of an assignment""" + offset = 1 + for tkn in reversed(node.block.tokens[: idx - 1]): + if tkn.kind == "SEMI" or tkn.kind == "LBRACE" or tkn.kind == "RBRACE": + return node.block.tokens[idx - offset : idx - 1] + offset += 1 + return [] + + refs: dict[lexer.Token, str | None] = {} + for idx, tkn in enumerate(node.block.tokens): + if tkn.kind != "IDENTIFIER" or tkn.text != "PyStackRef_FromPyObjectNew": + continue + + if idx == 0 or node.block.tokens[idx - 1].kind != "EQUALS": + raise analysis_error("Expected '=' before PyStackRef_FromPyObjectNew", tkn) + + lhs = find_assignment_target(idx) + if len(lhs) == 0: + raise analysis_error( + "PyStackRef_FromPyObjectNew() must be assigned to an output", tkn + ) + + if lhs[0].kind == "TIMES" or any( + t.kind == "ARROW" or t.kind == "LBRACKET" for t in lhs[1:] + ): + # Don't handle: *ptr = ..., ptr->field = ..., or ptr[field] = ... + # Assume that they are visible to the GC. + refs[tkn] = None + continue + + if len(lhs) != 1 or lhs[0].kind != "IDENTIFIER": + raise analysis_error( + "PyStackRef_FromPyObjectNew() must be assigned to an output", tkn + ) + + name = lhs[0].text + if not any(var.name == name for var in node.outputs): + raise analysis_error( + f"PyStackRef_FromPyObjectNew() must be assigned to an output, not '{name}'", + tkn, + ) + + refs[tkn] = name + + return refs + + def variable_used(node: parser.InstDef, name: str) -> bool: """Determine whether a variable with a given name is used in a node.""" return any( token.kind == "IDENTIFIER" and token.text == name for token in node.block.tokens ) + def oparg_used(node: parser.InstDef) -> bool: """Determine whether `oparg` is used in a node.""" return any( token.kind == "IDENTIFIER" and token.text == "oparg" for token in node.tokens ) + def tier_variable(node: parser.InstDef) -> int | None: """Determine whether a tier variable is used in a node.""" for token in node.tokens: @@ -374,6 +438,7 @@ def tier_variable(node: parser.InstDef) -> int | None: return int(token.text[-1]) return None + def has_error_with_pop(op: parser.InstDef) -> bool: return ( variable_used(op, "ERROR_IF") @@ -382,6 +447,7 @@ def has_error_with_pop(op: parser.InstDef) -> bool: or variable_used(op, "resume_with_error") ) + def has_error_without_pop(op: parser.InstDef) -> bool: return ( variable_used(op, "ERROR_NO_POP") @@ -564,8 +630,10 @@ def stack_effect_only_peeks(instr: parser.InstDef) -> bool: for s, other in zip(stack_inputs, instr.outputs) ) + OPARG_AND_1 = re.compile("\\(*oparg *& *1") + def effect_depends_on_oparg_1(op: parser.InstDef) -> bool: for effect in op.inputs: if isinstance(effect, parser.CacheEffect): @@ -581,6 +649,7 @@ def effect_depends_on_oparg_1(op: parser.InstDef) -> bool: return True return False + def compute_properties(op: parser.InstDef) -> Properties: has_free = ( variable_used(op, "PyCell_New") @@ -625,13 +694,19 @@ def compute_properties(op: parser.InstDef) -> Properties: ) -def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect], uops: dict[str, Uop]) -> Uop: +def make_uop( + name: str, + op: parser.InstDef, + inputs: list[parser.InputEffect], + uops: dict[str, Uop], +) -> Uop: result = Uop( name=name, context=op.context, annotations=op.annotations, stack=analyze_stack(op), caches=analyze_caches(inputs), + deferred_refs=analyze_deferred_refs(op), body=op.block.tokens, properties=compute_properties(op), ) @@ -642,13 +717,16 @@ def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect], uo properties = compute_properties(op) if properties.oparg: # May not need oparg anymore - properties.oparg = any(token.text == "oparg" for token in op.block.tokens) + properties.oparg = any( + token.text == "oparg" for token in op.block.tokens + ) rep = Uop( name=name_x, context=op.context, annotations=op.annotations, stack=analyze_stack(op, bit), caches=analyze_caches(inputs), + deferred_refs=analyze_deferred_refs(op), body=op.block.tokens, properties=properties, ) @@ -671,6 +749,7 @@ def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect], uo annotations=op.annotations, stack=analyze_stack(op), caches=analyze_caches(inputs), + deferred_refs=analyze_deferred_refs(op), body=op.block.tokens, properties=properties, ) @@ -691,8 +770,10 @@ def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: def add_instruction( - where: lexer.Token, name: str, parts: list[Part], - instructions: dict[str, Instruction] + where: lexer.Token, + name: str, + parts: list[Part], + instructions: dict[str, Instruction], ) -> None: instructions[name] = Instruction(where, name, parts, None) @@ -736,7 +817,9 @@ def add_macro( parts.append(Flush()) else: if part.name not in uops: - raise analysis_error(f"No Uop named {part.name}", macro.tokens[0]) + raise analysis_error( + f"No Uop named {part.name}", macro.tokens[0] + ) parts.append(uops[part.name]) case parser.CacheEffect(): parts.append(Skip(part.size)) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 2a339f8cd6bb66..6ed9d836cbbabe 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -6,6 +6,7 @@ Uop, Properties, StackItem, + analysis_error, ) from cwriter import CWriter from typing import Callable, Mapping, TextIO, Iterator @@ -57,12 +58,13 @@ def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: parens -= 1 out.emit(tkn) + ReplacementFunctionType = Callable[ [Token, Iterator[Token], Uop, Stack, Instruction | None], None ] -class Emitter: +class Emitter: out: CWriter _replacers: dict[str, ReplacementFunctionType] @@ -75,6 +77,7 @@ def __init__(self, out: CWriter): "DECREF_INPUTS": self.decref_inputs, "CHECK_EVAL_BREAKER": self.check_eval_breaker, "SYNC_SP": self.sync_sp, + "PyStackRef_FromPyObjectNew": self.py_stack_ref_from_py_object_new, } self.out = out @@ -174,7 +177,6 @@ def decref_inputs( else: self.out.emit(f"PyStackRef_CLOSE({var.name});\n") - def sync_sp( self, tkn: Token, @@ -188,7 +190,6 @@ def sync_sp( next(tkn_iter) stack.flush(self.out) - def check_eval_breaker( self, tkn: Token, @@ -203,6 +204,28 @@ def check_eval_breaker( if not uop.properties.ends_with_eval_breaker: self.out.emit_at("CHECK_EVAL_BREAKER();", tkn) + def py_stack_ref_from_py_object_new( + self, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, + ) -> None: + self.out.emit(tkn) + emit_to(self.out, tkn_iter, "SEMI") + self.out.emit(";\n") + + target = uop.deferred_refs[tkn] + if target is None: + # An assignment we don't handle, such as to a pointer or array. + return + + # Flush the assignment to the stack. Note that we don't flush the + # stack pointer here, and instead are currently relying on initializing + # unused portions of the stack to NULL. + stack.flush_single_var(self.out, target, uop.stack.outputs) + def emit_tokens( self, uop: Uop, @@ -223,6 +246,7 @@ def emit_tokens( def emit(self, txt: str | Token) -> None: self.out.emit(txt) + def cflags(p: Properties) -> str: flags: list[str] = [] if p.oparg: diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 13aee94f2b957c..d5831593215f76 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -242,7 +242,7 @@ def make_syntax_error( return SyntaxError(message, (filename, line, column, line_text)) -@dataclass(slots=True) +@dataclass(slots=True, frozen=True) class Token: filename: str kind: str diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 09b9d3d211eb24..9b1bc98b5c08d7 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -91,6 +91,7 @@ def emit_stack_effect_function( def generate_stack_effect_functions(analysis: Analysis, out: CWriter) -> None: popped_data: list[tuple[str, str]] = [] pushed_data: list[tuple[str, str]] = [] + def add(inst: Instruction | PseudoInstruction) -> None: stack = get_stack_effect(inst) popped = (-stack.base_offset).to_c() diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index e192b76b23319c..b74f627235ad84 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -88,8 +88,8 @@ def emit_default(out: CWriter, uop: Uop) -> None: else: out.emit(f"{var.name} = sym_new_not_null(ctx);\n") -class OptimizerEmitter(Emitter): +class OptimizerEmitter(Emitter): pass @@ -139,7 +139,7 @@ def write_uop( local = locals[var.name] else: local = Local.local(var) - out.emit(stack.push(local)) + stack.push(local) out.start_line() stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=True) except StackError as ex: @@ -161,8 +161,9 @@ def generate_abstract_interpreter( out.emit("\n") base_uop_names = set([uop.name for uop in base.uops.values()]) for abstract_uop_name in abstract.uops: - assert abstract_uop_name in base_uop_names,\ - f"All abstract uops should override base uops, but {abstract_uop_name} is not." + assert ( + abstract_uop_name in base_uop_names + ), f"All abstract uops should override base uops, but {abstract_uop_name} is not." for uop in base.uops.values(): override: Uop | None = None @@ -192,7 +193,7 @@ def generate_abstract_interpreter( def generate_tier2_abstract_from_files( - filenames: list[str], outfilename: str, debug: bool=False + filenames: list[str], outfilename: str, debug: bool = False ) -> None: assert len(filenames) == 2, "Need a base file and an abstract cases file." base = analyze_files([filenames[0]]) @@ -211,7 +212,7 @@ def generate_tier2_abstract_from_files( ) -arg_parser.add_argument("input", nargs='*', help="Abstract interpreter definition file") +arg_parser.add_argument("input", nargs="*", help="Abstract interpreter definition file") arg_parser.add_argument( "base", nargs="*", help="The base instruction definition file(s)" diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 5acad57f395ea6..ab5444d41ac6a9 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -66,6 +66,7 @@ def first_token(self) -> lx.Token: assert context is not None return context.owner.tokens[context.begin] + @dataclass class Block(Node): # This just holds a context which has the list of tokens. @@ -426,7 +427,9 @@ def pseudo_def(self) -> Pseudo | None: raise self.make_syntax_error("Expected {") if members := self.members(): if self.expect(lx.RBRACE) and self.expect(lx.SEMI): - return Pseudo(tkn.text, inp, outp, flags, members) + return Pseudo( + tkn.text, inp, outp, flags, members + ) return None def members(self) -> list[str] | None: diff --git a/Tools/cases_generator/py_metadata_generator.py b/Tools/cases_generator/py_metadata_generator.py index 3f7ffbc5523fd0..3ec06faf338488 100644 --- a/Tools/cases_generator/py_metadata_generator.py +++ b/Tools/cases_generator/py_metadata_generator.py @@ -18,7 +18,6 @@ from typing import TextIO - DEFAULT_OUTPUT = ROOT / "Lib/_opcode_metadata.py" diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index d2d598a120892d..34bf597f2f552d 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -38,9 +38,9 @@ def var_size(var: StackItem) -> str: else: return "1" + @dataclass class Local: - item: StackItem cached: bool in_memory: bool @@ -75,6 +75,7 @@ def condition(self) -> str | None: def is_array(self) -> bool: return self.item.is_array() + @dataclass class StackOffset: "The stack offset of the virtual base of the stack from the physical stack pointer" @@ -183,44 +184,37 @@ def pop(self, var: StackItem, extract_bits: bool = False) -> tuple[str, Local]: ) if var.name in UNUSED: if popped.name not in UNUSED and popped.name in self.defined: - raise StackError(f"Value is declared unused, but is already cached by prior operation") + raise StackError( + f"Value is declared unused, but is already cached by prior operation" + ) return "", popped if not var.used: return "", popped self.defined.add(var.name) - # Always define array variables as it is free, and their offset might have changed - if var.is_array(): - return ( - f"{var.name} = &stack_pointer[{self.top_offset.to_c()}];\n", - Local.redefinition(var, popped) - ) - if not popped.defined: - return ( - f"{var.name} = stack_pointer[{self.top_offset.to_c()}];\n", - Local.redefinition(var, popped) - ) - else: + if popped.defined: if popped.name == var.name: return "", popped else: - return ( - f"{var.name} = {popped.name};\n", - Local.redefinition(var, popped) - ) + defn = f"{var.name} = {popped.name};\n" + else: + if var.is_array(): + defn = f"{var.name} = &stack_pointer[{self.top_offset.to_c()}];\n" + else: + defn = f"{var.name} = stack_pointer[{self.top_offset.to_c()}];\n" + return defn, Local.redefinition(var, popped) + self.base_offset.pop(var) if var.name in UNUSED or not var.used: return "", Local.unused(var) self.defined.add(var.name) cast = f"({var.type})" if (not indirect and var.type) else "" bits = ".bits" if cast and not extract_bits else "" - assign = ( - f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}]{bits};" - ) + assign = f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}]{bits};" if var.condition: if var.condition == "1": assign = f"{assign}\n" elif var.condition == "0": - return "", Local.unused(var) + return "", Local.unused(var) else: assign = f"if ({var.condition}) {{ {assign} }}\n" else: @@ -228,21 +222,12 @@ def pop(self, var: StackItem, extract_bits: bool = False) -> tuple[str, Local]: in_memory = var.is_array() or var.peek return assign, Local(var, not var.is_array(), in_memory, True) - def push(self, var: Local) -> str: + def push(self, var: Local) -> None: self.variables.append(var) - if var.is_array() and not var.defined and var.item.used: - assert var.in_memory - assert not var.cached - c_offset = self.top_offset.to_c() - self.top_offset.push(var.item) + self.top_offset.push(var.item) + if var.item.used: self.defined.add(var.name) var.defined = True - return f"{var.name} = &stack_pointer[{c_offset}];\n" - else: - self.top_offset.push(var.item) - if var.item.used: - self.defined.add(var.name) - return "" def define_output_arrays(self, outputs: list[StackItem]) -> str: res = [] @@ -257,20 +242,39 @@ def define_output_arrays(self, outputs: list[StackItem]) -> str: return "\n".join(res) @staticmethod - def _do_flush(out: CWriter, variables: list[Local], base_offset: StackOffset, top_offset: StackOffset, - cast_type: str = "uintptr_t", extract_bits: bool = False) -> None: + def _do_emit( + out: CWriter, + var: StackItem, + base_offset: StackOffset, + cast_type: str = "uintptr_t", + extract_bits: bool = False, + ) -> None: + cast = f"({cast_type})" if var.type else "" + bits = ".bits" if cast and not extract_bits else "" + if var.condition == "0": + return + if var.condition and var.condition != "1": + out.emit(f"if ({var.condition}) ") + out.emit(f"stack_pointer[{base_offset.to_c()}]{bits} = {cast}{var.name};\n") + + @staticmethod + def _do_flush( + out: CWriter, + variables: list[Local], + base_offset: StackOffset, + top_offset: StackOffset, + cast_type: str = "uintptr_t", + extract_bits: bool = False, + ) -> None: out.start_line() for var in variables: - if var.cached and not var.in_memory and not var.item.peek and not var.name in UNUSED: - cast = f"({cast_type})" if var.item.type else "" - bits = ".bits" if cast and not extract_bits else "" - if var.condition == "0": - continue - if var.condition and var.condition != "1": - out.emit(f"if ({var.condition}) ") - out.emit( - f"stack_pointer[{base_offset.to_c()}]{bits} = {cast}{var.name};\n" - ) + if ( + var.cached + and not var.in_memory + and not var.item.peek + and not var.name in UNUSED + ): + Stack._do_emit(out, var.item, base_offset, cast_type, extract_bits) base_offset.push(var.item) if base_offset.to_c() != top_offset.to_c(): print("base", base_offset, "top", top_offset) @@ -281,15 +285,59 @@ def _do_flush(out: CWriter, variables: list[Local], base_offset: StackOffset, to out.emit("assert(WITHIN_STACK_BOUNDS());\n") out.start_line() - def flush_locally(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = False) -> None: - self._do_flush(out, self.variables[:], self.base_offset.copy(), self.top_offset.copy(), cast_type, extract_bits) + def flush_locally( + self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = False + ) -> None: + self._do_flush( + out, + self.variables[:], + self.base_offset.copy(), + self.top_offset.copy(), + cast_type, + extract_bits, + ) - def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = False) -> None: - self._do_flush(out, self.variables, self.base_offset, self.top_offset, cast_type, extract_bits) + def flush( + self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = False + ) -> None: + self._do_flush( + out, + self.variables, + self.base_offset, + self.top_offset, + cast_type, + extract_bits, + ) self.variables = [] self.base_offset.clear() self.top_offset.clear() + def flush_single_var( + self, + out: CWriter, + var_name: str, + outputs: list[StackItem], + cast_type: str = "uintptr_t", + extract_bits: bool = False, + ) -> None: + assert any(var.name == var_name for var in outputs) + base_offset = self.base_offset.copy() + top_offset = self.top_offset.copy() + for var in self.variables: + base_offset.push(var.item) + for output in outputs: + if any(output == v.item for v in self.variables): + # The variable is already on the stack, such as a peeked value + # in the tier1 generator + continue + if output.name == var_name: + Stack._do_emit(out, output, base_offset, cast_type, extract_bits) + base_offset.push(output) + top_offset.push(output) + if base_offset.to_c() != top_offset.to_c(): + print("base", base_offset, "top", top_offset) + assert False + def peek_offset(self) -> str: return self.top_offset.to_c() @@ -299,7 +347,8 @@ def as_comment(self) -> str: def get_stack_effect(inst: Instruction | PseudoInstruction) -> Stack: stack = Stack() - def stacks(inst : Instruction | PseudoInstruction) -> Iterator[StackEffect]: + + def stacks(inst: Instruction | PseudoInstruction) -> Iterator[StackEffect]: if isinstance(inst, Instruction): for uop in inst.parts: if isinstance(uop, Uop): diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 7f610bff6290e2..c5097b7584724c 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -30,6 +30,7 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: out.emit(target) out.emit("};\n") + arg_parser = argparse.ArgumentParser( description="Generate the file with dispatch targets.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 6c13d1f10b39f9..c3456cd39ffc3b 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -33,6 +33,7 @@ FOOTER = "#undef TIER_ONE\n" + def declare_variable(var: StackItem, out: CWriter) -> None: type, null = type_and_null(var) space = " " if type[-1].isalnum() else "" @@ -61,8 +62,14 @@ def declare_variables(inst: Instruction, out: CWriter) -> None: required.remove(var.name) declare_variable(var, out) + def write_uop( - uop: Part, emitter: Emitter, offset: int, stack: Stack, inst: Instruction, braces: bool + uop: Part, + emitter: Emitter, + offset: int, + stack: Stack, + inst: Instruction, + braces: bool, ) -> int: # out.emit(stack.as_comment() + "\n") if isinstance(uop, Skip): @@ -93,6 +100,16 @@ def write_uop( if braces: emitter.emit("{\n") emitter.out.emit(stack.define_output_arrays(uop.stack.outputs)) + outputs: list[Local] = [] + for var in uop.stack.outputs: + if not var.peek: + if var.name in locals: + local = locals[var.name] + elif var.name == "unused": + local = Local.unused(var) + else: + local = Local.local(var) + outputs.append(local) for cache in uop.caches: if cache.name != "unused": @@ -109,15 +126,11 @@ def write_uop( emitter.emit(f"(void){cache.name};\n") offset += cache.size emitter.emit_tokens(uop, stack, inst) - for i, var in enumerate(uop.stack.outputs): - if not var.peek: - if var.name in locals: - local = locals[var.name] - elif var.name == "unused": - local = Local.unused(var) - else: - local = Local.local(var) - emitter.emit(stack.push(local)) + for output in outputs: + if output.name in uop.deferred_refs.values(): + # We've already spilled this when emitting tokens + output.cached = False + stack.push(output) if braces: emitter.out.start_line() emitter.emit("}\n") diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 8c212f75878984..7ed937636ee855 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -19,7 +19,7 @@ emit_to, write_header, type_and_null, - Emitter + Emitter, ) from cwriter import CWriter from typing import TextIO, Iterator @@ -62,7 +62,6 @@ def declare_variables(uop: Uop, out: CWriter) -> None: class Tier2Emitter(Emitter): - def __init__(self, out: CWriter): super().__init__(out) self._replacers["oparg"] = self.oparg @@ -110,10 +109,10 @@ def deopt_if( next(tkn_iter) # Semi colon self.emit(") {\n") self.emit("UOP_STAT_INC(uopcode, miss);\n") - self.emit("JUMP_TO_JUMP_TARGET();\n"); + self.emit("JUMP_TO_JUMP_TARGET();\n") self.emit("}\n") - def exit_if( # type: ignore[override] + def exit_if( # type: ignore[override] self, tkn: Token, tkn_iter: Iterator[Token], @@ -150,6 +149,7 @@ def oparg( assert one.text == "1" self.out.emit_at(uop.name[-1], tkn) + def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> None: locals: dict[str, Local] = {} try: @@ -166,6 +166,13 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> None: if local.defined: locals[local.name] = local emitter.emit(stack.define_output_arrays(uop.stack.outputs)) + outputs: list[Local] = [] + for var in uop.stack.outputs: + if var.name in locals: + local = locals[var.name] + else: + local = Local.local(var) + outputs.append(local) for cache in uop.caches: if cache.name != "unused": if cache.size == 4: @@ -175,12 +182,11 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> None: cast = f"uint{cache.size*16}_t" emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") emitter.emit_tokens(uop, stack, None) - for i, var in enumerate(uop.stack.outputs): - if var.name in locals: - local = locals[var.name] - else: - local = Local.local(var) - emitter.emit(stack.push(local)) + for output in outputs: + if output.name in uop.deferred_refs.values(): + # We've already spilled this when emitting tokens + output.cached = False + stack.push(output) except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) from None @@ -213,7 +219,9 @@ def generate_tier2( continue why_not_viable = uop.why_not_viable() if why_not_viable is not None: - out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 because it {why_not_viable} */\n\n") + out.emit( + f"/* {uop.name} is not a viable micro-op for tier 2 because it {why_not_viable} */\n\n" + ) continue out.emit(f"case {uop.name}: {{\n") declare_variables(uop, out) diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index ab9b586693d01c..5ca3bd5cb6c3f1 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -915,8 +915,8 @@ def parse_parameter(self, line: str) -> None: f"invalid parameter declaration (**kwargs?): {line!r}") if function_args.vararg: - if any(p.is_vararg() for p in self.function.parameters.values()): - fail("Too many var args") + self.check_previous_star() + self.check_remaining_star() is_vararg = True parameter = function_args.vararg else: @@ -1124,6 +1124,9 @@ def bad_node(self, node: ast.AST) -> None: key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name self.function.parameters[key] = p + if is_vararg: + self.keyword_only = True + @staticmethod def parse_converter( annotation: ast.expr | None @@ -1165,8 +1168,6 @@ def parse_star(self, function: Function, version: VersionTuple | None) -> None: the marker will take effect (None means it is already in effect). """ if version is None: - if self.keyword_only: - fail(f"Function {function.name!r} uses '*' more than once.") self.check_previous_star() self.check_remaining_star() self.keyword_only = True @@ -1456,6 +1457,7 @@ def add_parameter(text: str) -> None: if p.is_vararg(): p_lines.append("*") + added_star = True name = p.converter.signature_name or p.name p_lines.append(name) @@ -1565,7 +1567,8 @@ def check_remaining_star(self, lineno: int | None = None) -> None: for p in reversed(self.function.parameters.values()): if self.keyword_only: - if p.kind == inspect.Parameter.KEYWORD_ONLY: + if (p.kind == inspect.Parameter.KEYWORD_ONLY or + p.kind == inspect.Parameter.VAR_POSITIONAL): return elif self.deprecated_positional: if p.deprecated_positional == self.deprecated_positional: @@ -1575,12 +1578,11 @@ def check_remaining_star(self, lineno: int | None = None) -> None: fail(f"Function {self.function.name!r} specifies {symbol!r} " f"without following parameters.", line_number=lineno) - def check_previous_star(self, lineno: int | None = None) -> None: + def check_previous_star(self) -> None: assert isinstance(self.function, Function) - for p in self.function.parameters.values(): - if p.kind == inspect.Parameter.VAR_POSITIONAL: - fail(f"Function {self.function.name!r} uses '*' more than once.") + if self.keyword_only: + fail(f"Function {self.function.name!r} uses '*' more than once.") def do_post_block_processing_cleanup(self, lineno: int) -> None: diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index 0f67901dd8609a..96c9b919bff811 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -262,7 +262,7 @@ def __init__(self, func: Function, codegen: CodeGen) -> None: if p.is_keyword_only(): assert not p.is_positional_only() if not p.is_optional(): - self.min_kw_only = i - self.max_pos + self.min_kw_only = i - self.max_pos - int(self.vararg != NO_VARARG) elif p.is_vararg(): self.pseudo_args += 1 self.vararg = i - 1 diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index cbf4072b500061..61556bec17dc59 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -4,4 +4,4 @@ mypy==1.11.1 # needed for peg_generator: types-psutil==6.0.0.20240621 -types-setuptools==70.1.0.20240627 +types-setuptools==71.1.0.20240726 diff --git a/configure b/configure index 39ab48fa4e2526..a0fbebcb0442b9 100755 --- a/configure +++ b/configure @@ -838,7 +838,7 @@ LIBPL PY_ENABLE_SHARED PLATLIBDIR BINLIBDEST -MODULE_LDFLAGS +LIBPYTHON MODULE_DEPS_SHARED EXT_SUFFIX ALT_SOABI @@ -24796,16 +24796,19 @@ LDVERSION='$(VERSION)$(ABIFLAGS)' { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDVERSION" >&5 printf "%s\n" "$LDVERSION" >&6; } -# Configure the flags and dependencies used when compiling shared modules +# Configure the flags and dependencies used when compiling shared modules. +# Do not rename LIBPYTHON - it's accessed via sysconfig by package build +# systems (e.g. Meson) to decide whether to link extension modules against +# libpython. MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' -MODULE_LDFLAGS='' +LIBPYTHON='' # On Android and Cygwin the shared libraries must be linked with libpython. if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" - MODULE_LDFLAGS="\$(BLDLIBRARY)" + LIBPYTHON="\$(BLDLIBRARY)" fi # On iOS the shared libraries must be linked with the Python framework @@ -28697,9 +28700,6 @@ fi # builtin hash modules default_hashlib_hashes="md5,sha1,sha2,sha3,blake2" - -printf "%s\n" "#define PY_BUILTIN_HASHLIB_HASHES /**/" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-builtin-hashlib-hashes" >&5 printf %s "checking for --with-builtin-hashlib-hashes... " >&6; } @@ -28724,6 +28724,7 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_builtin_hashlib_hashes" >&5 printf "%s\n" "$with_builtin_hashlib_hashes" >&6; } + printf "%s\n" "#define PY_BUILTIN_HASHLIB_HASHES \"$with_builtin_hashlib_hashes\"" >>confdefs.h diff --git a/configure.ac b/configure.ac index 62ed812991fc4e..9a17fc279e5a69 100644 --- a/configure.ac +++ b/configure.ac @@ -6224,16 +6224,19 @@ AC_MSG_CHECKING([LDVERSION]) LDVERSION='$(VERSION)$(ABIFLAGS)' AC_MSG_RESULT([$LDVERSION]) -# Configure the flags and dependencies used when compiling shared modules +# Configure the flags and dependencies used when compiling shared modules. +# Do not rename LIBPYTHON - it's accessed via sysconfig by package build +# systems (e.g. Meson) to decide whether to link extension modules against +# libpython. AC_SUBST([MODULE_DEPS_SHARED]) -AC_SUBST([MODULE_LDFLAGS]) +AC_SUBST([LIBPYTHON]) MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' -MODULE_LDFLAGS='' +LIBPYTHON='' # On Android and Cygwin the shared libraries must be linked with libpython. if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" - MODULE_LDFLAGS="\$(BLDLIBRARY)" + LIBPYTHON="\$(BLDLIBRARY)" fi # On iOS the shared libraries must be linked with the Python framework @@ -7403,8 +7406,6 @@ AC_DEFINE([PY_SSL_DEFAULT_CIPHERS], [1]) # builtin hash modules default_hashlib_hashes="md5,sha1,sha2,sha3,blake2" -AC_DEFINE([PY_BUILTIN_HASHLIB_HASHES], [], [enabled builtin hash modules] -) AC_MSG_CHECKING([for --with-builtin-hashlib-hashes]) AC_ARG_WITH( [builtin-hashlib-hashes], @@ -7421,7 +7422,8 @@ AC_ARG_WITH( AC_MSG_RESULT([$with_builtin_hashlib_hashes]) AC_DEFINE_UNQUOTED([PY_BUILTIN_HASHLIB_HASHES], - ["$with_builtin_hashlib_hashes"]) + ["$with_builtin_hashlib_hashes"], + [enabled builtin hash modules]) as_save_IFS=$IFS IFS=,